apt-pkg/indexcopy.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: indexcopy.cc,v 1.10 2002/03/26 07:38:58 jgg Exp $
   4 /* ######################################################################
   5
   6    Index Copying - Aid for copying and verifying the index files
   7
   8    This class helps apt-cache reconstruct a damaged index files.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "indexcopy.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/progress.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/tagfile.h>
  21 #include <apt-pkg/indexrecords.h>
  22 #include <apt-pkg/md5.h>
  23 #include <apt-pkg/cdrom.h>
  24 #include <apti18n.h>
  25
  26 #include <iostream>
  27 #include <sstream>
  28 #include <unistd.h>
  29 #include <sys/stat.h>
  30 #include <stdio.h>
  31                                                                         /*}}}*/
  32
  33 using namespace std;
  34
  35
  36
  37 // IndexCopy::CopyPackages - Copy the package files from the CD         /*{{{*/
  38 // ---------------------------------------------------------------------
  39 /* */
  40 bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
  41                              pkgCdromStatus *log)
  42 {
  43    OpProgress *Progress = NULL;
  44    if (List.size() == 0)
  45       return true;
  46
  47    if(log)
  48       Progress = log->GetOpProgress();
  49
  50    bool NoStat = _config->FindB("APT::CDROM::Fast",false);
  51    bool Debug = _config->FindB("Debug::aptcdrom",false);
  52
  53    // Prepare the progress indicator
  54    unsigned long TotalSize = 0;
  55    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  56    {
  57       struct stat Buf;
  58       if (stat(string(*I + GetFileName()).c_str(),&Buf) != 0 &&
  59           stat(string(*I + GetFileName() + ".gz").c_str(),&Buf) != 0)
  60          return _error->Errno("stat","Stat failed for %s",
  61                               string(*I + GetFileName()).c_str());
  62       TotalSize += Buf.st_size;
  63    }
  64
  65    unsigned long CurrentSize = 0;
  66    unsigned int NotFound = 0;
  67    unsigned int WrongSize = 0;
  68    unsigned int Packages = 0;
  69    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
  70    {
  71       string OrigPath = string(*I,CDROM.length());
  72       unsigned long FileSize = 0;
  73
  74       // Open the package file
  75       FileFd Pkg;
  76       if (FileExists(*I + GetFileName()) == true)
  77       {
  78          Pkg.Open(*I + GetFileName(),FileFd::ReadOnly);
  79          FileSize = Pkg.Size();
  80       }
  81       else
  82       {
  83          FileFd From(*I + GetFileName() + ".gz",FileFd::ReadOnly);
  84          if (_error->PendingError() == true)
  85             return false;
  86          FileSize = From.Size();
  87
  88          // Get a temp file
  89          FILE *tmp = tmpfile();
  90          if (tmp == 0)
  91             return _error->Errno("tmpfile","Unable to create a tmp file");
  92          Pkg.Fd(dup(fileno(tmp)));
  93          fclose(tmp);
  94
  95          // Fork gzip
  96          pid_t Process = fork();
  97          if (Process < 0)
  98             return _error->Errno("fork","Couldn't fork gzip");
  99
 100          // The child
 101          if (Process == 0)
 102          {
 103             dup2(From.Fd(),STDIN_FILENO);
 104             dup2(Pkg.Fd(),STDOUT_FILENO);
 105             SetCloseExec(STDIN_FILENO,false);
 106             SetCloseExec(STDOUT_FILENO,false);
 107
 108             const char *Args[3];
 109             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 110             Args[0] = Tmp.c_str();
 111             Args[1] = "-d";
 112             Args[2] = 0;
 113             execvp(Args[0],(char **)Args);
 114             exit(100);
 115          }
 116
 117          // Wait for gzip to finish
 118          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 119             return _error->Error("gzip failed, perhaps the disk is full.");
 120
 121          Pkg.Seek(0);
 122       }
 123       pkgTagFile Parser(&Pkg);
 124       if (_error->PendingError() == true)
 125          return false;
 126
 127       // Open the output file
 128       char S[400];
 129       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",Name.c_str(),
 130                (*I).c_str() + CDROM.length(),GetFileName());
 131       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 132       TargetF += URItoFileName(S);
 133       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 134          TargetF = "/dev/null";
 135       FileFd Target(TargetF,FileFd::WriteEmpty);
 136       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 137       if (_error->PendingError() == true)
 138          return false;
 139       if (TargetFl == 0)
 140          return _error->Errno("fdopen","Failed to reopen fd");
 141
 142       // Setup the progress meter
 143       if(Progress)
 144          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 145                                    string("Reading ") + Type() + " Indexes");
 146
 147       // Parse
 148       if(Progress)
 149          Progress->SubProgress(Pkg.Size());
 150       pkgTagSection Section;
 151       this->Section = &Section;
 152       string Prefix;
 153       unsigned long Hits = 0;
 154       unsigned long Chop = 0;
 155       while (Parser.Step(Section) == true)
 156       {
 157          if(Progress)
 158             Progress->Progress(Parser.Offset());
 159          string File;
 160          unsigned long Size;
 161          if (GetFile(File,Size) == false)
 162          {
 163             fclose(TargetFl);
 164             return false;
 165          }
 166
 167          if (Chop != 0)
 168             File = OrigPath + ChopDirs(File,Chop);
 169
 170          // See if the file exists
 171          bool Mangled = false;
 172          if (NoStat == false || Hits < 10)
 173          {
 174             // Attempt to fix broken structure
 175             if (Hits == 0)
 176             {
 177                if (ReconstructPrefix(Prefix,OrigPath,CDROM,File) == false &&
 178                    ReconstructChop(Chop,*I,File) == false)
 179                {
 180                   if (Debug == true)
 181                      clog << "Missed: " << File << endl;
 182                   NotFound++;
 183                   continue;
 184                }
 185                if (Chop != 0)
 186                   File = OrigPath + ChopDirs(File,Chop);
 187             }
 188
 189             // Get the size
 190             struct stat Buf;
 191             if (stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 192                 Buf.st_size == 0)
 193             {
 194                // Attempt to fix busted symlink support for one instance
 195                string OrigFile = File;
 196                string::size_type Start = File.find("binary-");
 197                string::size_type End = File.find("/",Start+3);
 198                if (Start != string::npos && End != string::npos)
 199                {
 200                   File.replace(Start,End-Start,"binary-all");
 201                   Mangled = true;
 202                }
 203
 204                if (Mangled == false ||
 205                    stat(string(CDROM + Prefix + File).c_str(),&Buf) != 0)
 206                {
 207                   if (Debug == true)
 208                      clog << "Missed(2): " << OrigFile << endl;
 209                   NotFound++;
 210                   continue;
 211                }
 212             }
 213
 214             // Size match
 215             if ((unsigned)Buf.st_size != Size)
 216             {
 217                if (Debug == true)
 218                   clog << "Wrong Size: " << File << endl;
 219                WrongSize++;
 220                continue;
 221             }
 222          }
 223
 224          Packages++;
 225          Hits++;
 226
 227          if (RewriteEntry(TargetFl,File) == false)
 228          {
 229             fclose(TargetFl);
 230             return false;
 231          }
 232       }
 233       fclose(TargetFl);
 234
 235       if (Debug == true)
 236          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 237
 238       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 239       {
 240          // Move out of the partial directory
 241          Target.Close();
 242          string FinalF = _config->FindDir("Dir::State::lists");
 243          FinalF += URItoFileName(S);
 244          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 245             return _error->Errno("rename","Failed to rename");
 246       }
 247
 248       /* Mangle the source to be in the proper notation with
 249          prefix dist [component] */
 250       *I = string(*I,Prefix.length());
 251       ConvertToSourceList(CDROM,*I);
 252       *I = Prefix + ' ' + *I;
 253
 254       CurrentSize += FileSize;
 255    }
 256    if(Progress)
 257       Progress->Done();
 258
 259    // Some stats
 260    if(log) {
 261       stringstream msg;
 262       if(NotFound == 0 && WrongSize == 0)
 263          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 264       else if (NotFound != 0 && WrongSize == 0)
 265          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 266                   Packages, NotFound);
 267       else if (NotFound == 0 && WrongSize != 0)
 268          ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 269                   Packages, WrongSize);
 270       if (NotFound != 0 && WrongSize != 0)
 271          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
 272    }
 273
 274    if (Packages == 0)
 275       _error->Warning("No valid records were found.");
 276
 277    if (NotFound + WrongSize > 10)
 278       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 279
 280
 281    return true;
 282 }
 283                                                                         /*}}}*/
 284 // IndexCopy::ChopDirs - Chop off the leading directory components      /*{{{*/
 285 // ---------------------------------------------------------------------
 286 /* */
 287 string IndexCopy::ChopDirs(string Path,unsigned int Depth)
 288 {
 289    string::size_type I = 0;
 290    do
 291    {
 292       I = Path.find('/',I+1);
 293       Depth--;
 294    }
 295    while (I != string::npos && Depth != 0);
 296
 297    if (I == string::npos)
 298       return string();
 299
 300    return string(Path,I+1);
 301 }
 302                                                                         /*}}}*/
 303 // IndexCopy::ReconstructPrefix - Fix strange prefixing                 /*{{{*/
 304 // ---------------------------------------------------------------------
 305 /* This prepends dir components from the path to the package files to
 306    the path to the deb until it is found */
 307 bool IndexCopy::ReconstructPrefix(string &Prefix,string OrigPath,string CD,
 308                                   string File)
 309 {
 310    bool Debug = _config->FindB("Debug::aptcdrom",false);
 311    unsigned int Depth = 1;
 312    string MyPrefix = Prefix;
 313    while (1)
 314    {
 315       struct stat Buf;
 316       if (stat(string(CD + MyPrefix + File).c_str(),&Buf) != 0)
 317       {
 318          if (Debug == true)
 319             cout << "Failed, " << CD + MyPrefix + File << endl;
 320          if (GrabFirst(OrigPath,MyPrefix,Depth++) == true)
 321             continue;
 322
 323          return false;
 324       }
 325       else
 326       {
 327          Prefix = MyPrefix;
 328          return true;
 329       }
 330    }
 331    return false;
 332 }
 333                                                                         /*}}}*/
 334 // IndexCopy::ReconstructChop - Fixes bad source paths                  /*{{{*/
 335 // ---------------------------------------------------------------------
 336 /* This removes path components from the filename and prepends the location
 337    of the package files until a file is found */
 338 bool IndexCopy::ReconstructChop(unsigned long &Chop,string Dir,string File)
 339 {
 340    // Attempt to reconstruct the filename
 341    unsigned long Depth = 0;
 342    while (1)
 343    {
 344       struct stat Buf;
 345       if (stat(string(Dir + File).c_str(),&Buf) != 0)
 346       {
 347          File = ChopDirs(File,1);
 348          Depth++;
 349          if (File.empty() == false)
 350             continue;
 351          return false;
 352       }
 353       else
 354       {
 355          Chop = Depth;
 356          return true;
 357       }
 358    }
 359    return false;
 360 }
 361                                                                         /*}}}*/
 362 // IndexCopy::ConvertToSourceList - Convert a Path to a sourcelist      /*{{{*/
 363 // ---------------------------------------------------------------------
 364 /* We look for things in dists/ notation and convert them to
 365    <dist> <component> form otherwise it is left alone. This also strips
 366    the CD path.
 367
 368    This implements a regex sort of like:
 369     (.*)/dists/([^/]*)/(.*)/binary-*
 370      ^          ^      ^- Component
 371      |          |-------- Distribution
 372      |------------------- Path
 373
 374    It was deciced to use only a single word for dist (rather than say
 375    unstable/non-us) to increase the chance that each CD gets a single
 376    line in sources.list.
 377  */
 378 void IndexCopy::ConvertToSourceList(string CD,string &Path)
 379 {
 380    char S[300];
 381    snprintf(S,sizeof(S),"binary-%s",_config->Find("Apt::Architecture").c_str());
 382
 383    // Strip the cdrom base path
 384    Path = string(Path,CD.length());
 385    if (Path.empty() == true)
 386       Path = "/";
 387
 388    // Too short to be a dists/ type
 389    if (Path.length() < strlen("dists/"))
 390       return;
 391
 392    // Not a dists type.
 393    if (stringcmp(Path.c_str(),Path.c_str()+strlen("dists/"),"dists/") != 0)
 394       return;
 395
 396    // Isolate the dist
 397    string::size_type Slash = strlen("dists/");
 398    string::size_type Slash2 = Path.find('/',Slash + 1);
 399    if (Slash2 == string::npos || Slash2 + 2 >= Path.length())
 400       return;
 401    string Dist = string(Path,Slash,Slash2 - Slash);
 402
 403    // Isolate the component
 404    Slash = Slash2;
 405    for (unsigned I = 0; I != 10; I++)
 406    {
 407       Slash = Path.find('/',Slash+1);
 408       if (Slash == string::npos || Slash + 2 >= Path.length())
 409          return;
 410       string Comp = string(Path,Slash2+1,Slash - Slash2-1);
 411
 412       // Verify the trailing binary- bit
 413       string::size_type BinSlash = Path.find('/',Slash + 1);
 414       if (Slash == string::npos)
 415          return;
 416       string Binary = string(Path,Slash+1,BinSlash - Slash-1);
 417
 418       if (Binary != S && Binary != "source")
 419          continue;
 420
 421       Path = Dist + ' ' + Comp;
 422       return;
 423    }
 424 }
 425                                                                         /*}}}*/
 426 // IndexCopy::GrabFirst - Return the first Depth path components        /*{{{*/
 427 // ---------------------------------------------------------------------
 428 /* */
 429 bool IndexCopy::GrabFirst(string Path,string &To,unsigned int Depth)
 430 {
 431    string::size_type I = 0;
 432    do
 433    {
 434       I = Path.find('/',I+1);
 435       Depth--;
 436    }
 437    while (I != string::npos && Depth != 0);
 438
 439    if (I == string::npos)
 440       return false;
 441
 442    To = string(Path,0,I+1);
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // PackageCopy::GetFile - Get the file information from the section     /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* */
 449 bool PackageCopy::GetFile(string &File,unsigned long &Size)
 450 {
 451    File = Section->FindS("Filename");
 452    Size = Section->FindI("Size");
 453    if (File.empty() || Size == 0)
 454       return _error->Error("Cannot find filename or size tag");
 455    return true;
 456 }
 457                                                                         /*}}}*/
 458 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename    /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* */
 461 bool PackageCopy::RewriteEntry(FILE *Target,string File)
 462 {
 463    TFRewriteData Changes[] = {{"Filename",File.c_str()},
 464                               {}};
 465
 466    if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
 467       return false;
 468    fputc('\n',Target);
 469    return true;
 470 }
 471                                                                         /*}}}*/
 472 // SourceCopy::GetFile - Get the file information from the section      /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* */
 475 bool SourceCopy::GetFile(string &File,unsigned long &Size)
 476 {
 477    string Files = Section->FindS("Files");
 478    if (Files.empty() == true)
 479       return false;
 480
 481    // Stash the / terminated directory prefix
 482    string Base = Section->FindS("Directory");
 483    if (Base.empty() == false && Base[Base.length()-1] != '/')
 484       Base += '/';
 485
 486    // Read the first file triplet
 487    const char *C = Files.c_str();
 488    string sSize;
 489    string MD5Hash;
 490
 491    // Parse each of the elements
 492    if (ParseQuoteWord(C,MD5Hash) == false ||
 493        ParseQuoteWord(C,sSize) == false ||
 494        ParseQuoteWord(C,File) == false)
 495       return _error->Error("Error parsing file record");
 496
 497    // Parse the size and append the directory
 498    Size = atoi(sSize.c_str());
 499    File = Base + File;
 500    return true;
 501 }
 502                                                                         /*}}}*/
 503 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename     /*{{{*/
 504 // ---------------------------------------------------------------------
 505 /* */
 506 bool SourceCopy::RewriteEntry(FILE *Target,string File)
 507 {
 508    string Dir(File,0,File.rfind('/'));
 509    TFRewriteData Changes[] = {{"Directory",Dir.c_str()},
 510                               {}};
 511
 512    if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
 513       return false;
 514    fputc('\n',Target);
 515    return true;
 516 }
 517                                                                         /*}}}*/
 518 // SigVerify::Verify - Verify a files md5sum against its metaindex      /*{{{*/
 519 // ---------------------------------------------------------------------
 520 /* */
 521 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 522 {
 523    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
 524
 525    // we skip non-existing files in the verifcation to support a cdrom
 526    // with no Packages file (just a Package.gz), see LP: #255545
 527    // (non-existing files are not considered a error)
 528    if(!FileExists(prefix+file))
 529    {
 530       _error->Warning("Skipping non-exisiting file %s", string(prefix+file).c_str());
 531       return true;
 532    }
 533
 534    if (!Record)
 535    {
 536       _error->Warning("Can't find authentication record for: %s",file.c_str());
 537       return false;
 538    }
 539
 540    if (!Record->Hash.VerifyFile(prefix+file))
 541    {
 542       _error->Warning("Hash mismatch for: %s",file.c_str());
 543       return false;
 544    }
 545
 546    if(_config->FindB("Debug::aptcdrom",false))
 547    {
 548       cout << "File: " << prefix+file << endl;
 549       cout << "Expected Hash " << Record->Hash.toStr() << endl;
 550    }
 551
 552    return true;
 553 }
 554
 555 bool SigVerify::CopyMetaIndex(string CDROM, string CDName,
 556                               string prefix, string file)
 557 {
 558       char S[400];
 559       snprintf(S,sizeof(S),"cdrom:[%s]/%s%s",CDName.c_str(),
 560                (prefix).c_str() + CDROM.length(),file.c_str());
 561       string TargetF = _config->FindDir("Dir::State::lists");
 562       TargetF += URItoFileName(S);
 563
 564       FileFd Target;
 565       FileFd Rel;
 566       Target.Open(TargetF,FileFd::WriteEmpty);
 567       Rel.Open(prefix + file,FileFd::ReadOnly);
 568       if (_error->PendingError() == true)
 569          return false;
 570       if (CopyFile(Rel,Target) == false)
 571          return false;
 572
 573       return true;
 574 }
 575
 576 bool SigVerify::CopyAndVerify(string CDROM,string Name,vector<string> &SigList,
 577                               vector<string> PkgList,vector<string> SrcList)
 578 {
 579    if (SigList.size() == 0)
 580       return true;
 581
 582    bool Debug = _config->FindB("Debug::aptcdrom",false);
 583
 584    // Read all Release files
 585    for (vector<string>::iterator I = SigList.begin(); I != SigList.end(); I++)
 586    {
 587       if(Debug)
 588          cout << "Signature verify for: " << *I << endl;
 589
 590       indexRecords *MetaIndex = new indexRecords;
 591       string prefix = *I;
 592
 593       // a Release.gpg without a Release should never happen
 594       if(!FileExists(*I+"Release"))
 595          continue;
 596
 597
 598       // verify the gpg signature of "Release"
 599       // gpg --verify "*I+Release.gpg", "*I+Release"
 600       const char *Args[400];
 601       unsigned int i = 0;
 602
 603       string gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv");
 604       string pubringpath = _config->Find("Apt::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg");
 605       string releasegpg = *I+"Release.gpg";
 606       string release = *I+"Release";
 607
 608       Args[i++] = gpgvpath.c_str();
 609       Args[i++] = "--keyring";
 610       Args[i++] = pubringpath.c_str();
 611       Configuration::Item const *Opts;
 612       Opts = _config->Tree("Acquire::gpgv::Options");
 613       if (Opts != 0)
 614       {
 615          Opts = Opts->Child;
 616          for (; Opts != 0; Opts = Opts->Next)
 617          {
 618             if (Opts->Value.empty() == true)
 619                continue;
 620             Args[i++] = Opts->Value.c_str();
 621             if(i >= 390) {
 622                _error->Error("Argument list from Acquire::gpgv::Options too long. Exiting.");
 623                return false;
 624             }
 625          }
 626       }
 627
 628       Args[i++] = releasegpg.c_str();
 629       Args[i++] = release.c_str();
 630       Args[i++] = NULL;
 631
 632       pid_t pid = ExecFork();
 633       if(pid < 0) {
 634          _error->Error("Fork failed");
 635          return false;
 636       }
 637       if(pid == 0) {
 638          execvp(gpgvpath.c_str(), (char**)Args);
 639       }
 640       if(!ExecWait(pid, "gpgv")) {
 641          _error->Warning("Signature verification failed for: %s",
 642                          string(*I+"Release.gpg").c_str());
 643          // something went wrong, don't copy the Release.gpg
 644          // FIXME: delete any existing gpg file?
 645          continue;
 646       }
 647
 648       // Open the Release file and add it to the MetaIndex
 649       if(!MetaIndex->Load(*I+"Release"))
 650       {
 651          _error->Error("%s",MetaIndex->ErrorText.c_str());
 652          return false;
 653       }
 654
 655       // go over the Indexfiles and see if they verify
 656       // if so, remove them from our copy of the lists
 657       vector<string> keys = MetaIndex->MetaKeys();
 658       for (vector<string>::iterator I = keys.begin(); I != keys.end(); I++)
 659       {
 660          if(!Verify(prefix,*I, MetaIndex)) {
 661             // something went wrong, don't copy the Release.gpg
 662             // FIXME: delete any existing gpg file?
 663             continue;
 664          }
 665       }
 666
 667       // we need a fresh one for the Release.gpg
 668       delete MetaIndex;
 669
 670       // everything was fine, copy the Release and Release.gpg file
 671       CopyMetaIndex(CDROM, Name, prefix, "Release");
 672       CopyMetaIndex(CDROM, Name, prefix, "Release.gpg");
 673    }
 674
 675    return true;
 676 }
 677
 678
 679 bool TranslationsCopy::CopyTranslations(string CDROM,string Name,vector<string> &List,
 680                              pkgCdromStatus *log)
 681 {
 682    OpProgress *Progress = NULL;
 683    if (List.size() == 0)
 684       return true;
 685
 686    if(log)
 687       Progress = log->GetOpProgress();
 688
 689    bool Debug = _config->FindB("Debug::aptcdrom",false);
 690
 691    // Prepare the progress indicator
 692    unsigned long TotalSize = 0;
 693    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
 694    {
 695       struct stat Buf;
 696       if (stat(string(*I).c_str(),&Buf) != 0 &&
 697           stat(string(*I + ".gz").c_str(),&Buf) != 0)
 698          return _error->Errno("stat","Stat failed for %s",
 699                               string(*I).c_str());
 700       TotalSize += Buf.st_size;
 701    }
 702
 703    unsigned long CurrentSize = 0;
 704    unsigned int NotFound = 0;
 705    unsigned int WrongSize = 0;
 706    unsigned int Packages = 0;
 707    for (vector<string>::iterator I = List.begin(); I != List.end(); I++)
 708    {
 709       string OrigPath = string(*I,CDROM.length());
 710       unsigned long FileSize = 0;
 711
 712       // Open the package file
 713       FileFd Pkg;
 714       if (FileExists(*I) == true)
 715       {
 716          Pkg.Open(*I,FileFd::ReadOnly);
 717          FileSize = Pkg.Size();
 718       }
 719       else
 720       {
 721          FileFd From(*I + ".gz",FileFd::ReadOnly);
 722          if (_error->PendingError() == true)
 723             return false;
 724          FileSize = From.Size();
 725
 726          // Get a temp file
 727          FILE *tmp = tmpfile();
 728          if (tmp == 0)
 729             return _error->Errno("tmpfile","Unable to create a tmp file");
 730          Pkg.Fd(dup(fileno(tmp)));
 731          fclose(tmp);
 732
 733          // Fork gzip
 734          pid_t Process = fork();
 735          if (Process < 0)
 736             return _error->Errno("fork","Couldn't fork gzip");
 737
 738          // The child
 739          if (Process == 0)
 740          {
 741             dup2(From.Fd(),STDIN_FILENO);
 742             dup2(Pkg.Fd(),STDOUT_FILENO);
 743             SetCloseExec(STDIN_FILENO,false);
 744             SetCloseExec(STDOUT_FILENO,false);
 745
 746             const char *Args[3];
 747             string Tmp =  _config->Find("Dir::bin::gzip","gzip");
 748             Args[0] = Tmp.c_str();
 749             Args[1] = "-d";
 750             Args[2] = 0;
 751             execvp(Args[0],(char **)Args);
 752             exit(100);
 753          }
 754
 755          // Wait for gzip to finish
 756          if (ExecWait(Process,_config->Find("Dir::bin::gzip","gzip").c_str(),false) == false)
 757             return _error->Error("gzip failed, perhaps the disk is full.");
 758
 759          Pkg.Seek(0);
 760       }
 761       pkgTagFile Parser(&Pkg);
 762       if (_error->PendingError() == true)
 763          return false;
 764
 765       // Open the output file
 766       char S[400];
 767       snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(),
 768                (*I).c_str() + CDROM.length());
 769       string TargetF = _config->FindDir("Dir::State::lists") + "partial/";
 770       TargetF += URItoFileName(S);
 771       if (_config->FindB("APT::CDROM::NoAct",false) == true)
 772          TargetF = "/dev/null";
 773       FileFd Target(TargetF,FileFd::WriteEmpty);
 774       FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
 775       if (_error->PendingError() == true)
 776          return false;
 777       if (TargetFl == 0)
 778          return _error->Errno("fdopen","Failed to reopen fd");
 779
 780       // Setup the progress meter
 781       if(Progress)
 782          Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
 783                                    string("Reading Translation Indexes"));
 784
 785       // Parse
 786       if(Progress)
 787          Progress->SubProgress(Pkg.Size());
 788       pkgTagSection Section;
 789       this->Section = &Section;
 790       string Prefix;
 791       unsigned long Hits = 0;
 792       unsigned long Chop = 0;
 793       while (Parser.Step(Section) == true)
 794       {
 795          if(Progress)
 796             Progress->Progress(Parser.Offset());
 797
 798          const char *Start;
 799          const char *Stop;
 800          Section.GetSection(Start,Stop);
 801          fwrite(Start,Stop-Start, 1, TargetFl);
 802          fputc('\n',TargetFl);
 803
 804          Packages++;
 805          Hits++;
 806       }
 807       fclose(TargetFl);
 808
 809       if (Debug == true)
 810          cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
 811
 812       if (_config->FindB("APT::CDROM::NoAct",false) == false)
 813       {
 814          // Move out of the partial directory
 815          Target.Close();
 816          string FinalF = _config->FindDir("Dir::State::lists");
 817          FinalF += URItoFileName(S);
 818          if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 819             return _error->Errno("rename","Failed to rename");
 820       }
 821
 822
 823       CurrentSize += FileSize;
 824    }
 825    if(Progress)
 826       Progress->Done();
 827
 828    // Some stats
 829    if(log) {
 830       stringstream msg;
 831       if(NotFound == 0 && WrongSize == 0)
 832          ioprintf(msg, _("Wrote %i records.\n"), Packages);
 833       else if (NotFound != 0 && WrongSize == 0)
 834          ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 835                   Packages, NotFound);
 836       else if (NotFound == 0 && WrongSize != 0)
 837          ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 838                   Packages, WrongSize);
 839       if (NotFound != 0 && WrongSize != 0)
 840          ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
 841    }
 842
 843    if (Packages == 0)
 844       _error->Warning("No valid records were found.");
 845
 846    if (NotFound + WrongSize > 10)
 847       _error->Warning("Alot of entries were discarded, something may be wrong.\n");
 848
 849
 850    return true;
 851 }