apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.27 1999/01/27 02:48:52 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25
  26 #include <sys/stat.h>
  27 #include <unistd.h>
  28                                                                         /*}}}*/
  29
  30 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  31 // ---------------------------------------------------------------------
  32 /* We set the diry flag and make sure that is written to the disk */
  33 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  34                     Map(Map), Cache(Map), Progress(Prog)
  35 {
  36    if (_error->PendingError() == true)
  37       return;
  38
  39    if (Map.Size() == 0)
  40    {
  41       Map.RawAllocate(sizeof(pkgCache::Header));
  42       *Cache.HeaderP = pkgCache::Header();
  43    }
  44    Cache.HeaderP->Dirty = true;
  45    Map.Sync(0,sizeof(pkgCache::Header));
  46    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  47 }
  48                                                                         /*}}}*/
  49 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  50 // ---------------------------------------------------------------------
  51 /* We sync the data then unset the dirty flag in two steps so as to
  52    advoid a problem during a crash */
  53 pkgCacheGenerator::~pkgCacheGenerator()
  54 {
  55    if (_error->PendingError() == true)
  56       return;
  57    if (Map.Sync() == false)
  58       return;
  59
  60    Cache.HeaderP->Dirty = false;
  61    Map.Sync(0,sizeof(pkgCache::Header));
  62 }
  63                                                                         /*}}}*/
  64 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* This provides the generation of the entries in the cache. Each loop
  67    goes through a single package record from the underlying parse engine. */
  68 bool pkgCacheGenerator::MergeList(ListParser &List)
  69 {
  70    List.Owner = this;
  71
  72    int Counter = 0;
  73    while (List.Step() == true)
  74    {
  75       // Get a pointer to the package structure
  76       string PackageName = List.Package();
  77       pkgCache::PkgIterator Pkg;
  78       if (NewPackage(Pkg,PackageName) == false)
  79          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  80       Counter++;
  81       if (Counter % 100 == 0)
  82          Progress.Progress(List.Offset());
  83
  84       /* Get a pointer to the version structure. We know the list is sorted
  85          so we use that fact in the search. Insertion of new versions is
  86          done with correct sorting */
  87       string Version = List.Version();
  88       if (Version.empty() == true)
  89       {
  90          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  91             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  92          continue;
  93       }
  94
  95       pkgCache::VerIterator Ver = Pkg.VersionList();
  96       __apt_ptrloc *Last = &Pkg->VersionList;
  97       int Res = 1;
  98       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
  99       {
 100          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 101                                  Ver.VerStr() + strlen(Ver.VerStr()));
 102          if (Res >= 0)
 103             break;
 104       }
 105
 106       /* We already have a version for this item, record that we
 107          saw it */
 108       if (Res == 0)
 109       {
 110          if (List.UsePackage(Pkg,Ver) == false)
 111             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 112
 113          if (NewFileVer(Ver,List) == false)
 114             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 115
 116          continue;
 117       }
 118
 119       // Add a new version
 120       *Last = NewVersion(Ver,Version,*Last);
 121       Ver->ParentPkg = Pkg.Index();
 122       if (List.NewVersion(Ver) == false)
 123          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 124
 125       if (List.UsePackage(Pkg,Ver) == false)
 126          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 127
 128       if (NewFileVer(Ver,List) == false)
 129          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 130    }
 131
 132    return true;
 133 }
 134                                                                         /*}}}*/
 135 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* This creates a new package structure and adds it to the hash table */
 138 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 139 {
 140    Pkg = Cache.FindPkg(Name);
 141    if (Pkg.end() == false)
 142       return true;
 143
 144    // Get a structure
 145    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 146    if (Package == 0)
 147       return false;
 148
 149    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 150
 151    // Insert it into the hash table
 152    unsigned long Hash = Cache.Hash(Name);
 153    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 154    Cache.HeaderP->HashTable[Hash] = Package;
 155
 156    // Set the name and the ID
 157    Pkg->Name = Map.WriteString(Name);
 158    if (Pkg->Name == 0)
 159       return false;
 160    Pkg->ID = Cache.HeaderP->PackageCount++;
 161
 162    return true;
 163 }
 164                                                                         /*}}}*/
 165 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* */
 168 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 169                                    ListParser &List)
 170 {
 171    // Get a structure
 172    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 173    if (VerFile == 0)
 174       return 0;
 175
 176    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 177    VF->File = CurrentFile - Cache.PkgFileP;
 178
 179    // Link it to the end of the list
 180    __apt_ptrloc *Last = &Ver->FileList;
 181    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 182       Last = &V->NextFile;
 183    VF->NextFile = *Last;
 184    *Last = VF.Index();
 185
 186    VF->Offset = List.Offset();
 187    VF->Size = List.Size();
 188    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 189       Cache.HeaderP->MaxVerFileSize = VF->Size;
 190    Cache.HeaderP->VerFileCount++;
 191
 192    return true;
 193 }
 194                                                                         /*}}}*/
 195 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 196 // ---------------------------------------------------------------------
 197 /* This puts a version structure in the linked list */
 198 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 199                                             string VerStr,
 200                                             unsigned long Next)
 201 {
 202    // Get a structure
 203    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 204    if (Version == 0)
 205       return 0;
 206
 207    // Fill it in
 208    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 209    Ver->NextVer = Next;
 210    Ver->ID = Cache.HeaderP->VersionCount++;
 211    Ver->VerStr = Map.WriteString(VerStr);
 212    if (Ver->VerStr == 0)
 213       return 0;
 214
 215    return Version;
 216 }
 217                                                                         /*}}}*/
 218 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 219 // ---------------------------------------------------------------------
 220 /* This creates a dependency element in the tree. It is linked to the
 221    version and to the package that it is pointing to. */
 222 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 223                                                string PackageName,
 224                                                string Version,
 225                                                unsigned int Op,
 226                                                unsigned int Type)
 227 {
 228    pkgCache &Cache = Owner->Cache;
 229
 230    // Get a structure
 231    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 232    if (Dependency == 0)
 233       return false;
 234
 235    // Fill it in
 236    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 237    Dep->ParentVer = Ver.Index();
 238    Dep->Type = Type;
 239    Dep->CompareOp = Op;
 240    Dep->ID = Cache.HeaderP->DependsCount++;
 241
 242    // Locate the target package
 243    pkgCache::PkgIterator Pkg;
 244    if (Owner->NewPackage(Pkg,PackageName) == false)
 245       return false;
 246
 247    // Probe the reverse dependency list for a version string that matches
 248    if (Version.empty() == false)
 249    {
 250       for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 251          if (I->Version != 0 && I.TargetVer() == Version)
 252             Dep->Version = I->Version;
 253       if (Dep->Version == 0)
 254          if ((Dep->Version = WriteString(Version)) == 0)
 255             return false;
 256    }
 257
 258    // Link it to the package
 259    Dep->Package = Pkg.Index();
 260    Dep->NextRevDepends = Pkg->RevDepends;
 261    Pkg->RevDepends = Dep.Index();
 262
 263    /* Link it to the version (at the end of the list)
 264       Caching the old end point speeds up generation substantially */
 265    static pkgCache::VerIterator OldVer(Cache);
 266    static __apt_ptrloc *OldLast;
 267    if (OldVer != Ver)
 268    {
 269       OldLast = &Ver->DependsList;
 270       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 271          OldLast = &D->NextDepends;
 272       OldVer = Ver;
 273    }
 274
 275    Dep->NextDepends = *OldLast;
 276    *OldLast = Dep.Index();
 277    OldLast = &Dep->NextDepends;
 278
 279    return true;
 280 }
 281                                                                         /*}}}*/
 282 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 283 // ---------------------------------------------------------------------
 284 /* */
 285 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 286                                                 string PackageName,
 287                                                 string Version)
 288 {
 289    pkgCache &Cache = Owner->Cache;
 290
 291    // We do not add self referencing provides
 292    if (Ver.ParentPkg().Name() == PackageName)
 293       return true;
 294
 295    // Get a structure
 296    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 297    if (Provides == 0)
 298       return false;
 299    Cache.HeaderP->ProvidesCount++;
 300
 301    // Fill it in
 302    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 303    Prv->Version = Ver.Index();
 304    Prv->NextPkgProv = Ver->ProvidesList;
 305    Ver->ProvidesList = Prv.Index();
 306    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 307       return false;
 308
 309    // Locate the target package
 310    pkgCache::PkgIterator Pkg;
 311    if (Owner->NewPackage(Pkg,PackageName) == false)
 312       return false;
 313
 314    // Link it to the package
 315    Prv->ParentPkg = Pkg.Index();
 316    Prv->NextProvides = Pkg->ProvidesList;
 317    Pkg->ProvidesList = Prv.Index();
 318
 319    return true;
 320 }
 321                                                                         /*}}}*/
 322 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 323 // ---------------------------------------------------------------------
 324 /* This is used to select which file is to be associated with all newly
 325    added versions. */
 326 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 327 {
 328    struct stat Buf;
 329    if (stat(File.c_str(),&Buf) == -1)
 330       return _error->Errno("stat","Couldn't stat ",File.c_str());
 331
 332    // Get some space for the structure
 333    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 334    if (CurrentFile == Cache.PkgFileP)
 335       return false;
 336
 337    // Fill it in
 338    CurrentFile->FileName = Map.WriteString(File);
 339    CurrentFile->Size = Buf.st_size;
 340    CurrentFile->mtime = Buf.st_mtime;
 341    CurrentFile->NextFile = Cache.HeaderP->FileList;
 342    CurrentFile->Flags = Flags;
 343    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 344    PkgFileName = File;
 345    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 346    Cache.HeaderP->PackageFileCount++;
 347
 348    if (CurrentFile->FileName == 0)
 349       return false;
 350
 351    Progress.SubProgress(Buf.st_size);
 352    return true;
 353 }
 354                                                                         /*}}}*/
 355 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 356 // ---------------------------------------------------------------------
 357 /* This is used to create handles to strings. Given the same text it
 358    always returns the same number */
 359 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 360                                                  unsigned int Size)
 361 {
 362    // Search for an insertion point
 363    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 364    int Res = 1;
 365    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 366    for (; I != Cache.StringItemP; Last = &I->NextItem,
 367         I = Cache.StringItemP + I->NextItem)
 368    {
 369       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 370       if (Res >= 0)
 371          break;
 372    }
 373
 374    // Match
 375    if (Res == 0)
 376       return I->String;
 377
 378    // Get a structure
 379    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 380    if (Item == 0)
 381       return 0;
 382
 383    // Fill in the structure
 384    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 385    ItemP->NextItem = I - Cache.StringItemP;
 386    *Last = Item;
 387    ItemP->String = Map.WriteString(S,Size);
 388    if (ItemP->String == 0)
 389       return 0;
 390
 391    return ItemP->String;
 392 }
 393                                                                         /*}}}*/
 394
 395 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 396 // ---------------------------------------------------------------------
 397 /* The source cache is checked against the source list and the files
 398    on disk, any difference results in a false. */
 399 bool pkgSrcCacheCheck(pkgSourceList &List)
 400 {
 401    if (_error->PendingError() == true)
 402       return false;
 403
 404    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 405    string ListDir = _config->FindDir("Dir::State::lists");
 406
 407    // Count the number of missing files
 408    int Missing = 0;
 409    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 410    {
 411       string File = ListDir + URItoFileName(I->PackagesURI());
 412       struct stat Buf;
 413       if (stat(File.c_str(),&Buf) != 0)
 414       {
 415          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 416                           I->PackagesInfo().c_str(),File.c_str());
 417          Missing++;
 418       }
 419    }
 420
 421    // Open the source package cache
 422    if (FileExists(CacheFile) == false)
 423       return false;
 424
 425    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 426    if (_error->PendingError() == true)
 427    {
 428       _error->Discard();
 429       return false;
 430    }
 431
 432    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 433    if (_error->PendingError() == true || Map.Size() == 0)
 434    {
 435       _error->Discard();
 436       return false;
 437    }
 438
 439    pkgCache Cache(Map);
 440    if (_error->PendingError() == true)
 441    {
 442       _error->Discard();
 443       return false;
 444    }
 445
 446    // They are certianly out of sync
 447    if (Cache.Head().PackageFileCount != List.size() - Missing)
 448       return false;
 449
 450    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 451    {
 452       // Search for a match in the source list
 453       bool Bad = true;
 454       for (pkgSourceList::const_iterator I = List.begin();
 455            I != List.end(); I++)
 456       {
 457          string File = ListDir + URItoFileName(I->PackagesURI());
 458          if (F.FileName() == File)
 459          {
 460             Bad = false;
 461             break;
 462          }
 463       }
 464
 465       // Check if the file matches what was cached
 466       Bad |= !F.IsOk();
 467       if (Bad == true)
 468          return false;
 469    }
 470
 471    return true;
 472 }
 473                                                                         /*}}}*/
 474 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 475 // ---------------------------------------------------------------------
 476 /* This does a simple check of all files used to compose the cache */
 477 bool pkgPkgCacheCheck(string CacheFile)
 478 {
 479    if (_error->PendingError() == true)
 480       return false;
 481
 482    // Open the source package cache
 483    if (FileExists(CacheFile) == false)
 484       return false;
 485
 486    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 487    if (_error->PendingError() == true)
 488    {
 489       _error->Discard();
 490       return false;
 491    }
 492
 493    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 494    if (_error->PendingError() == true || Map.Size() == 0)
 495    {
 496       _error->Discard();
 497       return false;
 498    }
 499
 500    pkgCache Cache(Map);
 501    if (_error->PendingError() == true)
 502    {
 503       _error->Discard();
 504       return false;
 505    }
 506
 507    // Status files that must be in the cache
 508    string Status[3];
 509    Status[0] = _config->FindFile("Dir::State::xstatus");
 510    Status[1]= _config->FindFile("Dir::State::userstatus");
 511    Status[2] = _config->FindFile("Dir::State::status");
 512
 513    // Cheack each file
 514    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 515    {
 516       if (F.IsOk() == false)
 517          return false;
 518
 519       // See if this is one of the status files
 520       for (int I = 0; I != 3; I++)
 521          if (F.FileName() == Status[I])
 522             Status[I] = string();
 523    }
 524
 525    // Make sure all the status files are loaded.
 526    for (int I = 0; I != 3; I++)
 527    {
 528       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 529          return false;
 530    }
 531
 532    return true;
 533 }
 534                                                                         /*}}}*/
 535 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 536 // ---------------------------------------------------------------------
 537 /* This adds the size of all the status files to the size counter */
 538 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 539 {
 540    // Grab the file names
 541    string xstatus = _config->FindFile("Dir::State::xstatus");
 542    string userstatus = _config->FindFile("Dir::State::userstatus");
 543    string status = _config->FindFile("Dir::State::status");
 544
 545    // Grab the sizes
 546    struct stat Buf;
 547    if (stat(xstatus.c_str(),&Buf) == 0)
 548       TotalSize += Buf.st_size;
 549    if (stat(userstatus.c_str(),&Buf) == 0)
 550       TotalSize += Buf.st_size;
 551    if (stat(status.c_str(),&Buf) != 0)
 552       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 553    TotalSize += Buf.st_size;
 554
 555    return true;
 556 }
 557                                                                         /*}}}*/
 558 // MergeStatus - Add the status files to the cache                      /*{{{*/
 559 // ---------------------------------------------------------------------
 560 /* This adds the status files to the map */
 561 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 562                            unsigned long &CurrentSize,unsigned long TotalSize)
 563 {
 564    // Grab the file names
 565    string Status[3];
 566    Status[0] = _config->FindFile("Dir::State::xstatus");
 567    Status[1]= _config->FindFile("Dir::State::userstatus");
 568    Status[2] = _config->FindFile("Dir::State::status");
 569
 570    for (int I = 0; I != 3; I++)
 571    {
 572       // Check if the file exists and it is not the primary status file.
 573       string File = Status[I];
 574       if (I != 2 && FileExists(File) == false)
 575          continue;
 576
 577       FileFd Pkg(File,FileFd::ReadOnly);
 578       debListParser Parser(Pkg);
 579       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 580       if (_error->PendingError() == true)
 581          return _error->Error("Problem opening %s",File.c_str());
 582       CurrentSize += Pkg.Size();
 583
 584       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 585       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 586          return _error->Error("Problem with SelectFile %s",File.c_str());
 587
 588       if (Gen.MergeList(Parser) == false)
 589          return _error->Error("Problem with MergeList %s",File.c_str());
 590       Progress.Progress(Pkg.Size());
 591    }
 592
 593    return true;
 594 }
 595                                                                         /*}}}*/
 596 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 597 // ---------------------------------------------------------------------
 598 /* This copies the package source cache and then merges the status and
 599    xstatus files into it. */
 600 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 601 {
 602    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 603
 604    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 605    bool SrcOk = pkgSrcCacheCheck(List);
 606    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 607
 608    // Rebuild the source and package caches
 609    if (SrcOk == false)
 610    {
 611       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 612       string ListDir = _config->FindDir("Dir::State::lists");
 613       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 614       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 615       DynamicMMap Map(CacheF,MMap::Public);
 616       if (_error->PendingError() == true)
 617          return false;
 618
 619       pkgCacheGenerator Gen(Map,Progress);
 620
 621       // Prepare the progress indicator
 622       unsigned long TotalSize = 0;
 623       struct stat Buf;
 624       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 625       {
 626          string File = ListDir + URItoFileName(I->PackagesURI());
 627          if (stat(File.c_str(),&Buf) != 0)
 628             continue;
 629          TotalSize += Buf.st_size;
 630       }
 631
 632       if (pkgAddSourcesSize(TotalSize) == false)
 633          return false;
 634
 635       // Generate the pkg source cache
 636       unsigned long CurrentSize = 0;
 637       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 638       {
 639          string File = ListDir + URItoFileName(I->PackagesURI());
 640
 641          if (FileExists(File) == false)
 642             continue;
 643
 644          FileFd Pkg(File,FileFd::ReadOnly);
 645          debListParser Parser(Pkg);
 646          Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 647          if (_error->PendingError() == true)
 648             return _error->Error("Problem opening %s",File.c_str());
 649          CurrentSize += Pkg.Size();
 650
 651          Progress.SubProgress(0,I->PackagesInfo());
 652          if (Gen.SelectFile(File) == false)
 653             return _error->Error("Problem with SelectFile %s",File.c_str());
 654
 655          if (Gen.MergeList(Parser) == false)
 656             return _error->Error("Problem with MergeList %s",File.c_str());
 657
 658          // Check the release file
 659          string RFile = ListDir + URItoFileName(I->ReleaseURI());
 660          if (FileExists(RFile) == true)
 661          {
 662             FileFd Rel(RFile,FileFd::ReadOnly);
 663             if (_error->PendingError() == true)
 664                return false;
 665             Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 666          }
 667       }
 668
 669       // Write the src cache
 670       Gen.GetCache().HeaderP->Dirty = false;
 671       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 672          return _error->Error("IO Error saving source cache");
 673       Gen.GetCache().HeaderP->Dirty = true;
 674
 675       // Merge in the source caches
 676       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 677    }
 678
 679    if (PkgOk == true)
 680    {
 681       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 682       return true;
 683    }
 684
 685    // We use the source cache to generate the package cache
 686    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 687
 688    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 689    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 690    DynamicMMap Map(CacheF,MMap::Public);
 691    if (_error->PendingError() == true)
 692       return false;
 693
 694    // Preload the map with the source cache
 695    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 696                     SCacheF.Size()) == false)
 697       return false;
 698
 699    pkgCacheGenerator Gen(Map,Progress);
 700
 701    // Compute the progress
 702    unsigned long TotalSize = 0;
 703    if (pkgAddSourcesSize(TotalSize) == false)
 704       return false;
 705
 706    unsigned long CurrentSize = 0;
 707    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 708 }
 709                                                                         /*}}}*/