apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.29 1999/02/23 06:46:24 jgg Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-pkg/pkgcachegen.h"
  15 #endif
  16
  17 #include <apt-pkg/pkgcachegen.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/version.h>
  20 #include <apt-pkg/progress.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/configuration.h>
  23 #include <apt-pkg/deblistparser.h>
  24 #include <apt-pkg/strutl.h>
  25 #include <system.h>
  26
  27 #include <sys/stat.h>
  28 #include <unistd.h>
  29                                                                         /*}}}*/
  30
  31 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  32 // ---------------------------------------------------------------------
  33 /* We set the diry flag and make sure that is written to the disk */
  34 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap &Map,OpProgress &Prog) :
  35                     Map(Map), Cache(Map), Progress(Prog)
  36 {
  37    if (_error->PendingError() == true)
  38       return;
  39
  40    if (Map.Size() == 0)
  41    {
  42       Map.RawAllocate(sizeof(pkgCache::Header));
  43       *Cache.HeaderP = pkgCache::Header();
  44    }
  45    Cache.HeaderP->Dirty = true;
  46    Map.Sync(0,sizeof(pkgCache::Header));
  47    Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  48    memset(UniqHash,0,sizeof(UniqHash));
  49 }
  50                                                                         /*}}}*/
  51 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* We sync the data then unset the dirty flag in two steps so as to
  54    advoid a problem during a crash */
  55 pkgCacheGenerator::~pkgCacheGenerator()
  56 {
  57    if (_error->PendingError() == true)
  58       return;
  59    if (Map.Sync() == false)
  60       return;
  61
  62    Cache.HeaderP->Dirty = false;
  63    Map.Sync(0,sizeof(pkgCache::Header));
  64 }
  65                                                                         /*}}}*/
  66 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  67 // ---------------------------------------------------------------------
  68 /* This provides the generation of the entries in the cache. Each loop
  69    goes through a single package record from the underlying parse engine. */
  70 bool pkgCacheGenerator::MergeList(ListParser &List)
  71 {
  72    List.Owner = this;
  73
  74    unsigned int Counter = 0;
  75    while (List.Step() == true)
  76    {
  77       // Get a pointer to the package structure
  78       string PackageName = List.Package();
  79       Pkgs++;
  80       if (PackageName.empty() == true)
  81          return false;
  82
  83       pkgCache::PkgIterator Pkg;
  84       if (NewPackage(Pkg,PackageName) == false)
  85          return _error->Error("Error occured while processing %s (NewPackage)",PackageName.c_str());
  86       Counter++;
  87       if (Counter % 100 == 0)
  88          Progress.Progress(List.Offset());
  89
  90       /* Get a pointer to the version structure. We know the list is sorted
  91          so we use that fact in the search. Insertion of new versions is
  92          done with correct sorting */
  93       string Version = List.Version();
  94       if (Version.empty() == true)
  95       {
  96          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
  97             return _error->Error("Error occured while processing %s (UsePackage1)",PackageName.c_str());
  98          continue;
  99       }
 100
 101       pkgCache::VerIterator Ver = Pkg.VersionList();
 102       __apt_ptrloc *Last = &Pkg->VersionList;
 103       int Res = 1;
 104       for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
 105       {
 106          Cmps++;
 107          Res = pkgVersionCompare(Version.begin(),Version.end(),Ver.VerStr(),
 108                                  Ver.VerStr() + strlen(Ver.VerStr()));
 109          if (Res >= 0)
 110             break;
 111       }
 112
 113       /* We already have a version for this item, record that we
 114          saw it */
 115       if (Res == 0)
 116       {
 117          if (List.UsePackage(Pkg,Ver) == false)
 118             return _error->Error("Error occured while processing %s (UsePackage2)",PackageName.c_str());
 119
 120          if (NewFileVer(Ver,List) == false)
 121             return _error->Error("Error occured while processing %s (NewFileVer1)",PackageName.c_str());
 122
 123          continue;
 124       }
 125
 126       // Add a new version
 127       Vers++;
 128       *Last = NewVersion(Ver,Version,*Last);
 129       Ver->ParentPkg = Pkg.Index();
 130       if (List.NewVersion(Ver) == false)
 131          return _error->Error("Error occured while processing %s (NewVersion1)",PackageName.c_str());
 132
 133       if (List.UsePackage(Pkg,Ver) == false)
 134          return _error->Error("Error occured while processing %s (UsePackage3)",PackageName.c_str());
 135
 136       if (NewFileVer(Ver,List) == false)
 137          return _error->Error("Error occured while processing %s (NewVersion2)",PackageName.c_str());
 138    }
 139
 140    return true;
 141 }
 142                                                                         /*}}}*/
 143 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 144 // ---------------------------------------------------------------------
 145 /* This creates a new package structure and adds it to the hash table */
 146 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
 147 {
 148    Pkg = Cache.FindPkg(Name);
 149    if (Pkg.end() == false)
 150       return true;
 151
 152    // Get a structure
 153    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 154    if (Package == 0)
 155       return false;
 156
 157    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 158
 159    // Insert it into the hash table
 160    unsigned long Hash = Cache.Hash(Name);
 161    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 162    Cache.HeaderP->HashTable[Hash] = Package;
 163
 164    // Set the name and the ID
 165    Pkg->Name = Map.WriteString(Name);
 166    if (Pkg->Name == 0)
 167       return false;
 168    Pkg->ID = Cache.HeaderP->PackageCount++;
 169
 170    return true;
 171 }
 172                                                                         /*}}}*/
 173 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 174 // ---------------------------------------------------------------------
 175 /* */
 176 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 177                                    ListParser &List)
 178 {
 179    // Get a structure
 180    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 181    if (VerFile == 0)
 182       return 0;
 183
 184    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 185    VF->File = CurrentFile - Cache.PkgFileP;
 186
 187    // Link it to the end of the list
 188    __apt_ptrloc *Last = &Ver->FileList;
 189    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 190       Last = &V->NextFile;
 191    VF->NextFile = *Last;
 192    *Last = VF.Index();
 193
 194    VF->Offset = List.Offset();
 195    VF->Size = List.Size();
 196    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 197       Cache.HeaderP->MaxVerFileSize = VF->Size;
 198    Cache.HeaderP->VerFileCount++;
 199
 200    return true;
 201 }
 202                                                                         /*}}}*/
 203 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 204 // ---------------------------------------------------------------------
 205 /* This puts a version structure in the linked list */
 206 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 207                                             string VerStr,
 208                                             unsigned long Next)
 209 {
 210    // Get a structure
 211    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 212    if (Version == 0)
 213       return 0;
 214
 215    // Fill it in
 216    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 217    Ver->NextVer = Next;
 218    Ver->ID = Cache.HeaderP->VersionCount++;
 219    Ver->VerStr = Map.WriteString(VerStr);
 220    if (Ver->VerStr == 0)
 221       return 0;
 222
 223    return Version;
 224 }
 225                                                                         /*}}}*/
 226 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 227 // ---------------------------------------------------------------------
 228 /* This creates a dependency element in the tree. It is linked to the
 229    version and to the package that it is pointing to. */
 230 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 231                                                string PackageName,
 232                                                string Version,
 233                                                unsigned int Op,
 234                                                unsigned int Type)
 235 {
 236    pkgCache &Cache = Owner->Cache;
 237
 238    // Get a structure
 239    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 240    if (Dependency == 0)
 241       return false;
 242
 243    // Fill it in
 244    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 245    Dep->ParentVer = Ver.Index();
 246    Dep->Type = Type;
 247    Dep->CompareOp = Op;
 248    Dep->ID = Cache.HeaderP->DependsCount++;
 249
 250    // Locate the target package
 251    pkgCache::PkgIterator Pkg;
 252    if (Owner->NewPackage(Pkg,PackageName) == false)
 253       return false;
 254
 255    // Probe the reverse dependency list for a version string that matches
 256    if (Version.empty() == false)
 257    {
 258 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++, Hit++)
 259          if (I->Version != 0 && I.TargetVer() == Version)
 260             Dep->Version = I->Version;*/
 261       if (Dep->Version == 0)
 262          if ((Dep->Version = WriteString(Version)) == 0)
 263             return false;
 264    }
 265
 266    // Link it to the package
 267    Dep->Package = Pkg.Index();
 268    Dep->NextRevDepends = Pkg->RevDepends;
 269    Pkg->RevDepends = Dep.Index();
 270
 271    /* Link it to the version (at the end of the list)
 272       Caching the old end point speeds up generation substantially */
 273    if (OldDepVer != Ver)
 274    {
 275       OldDepLast = &Ver->DependsList;
 276       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 277          OldDepLast = &D->NextDepends;
 278       OldDepVer = Ver;
 279    }
 280
 281    Dep->NextDepends = *OldDepLast;
 282    *OldDepLast = Dep.Index();
 283    OldDepLast = &Dep->NextDepends;
 284
 285    return true;
 286 }
 287                                                                         /*}}}*/
 288 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 289 // ---------------------------------------------------------------------
 290 /* */
 291 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 292                                                 string PackageName,
 293                                                 string Version)
 294 {
 295    pkgCache &Cache = Owner->Cache;
 296
 297    // We do not add self referencing provides
 298    if (Ver.ParentPkg().Name() == PackageName)
 299       return true;
 300
 301    // Get a structure
 302    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 303    if (Provides == 0)
 304       return false;
 305    Cache.HeaderP->ProvidesCount++;
 306
 307    // Fill it in
 308    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 309    Prv->Version = Ver.Index();
 310    Prv->NextPkgProv = Ver->ProvidesList;
 311    Ver->ProvidesList = Prv.Index();
 312    if (Version.empty() == false && (Prv->Version = WriteString(Version)) == 0)
 313       return false;
 314
 315    // Locate the target package
 316    pkgCache::PkgIterator Pkg;
 317    if (Owner->NewPackage(Pkg,PackageName) == false)
 318       return false;
 319
 320    // Link it to the package
 321    Prv->ParentPkg = Pkg.Index();
 322    Prv->NextProvides = Pkg->ProvidesList;
 323    Pkg->ProvidesList = Prv.Index();
 324
 325    return true;
 326 }
 327                                                                         /*}}}*/
 328 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 329 // ---------------------------------------------------------------------
 330 /* This is used to select which file is to be associated with all newly
 331    added versions. */
 332 bool pkgCacheGenerator::SelectFile(string File,unsigned long Flags)
 333 {
 334    struct stat Buf;
 335    if (stat(File.c_str(),&Buf) == -1)
 336       return _error->Errno("stat","Couldn't stat ",File.c_str());
 337
 338    // Get some space for the structure
 339    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 340    if (CurrentFile == Cache.PkgFileP)
 341       return false;
 342
 343    // Fill it in
 344    CurrentFile->FileName = Map.WriteString(File);
 345    CurrentFile->Size = Buf.st_size;
 346    CurrentFile->mtime = Buf.st_mtime;
 347    CurrentFile->NextFile = Cache.HeaderP->FileList;
 348    CurrentFile->Flags = Flags;
 349    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 350    PkgFileName = File;
 351    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 352    Cache.HeaderP->PackageFileCount++;
 353
 354    if (CurrentFile->FileName == 0)
 355       return false;
 356
 357    Progress.SubProgress(Buf.st_size);
 358    return true;
 359 }
 360                                                                         /*}}}*/
 361 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 362 // ---------------------------------------------------------------------
 363 /* This is used to create handles to strings. Given the same text it
 364    always returns the same number */
 365 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 366                                                  unsigned int Size)
 367 {
 368    /* We use a very small transient hash table here, this speeds up generation
 369       by a fair amount on slower machines */
 370    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 371    if (Bucket != 0 &&
 372        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 373       return Bucket->String;
 374
 375    // Search for an insertion point
 376    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 377    int Res = 1;
 378    __apt_ptrloc *Last = &Cache.HeaderP->StringList;
 379    for (; I != Cache.StringItemP; Last = &I->NextItem,
 380         I = Cache.StringItemP + I->NextItem)
 381    {
 382       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 383       if (Res >= 0)
 384          break;
 385    }
 386
 387    // Match
 388    if (Res == 0)
 389    {
 390       Bucket = I;
 391       return I->String;
 392    }
 393
 394    // Get a structure
 395    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 396    if (Item == 0)
 397       return 0;
 398
 399    // Fill in the structure
 400    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 401    ItemP->NextItem = I - Cache.StringItemP;
 402    *Last = Item;
 403    ItemP->String = Map.WriteString(S,Size);
 404    if (ItemP->String == 0)
 405       return 0;
 406
 407    Bucket = ItemP;
 408    return ItemP->String;
 409 }
 410                                                                         /*}}}*/
 411
 412 // SrcCacheCheck - Check if the source package cache is uptodate        /*{{{*/
 413 // ---------------------------------------------------------------------
 414 /* The source cache is checked against the source list and the files
 415    on disk, any difference results in a false. */
 416 bool pkgSrcCacheCheck(pkgSourceList &List)
 417 {
 418    if (_error->PendingError() == true)
 419       return false;
 420
 421    string CacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 422    string ListDir = _config->FindDir("Dir::State::lists");
 423
 424    // Count the number of missing files
 425    int Missing = 0;
 426    for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 427    {
 428       string File = ListDir + URItoFileName(I->PackagesURI());
 429       struct stat Buf;
 430       if (stat(File.c_str(),&Buf) != 0)
 431       {
 432          _error->WarningE("stat","Couldn't stat source package list '%s' (%s)",
 433                           I->PackagesInfo().c_str(),File.c_str());
 434          Missing++;
 435       }
 436    }
 437
 438    // Open the source package cache
 439    if (FileExists(CacheFile) == false)
 440       return false;
 441
 442    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 443    if (_error->PendingError() == true)
 444    {
 445       _error->Discard();
 446       return false;
 447    }
 448
 449    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 450    if (_error->PendingError() == true || Map.Size() == 0)
 451    {
 452       _error->Discard();
 453       return false;
 454    }
 455
 456    pkgCache Cache(Map);
 457    if (_error->PendingError() == true)
 458    {
 459       _error->Discard();
 460       return false;
 461    }
 462
 463    // They are certianly out of sync
 464    if (Cache.Head().PackageFileCount != List.size() - Missing)
 465       return false;
 466
 467    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 468    {
 469       // Search for a match in the source list
 470       bool Bad = true;
 471       for (pkgSourceList::const_iterator I = List.begin();
 472            I != List.end(); I++)
 473       {
 474          string File = ListDir + URItoFileName(I->PackagesURI());
 475          if (F.FileName() == File)
 476          {
 477             Bad = false;
 478             break;
 479          }
 480       }
 481
 482       // Check if the file matches what was cached
 483       Bad |= !F.IsOk();
 484       if (Bad == true)
 485          return false;
 486    }
 487
 488    return true;
 489 }
 490                                                                         /*}}}*/
 491 // PkgCacheCheck - Check if the package cache is uptodate               /*{{{*/
 492 // ---------------------------------------------------------------------
 493 /* This does a simple check of all files used to compose the cache */
 494 bool pkgPkgCacheCheck(string CacheFile)
 495 {
 496    if (_error->PendingError() == true)
 497       return false;
 498
 499    // Open the source package cache
 500    if (FileExists(CacheFile) == false)
 501       return false;
 502
 503    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 504    if (_error->PendingError() == true)
 505    {
 506       _error->Discard();
 507       return false;
 508    }
 509
 510    MMap Map(CacheF,MMap::Public | MMap::ReadOnly);
 511    if (_error->PendingError() == true || Map.Size() == 0)
 512    {
 513       _error->Discard();
 514       return false;
 515    }
 516
 517    pkgCache Cache(Map);
 518    if (_error->PendingError() == true)
 519    {
 520       _error->Discard();
 521       return false;
 522    }
 523
 524    // Status files that must be in the cache
 525    string Status[3];
 526    Status[0] = _config->FindFile("Dir::State::xstatus");
 527    Status[1]= _config->FindFile("Dir::State::userstatus");
 528    Status[2] = _config->FindFile("Dir::State::status");
 529
 530    // Cheack each file
 531    for (pkgCache::PkgFileIterator F(Cache); F.end() == false; F++)
 532    {
 533       if (F.IsOk() == false)
 534          return false;
 535
 536       // See if this is one of the status files
 537       for (int I = 0; I != 3; I++)
 538          if (F.FileName() == Status[I])
 539             Status[I] = string();
 540    }
 541
 542    // Make sure all the status files are loaded.
 543    for (int I = 0; I != 3; I++)
 544    {
 545       if (Status[I].empty() == false && FileExists(Status[I]) == true)
 546          return false;
 547    }
 548
 549    return true;
 550 }
 551                                                                         /*}}}*/
 552 // AddSourcesSize - Add the size of the status files                    /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* This adds the size of all the status files to the size counter */
 555 static bool pkgAddSourcesSize(unsigned long &TotalSize)
 556 {
 557    // Grab the file names
 558    string xstatus = _config->FindFile("Dir::State::xstatus");
 559    string userstatus = _config->FindFile("Dir::State::userstatus");
 560    string status = _config->FindFile("Dir::State::status");
 561
 562    // Grab the sizes
 563    struct stat Buf;
 564    if (stat(xstatus.c_str(),&Buf) == 0)
 565       TotalSize += Buf.st_size;
 566    if (stat(userstatus.c_str(),&Buf) == 0)
 567       TotalSize += Buf.st_size;
 568    if (stat(status.c_str(),&Buf) != 0)
 569       return _error->Errno("stat","Couldn't stat the status file %s",status.c_str());
 570    TotalSize += Buf.st_size;
 571
 572    return true;
 573 }
 574                                                                         /*}}}*/
 575 // MergeStatus - Add the status files to the cache                      /*{{{*/
 576 // ---------------------------------------------------------------------
 577 /* This adds the status files to the map */
 578 static bool pkgMergeStatus(OpProgress &Progress,pkgCacheGenerator &Gen,
 579                            unsigned long &CurrentSize,unsigned long TotalSize)
 580 {
 581    // Grab the file names
 582    string Status[3];
 583    Status[0] = _config->FindFile("Dir::State::xstatus");
 584    Status[1]= _config->FindFile("Dir::State::userstatus");
 585    Status[2] = _config->FindFile("Dir::State::status");
 586
 587    for (int I = 0; I != 3; I++)
 588    {
 589       // Check if the file exists and it is not the primary status file.
 590       string File = Status[I];
 591       if (I != 2 && FileExists(File) == false)
 592          continue;
 593
 594       FileFd Pkg(File,FileFd::ReadOnly);
 595       debListParser Parser(Pkg);
 596       Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 597       if (_error->PendingError() == true)
 598          return _error->Error("Problem opening %s",File.c_str());
 599       CurrentSize += Pkg.Size();
 600
 601       Progress.SubProgress(0,"Local Package State - " + flNotDir(File));
 602       if (Gen.SelectFile(File,pkgCache::Flag::NotSource) == false)
 603          return _error->Error("Problem with SelectFile %s",File.c_str());
 604
 605       if (Gen.MergeList(Parser) == false)
 606          return _error->Error("Problem with MergeList %s",File.c_str());
 607       Progress.Progress(Pkg.Size());
 608    }
 609
 610    return true;
 611 }
 612                                                                         /*}}}*/
 613 // MakeStatusCache - Generates a cache that includes the status files   /*{{{*/
 614 // ---------------------------------------------------------------------
 615 /* This copies the package source cache and then merges the status and
 616    xstatus files into it. */
 617 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress)
 618 {
 619    Progress.OverallProgress(0,1,1,"Reading Package Lists");
 620
 621    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 622    bool SrcOk = pkgSrcCacheCheck(List);
 623    bool PkgOk = SrcOk && pkgPkgCacheCheck(CacheFile);
 624
 625    // Rebuild the source and package caches
 626    if (SrcOk == false)
 627    {
 628       string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 629       string ListDir = _config->FindDir("Dir::State::lists");
 630       FileFd SCacheF(SCacheFile,FileFd::WriteEmpty);
 631       FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 632       DynamicMMap Map(CacheF,MMap::Public);
 633       if (_error->PendingError() == true)
 634          return false;
 635
 636       pkgCacheGenerator Gen(Map,Progress);
 637
 638       // Prepare the progress indicator
 639       unsigned long TotalSize = 0;
 640       struct stat Buf;
 641       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 642       {
 643          string File = ListDir + URItoFileName(I->PackagesURI());
 644          if (stat(File.c_str(),&Buf) != 0)
 645             continue;
 646          TotalSize += Buf.st_size;
 647       }
 648
 649       if (pkgAddSourcesSize(TotalSize) == false)
 650          return false;
 651
 652       // Generate the pkg source cache
 653       unsigned long CurrentSize = 0;
 654       for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); I++)
 655       {
 656          string File = ListDir + URItoFileName(I->PackagesURI());
 657
 658          if (FileExists(File) == false)
 659             continue;
 660
 661          FileFd Pkg(File,FileFd::ReadOnly);
 662          debListParser Parser(Pkg);
 663          Progress.OverallProgress(CurrentSize,TotalSize,Pkg.Size(),"Reading Package Lists");
 664          if (_error->PendingError() == true)
 665             return _error->Error("Problem opening %s",File.c_str());
 666          CurrentSize += Pkg.Size();
 667
 668          Progress.SubProgress(0,I->PackagesInfo());
 669          if (Gen.SelectFile(File) == false)
 670             return _error->Error("Problem with SelectFile %s",File.c_str());
 671
 672          if (Gen.MergeList(Parser) == false)
 673             return _error->Error("Problem with MergeList %s",File.c_str());
 674
 675          // Check the release file
 676          string RFile = ListDir + URItoFileName(I->ReleaseURI());
 677          if (FileExists(RFile) == true)
 678          {
 679             FileFd Rel(RFile,FileFd::ReadOnly);
 680             if (_error->PendingError() == true)
 681                return false;
 682             Parser.LoadReleaseInfo(Gen.GetCurFile(),Rel);
 683          }
 684       }
 685
 686       // Write the src cache
 687       Gen.GetCache().HeaderP->Dirty = false;
 688       if (SCacheF.Write(Map.Data(),Map.Size()) == false)
 689          return _error->Error("IO Error saving source cache");
 690       Gen.GetCache().HeaderP->Dirty = true;
 691
 692       // Merge in the source caches
 693       return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 694    }
 695
 696    if (PkgOk == true)
 697    {
 698       Progress.OverallProgress(1,1,1,"Reading Package Lists");
 699       return true;
 700    }
 701
 702    // We use the source cache to generate the package cache
 703    string SCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 704
 705    FileFd SCacheF(SCacheFile,FileFd::ReadOnly);
 706    FileFd CacheF(CacheFile,FileFd::WriteEmpty);
 707    DynamicMMap Map(CacheF,MMap::Public);
 708    if (_error->PendingError() == true)
 709       return false;
 710
 711    // Preload the map with the source cache
 712    if (SCacheF.Read((unsigned char *)Map.Data() + Map.RawAllocate(SCacheF.Size()),
 713                     SCacheF.Size()) == false)
 714       return false;
 715
 716    pkgCacheGenerator Gen(Map,Progress);
 717
 718    // Compute the progress
 719    unsigned long TotalSize = 0;
 720    if (pkgAddSourcesSize(TotalSize) == false)
 721       return false;
 722
 723    unsigned long CurrentSize = 0;
 724    return pkgMergeStatus(Progress,Gen,CurrentSize,TotalSize);
 725 }
 726                                                                         /*}}}*/