apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24
  25 #include <apt-pkg/tagfile.h>
  26
  27 #include <apti18n.h>
  28
  29 #include <vector>
  30
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35 #include <system.h>
  36                                                                         /*}}}*/
  37 typedef vector<pkgIndexFile *>::iterator FileIterator;
  38
  39 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* We set the diry flag and make sure that is written to the disk */
  42 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  43                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  44                     FoundFileDeps(0)
  45 {
  46    CurrentFile = 0;
  47    memset(UniqHash,0,sizeof(UniqHash));
  48
  49    if (_error->PendingError() == true)
  50       return;
  51
  52    if (Map.Size() == 0)
  53    {
  54       // Setup the map interface..
  55       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  56       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  57          return;
  58
  59       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  60
  61       // Starting header
  62       *Cache.HeaderP = pkgCache::Header();
  63       Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
  64       Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
  65       Cache.ReMap();
  66    }
  67    else
  68    {
  69       // Map directly from the existing file
  70       Cache.ReMap();
  71       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  72       if (Cache.VS != _system->VS)
  73       {
  74          _error->Error(_("Cache has an incompatible versioning system"));
  75          return;
  76       }
  77    }
  78
  79    Cache.HeaderP->Dirty = true;
  80    Map.Sync(0,sizeof(pkgCache::Header));
  81 }
  82                                                                         /*}}}*/
  83 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* We sync the data then unset the dirty flag in two steps so as to
  86    advoid a problem during a crash */
  87 pkgCacheGenerator::~pkgCacheGenerator()
  88 {
  89    if (_error->PendingError() == true)
  90       return;
  91    if (Map.Sync() == false)
  92       return;
  93
  94    Cache.HeaderP->Dirty = false;
  95    Map.Sync(0,sizeof(pkgCache::Header));
  96 }
  97                                                                         /*}}}*/
  98 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* This provides the generation of the entries in the cache. Each loop
 101    goes through a single package record from the underlying parse engine. */
 102 bool pkgCacheGenerator::MergeList(ListParser &List,
 103                                   pkgCache::VerIterator *OutVer)
 104 {
 105    List.Owner = this;
 106
 107    unsigned int Counter = 0;
 108    while (List.Step() == true)
 109    {
 110       // Get a pointer to the package structure
 111       string PackageName = List.Package();
 112       if (PackageName.empty() == true)
 113          return false;
 114
 115       pkgCache::PkgIterator Pkg;
 116       if (NewPackage(Pkg,PackageName) == false)
 117          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 118       Counter++;
 119       if (Counter % 100 == 0 && Progress != 0)
 120          Progress->Progress(List.Offset());
 121
 122       /* Get a pointer to the version structure. We know the list is sorted
 123          so we use that fact in the search. Insertion of new versions is
 124          done with correct sorting */
 125       string Version = List.Version();
 126       if (Version.empty() == true)
 127       {
 128          // we first process the package, then the descriptions
 129          // (this has the bonus that we get MMap error when we run out
 130          //  of MMap space)
 131          if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
 132             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 133                                  PackageName.c_str());
 134
 135          // Find the right version to write the description
 136          MD5SumValue CurMd5 = List.Description_md5();
 137          pkgCache::VerIterator Ver = Pkg.VersionList();
 138          map_ptrloc *LastVer = &Pkg->VersionList;
 139
 140          for (; Ver != 0 && Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 141          {
 142             pkgCache::DescIterator Desc = Ver.DescriptionList();
 143             map_ptrloc *LastDesc = &Ver->DescriptionList;
 144             bool duplicate=false;
 145
 146             // don't add a new description if we have one for the given
 147             // md5 && language
 148             for ( ; Desc != 0 && Desc.end() == false; Desc++)
 149                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 150                    Desc.LanguageCode() == List.DescriptionLanguage())
 151                   duplicate=true;
 152             if(duplicate)
 153                continue;
 154
 155             for (Desc = Ver.DescriptionList();
 156                  Desc != 0 && Desc.end() == false;
 157                  LastDesc = &Desc->NextDesc, Desc++)
 158             {
 159                if (MD5SumValue(Desc.md5()) == CurMd5)
 160                {
 161                   // Add new description
 162                   *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 163                   Desc->ParentPkg = Pkg.Index();
 164
 165                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 166                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 167                   break;
 168                }
 169             }
 170          }
 171
 172          continue;
 173       }
 174
 175       pkgCache::VerIterator Ver = Pkg.VersionList();
 176       map_ptrloc *LastVer = &Pkg->VersionList;
 177       int Res = 1;
 178       for (; Ver != 0 && Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 179       {
 180          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 181          if (Res >= 0)
 182             break;
 183       }
 184
 185       /* We already have a version for this item, record that we
 186          saw it */
 187       unsigned long Hash = List.VersionHash();
 188       if (Res == 0 && Ver->Hash == Hash)
 189       {
 190          if (List.UsePackage(Pkg,Ver) == false)
 191             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 192                                  PackageName.c_str());
 193
 194          if (NewFileVer(Ver,List) == false)
 195             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 196                                  PackageName.c_str());
 197
 198          // Read only a single record and return
 199          if (OutVer != 0)
 200          {
 201             *OutVer = Ver;
 202             FoundFileDeps |= List.HasFileDeps();
 203             return true;
 204          }
 205
 206          continue;
 207       }
 208
 209       // Skip to the end of the same version set.
 210       if (Res == 0)
 211       {
 212          for (; Ver != 0 && Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 213          {
 214             Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 215             if (Res != 0)
 216                break;
 217          }
 218       }
 219
 220       // Add a new version
 221       *LastVer = NewVersion(Ver,Version,*LastVer);
 222       Ver->ParentPkg = Pkg.Index();
 223       Ver->Hash = Hash;
 224
 225       if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false)
 226          return _error->Error(_("Error occurred while processing %s (NewVersion1)"),
 227                               PackageName.c_str());
 228
 229       if (List.UsePackage(Pkg,Ver) == false)
 230          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 231                               PackageName.c_str());
 232
 233       if (NewFileVer(Ver,List) == false)
 234          return _error->Error(_("Error occurred while processing %s (NewVersion2)"),
 235                               PackageName.c_str());
 236
 237       // Read only a single record and return
 238       if (OutVer != 0)
 239       {
 240          *OutVer = Ver;
 241          FoundFileDeps |= List.HasFileDeps();
 242          return true;
 243       }
 244
 245       /* Record the Description data. Description data always exist in
 246          Packages and Translation-* files. */
 247       pkgCache::DescIterator Desc = Ver.DescriptionList();
 248       map_ptrloc *LastDesc = &Ver->DescriptionList;
 249
 250       // Skip to the end of description set
 251       for (; Desc != 0 && Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 252
 253       // Add new description
 254       *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 255       Desc->ParentPkg = Pkg.Index();
 256
 257       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 258          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 259    }
 260
 261    FoundFileDeps |= List.HasFileDeps();
 262
 263    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 264       return _error->Error(_("Wow, you exceeded the number of package "
 265                              "names this APT is capable of."));
 266    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 267       return _error->Error(_("Wow, you exceeded the number of versions "
 268                              "this APT is capable of."));
 269    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 270       return _error->Error(_("Wow, you exceeded the number of descriptions "
 271                              "this APT is capable of."));
 272    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 273       return _error->Error(_("Wow, you exceeded the number of dependencies "
 274                              "this APT is capable of."));
 275    return true;
 276 }
 277                                                                         /*}}}*/
 278 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 279 // ---------------------------------------------------------------------
 280 /* If we found any file depends while parsing the main list we need to
 281    resolve them. Since it is undesired to load the entire list of files
 282    into the cache as virtual packages we do a two stage effort. MergeList
 283    identifies the file depends and this creates Provdies for them by
 284    re-parsing all the indexs. */
 285 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 286 {
 287    List.Owner = this;
 288
 289    unsigned int Counter = 0;
 290    while (List.Step() == true)
 291    {
 292       string PackageName = List.Package();
 293       if (PackageName.empty() == true)
 294          return false;
 295       string Version = List.Version();
 296       if (Version.empty() == true)
 297          continue;
 298
 299       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 300       if (Pkg.end() == true)
 301          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 302                                 PackageName.c_str());
 303       Counter++;
 304       if (Counter % 100 == 0 && Progress != 0)
 305          Progress->Progress(List.Offset());
 306
 307       unsigned long Hash = List.VersionHash();
 308       pkgCache::VerIterator Ver = Pkg.VersionList();
 309       for (; Ver != 0 && Ver.end() == false; Ver++)
 310       {
 311          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 312          {
 313             if (List.CollectFileProvides(Cache,Ver) == false)
 314                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 315             break;
 316          }
 317       }
 318
 319       if (Ver.end() == true)
 320          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 321    }
 322
 323    return true;
 324 }
 325                                                                         /*}}}*/
 326 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* This creates a new package structure and adds it to the hash table */
 329 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name)
 330 {
 331    Pkg = Cache.FindPkg(Name);
 332    if (Pkg.end() == false)
 333       return true;
 334
 335    // Get a structure
 336    unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
 337    if (Package == 0)
 338       return false;
 339
 340    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 341
 342    // Insert it into the hash table
 343    unsigned long Hash = Cache.Hash(Name);
 344    Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
 345    Cache.HeaderP->HashTable[Hash] = Package;
 346
 347    // Set the name and the ID
 348    Pkg->Name = Map.WriteString(Name);
 349    if (Pkg->Name == 0)
 350       return false;
 351    Pkg->ID = Cache.HeaderP->PackageCount++;
 352
 353    return true;
 354 }
 355                                                                         /*}}}*/
 356 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 357 // ---------------------------------------------------------------------
 358 /* */
 359 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 360                                    ListParser &List)
 361 {
 362    if (CurrentFile == 0)
 363       return true;
 364
 365    // Get a structure
 366    unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
 367    if (VerFile == 0)
 368       return 0;
 369
 370    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 371    VF->File = CurrentFile - Cache.PkgFileP;
 372
 373    // Link it to the end of the list
 374    map_ptrloc *Last = &Ver->FileList;
 375    for (pkgCache::VerFileIterator V = Ver.FileList(); V != 0 && V.end() == false; V++)
 376       Last = &V->NextFile;
 377    VF->NextFile = *Last;
 378    *Last = VF.Index();
 379
 380    VF->Offset = List.Offset();
 381    VF->Size = List.Size();
 382    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 383       Cache.HeaderP->MaxVerFileSize = VF->Size;
 384    Cache.HeaderP->VerFileCount++;
 385
 386    return true;
 387 }
 388                                                                         /*}}}*/
 389 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* This puts a version structure in the linked list */
 392 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 393                                             const string &VerStr,
 394                                             unsigned long Next)
 395 {
 396    // Get a structure
 397    unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
 398    if (Version == 0)
 399       return 0;
 400
 401    // Fill it in
 402    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 403    Ver->NextVer = Next;
 404    Ver->ID = Cache.HeaderP->VersionCount++;
 405    Ver->VerStr = Map.WriteString(VerStr);
 406    if (Ver->VerStr == 0)
 407       return 0;
 408
 409    return Version;
 410 }
 411                                                                         /*}}}*/
 412 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 413 // ---------------------------------------------------------------------
 414 /* */
 415 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 416                                    ListParser &List)
 417 {
 418    if (CurrentFile == 0)
 419       return true;
 420
 421    // Get a structure
 422    unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile));
 423    if (DescFile == 0)
 424       return false;
 425
 426    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 427    DF->File = CurrentFile - Cache.PkgFileP;
 428
 429    // Link it to the end of the list
 430    map_ptrloc *Last = &Desc->FileList;
 431    for (pkgCache::DescFileIterator D = Desc.FileList(); D != 0 && D.end() == false; D++)
 432       Last = &D->NextFile;
 433
 434    DF->NextFile = *Last;
 435    *Last = DF.Index();
 436
 437    DF->Offset = List.Offset();
 438    DF->Size = List.Size();
 439    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 440       Cache.HeaderP->MaxDescFileSize = DF->Size;
 441    Cache.HeaderP->DescFileCount++;
 442
 443    return true;
 444 }
 445                                                                         /*}}}*/
 446 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 447 // ---------------------------------------------------------------------
 448 /* This puts a description structure in the linked list */
 449 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 450                                             const string &Lang,
 451                                             const MD5SumValue &md5sum,
 452                                             map_ptrloc Next)
 453 {
 454    // Get a structure
 455    map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description));
 456    if (Description == 0)
 457       return 0;
 458
 459    // Fill it in
 460    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 461    Desc->NextDesc = Next;
 462    Desc->ID = Cache.HeaderP->DescriptionCount++;
 463    Desc->language_code = Map.WriteString(Lang);
 464    Desc->md5sum = Map.WriteString(md5sum.Value());
 465    if (Desc->language_code == 0 || Desc->md5sum == 0)
 466       return 0;
 467
 468    return Description;
 469 }
 470                                                                         /*}}}*/
 471 // ListParser::NewDepends - Create a dependency element                 /*{{{*/
 472 // ---------------------------------------------------------------------
 473 /* This creates a dependency element in the tree. It is linked to the
 474    version and to the package that it is pointing to. */
 475 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
 476                                                const string &PackageName,
 477                                                const string &Version,
 478                                                unsigned int Op,
 479                                                unsigned int Type)
 480 {
 481    pkgCache &Cache = Owner->Cache;
 482
 483    // Get a structure
 484    unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
 485    if (Dependency == 0)
 486       return false;
 487
 488    // Fill it in
 489    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 490    Dep->ParentVer = Ver.Index();
 491    Dep->Type = Type;
 492    Dep->CompareOp = Op;
 493    Dep->ID = Cache.HeaderP->DependsCount++;
 494
 495    // Locate the target package
 496    pkgCache::PkgIterator Pkg;
 497    if (Owner->NewPackage(Pkg,PackageName) == false)
 498       return false;
 499
 500    // Probe the reverse dependency list for a version string that matches
 501    if (Version.empty() == false)
 502    {
 503 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 504          if (I->Version != 0 && I.TargetVer() == Version)
 505             Dep->Version = I->Version;*/
 506       if (Dep->Version == 0)
 507          if ((Dep->Version = WriteString(Version)) == 0)
 508             return false;
 509    }
 510
 511    // Link it to the package
 512    Dep->Package = Pkg.Index();
 513    Dep->NextRevDepends = Pkg->RevDepends;
 514    Pkg->RevDepends = Dep.Index();
 515
 516    /* Link it to the version (at the end of the list)
 517       Caching the old end point speeds up generation substantially */
 518    if (OldDepVer != Ver)
 519    {
 520       OldDepLast = &Ver->DependsList;
 521       for (pkgCache::DepIterator D = Ver.DependsList(); D != 0 && D.end() == false; D++)
 522          OldDepLast = &D->NextDepends;
 523       OldDepVer = Ver;
 524    }
 525
 526    // Is it a file dependency?
 527    if (PackageName[0] == '/')
 528       FoundFileDeps = true;
 529
 530    Dep->NextDepends = *OldDepLast;
 531    *OldDepLast = Dep.Index();
 532    OldDepLast = &Dep->NextDepends;
 533
 534    return true;
 535 }
 536                                                                         /*}}}*/
 537 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 538 // ---------------------------------------------------------------------
 539 /* */
 540 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
 541                                                 const string &PackageName,
 542                                                 const string &Version)
 543 {
 544    pkgCache &Cache = Owner->Cache;
 545
 546    // We do not add self referencing provides
 547    if (Ver.ParentPkg().Name() == PackageName)
 548       return true;
 549
 550    // Get a structure
 551    unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
 552    if (Provides == 0)
 553       return false;
 554    Cache.HeaderP->ProvidesCount++;
 555
 556    // Fill it in
 557    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 558    Prv->Version = Ver.Index();
 559    Prv->NextPkgProv = Ver->ProvidesList;
 560    Ver->ProvidesList = Prv.Index();
 561    if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
 562       return false;
 563
 564    // Locate the target package
 565    pkgCache::PkgIterator Pkg;
 566    if (Owner->NewPackage(Pkg,PackageName) == false)
 567       return false;
 568
 569    // Link it to the package
 570    Prv->ParentPkg = Pkg.Index();
 571    Prv->NextProvides = Pkg->ProvidesList;
 572    Pkg->ProvidesList = Prv.Index();
 573
 574    return true;
 575 }
 576                                                                         /*}}}*/
 577 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 578 // ---------------------------------------------------------------------
 579 /* This is used to select which file is to be associated with all newly
 580    added versions. The caller is responsible for setting the IMS fields. */
 581 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 582                                    const pkgIndexFile &Index,
 583                                    unsigned long Flags)
 584 {
 585    // Get some space for the structure
 586    CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
 587    if (CurrentFile == Cache.PkgFileP)
 588       return false;
 589
 590    // Fill it in
 591    CurrentFile->FileName = Map.WriteString(File);
 592    CurrentFile->Site = WriteUniqString(Site);
 593    CurrentFile->NextFile = Cache.HeaderP->FileList;
 594    CurrentFile->Flags = Flags;
 595    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 596    CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
 597    PkgFileName = File;
 598    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 599    Cache.HeaderP->PackageFileCount++;
 600
 601    if (CurrentFile->FileName == 0)
 602       return false;
 603
 604    if (Progress != 0)
 605       Progress->SubProgress(Index.Size());
 606    return true;
 607 }
 608                                                                         /*}}}*/
 609 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 610 // ---------------------------------------------------------------------
 611 /* This is used to create handles to strings. Given the same text it
 612    always returns the same number */
 613 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 614                                                  unsigned int Size)
 615 {
 616    /* We use a very small transient hash table here, this speeds up generation
 617       by a fair amount on slower machines */
 618    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 619    if (Bucket != 0 &&
 620        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 621       return Bucket->String;
 622
 623    // Search for an insertion point
 624    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 625    int Res = 1;
 626    map_ptrloc *Last = &Cache.HeaderP->StringList;
 627    for (; I != Cache.StringItemP; Last = &I->NextItem,
 628         I = Cache.StringItemP + I->NextItem)
 629    {
 630       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 631       if (Res >= 0)
 632          break;
 633    }
 634
 635    // Match
 636    if (Res == 0)
 637    {
 638       Bucket = I;
 639       return I->String;
 640    }
 641
 642    // Get a structure
 643    unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
 644    if (Item == 0)
 645       return 0;
 646
 647    // Fill in the structure
 648    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 649    ItemP->NextItem = I - Cache.StringItemP;
 650    *Last = Item;
 651    ItemP->String = Map.WriteString(S,Size);
 652    if (ItemP->String == 0)
 653       return 0;
 654
 655    Bucket = ItemP;
 656    return ItemP->String;
 657 }
 658                                                                         /*}}}*/
 659 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 660 // ---------------------------------------------------------------------
 661 /* This just verifies that each file in the list of index files exists,
 662    has matching attributes with the cache and the cache does not have
 663    any extra files. */
 664 static bool CheckValidity(const string &CacheFile, FileIterator Start,
 665                           FileIterator End,MMap **OutMap = 0)
 666 {
 667    // No file, certainly invalid
 668    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 669       return false;
 670
 671    // Map it
 672    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 673    SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
 674    pkgCache Cache(Map);
 675    if (_error->PendingError() == true || Map->Size() == 0)
 676    {
 677       _error->Discard();
 678       return false;
 679    }
 680
 681    /* Now we check every index file, see if it is in the cache,
 682       verify the IMS data and check that it is on the disk too.. */
 683    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 684    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 685    for (; Start != End; Start++)
 686    {
 687       if ((*Start)->HasPackages() == false)
 688          continue;
 689
 690       if ((*Start)->Exists() == false)
 691       {
 692 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 693          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 694                           (*Start)->Describe().c_str());
 695 #endif
 696          continue;
 697       }
 698
 699       // FindInCache is also expected to do an IMS check.
 700       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 701       if (File.end() == true)
 702          return false;
 703
 704       Visited[File->ID] = true;
 705    }
 706
 707    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 708       if (Visited[I] == false)
 709          return false;
 710
 711    if (_error->PendingError() == true)
 712    {
 713       _error->Discard();
 714       return false;
 715    }
 716
 717    if (OutMap != 0)
 718       *OutMap = Map.UnGuard();
 719    return true;
 720 }
 721                                                                         /*}}}*/
 722 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
 723 // ---------------------------------------------------------------------
 724 /* Size is kind of an abstract notion that is only used for the progress
 725    meter */
 726 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
 727 {
 728    unsigned long TotalSize = 0;
 729    for (; Start != End; Start++)
 730    {
 731       if ((*Start)->HasPackages() == false)
 732          continue;
 733       TotalSize += (*Start)->Size();
 734    }
 735    return TotalSize;
 736 }
 737                                                                         /*}}}*/
 738 // BuildCache - Merge the list of index files into the cache            /*{{{*/
 739 // ---------------------------------------------------------------------
 740 /* */
 741 static bool BuildCache(pkgCacheGenerator &Gen,
 742                        OpProgress &Progress,
 743                        unsigned long &CurrentSize,unsigned long TotalSize,
 744                        FileIterator Start, FileIterator End)
 745 {
 746    FileIterator I;
 747    for (I = Start; I != End; I++)
 748    {
 749       if ((*I)->HasPackages() == false)
 750          continue;
 751
 752       if ((*I)->Exists() == false)
 753          continue;
 754
 755       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
 756       {
 757          _error->Warning("Duplicate sources.list entry %s",
 758                          (*I)->Describe().c_str());
 759          continue;
 760       }
 761
 762       unsigned long Size = (*I)->Size();
 763       Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
 764       CurrentSize += Size;
 765
 766       if ((*I)->Merge(Gen,Progress) == false)
 767          return false;
 768    }
 769
 770    if (Gen.HasFileDeps() == true)
 771    {
 772       Progress.Done();
 773       TotalSize = ComputeSize(Start, End);
 774       CurrentSize = 0;
 775       for (I = Start; I != End; I++)
 776       {
 777          unsigned long Size = (*I)->Size();
 778          Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
 779          CurrentSize += Size;
 780          if ((*I)->MergeFileProvides(Gen,Progress) == false)
 781             return false;
 782       }
 783    }
 784
 785    return true;
 786 }
 787                                                                         /*}}}*/
 788 // MakeStatusCache - Construct the status cache                         /*{{{*/
 789 // ---------------------------------------------------------------------
 790 /* This makes sure that the status cache (the cache that has all
 791    index files from the sources list and all local ones) is ready
 792    to be mmaped. If OutMap is not zero then a MMap object representing
 793    the cache will be stored there. This is pretty much mandetory if you
 794    are using AllowMem. AllowMem lets the function be run as non-root
 795    where it builds the cache 'fast' into a memory buffer. */
 796 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 797                         MMap **OutMap,bool AllowMem)
 798 {
 799    unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
 800
 801    vector<pkgIndexFile *> Files;
 802    for (vector<metaIndex *>::const_iterator i = List.begin();
 803         i != List.end();
 804         i++)
 805    {
 806       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
 807       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
 808            j != Indexes->end();
 809            j++)
 810          Files.push_back (*j);
 811    }
 812
 813    unsigned long EndOfSource = Files.size();
 814    if (_system->AddStatusFiles(Files) == false)
 815       return false;
 816
 817    // Decide if we can write to the files..
 818    string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
 819    string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
 820
 821    // Decide if we can write to the cache
 822    bool Writeable = false;
 823    if (CacheFile.empty() == false)
 824       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
 825    else
 826       if (SrcCacheFile.empty() == false)
 827          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
 828
 829    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
 830       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
 831
 832    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 833
 834    // Cache is OK, Fin.
 835    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
 836    {
 837       Progress.OverallProgress(1,1,1,_("Reading package lists"));
 838       return true;
 839    }
 840
 841    /* At this point we know we need to reconstruct the package cache,
 842       begin. */
 843    SPtr<FileFd> CacheF;
 844    SPtr<DynamicMMap> Map;
 845    if (Writeable == true && CacheFile.empty() == false)
 846    {
 847       unlink(CacheFile.c_str());
 848       CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
 849       fchmod(CacheF->Fd(),0644);
 850       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
 851       if (_error->PendingError() == true)
 852          return false;
 853    }
 854    else
 855    {
 856       // Just build it in memory..
 857       Map = new DynamicMMap(MMap::Public,MapSize);
 858    }
 859
 860    // Lets try the source cache.
 861    unsigned long CurrentSize = 0;
 862    unsigned long TotalSize = 0;
 863    if (CheckValidity(SrcCacheFile,Files.begin(),
 864                      Files.begin()+EndOfSource) == true)
 865    {
 866       // Preload the map with the source cache
 867       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
 868       unsigned long alloc = Map->RawAllocate(SCacheF.Size());
 869       if (alloc == 0 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
 870                                      SCacheF.Size()) == false)
 871          return false;
 872
 873       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 874
 875       // Build the status cache
 876       pkgCacheGenerator Gen(Map.Get(),&Progress);
 877       if (_error->PendingError() == true)
 878          return false;
 879       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 880                      Files.begin()+EndOfSource,Files.end()) == false)
 881          return false;
 882    }
 883    else
 884    {
 885       TotalSize = ComputeSize(Files.begin(),Files.end());
 886
 887       // Build the source cache
 888       pkgCacheGenerator Gen(Map.Get(),&Progress);
 889       if (_error->PendingError() == true)
 890          return false;
 891       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 892                      Files.begin(),Files.begin()+EndOfSource) == false)
 893          return false;
 894
 895       // Write it back
 896       if (Writeable == true && SrcCacheFile.empty() == false)
 897       {
 898          FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
 899          if (_error->PendingError() == true)
 900             return false;
 901
 902          fchmod(SCacheF.Fd(),0644);
 903
 904          // Write out the main data
 905          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
 906             return _error->Error(_("IO Error saving source cache"));
 907          SCacheF.Sync();
 908
 909          // Write out the proper header
 910          Gen.GetCache().HeaderP->Dirty = false;
 911          if (SCacheF.Seek(0) == false ||
 912              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
 913             return _error->Error(_("IO Error saving source cache"));
 914          Gen.GetCache().HeaderP->Dirty = true;
 915          SCacheF.Sync();
 916       }
 917
 918       // Build the status cache
 919       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 920                      Files.begin()+EndOfSource,Files.end()) == false)
 921          return false;
 922    }
 923
 924    if (_error->PendingError() == true)
 925       return false;
 926    if (OutMap != 0)
 927    {
 928       if (CacheF != 0)
 929       {
 930          delete Map.UnGuard();
 931          *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
 932       }
 933       else
 934       {
 935          *OutMap = Map.UnGuard();
 936       }
 937    }
 938
 939    return true;
 940 }
 941                                                                         /*}}}*/
 942 // MakeOnlyStatusCache - Build a cache with just the status files       /*{{{*/
 943 // ---------------------------------------------------------------------
 944 /* */
 945 bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
 946 {
 947    unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024);
 948    vector<pkgIndexFile *> Files;
 949    unsigned long EndOfSource = Files.size();
 950    if (_system->AddStatusFiles(Files) == false)
 951       return false;
 952
 953    SPtr<DynamicMMap> Map;
 954    Map = new DynamicMMap(MMap::Public,MapSize);
 955    unsigned long CurrentSize = 0;
 956    unsigned long TotalSize = 0;
 957
 958    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
 959
 960    // Build the status cache
 961    Progress.OverallProgress(0,1,1,_("Reading package lists"));
 962    pkgCacheGenerator Gen(Map.Get(),&Progress);
 963    if (_error->PendingError() == true)
 964       return false;
 965    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
 966                   Files.begin()+EndOfSource,Files.end()) == false)
 967       return false;
 968
 969    if (_error->PendingError() == true)
 970       return false;
 971    *OutMap = Map.UnGuard();
 972
 973    return true;
 974 }
 975                                                                         /*}}}*/