apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/metaindex.h>
  28 #include <apt-pkg/fileutl.h>
  29
  30 #include <vector>
  31 #include <sys/stat.h>
  32 #include <unistd.h>
  33 #include <errno.h>
  34 #include <stdio.h>
  35
  36 #include <apti18n.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  42                             MD5SumValue const &CurMd5, std::string const &CurLang);
  43
  44 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  45 // ---------------------------------------------------------------------
  46 /* We set the dirty flag and make sure that is written to the disk */
  47 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  48                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  49                     FoundFileDeps(0)
  50 {
  51    CurrentFile = 0;
  52    memset(UniqHash,0,sizeof(UniqHash));
  53
  54    if (_error->PendingError() == true)
  55       return;
  56
  57    if (Map.Size() == 0)
  58    {
  59       // Setup the map interface..
  60       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  61       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  62          return;
  63
  64       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  65
  66       // Starting header
  67       *Cache.HeaderP = pkgCache::Header();
  68       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  69       Cache.HeaderP->VerSysName = idxVerSysName;
  70       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  71       Cache.HeaderP->Architecture = idxArchitecture;
  72       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  73          return;
  74       Cache.ReMap();
  75    }
  76    else
  77    {
  78       // Map directly from the existing file
  79       Cache.ReMap();
  80       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  81       if (Cache.VS != _system->VS)
  82       {
  83          _error->Error(_("Cache has an incompatible versioning system"));
  84          return;
  85       }
  86    }
  87
  88    Cache.HeaderP->Dirty = true;
  89    Map.Sync(0,sizeof(pkgCache::Header));
  90 }
  91                                                                         /*}}}*/
  92 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  93 // ---------------------------------------------------------------------
  94 /* We sync the data then unset the dirty flag in two steps so as to
  95    advoid a problem during a crash */
  96 pkgCacheGenerator::~pkgCacheGenerator()
  97 {
  98    if (_error->PendingError() == true)
  99       return;
 100    if (Map.Sync() == false)
 101       return;
 102
 103    Cache.HeaderP->Dirty = false;
 104    Cache.HeaderP->CacheFileSize = Map.Size();
 105    Map.Sync(0,sizeof(pkgCache::Header));
 106 }
 107                                                                         /*}}}*/
 108 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 109    if (oldMap == newMap)
 110       return;
 111
 112    if (_config->FindB("Debug::pkgCacheGen", false))
 113       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 114
 115    Cache.ReMap(false);
 116
 117    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 118
 119    for (size_t i = 0; i < _count(UniqHash); ++i)
 120       if (UniqHash[i] != 0)
 121          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 122
 123    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 124         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 125       (*i)->ReMap(oldMap, newMap);
 126    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 127         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 128       (*i)->ReMap(oldMap, newMap);
 129    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 130         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 131       (*i)->ReMap(oldMap, newMap);
 132    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 133         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 134       (*i)->ReMap(oldMap, newMap);
 135    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 136         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 137       (*i)->ReMap(oldMap, newMap);
 138    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 139         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 140       (*i)->ReMap(oldMap, newMap);
 141    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 142         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 143       (*i)->ReMap(oldMap, newMap);
 144 }                                                                       /*}}}*/
 145 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 146 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 147                                         const unsigned long &Len) {
 148    void const * const oldMap = Map.Data();
 149    map_ptrloc const index = Map.WriteString(String, Len);
 150    if (index != 0)
 151       ReMap(oldMap, Map.Data());
 152    return index;
 153 }
 154                                                                         /*}}}*/
 155 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 156 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 157    void const * const oldMap = Map.Data();
 158    map_ptrloc const index = Map.WriteString(String);
 159    if (index != 0)
 160       ReMap(oldMap, Map.Data());
 161    return index;
 162 }
 163                                                                         /*}}}*/
 164 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 165    void const * const oldMap = Map.Data();
 166    map_ptrloc const index = Map.Allocate(size);
 167    if (index != 0)
 168       ReMap(oldMap, Map.Data());
 169    return index;
 170 }
 171                                                                         /*}}}*/
 172 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 173 // ---------------------------------------------------------------------
 174 /* This provides the generation of the entries in the cache. Each loop
 175    goes through a single package record from the underlying parse engine. */
 176 bool pkgCacheGenerator::MergeList(ListParser &List,
 177                                   pkgCache::VerIterator *OutVer)
 178 {
 179    List.Owner = this;
 180
 181    unsigned int Counter = 0;
 182    while (List.Step() == true)
 183    {
 184       string const PackageName = List.Package();
 185       if (PackageName.empty() == true)
 186          return false;
 187
 188       Counter++;
 189       if (Counter % 100 == 0 && Progress != 0)
 190          Progress->Progress(List.Offset());
 191
 192       string Arch = List.Architecture();
 193       string const Version = List.Version();
 194       if (Version.empty() == true && Arch.empty() == true)
 195       {
 196          if (MergeListGroup(List, PackageName) == false)
 197             return false;
 198       }
 199
 200       if (Arch.empty() == true)
 201          Arch = _config->Find("APT::Architecture");
 202
 203       // Get a pointer to the package structure
 204       pkgCache::PkgIterator Pkg;
 205       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 206       if (NewPackage(Pkg, PackageName, Arch) == false)
 207          // TRANSLATOR: The first placeholder is a package name,
 208          // the other two should be copied verbatim as they include debug info
 209          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 210                               PackageName.c_str(), "NewPackage", 1);
 211
 212
 213       if (Version.empty() == true)
 214       {
 215          if (MergeListPackage(List, Pkg) == false)
 216             return false;
 217       }
 218       else
 219       {
 220          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 221             return false;
 222       }
 223
 224       if (OutVer != 0)
 225       {
 226          FoundFileDeps |= List.HasFileDeps();
 227          return true;
 228       }
 229    }
 230
 231    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 232       return _error->Error(_("Wow, you exceeded the number of package "
 233                              "names this APT is capable of."));
 234    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 235       return _error->Error(_("Wow, you exceeded the number of versions "
 236                              "this APT is capable of."));
 237    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 238       return _error->Error(_("Wow, you exceeded the number of descriptions "
 239                              "this APT is capable of."));
 240    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 241       return _error->Error(_("Wow, you exceeded the number of dependencies "
 242                              "this APT is capable of."));
 243
 244    FoundFileDeps |= List.HasFileDeps();
 245    return true;
 246 }
 247 // CacheGenerator::MergeListGroup                                       /*{{{*/
 248 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 249 {
 250    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 251    // a group has no data on it's own, only packages have it but these
 252    // stanzas like this come from Translation- files to add descriptions,
 253    // but without a version we don't need a description for it…
 254    if (Grp.end() == true)
 255       return true;
 256    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 257
 258    pkgCache::PkgIterator Pkg;
 259    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 260    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 261       if (MergeListPackage(List, Pkg) == false)
 262          return false;
 263
 264    return true;
 265 }
 266                                                                         /*}}}*/
 267 // CacheGenerator::MergeListPackage                                     /*{{{*/
 268 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 269 {
 270    // we first process the package, then the descriptions
 271    // (for deb this package processing is in fact a no-op)
 272    pkgCache::VerIterator Ver(Cache);
 273    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 274    if (List.UsePackage(Pkg, Ver) == false)
 275       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 276                            Pkg.Name(), "UsePackage", 1);
 277
 278    // Find the right version to write the description
 279    MD5SumValue CurMd5 = List.Description_md5();
 280    std::string CurLang = List.DescriptionLanguage();
 281
 282    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 283    {
 284       pkgCache::DescIterator Desc = Ver.DescriptionList();
 285
 286       // a version can only have one md5 describing it
 287       if (MD5SumValue(Desc.md5()) != CurMd5)
 288          continue;
 289
 290       // don't add a new description if we have one for the given
 291       // md5 && language
 292       if (IsDuplicateDescription(Desc, CurMd5, CurLang) == true)
 293          continue;
 294
 295       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 296       // we add at the end, so that the start is constant as we need
 297       // that to be able to efficiently share these lists
 298       map_ptrloc *LastDesc = &Ver->DescriptionList;
 299       for (;Desc.end() == false && Desc->NextDesc != 0; ++Desc);
 300       if (Desc.end() == false)
 301          LastDesc = &Desc->NextDesc;
 302
 303       void const * const oldMap = Map.Data();
 304       map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, *LastDesc);
 305       if (oldMap != Map.Data())
 306          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 307       *LastDesc = descindex;
 308       Desc->ParentPkg = Pkg.Index();
 309
 310       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 311          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 312                               Pkg.Name(), "NewFileDesc", 1);
 313
 314       // we can stop here as all "same" versions will share the description
 315       break;
 316    }
 317
 318    return true;
 319 }
 320                                                                         /*}}}*/
 321 // CacheGenerator::MergeListVersion                                     /*{{{*/
 322 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 323                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 324 {
 325    pkgCache::VerIterator Ver = Pkg.VersionList();
 326    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 327    map_ptrloc *LastVer = &Pkg->VersionList;
 328    void const * oldMap = Map.Data();
 329
 330    unsigned long const Hash = List.VersionHash();
 331    if (Ver.end() == false)
 332    {
 333       /* We know the list is sorted so we use that fact in the search.
 334          Insertion of new versions is done with correct sorting */
 335       int Res = 1;
 336       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 337       {
 338          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 339          // Version is higher as current version - insert here
 340          if (Res > 0)
 341             break;
 342          // Versionstrings are equal - is hash also equal?
 343          if (Res == 0 && Ver->Hash == Hash)
 344             break;
 345          // proceed with the next till we have either the right
 346          // or we found another version (which will be lower)
 347       }
 348
 349       /* We already have a version for this item, record that we saw it */
 350       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 351       {
 352          if (List.UsePackage(Pkg,Ver) == false)
 353             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 354                                  Pkg.Name(), "UsePackage", 2);
 355
 356          if (NewFileVer(Ver,List) == false)
 357             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 358                                  Pkg.Name(), "NewFileVer", 1);
 359
 360          // Read only a single record and return
 361          if (OutVer != 0)
 362          {
 363             *OutVer = Ver;
 364             return true;
 365          }
 366
 367          return true;
 368       }
 369    }
 370
 371    // Add a new version
 372    map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 373    if (verindex == 0 && _error->PendingError())
 374       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 375                            Pkg.Name(), "NewVersion", 1);
 376
 377    if (oldMap != Map.Data())
 378          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 379    *LastVer = verindex;
 380    Ver->ParentPkg = Pkg.Index();
 381    Ver->Hash = Hash;
 382
 383    if (unlikely(List.NewVersion(Ver) == false))
 384       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 385                            Pkg.Name(), "NewVersion", 2);
 386
 387    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 388       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 389                            Pkg.Name(), "UsePackage", 3);
 390
 391    if (unlikely(NewFileVer(Ver,List) == false))
 392       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 393                            Pkg.Name(), "NewFileVer", 2);
 394
 395    pkgCache::GrpIterator Grp = Pkg.Group();
 396    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 397
 398    /* If it is the first version of this package we need to add implicit
 399       Multi-Arch dependencies to all other package versions in the group now -
 400       otherwise we just add them for this new version */
 401    if (Pkg.VersionList()->NextVer == 0)
 402    {
 403       pkgCache::PkgIterator P = Grp.PackageList();
 404       Dynamic<pkgCache::PkgIterator> DynP(P);
 405       for (; P.end() != true; P = Grp.NextPkg(P))
 406       {
 407          if (P->ID == Pkg->ID)
 408             continue;
 409          pkgCache::VerIterator V = P.VersionList();
 410          Dynamic<pkgCache::VerIterator> DynV(V);
 411          for (; V.end() != true; ++V)
 412             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 413                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 414                                     Pkg.Name(), "AddImplicitDepends", 1);
 415       }
 416    }
 417    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "AddImplicitDepends", 2);
 420
 421    // Read only a single record and return
 422    if (OutVer != 0)
 423    {
 424       *OutVer = Ver;
 425       return true;
 426    }
 427
 428    /* Record the Description (it is not translated) */
 429    MD5SumValue CurMd5 = List.Description_md5();
 430    if (CurMd5.Value().empty() == true)
 431       return true;
 432    std::string CurLang = List.DescriptionLanguage();
 433
 434    /* Before we add a new description we first search in the group for
 435       a version with a description of the same MD5 - if so we reuse this
 436       description group instead of creating our own for this version */
 437    for (pkgCache::PkgIterator P = Grp.PackageList();
 438         P.end() == false; P = Grp.NextPkg(P))
 439    {
 440       for (pkgCache::VerIterator V = P.VersionList();
 441            V.end() == false; ++V)
 442       {
 443          if (IsDuplicateDescription(V.DescriptionList(), CurMd5, "") == false)
 444             continue;
 445          Ver->DescriptionList = V->DescriptionList;
 446          return true;
 447       }
 448    }
 449
 450    // We haven't found reusable descriptions, so add the first description
 451    pkgCache::DescIterator Desc = Ver.DescriptionList();
 452    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 453    map_ptrloc *LastDesc = &Ver->DescriptionList;
 454
 455    oldMap = Map.Data();
 456    map_ptrloc const descindex = NewDescription(Desc, CurLang, CurMd5, *LastDesc);
 457    if (oldMap != Map.Data())
 458        LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 459    *LastDesc = descindex;
 460    Desc->ParentPkg = Pkg.Index();
 461
 462    if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 463       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 464                            Pkg.Name(), "NewFileDesc", 2);
 465
 466    return true;
 467 }
 468                                                                         /*}}}*/
 469                                                                         /*}}}*/
 470 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 471 // ---------------------------------------------------------------------
 472 /* If we found any file depends while parsing the main list we need to
 473    resolve them. Since it is undesired to load the entire list of files
 474    into the cache as virtual packages we do a two stage effort. MergeList
 475    identifies the file depends and this creates Provdies for them by
 476    re-parsing all the indexs. */
 477 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 478 {
 479    List.Owner = this;
 480
 481    unsigned int Counter = 0;
 482    while (List.Step() == true)
 483    {
 484       string PackageName = List.Package();
 485       if (PackageName.empty() == true)
 486          return false;
 487       string Version = List.Version();
 488       if (Version.empty() == true)
 489          continue;
 490
 491       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 492       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 493       if (Pkg.end() == true)
 494          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 495                                 PackageName.c_str(), "FindPkg", 1);
 496       Counter++;
 497       if (Counter % 100 == 0 && Progress != 0)
 498          Progress->Progress(List.Offset());
 499
 500       unsigned long Hash = List.VersionHash();
 501       pkgCache::VerIterator Ver = Pkg.VersionList();
 502       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 503       for (; Ver.end() == false; ++Ver)
 504       {
 505          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 506          {
 507             if (List.CollectFileProvides(Cache,Ver) == false)
 508                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 509                                     PackageName.c_str(), "CollectFileProvides", 1);
 510             break;
 511          }
 512       }
 513
 514       if (Ver.end() == true)
 515          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 516    }
 517
 518    return true;
 519 }
 520                                                                         /*}}}*/
 521 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 522 // ---------------------------------------------------------------------
 523 /* This creates a new group structure and adds it to the hash table */
 524 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 525 {
 526    Grp = Cache.FindGrp(Name);
 527    if (Grp.end() == false)
 528       return true;
 529
 530    // Get a structure
 531    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 532    if (unlikely(Group == 0))
 533       return false;
 534
 535    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 536    map_ptrloc const idxName = WriteStringInMap(Name);
 537    if (unlikely(idxName == 0))
 538       return false;
 539    Grp->Name = idxName;
 540
 541    // Insert it into the hash table
 542    unsigned long const Hash = Cache.Hash(Name);
 543    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 544    Cache.HeaderP->GrpHashTable[Hash] = Group;
 545
 546    Grp->ID = Cache.HeaderP->GroupCount++;
 547    return true;
 548 }
 549                                                                         /*}}}*/
 550 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* This creates a new package structure and adds it to the hash table */
 553 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 554                                         const string &Arch) {
 555    pkgCache::GrpIterator Grp;
 556    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 557    if (unlikely(NewGroup(Grp, Name) == false))
 558       return false;
 559
 560    Pkg = Grp.FindPkg(Arch);
 561       if (Pkg.end() == false)
 562          return true;
 563
 564    // Get a structure
 565    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 566    if (unlikely(Package == 0))
 567       return false;
 568    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 569
 570    // Insert the package into our package list
 571    if (Grp->FirstPackage == 0) // the group is new
 572    {
 573       // Insert it into the hash table
 574       unsigned long const Hash = Cache.Hash(Name);
 575       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 576       Cache.HeaderP->PkgHashTable[Hash] = Package;
 577       Grp->FirstPackage = Package;
 578    }
 579    else // Group the Packages together
 580    {
 581       // this package is the new last package
 582       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 583       Pkg->NextPackage = LastPkg->NextPackage;
 584       LastPkg->NextPackage = Package;
 585    }
 586    Grp->LastPackage = Package;
 587
 588    // Set the name, arch and the ID
 589    Pkg->Name = Grp->Name;
 590    Pkg->Group = Grp.Index();
 591    // all is mapped to the native architecture
 592    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 593    if (unlikely(idxArch == 0))
 594       return false;
 595    Pkg->Arch = idxArch;
 596    Pkg->ID = Cache.HeaderP->PackageCount++;
 597
 598    return true;
 599 }
 600                                                                         /*}}}*/
 601 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 602 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 603                                            pkgCache::PkgIterator &P,
 604                                            pkgCache::VerIterator &V)
 605 {
 606    // copy P.Arch() into a string here as a cache remap
 607    // in NewDepends() later may alter the pointer location
 608    string Arch = P.Arch() == NULL ? "" : P.Arch();
 609    map_ptrloc *OldDepLast = NULL;
 610    /* MultiArch handling introduces a lot of implicit Dependencies:
 611       - MultiArch: same → Co-Installable if they have the same version
 612       - All others conflict with all other group members */
 613    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 614    pkgCache::PkgIterator D = G.PackageList();
 615    Dynamic<pkgCache::PkgIterator> DynD(D);
 616    for (; D.end() != true; D = G.NextPkg(D))
 617    {
 618       if (Arch == D.Arch() || D->VersionList == 0)
 619          continue;
 620       /* We allow only one installed arch at the time
 621          per group, therefore each group member conflicts
 622          with all other group members */
 623       if (coInstall == true)
 624       {
 625          // Replaces: ${self}:other ( << ${binary:Version})
 626          NewDepends(D, V, V.VerStr(),
 627                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 628                     OldDepLast);
 629          // Breaks: ${self}:other (!= ${binary:Version})
 630          NewDepends(D, V, V.VerStr(),
 631                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 632                     OldDepLast);
 633       } else {
 634          // Conflicts: ${self}:other
 635          NewDepends(D, V, "",
 636                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 637                     OldDepLast);
 638       }
 639    }
 640    return true;
 641 }
 642 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 643                                            pkgCache::PkgIterator &D)
 644 {
 645    /* MultiArch handling introduces a lot of implicit Dependencies:
 646       - MultiArch: same → Co-Installable if they have the same version
 647       - All others conflict with all other group members */
 648    map_ptrloc *OldDepLast = NULL;
 649    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 650    if (coInstall == true)
 651    {
 652       // Replaces: ${self}:other ( << ${binary:Version})
 653       NewDepends(D, V, V.VerStr(),
 654                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 655                  OldDepLast);
 656       // Breaks: ${self}:other (!= ${binary:Version})
 657       NewDepends(D, V, V.VerStr(),
 658                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 659                  OldDepLast);
 660    } else {
 661       // Conflicts: ${self}:other
 662       NewDepends(D, V, "",
 663                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 664                  OldDepLast);
 665    }
 666    return true;
 667 }
 668
 669                                                                         /*}}}*/
 670 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* */
 673 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 674                                    ListParser &List)
 675 {
 676    if (CurrentFile == 0)
 677       return true;
 678
 679    // Get a structure
 680    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 681    if (VerFile == 0)
 682       return 0;
 683
 684    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 685    VF->File = CurrentFile - Cache.PkgFileP;
 686
 687    // Link it to the end of the list
 688    map_ptrloc *Last = &Ver->FileList;
 689    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 690       Last = &V->NextFile;
 691    VF->NextFile = *Last;
 692    *Last = VF.Index();
 693
 694    VF->Offset = List.Offset();
 695    VF->Size = List.Size();
 696    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 697       Cache.HeaderP->MaxVerFileSize = VF->Size;
 698    Cache.HeaderP->VerFileCount++;
 699
 700    return true;
 701 }
 702                                                                         /*}}}*/
 703 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 704 // ---------------------------------------------------------------------
 705 /* This puts a version structure in the linked list */
 706 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 707                                             const string &VerStr,
 708                                             unsigned long Next)
 709 {
 710    // Get a structure
 711    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 712    if (Version == 0)
 713       return 0;
 714
 715    // Fill it in
 716    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 717    Ver->NextVer = Next;
 718    Ver->ID = Cache.HeaderP->VersionCount++;
 719    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 720    if (unlikely(idxVerStr == 0))
 721       return 0;
 722    Ver->VerStr = idxVerStr;
 723
 724    return Version;
 725 }
 726                                                                         /*}}}*/
 727 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 728 // ---------------------------------------------------------------------
 729 /* */
 730 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 731                                    ListParser &List)
 732 {
 733    if (CurrentFile == 0)
 734       return true;
 735
 736    // Get a structure
 737    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 738    if (DescFile == 0)
 739       return false;
 740
 741    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 742    DF->File = CurrentFile - Cache.PkgFileP;
 743
 744    // Link it to the end of the list
 745    map_ptrloc *Last = &Desc->FileList;
 746    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 747       Last = &D->NextFile;
 748
 749    DF->NextFile = *Last;
 750    *Last = DF.Index();
 751
 752    DF->Offset = List.Offset();
 753    DF->Size = List.Size();
 754    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 755       Cache.HeaderP->MaxDescFileSize = DF->Size;
 756    Cache.HeaderP->DescFileCount++;
 757
 758    return true;
 759 }
 760                                                                         /*}}}*/
 761 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* This puts a description structure in the linked list */
 764 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 765                                             const string &Lang,
 766                                             const MD5SumValue &md5sum,
 767                                             map_ptrloc Next)
 768 {
 769    // Get a structure
 770    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 771    if (Description == 0)
 772       return 0;
 773
 774    // Fill it in
 775    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 776    Desc->NextDesc = Next;
 777    Desc->ID = Cache.HeaderP->DescriptionCount++;
 778    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 779    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 780    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 781       return 0;
 782    Desc->language_code = idxlanguage_code;
 783    Desc->md5sum = idxmd5sum;
 784
 785    return Description;
 786 }
 787                                                                         /*}}}*/
 788 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 789 // ---------------------------------------------------------------------
 790 /* This creates a dependency element in the tree. It is linked to the
 791    version and to the package that it is pointing to. */
 792 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 793                                    pkgCache::VerIterator &Ver,
 794                                    string const &Version,
 795                                    unsigned int const &Op,
 796                                    unsigned int const &Type,
 797                                    map_ptrloc* &OldDepLast)
 798 {
 799    void const * const oldMap = Map.Data();
 800    // Get a structure
 801    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 802    if (unlikely(Dependency == 0))
 803       return false;
 804
 805    // Fill it in
 806    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 807    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 808    Dep->ParentVer = Ver.Index();
 809    Dep->Type = Type;
 810    Dep->CompareOp = Op;
 811    Dep->ID = Cache.HeaderP->DependsCount++;
 812
 813    // Probe the reverse dependency list for a version string that matches
 814    if (Version.empty() == false)
 815    {
 816 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 817          if (I->Version != 0 && I.TargetVer() == Version)
 818             Dep->Version = I->Version;*/
 819       if (Dep->Version == 0) {
 820          map_ptrloc const index = WriteStringInMap(Version);
 821          if (unlikely(index == 0))
 822             return false;
 823          Dep->Version = index;
 824       }
 825    }
 826
 827    // Link it to the package
 828    Dep->Package = Pkg.Index();
 829    Dep->NextRevDepends = Pkg->RevDepends;
 830    Pkg->RevDepends = Dep.Index();
 831
 832    // Do we know where to link the Dependency to?
 833    if (OldDepLast == NULL)
 834    {
 835       OldDepLast = &Ver->DependsList;
 836       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 837          OldDepLast = &D->NextDepends;
 838    } else if (oldMap != Map.Data())
 839       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 840
 841    Dep->NextDepends = *OldDepLast;
 842    *OldDepLast = Dep.Index();
 843    OldDepLast = &Dep->NextDepends;
 844
 845    return true;
 846 }
 847                                                                         /*}}}*/
 848 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* This creates a Group and the Package to link this dependency to if
 851    needed and handles also the caching of the old endpoint */
 852 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 853                                                const string &PackageName,
 854                                                const string &Arch,
 855                                                const string &Version,
 856                                                unsigned int Op,
 857                                                unsigned int Type)
 858 {
 859    pkgCache::GrpIterator Grp;
 860    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 861    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 862       return false;
 863
 864    // Locate the target package
 865    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 866    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 867    if (Pkg.end() == true) {
 868       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 869          return false;
 870    }
 871
 872    // Is it a file dependency?
 873    if (unlikely(PackageName[0] == '/'))
 874       FoundFileDeps = true;
 875
 876    /* Caching the old end point speeds up generation substantially */
 877    if (OldDepVer != Ver) {
 878       OldDepLast = NULL;
 879       OldDepVer = Ver;
 880    }
 881
 882    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 883 }
 884                                                                         /*}}}*/
 885 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 886 // ---------------------------------------------------------------------
 887 /* */
 888 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 889                                                 const string &PkgName,
 890                                                 const string &PkgArch,
 891                                                 const string &Version)
 892 {
 893    pkgCache &Cache = Owner->Cache;
 894
 895    // We do not add self referencing provides
 896    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 897         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 898       return true;
 899
 900    // Get a structure
 901    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 902    if (unlikely(Provides == 0))
 903       return false;
 904    Cache.HeaderP->ProvidesCount++;
 905
 906    // Fill it in
 907    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 908    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 909    Prv->Version = Ver.Index();
 910    Prv->NextPkgProv = Ver->ProvidesList;
 911    Ver->ProvidesList = Prv.Index();
 912    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 913       return false;
 914
 915    // Locate the target package
 916    pkgCache::PkgIterator Pkg;
 917    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 918    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 919       return false;
 920
 921    // Link it to the package
 922    Prv->ParentPkg = Pkg.Index();
 923    Prv->NextProvides = Pkg->ProvidesList;
 924    Pkg->ProvidesList = Prv.Index();
 925
 926    return true;
 927 }
 928                                                                         /*}}}*/
 929 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 930 // ---------------------------------------------------------------------
 931 /* This is used to select which file is to be associated with all newly
 932    added versions. The caller is responsible for setting the IMS fields. */
 933 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 934                                    const pkgIndexFile &Index,
 935                                    unsigned long Flags)
 936 {
 937    // Get some space for the structure
 938    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 939    if (unlikely(idxFile == 0))
 940       return false;
 941    CurrentFile = Cache.PkgFileP + idxFile;
 942
 943    // Fill it in
 944    map_ptrloc const idxFileName = WriteStringInMap(File);
 945    map_ptrloc const idxSite = WriteUniqString(Site);
 946    if (unlikely(idxFileName == 0 || idxSite == 0))
 947       return false;
 948    CurrentFile->FileName = idxFileName;
 949    CurrentFile->Site = idxSite;
 950    CurrentFile->NextFile = Cache.HeaderP->FileList;
 951    CurrentFile->Flags = Flags;
 952    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 953    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 954    if (unlikely(idxIndexType == 0))
 955       return false;
 956    CurrentFile->IndexType = idxIndexType;
 957    PkgFileName = File;
 958    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 959    Cache.HeaderP->PackageFileCount++;
 960
 961    if (Progress != 0)
 962       Progress->SubProgress(Index.Size());
 963    return true;
 964 }
 965                                                                         /*}}}*/
 966 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 967 // ---------------------------------------------------------------------
 968 /* This is used to create handles to strings. Given the same text it
 969    always returns the same number */
 970 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 971                                                  unsigned int Size)
 972 {
 973    /* We use a very small transient hash table here, this speeds up generation
 974       by a fair amount on slower machines */
 975    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 976    if (Bucket != 0 &&
 977        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 978       return Bucket->String;
 979
 980    // Search for an insertion point
 981    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 982    int Res = 1;
 983    map_ptrloc *Last = &Cache.HeaderP->StringList;
 984    for (; I != Cache.StringItemP; Last = &I->NextItem,
 985         I = Cache.StringItemP + I->NextItem)
 986    {
 987       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 988       if (Res >= 0)
 989          break;
 990    }
 991
 992    // Match
 993    if (Res == 0)
 994    {
 995       Bucket = I;
 996       return I->String;
 997    }
 998
 999    // Get a structure
1000    void const * const oldMap = Map.Data();
1001    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
1002    if (Item == 0)
1003       return 0;
1004
1005    map_ptrloc const idxString = WriteStringInMap(S,Size);
1006    if (unlikely(idxString == 0))
1007       return 0;
1008    if (oldMap != Map.Data()) {
1009       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
1010       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
1011    }
1012    *Last = Item;
1013
1014    // Fill in the structure
1015    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
1016    ItemP->NextItem = I - Cache.StringItemP;
1017    ItemP->String = idxString;
1018
1019    Bucket = ItemP;
1020    return ItemP->String;
1021 }
1022                                                                         /*}}}*/
1023 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1024 // ---------------------------------------------------------------------
1025 /* This just verifies that each file in the list of index files exists,
1026    has matching attributes with the cache and the cache does not have
1027    any extra files. */
1028 static bool CheckValidity(const string &CacheFile,
1029                           pkgSourceList &List,
1030                           FileIterator Start,
1031                           FileIterator End,
1032                           MMap **OutMap = 0)
1033 {
1034    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1035    // No file, certainly invalid
1036    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1037    {
1038       if (Debug == true)
1039          std::clog << "CacheFile doesn't exist" << std::endl;
1040       return false;
1041    }
1042
1043    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1044    {
1045       if (Debug == true)
1046          std::clog << "sources.list is newer than the cache" << std::endl;
1047       return false;
1048    }
1049
1050    // Map it
1051    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1052    SPtr<MMap> Map = new MMap(CacheF,0);
1053    pkgCache Cache(Map);
1054    if (_error->PendingError() == true || Map->Size() == 0)
1055    {
1056       if (Debug == true)
1057          std::clog << "Errors are pending or Map is empty()" << std::endl;
1058       _error->Discard();
1059       return false;
1060    }
1061
1062    /* Now we check every index file, see if it is in the cache,
1063       verify the IMS data and check that it is on the disk too.. */
1064    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1065    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1066    for (; Start != End; ++Start)
1067    {
1068       if (Debug == true)
1069          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1070       if ((*Start)->HasPackages() == false)
1071       {
1072          if (Debug == true)
1073             std::clog << "Has NO packages" << std::endl;
1074          continue;
1075       }
1076
1077       if ((*Start)->Exists() == false)
1078       {
1079 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1080          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1081                           (*Start)->Describe().c_str());
1082 #endif
1083          if (Debug == true)
1084             std::clog << "file doesn't exist" << std::endl;
1085          continue;
1086       }
1087
1088       // FindInCache is also expected to do an IMS check.
1089       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1090       if (File.end() == true)
1091       {
1092          if (Debug == true)
1093             std::clog << "FindInCache returned end-Pointer" << std::endl;
1094          return false;
1095       }
1096
1097       Visited[File->ID] = true;
1098       if (Debug == true)
1099          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1100    }
1101
1102    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1103       if (Visited[I] == false)
1104       {
1105          if (Debug == true)
1106             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1107          return false;
1108       }
1109
1110    if (_error->PendingError() == true)
1111    {
1112       if (Debug == true)
1113       {
1114          std::clog << "Validity failed because of pending errors:" << std::endl;
1115          _error->DumpErrors();
1116       }
1117       _error->Discard();
1118       return false;
1119    }
1120
1121    if (OutMap != 0)
1122       *OutMap = Map.UnGuard();
1123    return true;
1124 }
1125                                                                         /*}}}*/
1126 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1127 // ---------------------------------------------------------------------
1128 /* Size is kind of an abstract notion that is only used for the progress
1129    meter */
1130 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1131 {
1132    unsigned long TotalSize = 0;
1133    for (; Start != End; ++Start)
1134    {
1135       if ((*Start)->HasPackages() == false)
1136          continue;
1137       TotalSize += (*Start)->Size();
1138    }
1139    return TotalSize;
1140 }
1141                                                                         /*}}}*/
1142 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1143 // ---------------------------------------------------------------------
1144 /* */
1145 static bool BuildCache(pkgCacheGenerator &Gen,
1146                        OpProgress *Progress,
1147                        unsigned long &CurrentSize,unsigned long TotalSize,
1148                        FileIterator Start, FileIterator End)
1149 {
1150    FileIterator I;
1151    for (I = Start; I != End; ++I)
1152    {
1153       if ((*I)->HasPackages() == false)
1154          continue;
1155
1156       if ((*I)->Exists() == false)
1157          continue;
1158
1159       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1160       {
1161          _error->Warning("Duplicate sources.list entry %s",
1162                          (*I)->Describe().c_str());
1163          continue;
1164       }
1165
1166       unsigned long Size = (*I)->Size();
1167       if (Progress != NULL)
1168          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1169       CurrentSize += Size;
1170
1171       if ((*I)->Merge(Gen,Progress) == false)
1172          return false;
1173    }
1174
1175    if (Gen.HasFileDeps() == true)
1176    {
1177       if (Progress != NULL)
1178          Progress->Done();
1179       TotalSize = ComputeSize(Start, End);
1180       CurrentSize = 0;
1181       for (I = Start; I != End; ++I)
1182       {
1183          unsigned long Size = (*I)->Size();
1184          if (Progress != NULL)
1185             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1186          CurrentSize += Size;
1187          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1188             return false;
1189       }
1190    }
1191
1192    return true;
1193 }
1194                                                                         /*}}}*/
1195 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1196 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1197    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1198    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1199    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1200    Flags |= MMap::Moveable;
1201    if (_config->FindB("APT::Cache-Fallback", false) == true)
1202       Flags |= MMap::Fallback;
1203    if (CacheF != NULL)
1204       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1205    else
1206       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1207 }
1208                                                                         /*}}}*/
1209 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1210 // ---------------------------------------------------------------------
1211 /* This makes sure that the status cache (the cache that has all
1212    index files from the sources list and all local ones) is ready
1213    to be mmaped. If OutMap is not zero then a MMap object representing
1214    the cache will be stored there. This is pretty much mandetory if you
1215    are using AllowMem. AllowMem lets the function be run as non-root
1216    where it builds the cache 'fast' into a memory buffer. */
1217 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1218                         MMap **OutMap, bool AllowMem)
1219    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1220 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1221                         MMap **OutMap,bool AllowMem)
1222 {
1223    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1224
1225    vector<pkgIndexFile *> Files;
1226    for (vector<metaIndex *>::const_iterator i = List.begin();
1227         i != List.end();
1228         ++i)
1229    {
1230       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1231       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1232            j != Indexes->end();
1233            ++j)
1234          Files.push_back (*j);
1235    }
1236
1237    unsigned long const EndOfSource = Files.size();
1238    if (_system->AddStatusFiles(Files) == false)
1239       return false;
1240
1241    // Decide if we can write to the files..
1242    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1243    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1244
1245    // ensure the cache directory exists
1246    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1247    {
1248       string dir = _config->FindDir("Dir::Cache");
1249       size_t const len = dir.size();
1250       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1251          dir = dir.substr(0, len - 5);
1252       if (CacheFile.empty() == false)
1253          CreateDirectory(dir, flNotFile(CacheFile));
1254       if (SrcCacheFile.empty() == false)
1255          CreateDirectory(dir, flNotFile(SrcCacheFile));
1256    }
1257
1258    // Decide if we can write to the cache
1259    bool Writeable = false;
1260    if (CacheFile.empty() == false)
1261       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1262    else
1263       if (SrcCacheFile.empty() == false)
1264          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1265    if (Debug == true)
1266       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1267
1268    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1269       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1270
1271    if (Progress != NULL)
1272       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1273
1274    // Cache is OK, Fin.
1275    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1276    {
1277       if (Progress != NULL)
1278          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1279       if (Debug == true)
1280          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1281       return true;
1282    }
1283    else if (Debug == true)
1284          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1285
1286    /* At this point we know we need to reconstruct the package cache,
1287       begin. */
1288    SPtr<FileFd> CacheF;
1289    SPtr<DynamicMMap> Map;
1290    if (Writeable == true && CacheFile.empty() == false)
1291    {
1292       _error->PushToStack();
1293       unlink(CacheFile.c_str());
1294       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1295       fchmod(CacheF->Fd(),0644);
1296       Map = CreateDynamicMMap(CacheF, MMap::Public);
1297       if (_error->PendingError() == true)
1298       {
1299          delete CacheF.UnGuard();
1300          delete Map.UnGuard();
1301          if (Debug == true)
1302             std::clog << "Open filebased MMap FAILED" << std::endl;
1303          Writeable = false;
1304          if (AllowMem == false)
1305          {
1306             _error->MergeWithStack();
1307             return false;
1308          }
1309          _error->RevertToStack();
1310       }
1311       else if (Debug == true)
1312       {
1313          _error->MergeWithStack();
1314          std::clog << "Open filebased MMap" << std::endl;
1315       }
1316    }
1317    if (Writeable == false || CacheFile.empty() == true)
1318    {
1319       // Just build it in memory..
1320       Map = CreateDynamicMMap(NULL);
1321       if (Debug == true)
1322          std::clog << "Open memory Map (not filebased)" << std::endl;
1323    }
1324
1325    // Lets try the source cache.
1326    unsigned long CurrentSize = 0;
1327    unsigned long TotalSize = 0;
1328    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1329                      Files.begin()+EndOfSource) == true)
1330    {
1331       if (Debug == true)
1332          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1333       // Preload the map with the source cache
1334       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1335       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1336       if ((alloc == 0 && _error->PendingError())
1337                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1338                                 SCacheF.Size()) == false)
1339          return false;
1340
1341       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1342
1343       // Build the status cache
1344       pkgCacheGenerator Gen(Map.Get(),Progress);
1345       if (_error->PendingError() == true)
1346          return false;
1347       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1348                      Files.begin()+EndOfSource,Files.end()) == false)
1349          return false;
1350    }
1351    else
1352    {
1353       if (Debug == true)
1354          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1355       TotalSize = ComputeSize(Files.begin(),Files.end());
1356
1357       // Build the source cache
1358       pkgCacheGenerator Gen(Map.Get(),Progress);
1359       if (_error->PendingError() == true)
1360          return false;
1361       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1362                      Files.begin(),Files.begin()+EndOfSource) == false)
1363          return false;
1364
1365       // Write it back
1366       if (Writeable == true && SrcCacheFile.empty() == false)
1367       {
1368          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1369          if (_error->PendingError() == true)
1370             return false;
1371
1372          fchmod(SCacheF.Fd(),0644);
1373
1374          // Write out the main data
1375          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1376             return _error->Error(_("IO Error saving source cache"));
1377          SCacheF.Sync();
1378
1379          // Write out the proper header
1380          Gen.GetCache().HeaderP->Dirty = false;
1381          if (SCacheF.Seek(0) == false ||
1382              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1383             return _error->Error(_("IO Error saving source cache"));
1384          Gen.GetCache().HeaderP->Dirty = true;
1385          SCacheF.Sync();
1386       }
1387
1388       // Build the status cache
1389       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1390                      Files.begin()+EndOfSource,Files.end()) == false)
1391          return false;
1392    }
1393    if (Debug == true)
1394       std::clog << "Caches are ready for shipping" << std::endl;
1395
1396    if (_error->PendingError() == true)
1397       return false;
1398    if (OutMap != 0)
1399    {
1400       if (CacheF != 0)
1401       {
1402          delete Map.UnGuard();
1403          *OutMap = new MMap(*CacheF,0);
1404       }
1405       else
1406       {
1407          *OutMap = Map.UnGuard();
1408       }
1409    }
1410
1411    return true;
1412 }
1413                                                                         /*}}}*/
1414 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1415 // ---------------------------------------------------------------------
1416 /* */
1417 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1418    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1419 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1420 {
1421    vector<pkgIndexFile *> Files;
1422    unsigned long EndOfSource = Files.size();
1423    if (_system->AddStatusFiles(Files) == false)
1424       return false;
1425
1426    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1427    unsigned long CurrentSize = 0;
1428    unsigned long TotalSize = 0;
1429
1430    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1431
1432    // Build the status cache
1433    if (Progress != NULL)
1434       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1435    pkgCacheGenerator Gen(Map.Get(),Progress);
1436    if (_error->PendingError() == true)
1437       return false;
1438    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1439                   Files.begin()+EndOfSource,Files.end()) == false)
1440       return false;
1441
1442    if (_error->PendingError() == true)
1443       return false;
1444    *OutMap = Map.UnGuard();
1445
1446    return true;
1447 }
1448                                                                         /*}}}*/
1449 // IsDuplicateDescription                                               /*{{{*/
1450 bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1451                             MD5SumValue const &CurMd5, std::string const &CurLang)
1452 {
1453    // Descriptions in the same link-list have all the same md5
1454    if (MD5SumValue(Desc.md5()) != CurMd5)
1455       return false;
1456    for (; Desc.end() == false; ++Desc)
1457       if (Desc.LanguageCode() == CurLang)
1458          return true;
1459    return false;
1460 }
1461                                                                         /*}}}*/
1462 // CacheGenerator::FinishCache                                          /*{{{*/
1463 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
1464 {
1465    return true;
1466 }
1467                                                                         /*}}}*/