apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/sptr.h>
  23 #include <apt-pkg/pkgsystem.h>
  24 #include <apt-pkg/macros.h>
  25 #include <apt-pkg/metaindex.h>
  26 #include <apt-pkg/fileutl.h>
  27 #include <apt-pkg/hashsum_template.h>
  28 #include <apt-pkg/indexfile.h>
  29 #include <apt-pkg/md5.h>
  30 #include <apt-pkg/mmap.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33
  34 #include <stddef.h>
  35 #include <string.h>
  36 #include <iostream>
  37 #include <string>
  38 #include <vector>
  39 #include <sys/stat.h>
  40 #include <unistd.h>
  41
  42 #include <apti18n.h>
  43                                                                         /*}}}*/
  44 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
  45 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  46
  47 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
  48                             MD5SumValue const &CurMd5, std::string const &CurLang);
  49
  50 using std::string;
  51
  52 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* We set the dirty flag and make sure that is written to the disk */
  55 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  56                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  57                     FoundFileDeps(0)
  58 {
  59    CurrentFile = 0;
  60    memset(UniqHash,0,sizeof(UniqHash));
  61
  62    if (_error->PendingError() == true)
  63       return;
  64
  65    if (Map.Size() == 0)
  66    {
  67       // Setup the map interface..
  68       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  69       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  70          return;
  71
  72       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  73
  74       // Starting header
  75       *Cache.HeaderP = pkgCache::Header();
  76
  77       // make room for the hashtables for packages and groups
  78       if (Map.RawAllocate(2 * (Cache.HeaderP->HashTableSize * sizeof(map_pointer_t))) == 0)
  79          return;
  80
  81       map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label);
  82       if (unlikely(idxVerSysName == 0))
  83          return;
  84       Cache.HeaderP->VerSysName = idxVerSysName;
  85       // this pointer is set in ReMap, but we need it now for WriteUniqString
  86       Cache.StringItemP = (pkgCache::StringItem *)Map.Data();
  87       map_stringitem_t const idxArchitecture = WriteUniqString(_config->Find("APT::Architecture"));
  88       if (unlikely(idxArchitecture == 0))
  89          return;
  90       Cache.HeaderP->Architecture = idxArchitecture;
  91
  92       std::vector<std::string> archs = APT::Configuration::getArchitectures();
  93       if (archs.size() > 1)
  94       {
  95          std::vector<std::string>::const_iterator a = archs.begin();
  96          std::string list = *a;
  97          for (++a; a != archs.end(); ++a)
  98             list.append(",").append(*a);
  99          map_stringitem_t const idxArchitectures = WriteStringInMap(list);
 100          if (unlikely(idxArchitectures == 0))
 101             return;
 102          Cache.HeaderP->Architectures = idxArchitectures;
 103       }
 104       else
 105          Cache.HeaderP->Architectures = idxArchitecture;
 106
 107       Cache.ReMap();
 108    }
 109    else
 110    {
 111       // Map directly from the existing file
 112       Cache.ReMap();
 113       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
 114       if (Cache.VS != _system->VS)
 115       {
 116          _error->Error(_("Cache has an incompatible versioning system"));
 117          return;
 118       }
 119    }
 120
 121    Cache.HeaderP->Dirty = true;
 122    Map.Sync(0,sizeof(pkgCache::Header));
 123 }
 124                                                                         /*}}}*/
 125 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* We sync the data then unset the dirty flag in two steps so as to
 128    advoid a problem during a crash */
 129 pkgCacheGenerator::~pkgCacheGenerator()
 130 {
 131    if (_error->PendingError() == true)
 132       return;
 133    if (Map.Sync() == false)
 134       return;
 135
 136    Cache.HeaderP->Dirty = false;
 137    Cache.HeaderP->CacheFileSize = Map.Size();
 138    Map.Sync(0,sizeof(pkgCache::Header));
 139 }
 140                                                                         /*}}}*/
 141 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 142    if (oldMap == newMap)
 143       return;
 144
 145    if (_config->FindB("Debug::pkgCacheGen", false))
 146       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 147
 148    Cache.ReMap(false);
 149
 150    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
 151
 152    for (size_t i = 0; i < _count(UniqHash); ++i)
 153       if (UniqHash[i] != 0)
 154          UniqHash[i] += (pkgCache::StringItem const * const) newMap - (pkgCache::StringItem const * const) oldMap;
 155
 156    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 157         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 158       (*i)->ReMap(oldMap, newMap);
 159    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 160         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 161       (*i)->ReMap(oldMap, newMap);
 162    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 163         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 164       (*i)->ReMap(oldMap, newMap);
 165    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 166         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 167       (*i)->ReMap(oldMap, newMap);
 168    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 169         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 170       (*i)->ReMap(oldMap, newMap);
 171    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 172         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 173       (*i)->ReMap(oldMap, newMap);
 174    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 175         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 176       (*i)->ReMap(oldMap, newMap);
 177 }                                                                       /*}}}*/
 178 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 179 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
 180                                         const unsigned long &Len) {
 181    void const * const oldMap = Map.Data();
 182    map_stringitem_t const index = Map.WriteString(String, Len);
 183    if (index != 0)
 184       ReMap(oldMap, Map.Data());
 185    return index;
 186 }
 187                                                                         /*}}}*/
 188 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 189 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) {
 190    void const * const oldMap = Map.Data();
 191    map_stringitem_t const index = Map.WriteString(String);
 192    if (index != 0)
 193       ReMap(oldMap, Map.Data());
 194    return index;
 195 }
 196                                                                         /*}}}*/
 197 map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 198    void const * const oldMap = Map.Data();
 199    map_pointer_t const index = Map.Allocate(size);
 200    if (index != 0)
 201       ReMap(oldMap, Map.Data());
 202    return index;
 203 }
 204                                                                         /*}}}*/
 205 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 206 // ---------------------------------------------------------------------
 207 /* This provides the generation of the entries in the cache. Each loop
 208    goes through a single package record from the underlying parse engine. */
 209 bool pkgCacheGenerator::MergeList(ListParser &List,
 210                                   pkgCache::VerIterator *OutVer)
 211 {
 212    List.Owner = this;
 213
 214    unsigned int Counter = 0;
 215    while (List.Step() == true)
 216    {
 217       string const PackageName = List.Package();
 218       if (PackageName.empty() == true)
 219          return false;
 220
 221       Counter++;
 222       if (Counter % 100 == 0 && Progress != 0)
 223          Progress->Progress(List.Offset());
 224
 225       string Arch = List.Architecture();
 226       string const Version = List.Version();
 227       if (Version.empty() == true && Arch.empty() == true)
 228       {
 229          // package descriptions
 230          if (MergeListGroup(List, PackageName) == false)
 231             return false;
 232          continue;
 233       }
 234
 235       if (Arch.empty() == true)
 236       {
 237          // use the pseudo arch 'none' for arch-less packages
 238          Arch = "none";
 239          /* We might built a SingleArchCache here, which we don't want to blow up
 240             just for these :none packages to a proper MultiArchCache, so just ensure
 241             that we have always a native package structure first for SingleArch */
 242          pkgCache::PkgIterator NP;
 243          Dynamic<pkgCache::PkgIterator> DynPkg(NP);
 244          if (NewPackage(NP, PackageName, _config->Find("APT::Architecture")) == false)
 245          // TRANSLATOR: The first placeholder is a package name,
 246          // the other two should be copied verbatim as they include debug info
 247          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 248                               PackageName.c_str(), "NewPackage", 0);
 249       }
 250
 251       // Get a pointer to the package structure
 252       pkgCache::PkgIterator Pkg;
 253       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 254       if (NewPackage(Pkg, PackageName, Arch) == false)
 255          // TRANSLATOR: The first placeholder is a package name,
 256          // the other two should be copied verbatim as they include debug info
 257          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 258                               PackageName.c_str(), "NewPackage", 1);
 259
 260
 261       if (Version.empty() == true)
 262       {
 263          if (MergeListPackage(List, Pkg) == false)
 264             return false;
 265       }
 266       else
 267       {
 268          if (MergeListVersion(List, Pkg, Version, OutVer) == false)
 269             return false;
 270       }
 271
 272       if (OutVer != 0)
 273       {
 274          FoundFileDeps |= List.HasFileDeps();
 275          return true;
 276       }
 277    }
 278
 279    if (Cache.HeaderP->PackageCount >= std::numeric_limits<map_id_t>::max())
 280       return _error->Error(_("Wow, you exceeded the number of package "
 281                              "names this APT is capable of."));
 282    if (Cache.HeaderP->VersionCount >= std::numeric_limits<map_id_t>::max())
 283       return _error->Error(_("Wow, you exceeded the number of versions "
 284                              "this APT is capable of."));
 285    if (Cache.HeaderP->DescriptionCount >= std::numeric_limits<map_id_t>::max())
 286       return _error->Error(_("Wow, you exceeded the number of descriptions "
 287                              "this APT is capable of."));
 288    if (Cache.HeaderP->DependsCount >= std::numeric_limits<map_id_t>::max())
 289       return _error->Error(_("Wow, you exceeded the number of dependencies "
 290                              "this APT is capable of."));
 291
 292    FoundFileDeps |= List.HasFileDeps();
 293    return true;
 294 }
 295 // CacheGenerator::MergeListGroup                                       /*{{{*/
 296 bool pkgCacheGenerator::MergeListGroup(ListParser &List, std::string const &GrpName)
 297 {
 298    pkgCache::GrpIterator Grp = Cache.FindGrp(GrpName);
 299    // a group has no data on it's own, only packages have it but these
 300    // stanzas like this come from Translation- files to add descriptions,
 301    // but without a version we don't need a description for it…
 302    if (Grp.end() == true)
 303       return true;
 304    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 305
 306    pkgCache::PkgIterator Pkg;
 307    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 308    for (Pkg = Grp.PackageList(); Pkg.end() == false; Pkg = Grp.NextPkg(Pkg))
 309       if (MergeListPackage(List, Pkg) == false)
 310          return false;
 311
 312    return true;
 313 }
 314                                                                         /*}}}*/
 315 // CacheGenerator::MergeListPackage                                     /*{{{*/
 316 bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg)
 317 {
 318    // we first process the package, then the descriptions
 319    // (for deb this package processing is in fact a no-op)
 320    pkgCache::VerIterator Ver(Cache);
 321    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 322    if (List.UsePackage(Pkg, Ver) == false)
 323       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 324                            Pkg.Name(), "UsePackage", 1);
 325
 326    // Find the right version to write the description
 327    MD5SumValue CurMd5 = List.Description_md5();
 328    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 329       return true;
 330    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 331    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
 332    {
 333       pkgCache::DescIterator VerDesc = Ver.DescriptionList();
 334
 335       // a version can only have one md5 describing it
 336       if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5)
 337          continue;
 338
 339       map_stringitem_t md5idx = VerDesc->md5sum;
 340       for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 341       {
 342          // don't add a new description if we have one for the given
 343          // md5 && language
 344          if (IsDuplicateDescription(VerDesc, CurMd5, *CurLang) == true)
 345             continue;
 346
 347          AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx);
 348       }
 349
 350       // we can stop here as all "same" versions will share the description
 351       break;
 352    }
 353
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357 // CacheGenerator::MergeListVersion                                     /*{{{*/
 358 bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
 359                                          std::string const &Version, pkgCache::VerIterator* &OutVer)
 360 {
 361    pkgCache::VerIterator Ver = Pkg.VersionList();
 362    Dynamic<pkgCache::VerIterator> DynVer(Ver);
 363    map_pointer_t *LastVer = &Pkg->VersionList;
 364    void const * oldMap = Map.Data();
 365
 366    unsigned short const Hash = List.VersionHash();
 367    if (Ver.end() == false)
 368    {
 369       /* We know the list is sorted so we use that fact in the search.
 370          Insertion of new versions is done with correct sorting */
 371       int Res = 1;
 372       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 373       {
 374          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 375          // Version is higher as current version - insert here
 376          if (Res > 0)
 377             break;
 378          // Versionstrings are equal - is hash also equal?
 379          if (Res == 0 && List.SameVersion(Hash, Ver) == true)
 380             break;
 381          // proceed with the next till we have either the right
 382          // or we found another version (which will be lower)
 383       }
 384
 385       /* We already have a version for this item, record that we saw it */
 386       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 387       {
 388          if (List.UsePackage(Pkg,Ver) == false)
 389             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 390                                  Pkg.Name(), "UsePackage", 2);
 391
 392          if (NewFileVer(Ver,List) == false)
 393             return _error->Error(_("Error occurred while processing %s (%s%d)"),
 394                                  Pkg.Name(), "NewFileVer", 1);
 395
 396          // Read only a single record and return
 397          if (OutVer != 0)
 398          {
 399             *OutVer = Ver;
 400             return true;
 401          }
 402
 403          return true;
 404       }
 405    }
 406
 407    // Add a new version
 408    map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer);
 409    if (verindex == 0 && _error->PendingError())
 410       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 411                            Pkg.Name(), "NewVersion", 1);
 412
 413    if (oldMap != Map.Data())
 414          LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 415    *LastVer = verindex;
 416
 417    if (unlikely(List.NewVersion(Ver) == false))
 418       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 419                            Pkg.Name(), "NewVersion", 2);
 420
 421    if (unlikely(List.UsePackage(Pkg,Ver) == false))
 422       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 423                            Pkg.Name(), "UsePackage", 3);
 424
 425    if (unlikely(NewFileVer(Ver,List) == false))
 426       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 427                            Pkg.Name(), "NewFileVer", 2);
 428
 429    pkgCache::GrpIterator Grp = Pkg.Group();
 430    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 431
 432    /* If it is the first version of this package we need to add implicit
 433       Multi-Arch dependencies to all other package versions in the group now -
 434       otherwise we just add them for this new version */
 435    if (Pkg.VersionList()->NextVer == 0)
 436    {
 437       pkgCache::PkgIterator P = Grp.PackageList();
 438       Dynamic<pkgCache::PkgIterator> DynP(P);
 439       for (; P.end() != true; P = Grp.NextPkg(P))
 440       {
 441          if (P->ID == Pkg->ID)
 442             continue;
 443          pkgCache::VerIterator V = P.VersionList();
 444          Dynamic<pkgCache::VerIterator> DynV(V);
 445          for (; V.end() != true; ++V)
 446             if (unlikely(AddImplicitDepends(V, Pkg) == false))
 447                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 448                                     Pkg.Name(), "AddImplicitDepends", 1);
 449       }
 450       /* :none packages are packages without an architecture. They are forbidden by
 451          debian-policy, so usually they will only be in (old) dpkg status files -
 452          and dpkg will complain about them - and are pretty rare. We therefore do
 453          usually not create conflicts while the parent is created, but only if a :none
 454          package (= the target) appears. This creates incorrect dependencies on :none
 455          for architecture-specific dependencies on the package we copy from, but we
 456          will ignore this bug as architecture-specific dependencies are only allowed
 457          in jessie and until then the :none packages should be extinct (hopefully).
 458          In other words: This should work long enough to allow graceful removal of
 459          these packages, it is not supposed to allow users to keep using them … */
 460       if (strcmp(Pkg.Arch(), "none") == 0)
 461       {
 462          pkgCache::PkgIterator M = Grp.FindPreferredPkg();
 463          if (M.end() == false && Pkg != M)
 464          {
 465             pkgCache::DepIterator D = M.RevDependsList();
 466             Dynamic<pkgCache::DepIterator> DynD(D);
 467             for (; D.end() == false; ++D)
 468             {
 469                if ((D->Type != pkgCache::Dep::Conflicts &&
 470                     D->Type != pkgCache::Dep::DpkgBreaks &&
 471                     D->Type != pkgCache::Dep::Replaces) ||
 472                    D.ParentPkg().Group() == Grp)
 473                   continue;
 474
 475                map_pointer_t *OldDepLast = NULL;
 476                pkgCache::VerIterator ConVersion = D.ParentVer();
 477                Dynamic<pkgCache::VerIterator> DynV(ConVersion);
 478                // duplicate the Conflicts/Breaks/Replaces for :none arch
 479                NewDepends(Pkg, ConVersion, D->Version,
 480                      D->CompareOp, D->Type, OldDepLast);
 481             }
 482          }
 483       }
 484    }
 485    if (unlikely(AddImplicitDepends(Grp, Pkg, Ver) == false))
 486       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 487                            Pkg.Name(), "AddImplicitDepends", 2);
 488
 489    // Read only a single record and return
 490    if (OutVer != 0)
 491    {
 492       *OutVer = Ver;
 493       return true;
 494    }
 495
 496    /* Record the Description(s) based on their master md5sum */
 497    MD5SumValue CurMd5 = List.Description_md5();
 498    if (CurMd5.Value().empty() == true && List.Description("").empty() == true)
 499       return true;
 500
 501    /* Before we add a new description we first search in the group for
 502       a version with a description of the same MD5 - if so we reuse this
 503       description group instead of creating our own for this version */
 504    for (pkgCache::PkgIterator P = Grp.PackageList();
 505         P.end() == false; P = Grp.NextPkg(P))
 506    {
 507       for (pkgCache::VerIterator V = P.VersionList();
 508            V.end() == false; ++V)
 509       {
 510          if (V->DescriptionList == 0 || MD5SumValue(V.DescriptionList().md5()) != CurMd5)
 511             continue;
 512          Ver->DescriptionList = V->DescriptionList;
 513       }
 514    }
 515
 516    // We haven't found reusable descriptions, so add the first description(s)
 517    map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum;
 518    std::vector<std::string> availDesc = List.AvailableDescriptionLanguages();
 519    for (std::vector<std::string>::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang)
 520       if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false)
 521          return false;
 522    return true;
 523 }
 524                                                                         /*}}}*/
 525 bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/
 526 {
 527    pkgCache::DescIterator Desc;
 528    Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 529
 530    map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx);
 531    if (unlikely(descindex == 0 && _error->PendingError()))
 532       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 533             Ver.ParentPkg().Name(), "NewDescription", 1);
 534
 535    md5idx = Desc->md5sum;
 536    Desc->ParentPkg = Ver.ParentPkg().Index();
 537
 538    // we add at the end, so that the start is constant as we need
 539    // that to be able to efficiently share these lists
 540    pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap
 541    for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc);
 542    map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc;
 543    *LastNextDesc = descindex;
 544
 545    if (NewFileDesc(Desc,List) == false)
 546       return _error->Error(_("Error occurred while processing %s (%s%d)"),
 547             Ver.ParentPkg().Name(), "NewFileDesc", 1);
 548
 549    return true;
 550 }
 551                                                                         /*}}}*/
 552                                                                         /*}}}*/
 553 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 554 // ---------------------------------------------------------------------
 555 /* If we found any file depends while parsing the main list we need to
 556    resolve them. Since it is undesired to load the entire list of files
 557    into the cache as virtual packages we do a two stage effort. MergeList
 558    identifies the file depends and this creates Provdies for them by
 559    re-parsing all the indexs. */
 560 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 561 {
 562    List.Owner = this;
 563
 564    unsigned int Counter = 0;
 565    while (List.Step() == true)
 566    {
 567       string PackageName = List.Package();
 568       if (PackageName.empty() == true)
 569          return false;
 570       string Version = List.Version();
 571       if (Version.empty() == true)
 572          continue;
 573
 574       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 575       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 576       if (Pkg.end() == true)
 577          return _error->Error(_("Error occurred while processing %s (%s%d)"),
 578                                 PackageName.c_str(), "FindPkg", 1);
 579       Counter++;
 580       if (Counter % 100 == 0 && Progress != 0)
 581          Progress->Progress(List.Offset());
 582
 583       unsigned short Hash = List.VersionHash();
 584       pkgCache::VerIterator Ver = Pkg.VersionList();
 585       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 586       for (; Ver.end() == false; ++Ver)
 587       {
 588          if (List.SameVersion(Hash, Ver) == true && Version == Ver.VerStr())
 589          {
 590             if (List.CollectFileProvides(Cache,Ver) == false)
 591                return _error->Error(_("Error occurred while processing %s (%s%d)"),
 592                                     PackageName.c_str(), "CollectFileProvides", 1);
 593             break;
 594          }
 595       }
 596
 597       if (Ver.end() == true)
 598          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 599    }
 600
 601    return true;
 602 }
 603                                                                         /*}}}*/
 604 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 605 // ---------------------------------------------------------------------
 606 /* This creates a new group structure and adds it to the hash table */
 607 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 608 {
 609    Grp = Cache.FindGrp(Name);
 610    if (Grp.end() == false)
 611       return true;
 612
 613    // Get a structure
 614    map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group));
 615    if (unlikely(Group == 0))
 616       return false;
 617
 618    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 619    map_pointer_t const idxName = WriteStringInMap(Name);
 620    if (unlikely(idxName == 0))
 621       return false;
 622    Grp->Name = idxName;
 623
 624    // Insert it into the hash table
 625    unsigned long const Hash = Cache.Hash(Name);
 626    map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTable()[Hash];
 627    while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0)
 628       insertAt = &(Cache.GrpP + *insertAt)->Next;
 629    Grp->Next = *insertAt;
 630    *insertAt = Group;
 631
 632    Grp->ID = Cache.HeaderP->GroupCount++;
 633    return true;
 634 }
 635                                                                         /*}}}*/
 636 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 637 // ---------------------------------------------------------------------
 638 /* This creates a new package structure and adds it to the hash table */
 639 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 640                                         const string &Arch) {
 641    pkgCache::GrpIterator Grp;
 642    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 643    if (unlikely(NewGroup(Grp, Name) == false))
 644       return false;
 645
 646    Pkg = Grp.FindPkg(Arch);
 647       if (Pkg.end() == false)
 648          return true;
 649
 650    // Get a structure
 651    map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package));
 652    if (unlikely(Package == 0))
 653       return false;
 654    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 655
 656    // Insert the package into our package list
 657    if (Grp->FirstPackage == 0) // the group is new
 658    {
 659       Grp->FirstPackage = Package;
 660       // Insert it into the hash table
 661       map_id_t const Hash = Cache.Hash(Name);
 662       map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTable()[Hash];
 663       while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.PkgP + *insertAt)->Name) > 0)
 664          insertAt = &(Cache.PkgP + *insertAt)->Next;
 665       Pkg->Next = *insertAt;
 666       *insertAt = Package;
 667    }
 668    else // Group the Packages together
 669    {
 670       // this package is the new last package
 671       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 672       Pkg->Next = LastPkg->Next;
 673       LastPkg->Next = Package;
 674    }
 675    Grp->LastPackage = Package;
 676
 677    // Set the name, arch and the ID
 678    Pkg->Name = Grp->Name;
 679    Pkg->Group = Grp.Index();
 680    // all is mapped to the native architecture
 681    map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 682    if (unlikely(idxArch == 0))
 683       return false;
 684    Pkg->Arch = idxArch;
 685    Pkg->ID = Cache.HeaderP->PackageCount++;
 686
 687    return true;
 688 }
 689                                                                         /*}}}*/
 690 // CacheGenerator::AddImplicitDepends                                   /*{{{*/
 691 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G,
 692                                            pkgCache::PkgIterator &P,
 693                                            pkgCache::VerIterator &V)
 694 {
 695    // copy P.Arch() into a string here as a cache remap
 696    // in NewDepends() later may alter the pointer location
 697    string Arch = P.Arch() == NULL ? "" : P.Arch();
 698    map_pointer_t *OldDepLast = NULL;
 699    /* MultiArch handling introduces a lot of implicit Dependencies:
 700       - MultiArch: same → Co-Installable if they have the same version
 701       - All others conflict with all other group members */
 702    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 703    pkgCache::PkgIterator D = G.PackageList();
 704    Dynamic<pkgCache::PkgIterator> DynD(D);
 705    map_stringitem_t const VerStrIdx = V->VerStr;
 706    for (; D.end() != true; D = G.NextPkg(D))
 707    {
 708       if (Arch == D.Arch() || D->VersionList == 0)
 709          continue;
 710       /* We allow only one installed arch at the time
 711          per group, therefore each group member conflicts
 712          with all other group members */
 713       if (coInstall == true)
 714       {
 715          // Replaces: ${self}:other ( << ${binary:Version})
 716          NewDepends(D, V, VerStrIdx,
 717                     pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 718                     OldDepLast);
 719          // Breaks: ${self}:other (!= ${binary:Version})
 720          NewDepends(D, V, VerStrIdx,
 721                     pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 722                     OldDepLast);
 723       } else {
 724          // Conflicts: ${self}:other
 725          NewDepends(D, V, 0,
 726                     pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 727                     OldDepLast);
 728       }
 729    }
 730    return true;
 731 }
 732 bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V,
 733                                            pkgCache::PkgIterator &D)
 734 {
 735    /* MultiArch handling introduces a lot of implicit Dependencies:
 736       - MultiArch: same → Co-Installable if they have the same version
 737       - All others conflict with all other group members */
 738    map_pointer_t *OldDepLast = NULL;
 739    bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 740    if (coInstall == true)
 741    {
 742       map_stringitem_t const VerStrIdx = V->VerStr;
 743       // Replaces: ${self}:other ( << ${binary:Version})
 744       NewDepends(D, V, VerStrIdx,
 745                  pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 746                  OldDepLast);
 747       // Breaks: ${self}:other (!= ${binary:Version})
 748       NewDepends(D, V, VerStrIdx,
 749                  pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 750                  OldDepLast);
 751    } else {
 752       // Conflicts: ${self}:other
 753       NewDepends(D, V, 0,
 754                  pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 755                  OldDepLast);
 756    }
 757    return true;
 758 }
 759
 760                                                                         /*}}}*/
 761 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 762 // ---------------------------------------------------------------------
 763 /* */
 764 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 765                                    ListParser &List)
 766 {
 767    if (CurrentFile == 0)
 768       return true;
 769
 770    // Get a structure
 771    map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 772    if (VerFile == 0)
 773       return 0;
 774
 775    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 776    VF->File = CurrentFile - Cache.PkgFileP;
 777
 778    // Link it to the end of the list
 779    map_pointer_t *Last = &Ver->FileList;
 780    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V)
 781       Last = &V->NextFile;
 782    VF->NextFile = *Last;
 783    *Last = VF.Index();
 784
 785    VF->Offset = List.Offset();
 786    VF->Size = List.Size();
 787    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 788       Cache.HeaderP->MaxVerFileSize = VF->Size;
 789    Cache.HeaderP->VerFileCount++;
 790
 791    return true;
 792 }
 793                                                                         /*}}}*/
 794 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 795 // ---------------------------------------------------------------------
 796 /* This puts a version structure in the linked list */
 797 map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 798                                             const string &VerStr,
 799                                             map_pointer_t const ParentPkg,
 800                                             unsigned short const Hash,
 801                                             map_pointer_t const Next)
 802 {
 803    // Get a structure
 804    map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version));
 805    if (Version == 0)
 806       return 0;
 807
 808    // Fill it in
 809    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 810    //Dynamic<pkgCache::VerIterator> DynV(Ver); // caller MergeListVersion already takes care of it
 811    Ver->NextVer = Next;
 812    Ver->ParentPkg = ParentPkg;
 813    Ver->Hash = Hash;
 814    Ver->ID = Cache.HeaderP->VersionCount++;
 815
 816    // try to find the version string in the group for reuse
 817    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
 818    pkgCache::GrpIterator Grp = Pkg.Group();
 819    if (Pkg.end() == false && Grp.end() == false)
 820    {
 821       for (pkgCache::PkgIterator P = Grp.PackageList(); P.end() == false; P = Grp.NextPkg(P))
 822       {
 823          if (Pkg == P)
 824             continue;
 825          for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V)
 826          {
 827             int const cmp = strcmp(V.VerStr(), VerStr.c_str());
 828             if (cmp == 0)
 829             {
 830                Ver->VerStr = V->VerStr;
 831                return Version;
 832             }
 833             else if (cmp < 0)
 834                break;
 835          }
 836       }
 837    }
 838    // haven't found the version string, so create
 839    map_stringitem_t const idxVerStr = WriteStringInMap(VerStr);
 840    if (unlikely(idxVerStr == 0))
 841       return 0;
 842    Ver->VerStr = idxVerStr;
 843    return Version;
 844 }
 845                                                                         /*}}}*/
 846 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* */
 849 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 850                                    ListParser &List)
 851 {
 852    if (CurrentFile == 0)
 853       return true;
 854
 855    // Get a structure
 856    map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 857    if (DescFile == 0)
 858       return false;
 859
 860    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 861    DF->File = CurrentFile - Cache.PkgFileP;
 862
 863    // Link it to the end of the list
 864    map_pointer_t *Last = &Desc->FileList;
 865    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D)
 866       Last = &D->NextFile;
 867
 868    DF->NextFile = *Last;
 869    *Last = DF.Index();
 870
 871    DF->Offset = List.Offset();
 872    DF->Size = List.Size();
 873    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 874       Cache.HeaderP->MaxDescFileSize = DF->Size;
 875    Cache.HeaderP->DescFileCount++;
 876
 877    return true;
 878 }
 879                                                                         /*}}}*/
 880 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 881 // ---------------------------------------------------------------------
 882 /* This puts a description structure in the linked list */
 883 map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 884                                             const string &Lang,
 885                                             const MD5SumValue &md5sum,
 886                                             map_stringitem_t const idxmd5str)
 887 {
 888    // Get a structure
 889    map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description));
 890    if (Description == 0)
 891       return 0;
 892
 893    // Fill it in
 894    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 895    Desc->ID = Cache.HeaderP->DescriptionCount++;
 896    map_stringitem_t const idxlanguage_code = WriteUniqString(Lang);
 897    if (unlikely(idxlanguage_code == 0))
 898       return 0;
 899    Desc->language_code = idxlanguage_code;
 900
 901    if (idxmd5str != 0)
 902       Desc->md5sum = idxmd5str;
 903    else
 904    {
 905       map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value());
 906       if (unlikely(idxmd5sum == 0))
 907          return 0;
 908       Desc->md5sum = idxmd5sum;
 909    }
 910
 911    return Description;
 912 }
 913                                                                         /*}}}*/
 914 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 915 // ---------------------------------------------------------------------
 916 /* This creates a dependency element in the tree. It is linked to the
 917    version and to the package that it is pointing to. */
 918 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 919                                    pkgCache::VerIterator &Ver,
 920                                    string const &Version,
 921                                    unsigned int const &Op,
 922                                    unsigned int const &Type,
 923                                    map_stringitem_t* &OldDepLast)
 924 {
 925    map_stringitem_t index = 0;
 926    if (Version.empty() == false)
 927    {
 928       int const CmpOp = Op & 0x0F;
 929       // =-deps are used (79:1) for lockstep on same-source packages (e.g. data-packages)
 930       if (CmpOp == pkgCache::Dep::Equals && strcmp(Version.c_str(), Ver.VerStr()) == 0)
 931          index = Ver->VerStr;
 932
 933       if (index == 0)
 934       {
 935          void const * const oldMap = Map.Data();
 936          index = WriteStringInMap(Version);
 937          if (unlikely(index == 0))
 938             return false;
 939          if (OldDepLast != 0 && oldMap != Map.Data())
 940             OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 941       }
 942    }
 943    return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast);
 944 }
 945 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 946                                    pkgCache::VerIterator &Ver,
 947                                    map_pointer_t const Version,
 948                                    unsigned int const &Op,
 949                                    unsigned int const &Type,
 950                                    map_pointer_t* &OldDepLast)
 951 {
 952    void const * const oldMap = Map.Data();
 953    // Get a structure
 954    map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 955    if (unlikely(Dependency == 0))
 956       return false;
 957
 958    // Fill it in
 959    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 960    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 961    Dep->ParentVer = Ver.Index();
 962    Dep->Type = Type;
 963    Dep->CompareOp = Op;
 964    Dep->Version = Version;
 965    Dep->ID = Cache.HeaderP->DependsCount++;
 966
 967    // Link it to the package
 968    Dep->Package = Pkg.Index();
 969    Dep->NextRevDepends = Pkg->RevDepends;
 970    Pkg->RevDepends = Dep.Index();
 971
 972    // Do we know where to link the Dependency to?
 973    if (OldDepLast == NULL)
 974    {
 975       OldDepLast = &Ver->DependsList;
 976       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 977          OldDepLast = &D->NextDepends;
 978    } else if (oldMap != Map.Data())
 979       OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
 980
 981    Dep->NextDepends = *OldDepLast;
 982    *OldDepLast = Dep.Index();
 983    OldDepLast = &Dep->NextDepends;
 984
 985    return true;
 986 }
 987                                                                         /*}}}*/
 988 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 989 // ---------------------------------------------------------------------
 990 /* This creates a Group and the Package to link this dependency to if
 991    needed and handles also the caching of the old endpoint */
 992 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 993                                                const string &PackageName,
 994                                                const string &Arch,
 995                                                const string &Version,
 996                                                unsigned int Op,
 997                                                unsigned int Type)
 998 {
 999    pkgCache::GrpIterator Grp;
1000    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
1001    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
1002       return false;
1003
1004    // Locate the target package
1005    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
1006    // we don't create 'none' packages and their dependencies if we can avoid it …
1007    if (Pkg.end() == true && Arch == "none" && strcmp(Ver.ParentPkg().Arch(), "none") != 0)
1008       return true;
1009    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1010    if (Pkg.end() == true) {
1011       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
1012          return false;
1013    }
1014
1015    // Is it a file dependency?
1016    if (unlikely(PackageName[0] == '/'))
1017       FoundFileDeps = true;
1018
1019    /* Caching the old end point speeds up generation substantially */
1020    if (OldDepVer != Ver) {
1021       OldDepLast = NULL;
1022       OldDepVer = Ver;
1023    }
1024
1025    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
1026 }
1027                                                                         /*}}}*/
1028 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
1029 // ---------------------------------------------------------------------
1030 /* */
1031 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
1032                                                 const string &PkgName,
1033                                                 const string &PkgArch,
1034                                                 const string &Version)
1035 {
1036    pkgCache &Cache = Owner->Cache;
1037
1038    // We do not add self referencing provides
1039    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
1040         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
1041       return true;
1042
1043    // Get a structure
1044    map_pointer_t const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
1045    if (unlikely(Provides == 0))
1046       return false;
1047    Cache.HeaderP->ProvidesCount++;
1048
1049    // Fill it in
1050    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
1051    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
1052    Prv->Version = Ver.Index();
1053    Prv->NextPkgProv = Ver->ProvidesList;
1054    Ver->ProvidesList = Prv.Index();
1055    if (Version.empty() == false) {
1056       map_stringitem_t const idxProvideVersion = WriteString(Version);
1057       Prv->ProvideVersion = idxProvideVersion;
1058       if (unlikely(idxProvideVersion == 0))
1059          return false;
1060    }
1061
1062    // Locate the target package
1063    pkgCache::PkgIterator Pkg;
1064    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
1065    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
1066       return false;
1067
1068    // Link it to the package
1069    Prv->ParentPkg = Pkg.Index();
1070    Prv->NextProvides = Pkg->ProvidesList;
1071    Pkg->ProvidesList = Prv.Index();
1072
1073    return true;
1074 }
1075                                                                         /*}}}*/
1076 bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/
1077       pkgCache::VerIterator const &Ver)
1078 {
1079    return Hash == Ver->Hash;
1080 }
1081                                                                         /*}}}*/
1082 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
1083 // ---------------------------------------------------------------------
1084 /* This is used to select which file is to be associated with all newly
1085    added versions. The caller is responsible for setting the IMS fields. */
1086 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
1087                                    const pkgIndexFile &Index,
1088                                    unsigned long Flags)
1089 {
1090    // Get some space for the structure
1091    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
1092    if (unlikely(idxFile == 0))
1093       return false;
1094    CurrentFile = Cache.PkgFileP + idxFile;
1095
1096    // Fill it in
1097    map_stringitem_t const idxFileName = WriteStringInMap(File);
1098    map_stringitem_t const idxSite = WriteUniqString(Site);
1099    if (unlikely(idxFileName == 0 || idxSite == 0))
1100       return false;
1101    CurrentFile->FileName = idxFileName;
1102    CurrentFile->Site = idxSite;
1103    CurrentFile->NextFile = Cache.HeaderP->FileList;
1104    CurrentFile->Flags = Flags;
1105    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
1106    map_stringitem_t const idxIndexType = WriteUniqString(Index.GetType()->Label);
1107    if (unlikely(idxIndexType == 0))
1108       return false;
1109    CurrentFile->IndexType = idxIndexType;
1110    PkgFileName = File;
1111    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
1112    Cache.HeaderP->PackageFileCount++;
1113
1114    if (Progress != 0)
1115       Progress->SubProgress(Index.Size());
1116    return true;
1117 }
1118                                                                         /*}}}*/
1119 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
1120 // ---------------------------------------------------------------------
1121 /* This is used to create handles to strings. Given the same text it
1122    always returns the same number */
1123 map_stringitem_t pkgCacheGenerator::WriteUniqString(const char *S,
1124                                                  unsigned int Size)
1125 {
1126    /* We use a very small transient hash table here, this speeds up generation
1127       by a fair amount on slower machines */
1128    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
1129    if (Bucket != 0 &&
1130        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
1131       return Bucket->String;
1132
1133    // Search for an insertion point
1134    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
1135    int Res = 1;
1136    map_stringitem_t *Last = &Cache.HeaderP->StringList;
1137    for (; I != Cache.StringItemP; Last = &I->NextItem,
1138         I = Cache.StringItemP + I->NextItem)
1139    {
1140       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
1141       if (Res >= 0)
1142          break;
1143    }
1144
1145    // Match
1146    if (Res == 0)
1147    {
1148       Bucket = I;
1149       return I->String;
1150    }
1151
1152    // Get a structure
1153    void const * const oldMap = Map.Data();
1154    map_pointer_t const Item = AllocateInMap(sizeof(pkgCache::StringItem));
1155    if (Item == 0)
1156       return 0;
1157
1158    map_stringitem_t const idxString = WriteStringInMap(S,Size);
1159    if (unlikely(idxString == 0))
1160       return 0;
1161    if (oldMap != Map.Data()) {
1162       Last += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap;
1163       I += (pkgCache::StringItem const * const) Map.Data() - (pkgCache::StringItem const * const) oldMap;
1164    }
1165    *Last = Item;
1166
1167    // Fill in the structure
1168    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
1169    ItemP->NextItem = I - Cache.StringItemP;
1170    ItemP->String = idxString;
1171
1172    Bucket = ItemP;
1173    return ItemP->String;
1174 }
1175                                                                         /*}}}*/
1176 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
1177 // ---------------------------------------------------------------------
1178 /* This just verifies that each file in the list of index files exists,
1179    has matching attributes with the cache and the cache does not have
1180    any extra files. */
1181 static bool CheckValidity(const string &CacheFile,
1182                           pkgSourceList &List,
1183                           FileIterator Start,
1184                           FileIterator End,
1185                           MMap **OutMap = 0)
1186 {
1187    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1188    // No file, certainly invalid
1189    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
1190    {
1191       if (Debug == true)
1192          std::clog << "CacheFile doesn't exist" << std::endl;
1193       return false;
1194    }
1195
1196    if (List.GetLastModifiedTime() > GetModificationTime(CacheFile))
1197    {
1198       if (Debug == true)
1199          std::clog << "sources.list is newer than the cache" << std::endl;
1200       return false;
1201    }
1202
1203    // Map it
1204    FileFd CacheF(CacheFile,FileFd::ReadOnly);
1205    SPtr<MMap> Map = new MMap(CacheF,0);
1206    pkgCache Cache(Map);
1207    if (_error->PendingError() == true || Map->Size() == 0)
1208    {
1209       if (Debug == true)
1210          std::clog << "Errors are pending or Map is empty()" << std::endl;
1211       _error->Discard();
1212       return false;
1213    }
1214
1215    /* Now we check every index file, see if it is in the cache,
1216       verify the IMS data and check that it is on the disk too.. */
1217    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
1218    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
1219    for (; Start != End; ++Start)
1220    {
1221       if (Debug == true)
1222          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
1223       if ((*Start)->HasPackages() == false)
1224       {
1225          if (Debug == true)
1226             std::clog << "Has NO packages" << std::endl;
1227          continue;
1228       }
1229
1230       if ((*Start)->Exists() == false)
1231       {
1232 #if 0 // mvo: we no longer give a message here (Default Sources spec)
1233          _error->WarningE("stat",_("Couldn't stat source package list %s"),
1234                           (*Start)->Describe().c_str());
1235 #endif
1236          if (Debug == true)
1237             std::clog << "file doesn't exist" << std::endl;
1238          continue;
1239       }
1240
1241       // FindInCache is also expected to do an IMS check.
1242       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
1243       if (File.end() == true)
1244       {
1245          if (Debug == true)
1246             std::clog << "FindInCache returned end-Pointer" << std::endl;
1247          return false;
1248       }
1249
1250       Visited[File->ID] = true;
1251       if (Debug == true)
1252          std::clog << "with ID " << File->ID << " is valid" << std::endl;
1253    }
1254
1255    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
1256       if (Visited[I] == false)
1257       {
1258          if (Debug == true)
1259             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1260          return false;
1261       }
1262
1263    if (_error->PendingError() == true)
1264    {
1265       if (Debug == true)
1266       {
1267          std::clog << "Validity failed because of pending errors:" << std::endl;
1268          _error->DumpErrors();
1269       }
1270       _error->Discard();
1271       return false;
1272    }
1273
1274    if (OutMap != 0)
1275       *OutMap = Map.UnGuard();
1276    return true;
1277 }
1278                                                                         /*}}}*/
1279 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1280 // ---------------------------------------------------------------------
1281 /* Size is kind of an abstract notion that is only used for the progress
1282    meter */
1283 static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
1284 {
1285    map_filesize_t TotalSize = 0;
1286    for (; Start < End; ++Start)
1287    {
1288       if ((*Start)->HasPackages() == false)
1289          continue;
1290       TotalSize += (*Start)->Size();
1291    }
1292    return TotalSize;
1293 }
1294                                                                         /*}}}*/
1295 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1296 // ---------------------------------------------------------------------
1297 /* */
1298 static bool BuildCache(pkgCacheGenerator &Gen,
1299                        OpProgress *Progress,
1300                        map_filesize_t &CurrentSize,map_filesize_t TotalSize,
1301                        FileIterator Start, FileIterator End)
1302 {
1303    FileIterator I;
1304    for (I = Start; I != End; ++I)
1305    {
1306       if ((*I)->HasPackages() == false)
1307          continue;
1308
1309       if ((*I)->Exists() == false)
1310          continue;
1311
1312       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1313       {
1314          _error->Warning("Duplicate sources.list entry %s",
1315                          (*I)->Describe().c_str());
1316          continue;
1317       }
1318
1319       map_filesize_t Size = (*I)->Size();
1320       if (Progress != NULL)
1321          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1322       CurrentSize += Size;
1323
1324       if ((*I)->Merge(Gen,Progress) == false)
1325          return false;
1326    }
1327
1328    if (Gen.HasFileDeps() == true)
1329    {
1330       if (Progress != NULL)
1331          Progress->Done();
1332       TotalSize = ComputeSize(Start, End);
1333       CurrentSize = 0;
1334       for (I = Start; I != End; ++I)
1335       {
1336          map_filesize_t Size = (*I)->Size();
1337          if (Progress != NULL)
1338             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1339          CurrentSize += Size;
1340          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1341             return false;
1342       }
1343    }
1344
1345    return true;
1346 }
1347                                                                         /*}}}*/
1348 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1349 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1350    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1351    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1352    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1353    Flags |= MMap::Moveable;
1354    if (_config->FindB("APT::Cache-Fallback", false) == true)
1355       Flags |= MMap::Fallback;
1356    if (CacheF != NULL)
1357       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1358    else
1359       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1360 }
1361                                                                         /*}}}*/
1362 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1363 // ---------------------------------------------------------------------
1364 /* This makes sure that the status cache (the cache that has all
1365    index files from the sources list and all local ones) is ready
1366    to be mmaped. If OutMap is not zero then a MMap object representing
1367    the cache will be stored there. This is pretty much mandetory if you
1368    are using AllowMem. AllowMem lets the function be run as non-root
1369    where it builds the cache 'fast' into a memory buffer. */
1370 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1371                         MMap **OutMap, bool AllowMem)
1372    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1373 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1374                         MMap **OutMap,bool AllowMem)
1375 {
1376    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1377
1378    std::vector<pkgIndexFile *> Files;
1379    for (std::vector<metaIndex *>::const_iterator i = List.begin();
1380         i != List.end();
1381         ++i)
1382    {
1383       std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1384       for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1385            j != Indexes->end();
1386            ++j)
1387          Files.push_back (*j);
1388    }
1389
1390    map_filesize_t const EndOfSource = Files.size();
1391    if (_system->AddStatusFiles(Files) == false)
1392       return false;
1393
1394    // Decide if we can write to the files..
1395    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1396    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1397
1398    // ensure the cache directory exists
1399    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1400    {
1401       string dir = _config->FindDir("Dir::Cache");
1402       size_t const len = dir.size();
1403       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1404          dir = dir.substr(0, len - 5);
1405       if (CacheFile.empty() == false)
1406          CreateDirectory(dir, flNotFile(CacheFile));
1407       if (SrcCacheFile.empty() == false)
1408          CreateDirectory(dir, flNotFile(SrcCacheFile));
1409    }
1410
1411    // Decide if we can write to the cache
1412    bool Writeable = false;
1413    if (CacheFile.empty() == false)
1414       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1415    else
1416       if (SrcCacheFile.empty() == false)
1417          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1418    if (Debug == true)
1419       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1420
1421    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1422       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1423
1424    if (Progress != NULL)
1425       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1426
1427    // Cache is OK, Fin.
1428    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1429    {
1430       if (Progress != NULL)
1431          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1432       if (Debug == true)
1433          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1434       return true;
1435    }
1436    else if (Debug == true)
1437          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1438
1439    /* At this point we know we need to reconstruct the package cache,
1440       begin. */
1441    SPtr<FileFd> CacheF;
1442    SPtr<DynamicMMap> Map;
1443    if (Writeable == true && CacheFile.empty() == false)
1444    {
1445       _error->PushToStack();
1446       unlink(CacheFile.c_str());
1447       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1448       fchmod(CacheF->Fd(),0644);
1449       Map = CreateDynamicMMap(CacheF, MMap::Public);
1450       if (_error->PendingError() == true)
1451       {
1452          delete CacheF.UnGuard();
1453          delete Map.UnGuard();
1454          if (Debug == true)
1455             std::clog << "Open filebased MMap FAILED" << std::endl;
1456          Writeable = false;
1457          if (AllowMem == false)
1458          {
1459             _error->MergeWithStack();
1460             return false;
1461          }
1462          _error->RevertToStack();
1463       }
1464       else
1465       {
1466          _error->MergeWithStack();
1467          if (Debug == true)
1468             std::clog << "Open filebased MMap" << std::endl;
1469       }
1470    }
1471    if (Writeable == false || CacheFile.empty() == true)
1472    {
1473       // Just build it in memory..
1474       Map = CreateDynamicMMap(NULL);
1475       if (Debug == true)
1476          std::clog << "Open memory Map (not filebased)" << std::endl;
1477    }
1478
1479    // Lets try the source cache.
1480    map_filesize_t CurrentSize = 0;
1481    map_filesize_t TotalSize = 0;
1482    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1483                      Files.begin()+EndOfSource) == true)
1484    {
1485       if (Debug == true)
1486          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1487       // Preload the map with the source cache
1488       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1489       map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
1490       if ((alloc == 0 && _error->PendingError())
1491                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1492                                 SCacheF.Size()) == false)
1493          return false;
1494
1495       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1496
1497       // Build the status cache
1498       pkgCacheGenerator Gen(Map.Get(),Progress);
1499       if (_error->PendingError() == true)
1500          return false;
1501       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1502                      Files.begin()+EndOfSource,Files.end()) == false)
1503          return false;
1504    }
1505    else
1506    {
1507       if (Debug == true)
1508          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1509       TotalSize = ComputeSize(Files.begin(),Files.end());
1510
1511       // Build the source cache
1512       pkgCacheGenerator Gen(Map.Get(),Progress);
1513       if (_error->PendingError() == true)
1514          return false;
1515       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1516                      Files.begin(),Files.begin()+EndOfSource) == false)
1517          return false;
1518
1519       // Write it back
1520       if (Writeable == true && SrcCacheFile.empty() == false)
1521       {
1522          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1523          if (_error->PendingError() == true)
1524             return false;
1525
1526          fchmod(SCacheF.Fd(),0644);
1527
1528          // Write out the main data
1529          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1530             return _error->Error(_("IO Error saving source cache"));
1531          SCacheF.Sync();
1532
1533          // Write out the proper header
1534          Gen.GetCache().HeaderP->Dirty = false;
1535          if (SCacheF.Seek(0) == false ||
1536              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1537             return _error->Error(_("IO Error saving source cache"));
1538          Gen.GetCache().HeaderP->Dirty = true;
1539          SCacheF.Sync();
1540       }
1541
1542       // Build the status cache
1543       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1544                      Files.begin()+EndOfSource,Files.end()) == false)
1545          return false;
1546    }
1547    if (Debug == true)
1548       std::clog << "Caches are ready for shipping" << std::endl;
1549
1550    if (_error->PendingError() == true)
1551       return false;
1552    if (OutMap != 0)
1553    {
1554       if (CacheF != 0)
1555       {
1556          delete Map.UnGuard();
1557          *OutMap = new MMap(*CacheF,0);
1558       }
1559       else
1560       {
1561          *OutMap = Map.UnGuard();
1562       }
1563    }
1564
1565    return true;
1566 }
1567                                                                         /*}}}*/
1568 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1569 // ---------------------------------------------------------------------
1570 /* */
1571 APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1572    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1573 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1574 {
1575    std::vector<pkgIndexFile *> Files;
1576    map_filesize_t EndOfSource = Files.size();
1577    if (_system->AddStatusFiles(Files) == false)
1578       return false;
1579
1580    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1581    map_filesize_t CurrentSize = 0;
1582    map_filesize_t TotalSize = 0;
1583
1584    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1585
1586    // Build the status cache
1587    if (Progress != NULL)
1588       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1589    pkgCacheGenerator Gen(Map.Get(),Progress);
1590    if (_error->PendingError() == true)
1591       return false;
1592    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1593                   Files.begin()+EndOfSource,Files.end()) == false)
1594       return false;
1595
1596    if (_error->PendingError() == true)
1597       return false;
1598    *OutMap = Map.UnGuard();
1599
1600    return true;
1601 }
1602                                                                         /*}}}*/
1603 // IsDuplicateDescription                                               /*{{{*/
1604 static bool IsDuplicateDescription(pkgCache::DescIterator Desc,
1605                             MD5SumValue const &CurMd5, std::string const &CurLang)
1606 {
1607    // Descriptions in the same link-list have all the same md5
1608    if (Desc.end() == true || MD5SumValue(Desc.md5()) != CurMd5)
1609       return false;
1610    for (; Desc.end() == false; ++Desc)
1611       if (Desc.LanguageCode() == CurLang)
1612          return true;
1613    return false;
1614 }
1615                                                                         /*}}}*/
1616 // CacheGenerator::FinishCache                                          /*{{{*/
1617 bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
1618 {
1619    return true;
1620 }
1621                                                                         /*}}}*/