apt-pkg/pkgcachegen.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: pkgcachegen.cc,v 1.53.2.1 2003/12/24 23:09:17 mdz Exp $
   4 /* ######################################################################
   5
   6    Package Cache Generator - Generator for the cache structure.
   7
   8    This builds the cache structure from the abstract package list parser.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #define APT_COMPATIBILITY 986
  14
  15 #include <apt-pkg/pkgcachegen.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/version.h>
  18 #include <apt-pkg/progress.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/aptconfiguration.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/sptr.h>
  24 #include <apt-pkg/pkgsystem.h>
  25 #include <apt-pkg/macros.h>
  26
  27 #include <apt-pkg/tagfile.h>
  28
  29 #include <apti18n.h>
  30
  31 #include <vector>
  32
  33 #include <sys/stat.h>
  34 #include <unistd.h>
  35 #include <errno.h>
  36 #include <stdio.h>
  37                                                                         /*}}}*/
  38 typedef vector<pkgIndexFile *>::iterator FileIterator;
  39 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
  40
  41 // CacheGenerator::pkgCacheGenerator - Constructor                      /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* We set the dirty flag and make sure that is written to the disk */
  44 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
  45                     Map(*pMap), Cache(pMap,false), Progress(Prog),
  46                     FoundFileDeps(0)
  47 {
  48    CurrentFile = 0;
  49    memset(UniqHash,0,sizeof(UniqHash));
  50
  51    if (_error->PendingError() == true)
  52       return;
  53
  54    if (Map.Size() == 0)
  55    {
  56       // Setup the map interface..
  57       Cache.HeaderP = (pkgCache::Header *)Map.Data();
  58       if (Map.RawAllocate(sizeof(pkgCache::Header)) == 0 && _error->PendingError() == true)
  59          return;
  60
  61       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  62
  63       // Starting header
  64       *Cache.HeaderP = pkgCache::Header();
  65       map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label);
  66       Cache.HeaderP->VerSysName = idxVerSysName;
  67       map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture"));
  68       Cache.HeaderP->Architecture = idxArchitecture;
  69       if (unlikely(idxVerSysName == 0 || idxArchitecture == 0))
  70          return;
  71       Cache.ReMap();
  72    }
  73    else
  74    {
  75       // Map directly from the existing file
  76       Cache.ReMap();
  77       Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
  78       if (Cache.VS != _system->VS)
  79       {
  80          _error->Error(_("Cache has an incompatible versioning system"));
  81          return;
  82       }
  83    }
  84
  85    Cache.HeaderP->Dirty = true;
  86    Map.Sync(0,sizeof(pkgCache::Header));
  87 }
  88                                                                         /*}}}*/
  89 // CacheGenerator::~pkgCacheGenerator - Destructor                      /*{{{*/
  90 // ---------------------------------------------------------------------
  91 /* We sync the data then unset the dirty flag in two steps so as to
  92    advoid a problem during a crash */
  93 pkgCacheGenerator::~pkgCacheGenerator()
  94 {
  95    if (_error->PendingError() == true)
  96       return;
  97    if (Map.Sync() == false)
  98       return;
  99
 100    Cache.HeaderP->Dirty = false;
 101    Map.Sync(0,sizeof(pkgCache::Header));
 102 }
 103                                                                         /*}}}*/
 104 void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/
 105    if (oldMap == newMap)
 106       return;
 107
 108    if (_config->FindB("Debug::pkgCacheGen", false))
 109       std::clog << "Remaping from " << oldMap << " to " << newMap << std::endl;
 110
 111    Cache.ReMap(false);
 112
 113    CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap;
 114
 115    for (size_t i = 0; i < _count(UniqHash); ++i)
 116       if (UniqHash[i] != 0)
 117          UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap;
 118
 119    for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
 120         i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
 121       (*i)->ReMap(oldMap, newMap);
 122    for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
 123         i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
 124       (*i)->ReMap(oldMap, newMap);
 125    for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
 126         i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
 127       (*i)->ReMap(oldMap, newMap);
 128    for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
 129         i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
 130       (*i)->ReMap(oldMap, newMap);
 131    for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
 132         i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
 133       (*i)->ReMap(oldMap, newMap);
 134    for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
 135         i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
 136       (*i)->ReMap(oldMap, newMap);
 137    for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
 138         i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
 139       (*i)->ReMap(oldMap, newMap);
 140 }                                                                       /*}}}*/
 141 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 142 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String,
 143                                         const unsigned long &Len) {
 144    void const * const oldMap = Map.Data();
 145    map_ptrloc const index = Map.WriteString(String, Len);
 146    if (index != 0)
 147       ReMap(oldMap, Map.Data());
 148    return index;
 149 }
 150                                                                         /*}}}*/
 151 // CacheGenerator::WriteStringInMap                                     /*{{{*/
 152 map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) {
 153    void const * const oldMap = Map.Data();
 154    map_ptrloc const index = Map.WriteString(String);
 155    if (index != 0)
 156       ReMap(oldMap, Map.Data());
 157    return index;
 158 }
 159                                                                         /*}}}*/
 160 map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/
 161    void const * const oldMap = Map.Data();
 162    map_ptrloc const index = Map.Allocate(size);
 163    if (index != 0)
 164       ReMap(oldMap, Map.Data());
 165    return index;
 166 }
 167                                                                         /*}}}*/
 168 // CacheGenerator::MergeList - Merge the package list                   /*{{{*/
 169 // ---------------------------------------------------------------------
 170 /* This provides the generation of the entries in the cache. Each loop
 171    goes through a single package record from the underlying parse engine. */
 172 bool pkgCacheGenerator::MergeList(ListParser &List,
 173                                   pkgCache::VerIterator *OutVer)
 174 {
 175    List.Owner = this;
 176
 177    unsigned int Counter = 0;
 178    while (List.Step() == true)
 179    {
 180       string const PackageName = List.Package();
 181       if (PackageName.empty() == true)
 182          return false;
 183
 184       string const Arch = List.Architecture();
 185
 186       // Get a pointer to the package structure
 187       pkgCache::PkgIterator Pkg;
 188       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 189       if (NewPackage(Pkg, PackageName, Arch) == false)
 190          return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str());
 191       Counter++;
 192       if (Counter % 100 == 0 && Progress != 0)
 193          Progress->Progress(List.Offset());
 194
 195       /* Get a pointer to the version structure. We know the list is sorted
 196          so we use that fact in the search. Insertion of new versions is
 197          done with correct sorting */
 198       string Version = List.Version();
 199       if (Version.empty() == true)
 200       {
 201          // we first process the package, then the descriptions
 202          // (this has the bonus that we get MMap error when we run out
 203          //  of MMap space)
 204          pkgCache::VerIterator Ver(Cache);
 205          Dynamic<pkgCache::VerIterator> DynVer(Ver);
 206          if (List.UsePackage(Pkg, Ver) == false)
 207             return _error->Error(_("Error occurred while processing %s (UsePackage1)"),
 208                                  PackageName.c_str());
 209
 210          // Find the right version to write the description
 211          MD5SumValue CurMd5 = List.Description_md5();
 212          Ver = Pkg.VersionList();
 213
 214          for (; Ver.end() == false; ++Ver)
 215          {
 216             pkgCache::DescIterator Desc = Ver.DescriptionList();
 217             Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 218             map_ptrloc *LastDesc = &Ver->DescriptionList;
 219             bool duplicate=false;
 220
 221             // don't add a new description if we have one for the given
 222             // md5 && language
 223             for ( ; Desc.end() == false; Desc++)
 224                if (MD5SumValue(Desc.md5()) == CurMd5 &&
 225                    Desc.LanguageCode() == List.DescriptionLanguage())
 226                   duplicate=true;
 227             if(duplicate)
 228                continue;
 229
 230             for (Desc = Ver.DescriptionList();
 231                  Desc.end() == false;
 232                  LastDesc = &Desc->NextDesc, Desc++)
 233             {
 234                if (MD5SumValue(Desc.md5()) == CurMd5)
 235                {
 236                   // Add new description
 237                   void const * const oldMap = Map.Data();
 238                   map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc);
 239                   if (oldMap != Map.Data())
 240                      LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 241                   *LastDesc = descindex;
 242                   Desc->ParentPkg = Pkg.Index();
 243
 244                   if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 245                      return _error->Error(_("Error occurred while processing %s (NewFileDesc1)"),PackageName.c_str());
 246                   break;
 247                }
 248             }
 249          }
 250
 251          continue;
 252       }
 253
 254       pkgCache::VerIterator Ver = Pkg.VersionList();
 255       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 256       map_ptrloc *LastVer = &Pkg->VersionList;
 257       void const * oldMap = Map.Data();
 258       int Res = 1;
 259       unsigned long const Hash = List.VersionHash();
 260       for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++)
 261       {
 262          Res = Cache.VS->CmpVersion(Version,Ver.VerStr());
 263          // Version is higher as current version - insert here
 264          if (Res > 0)
 265             break;
 266          // Versionstrings are equal - is hash also equal?
 267          if (Res == 0 && Ver->Hash == Hash)
 268             break;
 269          // proceed with the next till we have either the right
 270          // or we found another version (which will be lower)
 271       }
 272
 273       /* We already have a version for this item, record that we saw it */
 274       if (Res == 0 && Ver.end() == false && Ver->Hash == Hash)
 275       {
 276          if (List.UsePackage(Pkg,Ver) == false)
 277             return _error->Error(_("Error occurred while processing %s (UsePackage2)"),
 278                                  PackageName.c_str());
 279
 280          if (NewFileVer(Ver,List) == false)
 281             return _error->Error(_("Error occurred while processing %s (NewFileVer1)"),
 282                                  PackageName.c_str());
 283
 284          // Read only a single record and return
 285          if (OutVer != 0)
 286          {
 287             *OutVer = Ver;
 288             FoundFileDeps |= List.HasFileDeps();
 289             return true;
 290          }
 291
 292          continue;
 293       }
 294
 295       // Add a new version
 296       map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer);
 297       if (verindex == 0 && _error->PendingError())
 298          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 299                               PackageName.c_str(), 1);
 300
 301       if (oldMap != Map.Data())
 302          LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 303       *LastVer = verindex;
 304       Ver->ParentPkg = Pkg.Index();
 305       Ver->Hash = Hash;
 306
 307       if (List.NewVersion(Ver) == false)
 308          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 309                               PackageName.c_str(), 2);
 310
 311       if (List.UsePackage(Pkg,Ver) == false)
 312          return _error->Error(_("Error occurred while processing %s (UsePackage3)"),
 313                               PackageName.c_str());
 314
 315       if (NewFileVer(Ver,List) == false)
 316          return _error->Error(_("Error occurred while processing %s (NewVersion%d)"),
 317                               PackageName.c_str(), 3);
 318
 319       // Read only a single record and return
 320       if (OutVer != 0)
 321       {
 322          *OutVer = Ver;
 323          FoundFileDeps |= List.HasFileDeps();
 324          return true;
 325       }
 326
 327       /* Record the Description data. Description data always exist in
 328          Packages and Translation-* files. */
 329       pkgCache::DescIterator Desc = Ver.DescriptionList();
 330       Dynamic<pkgCache::DescIterator> DynDesc(Desc);
 331       map_ptrloc *LastDesc = &Ver->DescriptionList;
 332
 333       // Skip to the end of description set
 334       for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++);
 335
 336       // Add new description
 337       oldMap = Map.Data();
 338       map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc);
 339       if (oldMap != Map.Data())
 340          LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 341       *LastDesc = descindex;
 342       Desc->ParentPkg = Pkg.Index();
 343
 344       if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false)
 345          return _error->Error(_("Error occurred while processing %s (NewFileDesc2)"),PackageName.c_str());
 346    }
 347
 348    FoundFileDeps |= List.HasFileDeps();
 349
 350    if (Cache.HeaderP->PackageCount >= (1ULL<<sizeof(Cache.PkgP->ID)*8)-1)
 351       return _error->Error(_("Wow, you exceeded the number of package "
 352                              "names this APT is capable of."));
 353    if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1)
 354       return _error->Error(_("Wow, you exceeded the number of versions "
 355                              "this APT is capable of."));
 356    if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1)
 357       return _error->Error(_("Wow, you exceeded the number of descriptions "
 358                              "this APT is capable of."));
 359    if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL)
 360       return _error->Error(_("Wow, you exceeded the number of dependencies "
 361                              "this APT is capable of."));
 362    return true;
 363 }
 364                                                                         /*}}}*/
 365 // CacheGenerator::MergeFileProvides - Merge file provides              /*{{{*/
 366 // ---------------------------------------------------------------------
 367 /* If we found any file depends while parsing the main list we need to
 368    resolve them. Since it is undesired to load the entire list of files
 369    into the cache as virtual packages we do a two stage effort. MergeList
 370    identifies the file depends and this creates Provdies for them by
 371    re-parsing all the indexs. */
 372 bool pkgCacheGenerator::MergeFileProvides(ListParser &List)
 373 {
 374    List.Owner = this;
 375
 376    unsigned int Counter = 0;
 377    while (List.Step() == true)
 378    {
 379       string PackageName = List.Package();
 380       if (PackageName.empty() == true)
 381          return false;
 382       string Version = List.Version();
 383       if (Version.empty() == true)
 384          continue;
 385
 386       pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName);
 387       Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 388       if (Pkg.end() == true)
 389          return _error->Error(_("Error occurred while processing %s (FindPkg)"),
 390                                 PackageName.c_str());
 391       Counter++;
 392       if (Counter % 100 == 0 && Progress != 0)
 393          Progress->Progress(List.Offset());
 394
 395       unsigned long Hash = List.VersionHash();
 396       pkgCache::VerIterator Ver = Pkg.VersionList();
 397       Dynamic<pkgCache::VerIterator> DynVer(Ver);
 398       for (; Ver.end() == false; Ver++)
 399       {
 400          if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr())
 401          {
 402             if (List.CollectFileProvides(Cache,Ver) == false)
 403                return _error->Error(_("Error occurred while processing %s (CollectFileProvides)"),PackageName.c_str());
 404             break;
 405          }
 406       }
 407
 408       if (Ver.end() == true)
 409          _error->Warning(_("Package %s %s was not found while processing file dependencies"),PackageName.c_str(),Version.c_str());
 410    }
 411
 412    return true;
 413 }
 414                                                                         /*}}}*/
 415 // CacheGenerator::NewGroup - Add a new group                           /*{{{*/
 416 // ---------------------------------------------------------------------
 417 /* This creates a new group structure and adds it to the hash table */
 418 bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name)
 419 {
 420    Grp = Cache.FindGrp(Name);
 421    if (Grp.end() == false)
 422       return true;
 423
 424    // Get a structure
 425    map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group));
 426    if (unlikely(Group == 0))
 427       return false;
 428
 429    Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group);
 430    map_ptrloc const idxName = WriteStringInMap(Name);
 431    if (unlikely(idxName == 0))
 432       return false;
 433    Grp->Name = idxName;
 434
 435    // Insert it into the hash table
 436    unsigned long const Hash = Cache.Hash(Name);
 437    Grp->Next = Cache.HeaderP->GrpHashTable[Hash];
 438    Cache.HeaderP->GrpHashTable[Hash] = Group;
 439
 440    Grp->ID = Cache.HeaderP->GroupCount++;
 441    return true;
 442 }
 443                                                                         /*}}}*/
 444 // CacheGenerator::NewPackage - Add a new package                       /*{{{*/
 445 // ---------------------------------------------------------------------
 446 /* This creates a new package structure and adds it to the hash table */
 447 bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name,
 448                                         const string &Arch) {
 449    pkgCache::GrpIterator Grp;
 450    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 451    if (unlikely(NewGroup(Grp, Name) == false))
 452       return false;
 453
 454    Pkg = Grp.FindPkg(Arch);
 455       if (Pkg.end() == false)
 456          return true;
 457
 458    // Get a structure
 459    map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package));
 460    if (unlikely(Package == 0))
 461       return false;
 462    Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
 463
 464    // Insert the package into our package list
 465    if (Grp->FirstPackage == 0) // the group is new
 466    {
 467       // Insert it into the hash table
 468       unsigned long const Hash = Cache.Hash(Name);
 469       Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash];
 470       Cache.HeaderP->PkgHashTable[Hash] = Package;
 471       Grp->FirstPackage = Package;
 472    }
 473    else // Group the Packages together
 474    {
 475       // this package is the new last package
 476       pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage);
 477       Pkg->NextPackage = LastPkg->NextPackage;
 478       LastPkg->NextPackage = Package;
 479    }
 480    Grp->LastPackage = Package;
 481
 482    // Set the name, arch and the ID
 483    Pkg->Name = Grp->Name;
 484    Pkg->Group = Grp.Index();
 485    // all is mapped to the native architecture
 486    map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str());
 487    if (unlikely(idxArch == 0))
 488       return false;
 489    Pkg->Arch = idxArch;
 490    Pkg->ID = Cache.HeaderP->PackageCount++;
 491
 492    return true;
 493 }
 494                                                                         /*}}}*/
 495 // CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
 496 // ---------------------------------------------------------------------
 497 /* */
 498 bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
 499                                    ListParser &List)
 500 {
 501    if (CurrentFile == 0)
 502       return true;
 503
 504    // Get a structure
 505    map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile));
 506    if (VerFile == 0)
 507       return 0;
 508
 509    pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
 510    VF->File = CurrentFile - Cache.PkgFileP;
 511
 512    // Link it to the end of the list
 513    map_ptrloc *Last = &Ver->FileList;
 514    for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
 515       Last = &V->NextFile;
 516    VF->NextFile = *Last;
 517    *Last = VF.Index();
 518
 519    VF->Offset = List.Offset();
 520    VF->Size = List.Size();
 521    if (Cache.HeaderP->MaxVerFileSize < VF->Size)
 522       Cache.HeaderP->MaxVerFileSize = VF->Size;
 523    Cache.HeaderP->VerFileCount++;
 524
 525    return true;
 526 }
 527                                                                         /*}}}*/
 528 // CacheGenerator::NewVersion - Create a new Version                    /*{{{*/
 529 // ---------------------------------------------------------------------
 530 /* This puts a version structure in the linked list */
 531 unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
 532                                             const string &VerStr,
 533                                             unsigned long Next)
 534 {
 535    // Get a structure
 536    map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version));
 537    if (Version == 0)
 538       return 0;
 539
 540    // Fill it in
 541    Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
 542    Ver->NextVer = Next;
 543    Ver->ID = Cache.HeaderP->VersionCount++;
 544    map_ptrloc const idxVerStr = WriteStringInMap(VerStr);
 545    if (unlikely(idxVerStr == 0))
 546       return 0;
 547    Ver->VerStr = idxVerStr;
 548
 549    return Version;
 550 }
 551                                                                         /*}}}*/
 552 // CacheGenerator::NewFileDesc - Create a new File<->Desc association   /*{{{*/
 553 // ---------------------------------------------------------------------
 554 /* */
 555 bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc,
 556                                    ListParser &List)
 557 {
 558    if (CurrentFile == 0)
 559       return true;
 560
 561    // Get a structure
 562    map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile));
 563    if (DescFile == 0)
 564       return false;
 565
 566    pkgCache::DescFileIterator DF(Cache,Cache.DescFileP + DescFile);
 567    DF->File = CurrentFile - Cache.PkgFileP;
 568
 569    // Link it to the end of the list
 570    map_ptrloc *Last = &Desc->FileList;
 571    for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; D++)
 572       Last = &D->NextFile;
 573
 574    DF->NextFile = *Last;
 575    *Last = DF.Index();
 576
 577    DF->Offset = List.Offset();
 578    DF->Size = List.Size();
 579    if (Cache.HeaderP->MaxDescFileSize < DF->Size)
 580       Cache.HeaderP->MaxDescFileSize = DF->Size;
 581    Cache.HeaderP->DescFileCount++;
 582
 583    return true;
 584 }
 585                                                                         /*}}}*/
 586 // CacheGenerator::NewDescription - Create a new Description            /*{{{*/
 587 // ---------------------------------------------------------------------
 588 /* This puts a description structure in the linked list */
 589 map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc,
 590                                             const string &Lang,
 591                                             const MD5SumValue &md5sum,
 592                                             map_ptrloc Next)
 593 {
 594    // Get a structure
 595    map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description));
 596    if (Description == 0)
 597       return 0;
 598
 599    // Fill it in
 600    Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description);
 601    Desc->NextDesc = Next;
 602    Desc->ID = Cache.HeaderP->DescriptionCount++;
 603    map_ptrloc const idxlanguage_code = WriteStringInMap(Lang);
 604    map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value());
 605    if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0))
 606       return 0;
 607    Desc->language_code = idxlanguage_code;
 608    Desc->md5sum = idxmd5sum;
 609
 610    return Description;
 611 }
 612                                                                         /*}}}*/
 613 // CacheGenerator::FinishCache - do various finish operations           /*{{{*/
 614 // ---------------------------------------------------------------------
 615 /* This prepares the Cache for delivery */
 616 bool pkgCacheGenerator::FinishCache(OpProgress *Progress)
 617 {
 618    // FIXME: add progress reporting for this operation
 619    // Do we have different architectures in your groups ?
 620    vector<string> archs = APT::Configuration::getArchitectures();
 621    if (archs.size() > 1)
 622    {
 623       // Create Conflicts in between the group
 624       pkgCache::GrpIterator G = GetCache().GrpBegin();
 625       Dynamic<pkgCache::GrpIterator> DynG(G);
 626       for (; G.end() != true; G++)
 627       {
 628          string const PkgName = G.Name();
 629          pkgCache::PkgIterator P = G.PackageList();
 630          Dynamic<pkgCache::PkgIterator> DynP(P);
 631          for (; P.end() != true; P = G.NextPkg(P))
 632          {
 633             pkgCache::PkgIterator allPkg;
 634             Dynamic<pkgCache::PkgIterator> DynallPkg(allPkg);
 635             pkgCache::VerIterator V = P.VersionList();
 636             Dynamic<pkgCache::VerIterator> DynV(V);
 637             for (; V.end() != true; V++)
 638             {
 639                char const * const Arch = P.Arch();
 640                map_ptrloc *OldDepLast = NULL;
 641                /* MultiArch handling introduces a lot of implicit Dependencies:
 642                 - MultiArch: same → Co-Installable if they have the same version
 643                 - Architecture: all → Need to be Co-Installable for internal reasons
 644                 - All others conflict with all other group members */
 645                bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same);
 646                for (vector<string>::const_iterator A = archs.begin(); A != archs.end(); ++A)
 647                {
 648                   if (*A == Arch)
 649                      continue;
 650                   /* We allow only one installed arch at the time
 651                      per group, therefore each group member conflicts
 652                      with all other group members */
 653                   pkgCache::PkgIterator D = G.FindPkg(*A);
 654                   Dynamic<pkgCache::PkgIterator> DynD(D);
 655                   if (D.end() == true)
 656                      continue;
 657                   if (coInstall == true)
 658                   {
 659                      // Replaces: ${self}:other ( << ${binary:Version})
 660                      NewDepends(D, V, V.VerStr(),
 661                                 pkgCache::Dep::Less, pkgCache::Dep::Replaces,
 662                                 OldDepLast);
 663                      // Breaks: ${self}:other (!= ${binary:Version})
 664                      NewDepends(D, V, V.VerStr(),
 665                                 pkgCache::Dep::NotEquals, pkgCache::Dep::DpkgBreaks,
 666                                 OldDepLast);
 667                   } else {
 668                         // Conflicts: ${self}:other
 669                         NewDepends(D, V, "",
 670                                    pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts,
 671                                    OldDepLast);
 672                   }
 673                }
 674             }
 675          }
 676       }
 677    }
 678    return true;
 679 }
 680                                                                         /*}}}*/
 681 // CacheGenerator::NewDepends - Create a dependency element             /*{{{*/
 682 // ---------------------------------------------------------------------
 683 /* This creates a dependency element in the tree. It is linked to the
 684    version and to the package that it is pointing to. */
 685 bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
 686                                    pkgCache::VerIterator &Ver,
 687                                    string const &Version,
 688                                    unsigned int const &Op,
 689                                    unsigned int const &Type,
 690                                    map_ptrloc* &OldDepLast)
 691 {
 692    void const * const oldMap = Map.Data();
 693    // Get a structure
 694    map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency));
 695    if (unlikely(Dependency == 0))
 696       return false;
 697
 698    // Fill it in
 699    pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
 700    Dynamic<pkgCache::DepIterator> DynDep(Dep);
 701    Dep->ParentVer = Ver.Index();
 702    Dep->Type = Type;
 703    Dep->CompareOp = Op;
 704    Dep->ID = Cache.HeaderP->DependsCount++;
 705
 706    // Probe the reverse dependency list for a version string that matches
 707    if (Version.empty() == false)
 708    {
 709 /*      for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
 710          if (I->Version != 0 && I.TargetVer() == Version)
 711             Dep->Version = I->Version;*/
 712       if (Dep->Version == 0) {
 713          map_ptrloc const index = WriteStringInMap(Version);
 714          if (unlikely(index == 0))
 715             return false;
 716          Dep->Version = index;
 717       }
 718    }
 719
 720    // Link it to the package
 721    Dep->Package = Pkg.Index();
 722    Dep->NextRevDepends = Pkg->RevDepends;
 723    Pkg->RevDepends = Dep.Index();
 724
 725    // Do we know where to link the Dependency to?
 726    if (OldDepLast == NULL)
 727    {
 728       OldDepLast = &Ver->DependsList;
 729       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
 730          OldDepLast = &D->NextDepends;
 731    } else if (oldMap != Map.Data())
 732       OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 733
 734    Dep->NextDepends = *OldDepLast;
 735    *OldDepLast = Dep.Index();
 736    OldDepLast = &Dep->NextDepends;
 737
 738    return true;
 739 }
 740                                                                         /*}}}*/
 741 // ListParser::NewDepends - Create the environment for a new dependency /*{{{*/
 742 // ---------------------------------------------------------------------
 743 /* This creates a Group and the Package to link this dependency to if
 744    needed and handles also the caching of the old endpoint */
 745 bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver,
 746                                                const string &PackageName,
 747                                                const string &Arch,
 748                                                const string &Version,
 749                                                unsigned int Op,
 750                                                unsigned int Type)
 751 {
 752    pkgCache::GrpIterator Grp;
 753    Dynamic<pkgCache::GrpIterator> DynGrp(Grp);
 754    if (unlikely(Owner->NewGroup(Grp, PackageName) == false))
 755       return false;
 756
 757    // Locate the target package
 758    pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch);
 759    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 760    if (Pkg.end() == true) {
 761       if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false))
 762          return false;
 763    }
 764
 765    // Is it a file dependency?
 766    if (unlikely(PackageName[0] == '/'))
 767       FoundFileDeps = true;
 768
 769    /* Caching the old end point speeds up generation substantially */
 770    if (OldDepVer != Ver) {
 771       OldDepLast = NULL;
 772       OldDepVer = Ver;
 773    }
 774
 775    return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast);
 776 }
 777                                                                         /*}}}*/
 778 // ListParser::NewProvides - Create a Provides element                  /*{{{*/
 779 // ---------------------------------------------------------------------
 780 /* */
 781 bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver,
 782                                                 const string &PkgName,
 783                                                 const string &PkgArch,
 784                                                 const string &Version)
 785 {
 786    pkgCache &Cache = Owner->Cache;
 787
 788    // We do not add self referencing provides
 789    if (Ver.ParentPkg().Name() == PkgName && (PkgArch == Ver.ParentPkg().Arch() ||
 790         (PkgArch == "all" && strcmp((Cache.StrP + Cache.HeaderP->Architecture), Ver.ParentPkg().Arch()) == 0)))
 791       return true;
 792
 793    // Get a structure
 794    map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides));
 795    if (unlikely(Provides == 0))
 796       return false;
 797    Cache.HeaderP->ProvidesCount++;
 798
 799    // Fill it in
 800    pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
 801    Dynamic<pkgCache::PrvIterator> DynPrv(Prv);
 802    Prv->Version = Ver.Index();
 803    Prv->NextPkgProv = Ver->ProvidesList;
 804    Ver->ProvidesList = Prv.Index();
 805    if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0))
 806       return false;
 807
 808    // Locate the target package
 809    pkgCache::PkgIterator Pkg;
 810    Dynamic<pkgCache::PkgIterator> DynPkg(Pkg);
 811    if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false))
 812       return false;
 813
 814    // Link it to the package
 815    Prv->ParentPkg = Pkg.Index();
 816    Prv->NextProvides = Pkg->ProvidesList;
 817    Pkg->ProvidesList = Prv.Index();
 818
 819    return true;
 820 }
 821                                                                         /*}}}*/
 822 // CacheGenerator::SelectFile - Select the current file being parsed    /*{{{*/
 823 // ---------------------------------------------------------------------
 824 /* This is used to select which file is to be associated with all newly
 825    added versions. The caller is responsible for setting the IMS fields. */
 826 bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 827                                    const pkgIndexFile &Index,
 828                                    unsigned long Flags)
 829 {
 830    // Get some space for the structure
 831    map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile));
 832    if (unlikely(idxFile == 0))
 833       return false;
 834    CurrentFile = Cache.PkgFileP + idxFile;
 835
 836    // Fill it in
 837    map_ptrloc const idxFileName = WriteStringInMap(File);
 838    map_ptrloc const idxSite = WriteUniqString(Site);
 839    if (unlikely(idxFileName == 0 || idxSite == 0))
 840       return false;
 841    CurrentFile->FileName = idxFileName;
 842    CurrentFile->Site = idxSite;
 843    CurrentFile->NextFile = Cache.HeaderP->FileList;
 844    CurrentFile->Flags = Flags;
 845    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
 846    map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label);
 847    if (unlikely(idxIndexType == 0))
 848       return false;
 849    CurrentFile->IndexType = idxIndexType;
 850    PkgFileName = File;
 851    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
 852    Cache.HeaderP->PackageFileCount++;
 853
 854    if (Progress != 0)
 855       Progress->SubProgress(Index.Size());
 856    return true;
 857 }
 858                                                                         /*}}}*/
 859 // CacheGenerator::WriteUniqueString - Insert a unique string           /*{{{*/
 860 // ---------------------------------------------------------------------
 861 /* This is used to create handles to strings. Given the same text it
 862    always returns the same number */
 863 unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 864                                                  unsigned int Size)
 865 {
 866    /* We use a very small transient hash table here, this speeds up generation
 867       by a fair amount on slower machines */
 868    pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
 869    if (Bucket != 0 &&
 870        stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
 871       return Bucket->String;
 872
 873    // Search for an insertion point
 874    pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
 875    int Res = 1;
 876    map_ptrloc *Last = &Cache.HeaderP->StringList;
 877    for (; I != Cache.StringItemP; Last = &I->NextItem,
 878         I = Cache.StringItemP + I->NextItem)
 879    {
 880       Res = stringcmp(S,S+Size,Cache.StrP + I->String);
 881       if (Res >= 0)
 882          break;
 883    }
 884
 885    // Match
 886    if (Res == 0)
 887    {
 888       Bucket = I;
 889       return I->String;
 890    }
 891
 892    // Get a structure
 893    void const * const oldMap = Map.Data();
 894    map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem));
 895    if (Item == 0)
 896       return 0;
 897
 898    map_ptrloc const idxString = WriteStringInMap(S,Size);
 899    if (unlikely(idxString == 0))
 900       return 0;
 901    if (oldMap != Map.Data()) {
 902       Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap;
 903       I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap;
 904    }
 905    *Last = Item;
 906
 907    // Fill in the structure
 908    pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
 909    ItemP->NextItem = I - Cache.StringItemP;
 910    ItemP->String = idxString;
 911
 912    Bucket = ItemP;
 913    return ItemP->String;
 914 }
 915                                                                         /*}}}*/
 916 // CheckValidity - Check that a cache is up-to-date                     /*{{{*/
 917 // ---------------------------------------------------------------------
 918 /* This just verifies that each file in the list of index files exists,
 919    has matching attributes with the cache and the cache does not have
 920    any extra files. */
 921 static bool CheckValidity(const string &CacheFile,
 922                           pkgSourceList &List,
 923                           FileIterator Start,
 924                           FileIterator End,
 925                           MMap **OutMap = 0)
 926 {
 927    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 928    // No file, certainly invalid
 929    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
 930    {
 931       if (Debug == true)
 932          std::clog << "CacheFile doesn't exist" << std::endl;
 933       return false;
 934    }
 935
 936    if (List.GetLastModifiedTime() < GetModificationTime(CacheFile))
 937    {
 938       if (Debug == true)
 939          std::clog << "sources.list is newer than the cache" << std::endl;
 940       return false;
 941    }
 942
 943    // Map it
 944    FileFd CacheF(CacheFile,FileFd::ReadOnly);
 945    SPtr<MMap> Map = new MMap(CacheF,0);
 946    pkgCache Cache(Map);
 947    if (_error->PendingError() == true || Map->Size() == 0)
 948    {
 949       if (Debug == true)
 950          std::clog << "Errors are pending or Map is empty()" << std::endl;
 951       _error->Discard();
 952       return false;
 953    }
 954
 955    /* Now we check every index file, see if it is in the cache,
 956       verify the IMS data and check that it is on the disk too.. */
 957    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
 958    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
 959    for (; Start != End; Start++)
 960    {
 961       if (Debug == true)
 962          std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
 963       if ((*Start)->HasPackages() == false)
 964       {
 965          if (Debug == true)
 966             std::clog << "Has NO packages" << std::endl;
 967          continue;
 968       }
 969
 970       if ((*Start)->Exists() == false)
 971       {
 972 #if 0 // mvo: we no longer give a message here (Default Sources spec)
 973          _error->WarningE("stat",_("Couldn't stat source package list %s"),
 974                           (*Start)->Describe().c_str());
 975 #endif
 976          if (Debug == true)
 977             std::clog << "file doesn't exist" << std::endl;
 978          continue;
 979       }
 980
 981       // FindInCache is also expected to do an IMS check.
 982       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
 983       if (File.end() == true)
 984       {
 985          if (Debug == true)
 986             std::clog << "FindInCache returned end-Pointer" << std::endl;
 987          return false;
 988       }
 989
 990       Visited[File->ID] = true;
 991       if (Debug == true)
 992          std::clog << "with ID " << File->ID << " is valid" << std::endl;
 993    }
 994
 995    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
 996       if (Visited[I] == false)
 997       {
 998          if (Debug == true)
 999             std::clog << "File with ID" << I << " wasn't visited" << std::endl;
1000          return false;
1001       }
1002
1003    if (_error->PendingError() == true)
1004    {
1005       if (Debug == true)
1006       {
1007          std::clog << "Validity failed because of pending errors:" << std::endl;
1008          _error->DumpErrors();
1009       }
1010       _error->Discard();
1011       return false;
1012    }
1013
1014    if (OutMap != 0)
1015       *OutMap = Map.UnGuard();
1016    return true;
1017 }
1018                                                                         /*}}}*/
1019 // ComputeSize - Compute the total size of a bunch of files             /*{{{*/
1020 // ---------------------------------------------------------------------
1021 /* Size is kind of an abstract notion that is only used for the progress
1022    meter */
1023 static unsigned long ComputeSize(FileIterator Start,FileIterator End)
1024 {
1025    unsigned long TotalSize = 0;
1026    for (; Start != End; Start++)
1027    {
1028       if ((*Start)->HasPackages() == false)
1029          continue;
1030       TotalSize += (*Start)->Size();
1031    }
1032    return TotalSize;
1033 }
1034                                                                         /*}}}*/
1035 // BuildCache - Merge the list of index files into the cache            /*{{{*/
1036 // ---------------------------------------------------------------------
1037 /* */
1038 static bool BuildCache(pkgCacheGenerator &Gen,
1039                        OpProgress *Progress,
1040                        unsigned long &CurrentSize,unsigned long TotalSize,
1041                        FileIterator Start, FileIterator End)
1042 {
1043    FileIterator I;
1044    for (I = Start; I != End; I++)
1045    {
1046       if ((*I)->HasPackages() == false)
1047          continue;
1048
1049       if ((*I)->Exists() == false)
1050          continue;
1051
1052       if ((*I)->FindInCache(Gen.GetCache()).end() == false)
1053       {
1054          _error->Warning("Duplicate sources.list entry %s",
1055                          (*I)->Describe().c_str());
1056          continue;
1057       }
1058
1059       unsigned long Size = (*I)->Size();
1060       if (Progress != NULL)
1061          Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
1062       CurrentSize += Size;
1063
1064       if ((*I)->Merge(Gen,Progress) == false)
1065          return false;
1066    }
1067
1068    if (Gen.HasFileDeps() == true)
1069    {
1070       if (Progress != NULL)
1071          Progress->Done();
1072       TotalSize = ComputeSize(Start, End);
1073       CurrentSize = 0;
1074       for (I = Start; I != End; I++)
1075       {
1076          unsigned long Size = (*I)->Size();
1077          if (Progress != NULL)
1078             Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides"));
1079          CurrentSize += Size;
1080          if ((*I)->MergeFileProvides(Gen,Progress) == false)
1081             return false;
1082       }
1083    }
1084
1085    return true;
1086 }
1087                                                                         /*}}}*/
1088 // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options  /*{{{*/
1089 DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
1090    unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
1091    unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
1092    unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0);
1093    Flags |= MMap::Moveable;
1094    if (_config->FindB("APT::Cache-Fallback", false) == true)
1095       Flags |= MMap::Fallback;
1096    if (CacheF != NULL)
1097       return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit);
1098    else
1099       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
1100 }
1101                                                                         /*}}}*/
1102 // CacheGenerator::MakeStatusCache - Construct the status cache         /*{{{*/
1103 // ---------------------------------------------------------------------
1104 /* This makes sure that the status cache (the cache that has all
1105    index files from the sources list and all local ones) is ready
1106    to be mmaped. If OutMap is not zero then a MMap object representing
1107    the cache will be stored there. This is pretty much mandetory if you
1108    are using AllowMem. AllowMem lets the function be run as non-root
1109    where it builds the cache 'fast' into a memory buffer. */
1110 __deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
1111                         MMap **OutMap, bool AllowMem)
1112    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
1113 bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
1114                         MMap **OutMap,bool AllowMem)
1115 {
1116    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
1117
1118    vector<pkgIndexFile *> Files;
1119    for (vector<metaIndex *>::const_iterator i = List.begin();
1120         i != List.end();
1121         i++)
1122    {
1123       vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
1124       for (vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
1125            j != Indexes->end();
1126            j++)
1127          Files.push_back (*j);
1128    }
1129
1130    unsigned long const EndOfSource = Files.size();
1131    if (_system->AddStatusFiles(Files) == false)
1132       return false;
1133
1134    // Decide if we can write to the files..
1135    string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
1136    string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
1137
1138    // ensure the cache directory exists
1139    if (CacheFile.empty() == false || SrcCacheFile.empty() == false)
1140    {
1141       string dir = _config->FindDir("Dir::Cache");
1142       size_t const len = dir.size();
1143       if (len > 5 && dir.find("/apt/", len - 6, 5) == len - 5)
1144          dir = dir.substr(0, len - 5);
1145       if (CacheFile.empty() == false)
1146          CreateDirectory(dir, flNotFile(CacheFile));
1147       if (SrcCacheFile.empty() == false)
1148          CreateDirectory(dir, flNotFile(SrcCacheFile));
1149    }
1150
1151    // Decide if we can write to the cache
1152    bool Writeable = false;
1153    if (CacheFile.empty() == false)
1154       Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
1155    else
1156       if (SrcCacheFile.empty() == false)
1157          Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
1158    if (Debug == true)
1159       std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
1160
1161    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
1162       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
1163
1164    if (Progress != NULL)
1165       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1166
1167    // Cache is OK, Fin.
1168    if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
1169    {
1170       if (Progress != NULL)
1171          Progress->OverallProgress(1,1,1,_("Reading package lists"));
1172       if (Debug == true)
1173          std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
1174       return true;
1175    }
1176    else if (Debug == true)
1177          std::clog << "pkgcache.bin is NOT valid" << std::endl;
1178
1179    /* At this point we know we need to reconstruct the package cache,
1180       begin. */
1181    SPtr<FileFd> CacheF;
1182    SPtr<DynamicMMap> Map;
1183    if (Writeable == true && CacheFile.empty() == false)
1184    {
1185       _error->PushToStack();
1186       unlink(CacheFile.c_str());
1187       CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
1188       fchmod(CacheF->Fd(),0644);
1189       Map = CreateDynamicMMap(CacheF, MMap::Public);
1190       if (_error->PendingError() == true)
1191       {
1192          delete CacheF.UnGuard();
1193          delete Map.UnGuard();
1194          if (Debug == true)
1195             std::clog << "Open filebased MMap FAILED" << std::endl;
1196          Writeable = false;
1197          if (AllowMem == false)
1198          {
1199             _error->MergeWithStack();
1200             return false;
1201          }
1202          _error->RevertToStack();
1203       }
1204       else if (Debug == true)
1205       {
1206          _error->MergeWithStack();
1207          std::clog << "Open filebased MMap" << std::endl;
1208       }
1209    }
1210    if (Writeable == false || CacheFile.empty() == true)
1211    {
1212       // Just build it in memory..
1213       Map = CreateDynamicMMap(NULL);
1214       if (Debug == true)
1215          std::clog << "Open memory Map (not filebased)" << std::endl;
1216    }
1217
1218    // Lets try the source cache.
1219    unsigned long CurrentSize = 0;
1220    unsigned long TotalSize = 0;
1221    if (CheckValidity(SrcCacheFile, List, Files.begin(),
1222                      Files.begin()+EndOfSource) == true)
1223    {
1224       if (Debug == true)
1225          std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
1226       // Preload the map with the source cache
1227       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
1228       unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
1229       if ((alloc == 0 && _error->PendingError())
1230                 || SCacheF.Read((unsigned char *)Map->Data() + alloc,
1231                                 SCacheF.Size()) == false)
1232          return false;
1233
1234       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1235
1236       // Build the status cache
1237       pkgCacheGenerator Gen(Map.Get(),Progress);
1238       if (_error->PendingError() == true)
1239          return false;
1240       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1241                      Files.begin()+EndOfSource,Files.end()) == false)
1242          return false;
1243
1244       // FIXME: move me to a better place
1245       Gen.FinishCache(Progress);
1246    }
1247    else
1248    {
1249       if (Debug == true)
1250          std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
1251       TotalSize = ComputeSize(Files.begin(),Files.end());
1252
1253       // Build the source cache
1254       pkgCacheGenerator Gen(Map.Get(),Progress);
1255       if (_error->PendingError() == true)
1256          return false;
1257       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1258                      Files.begin(),Files.begin()+EndOfSource) == false)
1259          return false;
1260
1261       // Write it back
1262       if (Writeable == true && SrcCacheFile.empty() == false)
1263       {
1264          FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
1265          if (_error->PendingError() == true)
1266             return false;
1267
1268          fchmod(SCacheF.Fd(),0644);
1269
1270          // Write out the main data
1271          if (SCacheF.Write(Map->Data(),Map->Size()) == false)
1272             return _error->Error(_("IO Error saving source cache"));
1273          SCacheF.Sync();
1274
1275          // Write out the proper header
1276          Gen.GetCache().HeaderP->Dirty = false;
1277          if (SCacheF.Seek(0) == false ||
1278              SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
1279             return _error->Error(_("IO Error saving source cache"));
1280          Gen.GetCache().HeaderP->Dirty = true;
1281          SCacheF.Sync();
1282       }
1283
1284       // Build the status cache
1285       if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1286                      Files.begin()+EndOfSource,Files.end()) == false)
1287          return false;
1288
1289       // FIXME: move me to a better place
1290       Gen.FinishCache(Progress);
1291    }
1292    if (Debug == true)
1293       std::clog << "Caches are ready for shipping" << std::endl;
1294
1295    if (_error->PendingError() == true)
1296       return false;
1297    if (OutMap != 0)
1298    {
1299       if (CacheF != 0)
1300       {
1301          delete Map.UnGuard();
1302          *OutMap = new MMap(*CacheF,0);
1303       }
1304       else
1305       {
1306          *OutMap = Map.UnGuard();
1307       }
1308    }
1309
1310    return true;
1311 }
1312                                                                         /*}}}*/
1313 // CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/
1314 // ---------------------------------------------------------------------
1315 /* */
1316 __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
1317    { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); }
1318 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
1319 {
1320    vector<pkgIndexFile *> Files;
1321    unsigned long EndOfSource = Files.size();
1322    if (_system->AddStatusFiles(Files) == false)
1323       return false;
1324
1325    SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
1326    unsigned long CurrentSize = 0;
1327    unsigned long TotalSize = 0;
1328
1329    TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
1330
1331    // Build the status cache
1332    if (Progress != NULL)
1333       Progress->OverallProgress(0,1,1,_("Reading package lists"));
1334    pkgCacheGenerator Gen(Map.Get(),Progress);
1335    if (_error->PendingError() == true)
1336       return false;
1337    if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
1338                   Files.begin()+EndOfSource,Files.end()) == false)
1339       return false;
1340
1341    // FIXME: move me to a better place
1342    Gen.FinishCache(Progress);
1343
1344    if (_error->PendingError() == true)
1345       return false;
1346    *OutMap = Map.UnGuard();
1347
1348    return true;
1349 }
1350                                                                         /*}}}*/