Join with aliencode
[ntk/apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
b2e465d6 3// $Id: pkgcachegen.cc,v 1.46 2001/02/20 07:03:17 jgg Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
6c139d6e 13#ifdef __GNUG__
094a497d 14#pragma implementation "apt-pkg/pkgcachegen.h"
6c139d6e
AL
15#endif
16
b2e465d6
AL
17#define APT_COMPATIBILITY 986
18
094a497d
AL
19#include <apt-pkg/pkgcachegen.h>
20#include <apt-pkg/error.h>
21#include <apt-pkg/version.h>
b35d2f5f
AL
22#include <apt-pkg/progress.h>
23#include <apt-pkg/sourcelist.h>
24#include <apt-pkg/configuration.h>
cdcc6d34 25#include <apt-pkg/strutl.h>
b2e465d6
AL
26#include <apt-pkg/sptr.h>
27#include <apt-pkg/pkgsystem.h>
578bfd0a 28
b2e465d6
AL
29#include <apti18n.h>
30
578bfd0a
AL
31#include <sys/stat.h>
32#include <unistd.h>
803fafcb 33#include <errno.h>
7ef72446 34#include <stdio.h>
1ae93c94 35#include <system.h>
578bfd0a
AL
36 /*}}}*/
37
38// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
39// ---------------------------------------------------------------------
40/* We set the diry flag and make sure that is written to the disk */
b2e465d6
AL
41pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
42 Map(*pMap), Cache(pMap,false), Progress(Prog)
578bfd0a 43{
ddc1d8d0 44 CurrentFile = 0;
b2e465d6 45 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 46
578bfd0a
AL
47 if (_error->PendingError() == true)
48 return;
b2e465d6 49
578bfd0a
AL
50 if (Map.Size() == 0)
51 {
b2e465d6
AL
52 // Setup the map interface..
53 Cache.HeaderP = (pkgCache::Header *)Map.Data();
578bfd0a 54 Map.RawAllocate(sizeof(pkgCache::Header));
b2e465d6
AL
55 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
56
57 // Starting header
578bfd0a 58 *Cache.HeaderP = pkgCache::Header();
b2e465d6
AL
59 Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
60 Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
61 Cache.ReMap();
578bfd0a 62 }
b2e465d6
AL
63 else
64 {
65 // Map directly from the existing file
66 Cache.ReMap();
67 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
68 if (Cache.VS != _system->VS)
69 {
70 _error->Error(_("Cache has an incompatible versioning system"));
71 return;
72 }
73 }
74
578bfd0a
AL
75 Cache.HeaderP->Dirty = true;
76 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
77}
78 /*}}}*/
79// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
80// ---------------------------------------------------------------------
81/* We sync the data then unset the dirty flag in two steps so as to
82 advoid a problem during a crash */
83pkgCacheGenerator::~pkgCacheGenerator()
84{
85 if (_error->PendingError() == true)
86 return;
87 if (Map.Sync() == false)
88 return;
89
90 Cache.HeaderP->Dirty = false;
91 Map.Sync(0,sizeof(pkgCache::Header));
92}
93 /*}}}*/
94// CacheGenerator::MergeList - Merge the package list /*{{{*/
95// ---------------------------------------------------------------------
96/* This provides the generation of the entries in the cache. Each loop
97 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
98bool pkgCacheGenerator::MergeList(ListParser &List,
99 pkgCache::VerIterator *OutVer)
578bfd0a
AL
100{
101 List.Owner = this;
0149949b 102
f9eec0e7 103 unsigned int Counter = 0;
0149949b 104 while (List.Step() == true)
578bfd0a
AL
105 {
106 // Get a pointer to the package structure
9ddf7030 107 string PackageName = List.Package();
65a1e968
AL
108 if (PackageName.empty() == true)
109 return false;
110
9ddf7030 111 pkgCache::PkgIterator Pkg;
8efa2a3b 112 if (NewPackage(Pkg,PackageName) == false)
b2e465d6 113 return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 114 Counter++;
ddc1d8d0
AL
115 if (Counter % 100 == 0 && Progress != 0)
116 Progress->Progress(List.Offset());
8ce4327b 117
578bfd0a
AL
118 /* Get a pointer to the version structure. We know the list is sorted
119 so we use that fact in the search. Insertion of new versions is
120 done with correct sorting */
121 string Version = List.Version();
f55a958f
AL
122 if (Version.empty() == true)
123 {
124 if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
b2e465d6 125 return _error->Error(_("Error occured while processing %s (UsePackage1)"),PackageName.c_str());
f55a958f
AL
126 continue;
127 }
128
578bfd0a 129 pkgCache::VerIterator Ver = Pkg.VersionList();
349cd3b8 130 map_ptrloc *Last = &Pkg->VersionList;
2246928b 131 int Res = 1;
f55a958f 132 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
578bfd0a 133 {
b2e465d6 134 Res = Cache.VS->DoCmpVersion(Version.begin(),Version.end(),Ver.VerStr(),
578bfd0a
AL
135 Ver.VerStr() + strlen(Ver.VerStr()));
136 if (Res >= 0)
137 break;
138 }
139
140 /* We already have a version for this item, record that we
141 saw it */
204fbdcc
AL
142 unsigned long Hash = List.VersionHash();
143 if (Res == 0 && Ver->Hash == Hash)
578bfd0a 144 {
f55a958f 145 if (List.UsePackage(Pkg,Ver) == false)
b2e465d6 146 return _error->Error(_("Error occured while processing %s (UsePackage2)"),PackageName.c_str());
f78439bf 147
578bfd0a 148 if (NewFileVer(Ver,List) == false)
b2e465d6 149 return _error->Error(_("Error occured while processing %s (NewFileVer1)"),PackageName.c_str());
578bfd0a 150
ddc1d8d0
AL
151 // Read only a single record and return
152 if (OutVer != 0)
153 {
154 *OutVer = Ver;
155 return true;
156 }
157
578bfd0a
AL
158 continue;
159 }
160
204fbdcc
AL
161 // Skip to the end of the same version set.
162 if (Res == 0)
163 {
164 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
165 {
b2e465d6 166 Res = Cache.VS->DoCmpVersion(Version.begin(),Version.end(),Ver.VerStr(),
204fbdcc
AL
167 Ver.VerStr() + strlen(Ver.VerStr()));
168 if (Res != 0)
169 break;
170 }
171 }
172
578bfd0a 173 // Add a new version
f55a958f
AL
174 *Last = NewVersion(Ver,Version,*Last);
175 Ver->ParentPkg = Pkg.Index();
204fbdcc 176 Ver->Hash = Hash;
578bfd0a 177 if (List.NewVersion(Ver) == false)
b2e465d6 178 return _error->Error(_("Error occured while processing %s (NewVersion1)"),PackageName.c_str());
0149949b 179
f55a958f 180 if (List.UsePackage(Pkg,Ver) == false)
b2e465d6 181 return _error->Error(_("Error occured while processing %s (UsePackage3)"),PackageName.c_str());
f55a958f 182
578bfd0a 183 if (NewFileVer(Ver,List) == false)
b2e465d6 184 return _error->Error(_("Error occured while processing %s (NewVersion2)"),PackageName.c_str());
ddc1d8d0
AL
185
186 // Read only a single record and return
187 if (OutVer != 0)
188 {
189 *OutVer = Ver;
190 return true;
191 }
578bfd0a 192 }
0149949b 193
578bfd0a
AL
194 return true;
195}
196 /*}}}*/
197// CacheGenerator::NewPackage - Add a new package /*{{{*/
198// ---------------------------------------------------------------------
199/* This creates a new package structure and adds it to the hash table */
f55a958f 200bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
578bfd0a 201{
8efa2a3b
AL
202 Pkg = Cache.FindPkg(Name);
203 if (Pkg.end() == false)
204 return true;
205
578bfd0a
AL
206 // Get a structure
207 unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
208 if (Package == 0)
209 return false;
210
f55a958f 211 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
578bfd0a
AL
212
213 // Insert it into the hash table
f55a958f 214 unsigned long Hash = Cache.Hash(Name);
578bfd0a
AL
215 Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
216 Cache.HeaderP->HashTable[Hash] = Package;
217
218 // Set the name and the ID
219 Pkg->Name = Map.WriteString(Name);
220 if (Pkg->Name == 0)
221 return false;
222 Pkg->ID = Cache.HeaderP->PackageCount++;
223
224 return true;
225}
226 /*}}}*/
227// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
228// ---------------------------------------------------------------------
229/* */
f55a958f 230bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
231 ListParser &List)
232{
ddc1d8d0
AL
233 if (CurrentFile == 0)
234 return true;
235
dcb79bae
AL
236 // Get a structure
237 unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
238 if (VerFile == 0)
239 return 0;
240
241 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
242 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
243
244 // Link it to the end of the list
349cd3b8 245 map_ptrloc *Last = &Ver->FileList;
03e39e59
AL
246 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
247 Last = &V->NextFile;
248 VF->NextFile = *Last;
249 *Last = VF.Index();
250
dcb79bae
AL
251 VF->Offset = List.Offset();
252 VF->Size = List.Size();
ad00ae81
AL
253 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
254 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
255 Cache.HeaderP->VerFileCount++;
256
f55a958f 257 return true;
578bfd0a
AL
258}
259 /*}}}*/
260// CacheGenerator::NewVersion - Create a new Version /*{{{*/
261// ---------------------------------------------------------------------
f55a958f 262/* This puts a version structure in the linked list */
578bfd0a 263unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
f55a958f 264 string VerStr,
578bfd0a
AL
265 unsigned long Next)
266{
f55a958f
AL
267 // Get a structure
268 unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
269 if (Version == 0)
0149949b 270 return 0;
f55a958f
AL
271
272 // Fill it in
273 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
274 Ver->NextVer = Next;
275 Ver->ID = Cache.HeaderP->VersionCount++;
276 Ver->VerStr = Map.WriteString(VerStr);
277 if (Ver->VerStr == 0)
0149949b 278 return 0;
f55a958f 279
0149949b 280 return Version;
578bfd0a
AL
281}
282 /*}}}*/
dcb79bae
AL
283// ListParser::NewDepends - Create a dependency element /*{{{*/
284// ---------------------------------------------------------------------
285/* This creates a dependency element in the tree. It is linked to the
286 version and to the package that it is pointing to. */
287bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
9ddf7030
AL
288 string PackageName,
289 string Version,
dcb79bae
AL
290 unsigned int Op,
291 unsigned int Type)
292{
293 pkgCache &Cache = Owner->Cache;
294
295 // Get a structure
296 unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
297 if (Dependency == 0)
298 return false;
299
300 // Fill it in
301 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
302 Dep->ParentVer = Ver.Index();
303 Dep->Type = Type;
304 Dep->CompareOp = Op;
305 Dep->ID = Cache.HeaderP->DependsCount++;
306
307 // Locate the target package
8efa2a3b
AL
308 pkgCache::PkgIterator Pkg;
309 if (Owner->NewPackage(Pkg,PackageName) == false)
310 return false;
dcb79bae
AL
311
312 // Probe the reverse dependency list for a version string that matches
313 if (Version.empty() == false)
314 {
b2e465d6 315/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 316 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 317 Dep->Version = I->Version;*/
dcb79bae
AL
318 if (Dep->Version == 0)
319 if ((Dep->Version = WriteString(Version)) == 0)
320 return false;
321 }
c1a22377 322
dcb79bae
AL
323 // Link it to the package
324 Dep->Package = Pkg.Index();
325 Dep->NextRevDepends = Pkg->RevDepends;
326 Pkg->RevDepends = Dep.Index();
327
c1a22377
AL
328 /* Link it to the version (at the end of the list)
329 Caching the old end point speeds up generation substantially */
f9eec0e7 330 if (OldDepVer != Ver)
c1a22377 331 {
f9eec0e7 332 OldDepLast = &Ver->DependsList;
c1a22377 333 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7
AL
334 OldDepLast = &D->NextDepends;
335 OldDepVer = Ver;
c1a22377 336 }
dcb79bae 337
f9eec0e7
AL
338 Dep->NextDepends = *OldDepLast;
339 *OldDepLast = Dep.Index();
340 OldDepLast = &Dep->NextDepends;
c1a22377 341
dcb79bae
AL
342 return true;
343}
344 /*}}}*/
345// ListParser::NewProvides - Create a Provides element /*{{{*/
346// ---------------------------------------------------------------------
347/* */
348bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
8efa2a3b 349 string PackageName,
9ddf7030 350 string Version)
dcb79bae
AL
351{
352 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
353
354 // We do not add self referencing provides
355 if (Ver.ParentPkg().Name() == PackageName)
356 return true;
dcb79bae
AL
357
358 // Get a structure
359 unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
360 if (Provides == 0)
361 return false;
a7e66b17 362 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
363
364 // Fill it in
365 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
366 Prv->Version = Ver.Index();
367 Prv->NextPkgProv = Ver->ProvidesList;
368 Ver->ProvidesList = Prv.Index();
b2e465d6 369 if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
dcb79bae
AL
370 return false;
371
372 // Locate the target package
8efa2a3b
AL
373 pkgCache::PkgIterator Pkg;
374 if (Owner->NewPackage(Pkg,PackageName) == false)
375 return false;
dcb79bae
AL
376
377 // Link it to the package
378 Prv->ParentPkg = Pkg.Index();
379 Prv->NextProvides = Pkg->ProvidesList;
380 Pkg->ProvidesList = Prv.Index();
381
382 return true;
383}
384 /*}}}*/
578bfd0a
AL
385// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
386// ---------------------------------------------------------------------
387/* This is used to select which file is to be associated with all newly
b2e465d6
AL
388 added versions. The caller is responsible for setting the IMS fields. */
389bool pkgCacheGenerator::SelectFile(string File,string Site,
390 const pkgIndexFile &Index,
391 unsigned long Flags)
578bfd0a 392{
578bfd0a
AL
393 // Get some space for the structure
394 CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
395 if (CurrentFile == Cache.PkgFileP)
396 return false;
397
398 // Fill it in
399 CurrentFile->FileName = Map.WriteString(File);
b2e465d6 400 CurrentFile->Site = WriteUniqString(Site);
578bfd0a
AL
401 CurrentFile->NextFile = Cache.HeaderP->FileList;
402 CurrentFile->Flags = Flags;
e1b74f61 403 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
b2e465d6 404 CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
578bfd0a 405 PkgFileName = File;
ad00ae81 406 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 407 Cache.HeaderP->PackageFileCount++;
b2e465d6 408
578bfd0a
AL
409 if (CurrentFile->FileName == 0)
410 return false;
404ec98e 411
ddc1d8d0 412 if (Progress != 0)
b2e465d6 413 Progress->SubProgress(Index.Size());
8efa2a3b 414 return true;
578bfd0a
AL
415}
416 /*}}}*/
f55a958f
AL
417// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
418// ---------------------------------------------------------------------
419/* This is used to create handles to strings. Given the same text it
420 always returns the same number */
421unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
422 unsigned int Size)
423{
f9eec0e7
AL
424 /* We use a very small transient hash table here, this speeds up generation
425 by a fair amount on slower machines */
426 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
427 if (Bucket != 0 &&
428 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
429 return Bucket->String;
430
f55a958f
AL
431 // Search for an insertion point
432 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
433 int Res = 1;
349cd3b8 434 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
435 for (; I != Cache.StringItemP; Last = &I->NextItem,
436 I = Cache.StringItemP + I->NextItem)
437 {
9c14e3d6 438 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
439 if (Res >= 0)
440 break;
441 }
442
443 // Match
444 if (Res == 0)
f9eec0e7
AL
445 {
446 Bucket = I;
0149949b 447 return I->String;
f9eec0e7 448 }
f55a958f
AL
449
450 // Get a structure
451 unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
452 if (Item == 0)
0149949b
AL
453 return 0;
454
f55a958f
AL
455 // Fill in the structure
456 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
457 ItemP->NextItem = I - Cache.StringItemP;
458 *Last = Item;
459 ItemP->String = Map.WriteString(S,Size);
460 if (ItemP->String == 0)
0149949b 461 return 0;
f55a958f 462
f9eec0e7 463 Bucket = ItemP;
0149949b 464 return ItemP->String;
f55a958f
AL
465}
466 /*}}}*/
b35d2f5f 467
b2e465d6 468// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 469// ---------------------------------------------------------------------
b2e465d6
AL
470/* This just verifies that each file in the list of index files exists,
471 has matching attributes with the cache and the cache does not have
472 any extra files. */
473static bool CheckValidity(string CacheFile,pkgIndexFile **Start,
474 pkgIndexFile **End,MMap **OutMap = 0)
b35d2f5f 475{
b2e465d6
AL
476 // No file, certainly invalid
477 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
b35d2f5f
AL
478 return false;
479
b2e465d6 480 // Map it
b35d2f5f 481 FileFd CacheF(CacheFile,FileFd::ReadOnly);
b2e465d6 482 SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
b35d2f5f 483 pkgCache Cache(Map);
b2e465d6 484 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f
AL
485 {
486 _error->Discard();
487 return false;
488 }
b35d2f5f 489
b2e465d6
AL
490 /* Now we check every index file, see if it is in the cache,
491 verify the IMS data and check that it is on the disk too.. */
492 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
493 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
494 for (; Start != End; Start++)
495 {
496 if ((*Start)->HasPackages() == false)
497 continue;
498
499 if ((*Start)->Exists() == false)
b35d2f5f 500 {
b2e465d6
AL
501 _error->WarningE("stat",_("Couldn't stat source package list %s"),
502 (*Start)->Describe().c_str());
503 continue;
b35d2f5f 504 }
b2e465d6
AL
505
506 // FindInCache is also expected to do an IMS check.
507 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
508 if (File.end() == true)
b35d2f5f 509 return false;
b2e465d6
AL
510
511 Visited[File->ID] = true;
b35d2f5f
AL
512 }
513
b2e465d6
AL
514 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
515 if (Visited[I] == false)
516 return false;
b35d2f5f 517
b35d2f5f
AL
518 if (_error->PendingError() == true)
519 {
520 _error->Discard();
521 return false;
522 }
b35d2f5f 523
b2e465d6
AL
524 if (OutMap != 0)
525 *OutMap = Map.UnGuard();
b35d2f5f
AL
526 return true;
527}
528 /*}}}*/
b2e465d6 529// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 530// ---------------------------------------------------------------------
b2e465d6
AL
531/* Size is kind of an abstract notion that is only used for the progress
532 meter */
533static unsigned long ComputeSize(pkgIndexFile **Start,pkgIndexFile **End)
b35d2f5f 534{
b2e465d6
AL
535 unsigned long TotalSize = 0;
536 for (; Start != End; Start++)
b35d2f5f 537 {
b2e465d6
AL
538 if ((*Start)->HasPackages() == false)
539 continue;
540 TotalSize += (*Start)->Size();
b35d2f5f 541 }
b2e465d6 542 return TotalSize;
2d11135a
AL
543}
544 /*}}}*/
b2e465d6 545// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 546// ---------------------------------------------------------------------
b2e465d6
AL
547/* */
548static bool BuildCache(pkgCacheGenerator &Gen,
549 OpProgress &Progress,
550 unsigned long &CurrentSize,unsigned long TotalSize,
551 pkgIndexFile **Start,pkgIndexFile **End)
2d11135a 552{
b2e465d6 553 for (; Start != End; Start++)
2d11135a 554 {
b2e465d6 555 if ((*Start)->HasPackages() == false)
2d11135a
AL
556 continue;
557
b2e465d6 558 if ((*Start)->Exists() == false)
2d11135a 559 continue;
b2e465d6
AL
560
561 unsigned long Size = (*Start)->Size();
562 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading Package Lists"));
563 CurrentSize += Size;
2d11135a 564
b2e465d6
AL
565 if ((*Start)->Merge(Gen,Progress) == false)
566 return false;
567 }
2d11135a 568
b35d2f5f
AL
569 return true;
570}
571 /*}}}*/
b2e465d6 572// MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 573// ---------------------------------------------------------------------
b2e465d6
AL
574/* This makes sure that the status cache (the cache that has all
575 index files from the sources list and all local ones) is ready
576 to be mmaped. If OutMap is not zero then a MMap object representing
577 the cache will be stored there. This is pretty much mandetory if you
578 are using AllowMem. AllowMem lets the function be run as non-root
579 where it builds the cache 'fast' into a memory buffer. */
580bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
581 MMap **OutMap,bool AllowMem)
b35d2f5f 582{
67db871e
AL
583 unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
584
b2e465d6
AL
585 vector<pkgIndexFile *> Files(List.begin(),List.end());
586 unsigned long EndOfSource = Files.size();
587 if (_system->AddStatusFiles(Files) == false)
588 return false;
8ce4327b 589
b2e465d6 590 // Decide if we can write to the files..
3b5421b4 591 string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
b2e465d6
AL
592 string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
593
594 // Decide if we can write to the cache
595 bool Writeable = false;
596 if (CacheFile.empty() == false)
597 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
598 else
599 if (SrcCacheFile.empty() == false)
600 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
601
602 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
603 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
604
605 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
606
607 // Cache is OK, Fin.
608 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
609 {
610 Progress.OverallProgress(1,1,1,_("Reading Package Lists"));
611 return true;
612 }
613
614 /* At this point we know we need to reconstruct the package cache,
615 begin. */
616 SPtr<FileFd> CacheF;
617 SPtr<DynamicMMap> Map;
618 if (Writeable == true && CacheFile.empty() == false)
619 {
620 unlink(CacheFile.c_str());
621 CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
622 Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
b35d2f5f
AL
623 if (_error->PendingError() == true)
624 return false;
b35d2f5f 625 }
b2e465d6 626 else
8ce4327b 627 {
b2e465d6
AL
628 // Just build it in memory..
629 Map = new DynamicMMap(MMap::Public,MapSize);
8ce4327b 630 }
b35d2f5f 631
b2e465d6 632 // Lets try the source cache.
b35d2f5f 633 unsigned long CurrentSize = 0;
b2e465d6
AL
634 unsigned long TotalSize = 0;
635 if (CheckValidity(SrcCacheFile,Files.begin(),
636 Files.begin()+EndOfSource) == true)
2d11135a 637 {
b2e465d6
AL
638 // Preload the map with the source cache
639 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
640 if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
641 SCacheF.Size()) == false)
642 return false;
643
644 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
2d11135a 645
b2e465d6
AL
646 // Build the status cache
647 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 648 if (_error->PendingError() == true)
b2e465d6
AL
649 return false;
650 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
651 Files.begin()+EndOfSource,Files.end()) == false)
652 return false;
653 }
654 else
2d11135a 655 {
b2e465d6 656 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 657
b2e465d6
AL
658 // Build the source cache
659 pkgCacheGenerator Gen(Map.Get(),&Progress);
660 if (_error->PendingError() == true)
661 return false;
662 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
663 Files.begin(),Files.begin()+EndOfSource) == false)
664 return false;
2d11135a 665
b2e465d6
AL
666 // Write it back
667 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 668 {
b2e465d6
AL
669 FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
670 if (_error->PendingError() == true)
671 return false;
672 // Write out the main data
673 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
674 return _error->Error(_("IO Error saving source cache"));
675 SCacheF.Sync();
676
677 // Write out the proper header
678 Gen.GetCache().HeaderP->Dirty = false;
679 if (SCacheF.Seek(0) == false ||
680 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
681 return _error->Error(_("IO Error saving source cache"));
682 SCacheF.Sync();
683 Gen.GetCache().HeaderP->Dirty = true;
2d11135a
AL
684 }
685
b2e465d6
AL
686 // Build the status cache
687 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
688 Files.begin()+EndOfSource,Files.end()) == false)
689 return false;
2d11135a
AL
690 }
691
b2e465d6
AL
692 if (_error->PendingError() == true)
693 return false;
694 if (OutMap != 0)
2d11135a 695 {
b2e465d6 696 if (CacheF != 0)
2d11135a 697 {
b2e465d6
AL
698 delete Map.UnGuard();
699 *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
2d11135a 700 }
b2e465d6
AL
701 else
702 {
703 *OutMap = Map.UnGuard();
704 }
2d11135a
AL
705 }
706
b2e465d6
AL
707 return true;
708}
709 /*}}}*/
710// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/
711// ---------------------------------------------------------------------
712/* */
713bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
714{
715 unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
716 vector<pkgIndexFile *> Files;
717 unsigned long EndOfSource = Files.size();
718 if (_system->AddStatusFiles(Files) == false)
719 return false;
720
721 SPtr<DynamicMMap> Map;
722 Map = new DynamicMMap(MMap::Public,MapSize);
723 unsigned long CurrentSize = 0;
724 unsigned long TotalSize = 0;
725
726 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
727
728 // Build the status cache
729 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
730 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 731 if (_error->PendingError() == true)
b2e465d6
AL
732 return false;
733 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
734 Files.begin()+EndOfSource,Files.end()) == false)
735 return false;
2d11135a 736
b2e465d6
AL
737 if (_error->PendingError() == true)
738 return false;
739 *OutMap = Map.UnGuard();
2d11135a 740
b2e465d6 741 return true;
2d11135a
AL
742}
743 /*}}}*/