deblistparser.cc
[ntk/apt.git] / apt-pkg / pkgcachegen.cc
CommitLineData
578bfd0a
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
e7b470ee 3// $Id: pkgcachegen.cc,v 1.48 2001/05/27 05:36:04 jgg Exp $
578bfd0a
AL
4/* ######################################################################
5
6 Package Cache Generator - Generator for the cache structure.
7
8 This builds the cache structure from the abstract package list parser.
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
6c139d6e 13#ifdef __GNUG__
094a497d 14#pragma implementation "apt-pkg/pkgcachegen.h"
6c139d6e
AL
15#endif
16
b2e465d6
AL
17#define APT_COMPATIBILITY 986
18
094a497d
AL
19#include <apt-pkg/pkgcachegen.h>
20#include <apt-pkg/error.h>
21#include <apt-pkg/version.h>
b35d2f5f
AL
22#include <apt-pkg/progress.h>
23#include <apt-pkg/sourcelist.h>
24#include <apt-pkg/configuration.h>
cdcc6d34 25#include <apt-pkg/strutl.h>
b2e465d6
AL
26#include <apt-pkg/sptr.h>
27#include <apt-pkg/pkgsystem.h>
578bfd0a 28
b2e465d6 29#include <apti18n.h>
e7b470ee
AL
30
31#include <vector>
32
578bfd0a
AL
33#include <sys/stat.h>
34#include <unistd.h>
803fafcb 35#include <errno.h>
7ef72446 36#include <stdio.h>
1ae93c94 37#include <system.h>
578bfd0a 38 /*}}}*/
e7b470ee 39typedef vector<pkgIndexFile *>::iterator FileIterator;
578bfd0a
AL
40
41// CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/
42// ---------------------------------------------------------------------
43/* We set the diry flag and make sure that is written to the disk */
b2e465d6
AL
44pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
45 Map(*pMap), Cache(pMap,false), Progress(Prog)
578bfd0a 46{
ddc1d8d0 47 CurrentFile = 0;
b2e465d6 48 memset(UniqHash,0,sizeof(UniqHash));
ddc1d8d0 49
578bfd0a
AL
50 if (_error->PendingError() == true)
51 return;
b2e465d6 52
578bfd0a
AL
53 if (Map.Size() == 0)
54 {
b2e465d6
AL
55 // Setup the map interface..
56 Cache.HeaderP = (pkgCache::Header *)Map.Data();
578bfd0a 57 Map.RawAllocate(sizeof(pkgCache::Header));
b2e465d6
AL
58 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
59
60 // Starting header
578bfd0a 61 *Cache.HeaderP = pkgCache::Header();
b2e465d6
AL
62 Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label);
63 Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture"));
64 Cache.ReMap();
578bfd0a 65 }
b2e465d6
AL
66 else
67 {
68 // Map directly from the existing file
69 Cache.ReMap();
70 Map.UsePools(*Cache.HeaderP->Pools,sizeof(Cache.HeaderP->Pools)/sizeof(Cache.HeaderP->Pools[0]));
71 if (Cache.VS != _system->VS)
72 {
73 _error->Error(_("Cache has an incompatible versioning system"));
74 return;
75 }
76 }
77
578bfd0a
AL
78 Cache.HeaderP->Dirty = true;
79 Map.Sync(0,sizeof(pkgCache::Header));
578bfd0a
AL
80}
81 /*}}}*/
82// CacheGenerator::~pkgCacheGenerator - Destructor /*{{{*/
83// ---------------------------------------------------------------------
84/* We sync the data then unset the dirty flag in two steps so as to
85 advoid a problem during a crash */
86pkgCacheGenerator::~pkgCacheGenerator()
87{
88 if (_error->PendingError() == true)
89 return;
90 if (Map.Sync() == false)
91 return;
92
93 Cache.HeaderP->Dirty = false;
94 Map.Sync(0,sizeof(pkgCache::Header));
95}
96 /*}}}*/
97// CacheGenerator::MergeList - Merge the package list /*{{{*/
98// ---------------------------------------------------------------------
99/* This provides the generation of the entries in the cache. Each loop
100 goes through a single package record from the underlying parse engine. */
ddc1d8d0
AL
101bool pkgCacheGenerator::MergeList(ListParser &List,
102 pkgCache::VerIterator *OutVer)
578bfd0a
AL
103{
104 List.Owner = this;
0149949b 105
f9eec0e7 106 unsigned int Counter = 0;
0149949b 107 while (List.Step() == true)
578bfd0a
AL
108 {
109 // Get a pointer to the package structure
9ddf7030 110 string PackageName = List.Package();
65a1e968
AL
111 if (PackageName.empty() == true)
112 return false;
113
9ddf7030 114 pkgCache::PkgIterator Pkg;
8efa2a3b 115 if (NewPackage(Pkg,PackageName) == false)
b2e465d6 116 return _error->Error(_("Error occured while processing %s (NewPackage)"),PackageName.c_str());
a246f2dc 117 Counter++;
ddc1d8d0
AL
118 if (Counter % 100 == 0 && Progress != 0)
119 Progress->Progress(List.Offset());
8ce4327b 120
578bfd0a
AL
121 /* Get a pointer to the version structure. We know the list is sorted
122 so we use that fact in the search. Insertion of new versions is
123 done with correct sorting */
124 string Version = List.Version();
f55a958f
AL
125 if (Version.empty() == true)
126 {
127 if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false)
b2e465d6 128 return _error->Error(_("Error occured while processing %s (UsePackage1)"),PackageName.c_str());
f55a958f
AL
129 continue;
130 }
131
578bfd0a 132 pkgCache::VerIterator Ver = Pkg.VersionList();
349cd3b8 133 map_ptrloc *Last = &Pkg->VersionList;
2246928b 134 int Res = 1;
f55a958f 135 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
578bfd0a 136 {
e7b470ee 137 Res = Cache.VS->DoCmpVersion(Version.c_str(),Version.c_str()+Version.length(),Ver.VerStr(),
578bfd0a
AL
138 Ver.VerStr() + strlen(Ver.VerStr()));
139 if (Res >= 0)
140 break;
141 }
142
143 /* We already have a version for this item, record that we
144 saw it */
204fbdcc
AL
145 unsigned long Hash = List.VersionHash();
146 if (Res == 0 && Ver->Hash == Hash)
578bfd0a 147 {
f55a958f 148 if (List.UsePackage(Pkg,Ver) == false)
b2e465d6 149 return _error->Error(_("Error occured while processing %s (UsePackage2)"),PackageName.c_str());
f78439bf 150
578bfd0a 151 if (NewFileVer(Ver,List) == false)
b2e465d6 152 return _error->Error(_("Error occured while processing %s (NewFileVer1)"),PackageName.c_str());
578bfd0a 153
ddc1d8d0
AL
154 // Read only a single record and return
155 if (OutVer != 0)
156 {
157 *OutVer = Ver;
158 return true;
159 }
160
578bfd0a
AL
161 continue;
162 }
163
204fbdcc
AL
164 // Skip to the end of the same version set.
165 if (Res == 0)
166 {
167 for (; Ver.end() == false; Last = &Ver->NextVer, Ver++)
168 {
e7b470ee 169 Res = Cache.VS->DoCmpVersion(Version.c_str(),Version.c_str()+Version.length(),Ver.VerStr(),
204fbdcc
AL
170 Ver.VerStr() + strlen(Ver.VerStr()));
171 if (Res != 0)
172 break;
173 }
174 }
175
578bfd0a 176 // Add a new version
f55a958f
AL
177 *Last = NewVersion(Ver,Version,*Last);
178 Ver->ParentPkg = Pkg.Index();
204fbdcc 179 Ver->Hash = Hash;
578bfd0a 180 if (List.NewVersion(Ver) == false)
b2e465d6 181 return _error->Error(_("Error occured while processing %s (NewVersion1)"),PackageName.c_str());
0149949b 182
f55a958f 183 if (List.UsePackage(Pkg,Ver) == false)
b2e465d6 184 return _error->Error(_("Error occured while processing %s (UsePackage3)"),PackageName.c_str());
f55a958f 185
578bfd0a 186 if (NewFileVer(Ver,List) == false)
b2e465d6 187 return _error->Error(_("Error occured while processing %s (NewVersion2)"),PackageName.c_str());
ddc1d8d0
AL
188
189 // Read only a single record and return
190 if (OutVer != 0)
191 {
192 *OutVer = Ver;
193 return true;
194 }
578bfd0a 195 }
0149949b 196
578bfd0a
AL
197 return true;
198}
199 /*}}}*/
200// CacheGenerator::NewPackage - Add a new package /*{{{*/
201// ---------------------------------------------------------------------
202/* This creates a new package structure and adds it to the hash table */
f55a958f 203bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,string Name)
578bfd0a 204{
8efa2a3b
AL
205 Pkg = Cache.FindPkg(Name);
206 if (Pkg.end() == false)
207 return true;
208
578bfd0a
AL
209 // Get a structure
210 unsigned long Package = Map.Allocate(sizeof(pkgCache::Package));
211 if (Package == 0)
212 return false;
213
f55a958f 214 Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package);
578bfd0a
AL
215
216 // Insert it into the hash table
f55a958f 217 unsigned long Hash = Cache.Hash(Name);
578bfd0a
AL
218 Pkg->NextPackage = Cache.HeaderP->HashTable[Hash];
219 Cache.HeaderP->HashTable[Hash] = Package;
220
221 // Set the name and the ID
222 Pkg->Name = Map.WriteString(Name);
223 if (Pkg->Name == 0)
224 return false;
225 Pkg->ID = Cache.HeaderP->PackageCount++;
226
227 return true;
228}
229 /*}}}*/
230// CacheGenerator::NewFileVer - Create a new File<->Version association /*{{{*/
231// ---------------------------------------------------------------------
232/* */
f55a958f 233bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver,
578bfd0a
AL
234 ListParser &List)
235{
ddc1d8d0
AL
236 if (CurrentFile == 0)
237 return true;
238
dcb79bae
AL
239 // Get a structure
240 unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile));
241 if (VerFile == 0)
242 return 0;
243
244 pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile);
245 VF->File = CurrentFile - Cache.PkgFileP;
03e39e59
AL
246
247 // Link it to the end of the list
349cd3b8 248 map_ptrloc *Last = &Ver->FileList;
03e39e59
AL
249 for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; V++)
250 Last = &V->NextFile;
251 VF->NextFile = *Last;
252 *Last = VF.Index();
253
dcb79bae
AL
254 VF->Offset = List.Offset();
255 VF->Size = List.Size();
ad00ae81
AL
256 if (Cache.HeaderP->MaxVerFileSize < VF->Size)
257 Cache.HeaderP->MaxVerFileSize = VF->Size;
a7e66b17
AL
258 Cache.HeaderP->VerFileCount++;
259
f55a958f 260 return true;
578bfd0a
AL
261}
262 /*}}}*/
263// CacheGenerator::NewVersion - Create a new Version /*{{{*/
264// ---------------------------------------------------------------------
f55a958f 265/* This puts a version structure in the linked list */
578bfd0a 266unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver,
f55a958f 267 string VerStr,
578bfd0a
AL
268 unsigned long Next)
269{
f55a958f
AL
270 // Get a structure
271 unsigned long Version = Map.Allocate(sizeof(pkgCache::Version));
272 if (Version == 0)
0149949b 273 return 0;
f55a958f
AL
274
275 // Fill it in
276 Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version);
f55a958f
AL
277 Ver->NextVer = Next;
278 Ver->ID = Cache.HeaderP->VersionCount++;
279 Ver->VerStr = Map.WriteString(VerStr);
280 if (Ver->VerStr == 0)
0149949b 281 return 0;
f55a958f 282
0149949b 283 return Version;
578bfd0a
AL
284}
285 /*}}}*/
dcb79bae
AL
286// ListParser::NewDepends - Create a dependency element /*{{{*/
287// ---------------------------------------------------------------------
288/* This creates a dependency element in the tree. It is linked to the
289 version and to the package that it is pointing to. */
290bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver,
9ddf7030
AL
291 string PackageName,
292 string Version,
dcb79bae
AL
293 unsigned int Op,
294 unsigned int Type)
295{
296 pkgCache &Cache = Owner->Cache;
297
298 // Get a structure
299 unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency));
300 if (Dependency == 0)
301 return false;
302
303 // Fill it in
304 pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency);
305 Dep->ParentVer = Ver.Index();
306 Dep->Type = Type;
307 Dep->CompareOp = Op;
308 Dep->ID = Cache.HeaderP->DependsCount++;
309
310 // Locate the target package
8efa2a3b
AL
311 pkgCache::PkgIterator Pkg;
312 if (Owner->NewPackage(Pkg,PackageName) == false)
313 return false;
dcb79bae
AL
314
315 // Probe the reverse dependency list for a version string that matches
316 if (Version.empty() == false)
317 {
b2e465d6 318/* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++)
dcb79bae 319 if (I->Version != 0 && I.TargetVer() == Version)
f9eec0e7 320 Dep->Version = I->Version;*/
dcb79bae
AL
321 if (Dep->Version == 0)
322 if ((Dep->Version = WriteString(Version)) == 0)
323 return false;
324 }
c1a22377 325
dcb79bae
AL
326 // Link it to the package
327 Dep->Package = Pkg.Index();
328 Dep->NextRevDepends = Pkg->RevDepends;
329 Pkg->RevDepends = Dep.Index();
330
c1a22377
AL
331 /* Link it to the version (at the end of the list)
332 Caching the old end point speeds up generation substantially */
f9eec0e7 333 if (OldDepVer != Ver)
c1a22377 334 {
f9eec0e7 335 OldDepLast = &Ver->DependsList;
c1a22377 336 for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++)
f9eec0e7
AL
337 OldDepLast = &D->NextDepends;
338 OldDepVer = Ver;
c1a22377 339 }
dcb79bae 340
f9eec0e7
AL
341 Dep->NextDepends = *OldDepLast;
342 *OldDepLast = Dep.Index();
343 OldDepLast = &Dep->NextDepends;
c1a22377 344
dcb79bae
AL
345 return true;
346}
347 /*}}}*/
348// ListParser::NewProvides - Create a Provides element /*{{{*/
349// ---------------------------------------------------------------------
350/* */
351bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver,
8efa2a3b 352 string PackageName,
9ddf7030 353 string Version)
dcb79bae
AL
354{
355 pkgCache &Cache = Owner->Cache;
8efa2a3b
AL
356
357 // We do not add self referencing provides
358 if (Ver.ParentPkg().Name() == PackageName)
359 return true;
dcb79bae
AL
360
361 // Get a structure
362 unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides));
363 if (Provides == 0)
364 return false;
a7e66b17 365 Cache.HeaderP->ProvidesCount++;
dcb79bae
AL
366
367 // Fill it in
368 pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP);
369 Prv->Version = Ver.Index();
370 Prv->NextPkgProv = Ver->ProvidesList;
371 Ver->ProvidesList = Prv.Index();
b2e465d6 372 if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0)
dcb79bae
AL
373 return false;
374
375 // Locate the target package
8efa2a3b
AL
376 pkgCache::PkgIterator Pkg;
377 if (Owner->NewPackage(Pkg,PackageName) == false)
378 return false;
dcb79bae
AL
379
380 // Link it to the package
381 Prv->ParentPkg = Pkg.Index();
382 Prv->NextProvides = Pkg->ProvidesList;
383 Pkg->ProvidesList = Prv.Index();
384
385 return true;
386}
387 /*}}}*/
578bfd0a
AL
388// CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/
389// ---------------------------------------------------------------------
390/* This is used to select which file is to be associated with all newly
b2e465d6
AL
391 added versions. The caller is responsible for setting the IMS fields. */
392bool pkgCacheGenerator::SelectFile(string File,string Site,
393 const pkgIndexFile &Index,
394 unsigned long Flags)
578bfd0a 395{
578bfd0a
AL
396 // Get some space for the structure
397 CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile));
398 if (CurrentFile == Cache.PkgFileP)
399 return false;
400
401 // Fill it in
402 CurrentFile->FileName = Map.WriteString(File);
b2e465d6 403 CurrentFile->Site = WriteUniqString(Site);
578bfd0a
AL
404 CurrentFile->NextFile = Cache.HeaderP->FileList;
405 CurrentFile->Flags = Flags;
e1b74f61 406 CurrentFile->ID = Cache.HeaderP->PackageFileCount;
b2e465d6 407 CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label);
578bfd0a 408 PkgFileName = File;
ad00ae81 409 Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
b35d2f5f 410 Cache.HeaderP->PackageFileCount++;
b2e465d6 411
578bfd0a
AL
412 if (CurrentFile->FileName == 0)
413 return false;
404ec98e 414
ddc1d8d0 415 if (Progress != 0)
b2e465d6 416 Progress->SubProgress(Index.Size());
8efa2a3b 417 return true;
578bfd0a
AL
418}
419 /*}}}*/
f55a958f
AL
420// CacheGenerator::WriteUniqueString - Insert a unique string /*{{{*/
421// ---------------------------------------------------------------------
422/* This is used to create handles to strings. Given the same text it
423 always returns the same number */
424unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
425 unsigned int Size)
426{
f9eec0e7
AL
427 /* We use a very small transient hash table here, this speeds up generation
428 by a fair amount on slower machines */
429 pkgCache::StringItem *&Bucket = UniqHash[(S[0]*5 + S[1]) % _count(UniqHash)];
430 if (Bucket != 0 &&
431 stringcmp(S,S+Size,Cache.StrP + Bucket->String) == 0)
432 return Bucket->String;
433
f55a958f
AL
434 // Search for an insertion point
435 pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList;
436 int Res = 1;
349cd3b8 437 map_ptrloc *Last = &Cache.HeaderP->StringList;
f55a958f
AL
438 for (; I != Cache.StringItemP; Last = &I->NextItem,
439 I = Cache.StringItemP + I->NextItem)
440 {
9c14e3d6 441 Res = stringcmp(S,S+Size,Cache.StrP + I->String);
f55a958f
AL
442 if (Res >= 0)
443 break;
444 }
445
446 // Match
447 if (Res == 0)
f9eec0e7
AL
448 {
449 Bucket = I;
0149949b 450 return I->String;
f9eec0e7 451 }
f55a958f
AL
452
453 // Get a structure
454 unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem));
455 if (Item == 0)
0149949b
AL
456 return 0;
457
f55a958f
AL
458 // Fill in the structure
459 pkgCache::StringItem *ItemP = Cache.StringItemP + Item;
460 ItemP->NextItem = I - Cache.StringItemP;
461 *Last = Item;
462 ItemP->String = Map.WriteString(S,Size);
463 if (ItemP->String == 0)
0149949b 464 return 0;
f55a958f 465
f9eec0e7 466 Bucket = ItemP;
0149949b 467 return ItemP->String;
f55a958f
AL
468}
469 /*}}}*/
b35d2f5f 470
b2e465d6 471// CheckValidity - Check that a cache is up-to-date /*{{{*/
b35d2f5f 472// ---------------------------------------------------------------------
b2e465d6
AL
473/* This just verifies that each file in the list of index files exists,
474 has matching attributes with the cache and the cache does not have
475 any extra files. */
e7b470ee
AL
476static bool CheckValidity(string CacheFile, FileIterator Start,
477 FileIterator End,MMap **OutMap = 0)
b35d2f5f 478{
b2e465d6
AL
479 // No file, certainly invalid
480 if (CacheFile.empty() == true || FileExists(CacheFile) == false)
b35d2f5f
AL
481 return false;
482
b2e465d6 483 // Map it
b35d2f5f 484 FileFd CacheF(CacheFile,FileFd::ReadOnly);
b2e465d6 485 SPtr<MMap> Map = new MMap(CacheF,MMap::Public | MMap::ReadOnly);
b35d2f5f 486 pkgCache Cache(Map);
b2e465d6 487 if (_error->PendingError() == true || Map->Size() == 0)
b35d2f5f
AL
488 {
489 _error->Discard();
490 return false;
491 }
b35d2f5f 492
b2e465d6
AL
493 /* Now we check every index file, see if it is in the cache,
494 verify the IMS data and check that it is on the disk too.. */
495 SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
496 memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
497 for (; Start != End; Start++)
a77ad7c3 498 {
b2e465d6
AL
499 if ((*Start)->HasPackages() == false)
500 continue;
a77ad7c3 501
b2e465d6 502 if ((*Start)->Exists() == false)
b35d2f5f 503 {
b2e465d6
AL
504 _error->WarningE("stat",_("Couldn't stat source package list %s"),
505 (*Start)->Describe().c_str());
506 continue;
b35d2f5f 507 }
b2e465d6
AL
508
509 // FindInCache is also expected to do an IMS check.
510 pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
511 if (File.end() == true)
b35d2f5f 512 return false;
b2e465d6
AL
513
514 Visited[File->ID] = true;
b35d2f5f
AL
515 }
516
b2e465d6
AL
517 for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
518 if (Visited[I] == false)
519 return false;
b35d2f5f 520
b35d2f5f
AL
521 if (_error->PendingError() == true)
522 {
523 _error->Discard();
524 return false;
525 }
b35d2f5f 526
b2e465d6
AL
527 if (OutMap != 0)
528 *OutMap = Map.UnGuard();
b35d2f5f
AL
529 return true;
530}
531 /*}}}*/
b2e465d6 532// ComputeSize - Compute the total size of a bunch of files /*{{{*/
b35d2f5f 533// ---------------------------------------------------------------------
b2e465d6
AL
534/* Size is kind of an abstract notion that is only used for the progress
535 meter */
e7b470ee 536static unsigned long ComputeSize(FileIterator Start,FileIterator End)
b35d2f5f 537{
b2e465d6
AL
538 unsigned long TotalSize = 0;
539 for (; Start != End; Start++)
b35d2f5f 540 {
b2e465d6
AL
541 if ((*Start)->HasPackages() == false)
542 continue;
543 TotalSize += (*Start)->Size();
b35d2f5f 544 }
b2e465d6 545 return TotalSize;
2d11135a
AL
546}
547 /*}}}*/
b2e465d6 548// BuildCache - Merge the list of index files into the cache /*{{{*/
2d11135a 549// ---------------------------------------------------------------------
b2e465d6
AL
550/* */
551static bool BuildCache(pkgCacheGenerator &Gen,
552 OpProgress &Progress,
553 unsigned long &CurrentSize,unsigned long TotalSize,
e7b470ee 554 FileIterator Start, FileIterator End)
2d11135a 555{
b2e465d6 556 for (; Start != End; Start++)
2d11135a 557 {
b2e465d6 558 if ((*Start)->HasPackages() == false)
2d11135a
AL
559 continue;
560
b2e465d6 561 if ((*Start)->Exists() == false)
2d11135a 562 continue;
b2e465d6 563
a77ad7c3
AL
564 if ((*Start)->FindInCache(Gen.GetCache()).end() == false)
565 {
566 _error->Warning("Duplicate sources.list entry %s",
567 (*Start)->Describe().c_str());
568 continue;
569 }
570
b2e465d6
AL
571 unsigned long Size = (*Start)->Size();
572 Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading Package Lists"));
573 CurrentSize += Size;
2d11135a 574
b2e465d6
AL
575 if ((*Start)->Merge(Gen,Progress) == false)
576 return false;
577 }
2d11135a 578
b35d2f5f
AL
579 return true;
580}
581 /*}}}*/
b2e465d6 582// MakeStatusCache - Construct the status cache /*{{{*/
b35d2f5f 583// ---------------------------------------------------------------------
b2e465d6
AL
584/* This makes sure that the status cache (the cache that has all
585 index files from the sources list and all local ones) is ready
586 to be mmaped. If OutMap is not zero then a MMap object representing
587 the cache will be stored there. This is pretty much mandetory if you
588 are using AllowMem. AllowMem lets the function be run as non-root
589 where it builds the cache 'fast' into a memory buffer. */
590bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
591 MMap **OutMap,bool AllowMem)
b35d2f5f 592{
a77ad7c3 593 unsigned long MapSize = _config->FindI("APT::Cache-Limit",6*1024*1024);
67db871e 594
b2e465d6
AL
595 vector<pkgIndexFile *> Files(List.begin(),List.end());
596 unsigned long EndOfSource = Files.size();
597 if (_system->AddStatusFiles(Files) == false)
598 return false;
8ce4327b 599
b2e465d6 600 // Decide if we can write to the files..
3b5421b4 601 string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
b2e465d6
AL
602 string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
603
604 // Decide if we can write to the cache
605 bool Writeable = false;
606 if (CacheFile.empty() == false)
607 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
608 else
609 if (SrcCacheFile.empty() == false)
610 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
611
612 if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
613 return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
614
615 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
616
617 // Cache is OK, Fin.
618 if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
619 {
620 Progress.OverallProgress(1,1,1,_("Reading Package Lists"));
621 return true;
622 }
623
624 /* At this point we know we need to reconstruct the package cache,
625 begin. */
626 SPtr<FileFd> CacheF;
627 SPtr<DynamicMMap> Map;
628 if (Writeable == true && CacheFile.empty() == false)
629 {
630 unlink(CacheFile.c_str());
631 CacheF = new FileFd(CacheFile,FileFd::WriteEmpty);
632 Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
b35d2f5f
AL
633 if (_error->PendingError() == true)
634 return false;
b35d2f5f 635 }
b2e465d6 636 else
8ce4327b 637 {
b2e465d6
AL
638 // Just build it in memory..
639 Map = new DynamicMMap(MMap::Public,MapSize);
8ce4327b 640 }
b35d2f5f 641
b2e465d6 642 // Lets try the source cache.
b35d2f5f 643 unsigned long CurrentSize = 0;
b2e465d6
AL
644 unsigned long TotalSize = 0;
645 if (CheckValidity(SrcCacheFile,Files.begin(),
646 Files.begin()+EndOfSource) == true)
2d11135a 647 {
b2e465d6
AL
648 // Preload the map with the source cache
649 FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
650 if (SCacheF.Read((unsigned char *)Map->Data() + Map->RawAllocate(SCacheF.Size()),
651 SCacheF.Size()) == false)
652 return false;
653
654 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
2d11135a 655
b2e465d6
AL
656 // Build the status cache
657 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 658 if (_error->PendingError() == true)
b2e465d6
AL
659 return false;
660 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
661 Files.begin()+EndOfSource,Files.end()) == false)
662 return false;
663 }
664 else
2d11135a 665 {
b2e465d6 666 TotalSize = ComputeSize(Files.begin(),Files.end());
2d11135a 667
b2e465d6
AL
668 // Build the source cache
669 pkgCacheGenerator Gen(Map.Get(),&Progress);
670 if (_error->PendingError() == true)
671 return false;
672 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
673 Files.begin(),Files.begin()+EndOfSource) == false)
674 return false;
2d11135a 675
b2e465d6
AL
676 // Write it back
677 if (Writeable == true && SrcCacheFile.empty() == false)
2d11135a 678 {
b2e465d6
AL
679 FileFd SCacheF(SrcCacheFile,FileFd::WriteEmpty);
680 if (_error->PendingError() == true)
681 return false;
682 // Write out the main data
683 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
684 return _error->Error(_("IO Error saving source cache"));
685 SCacheF.Sync();
686
687 // Write out the proper header
688 Gen.GetCache().HeaderP->Dirty = false;
689 if (SCacheF.Seek(0) == false ||
690 SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
691 return _error->Error(_("IO Error saving source cache"));
692 SCacheF.Sync();
693 Gen.GetCache().HeaderP->Dirty = true;
2d11135a
AL
694 }
695
b2e465d6
AL
696 // Build the status cache
697 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
698 Files.begin()+EndOfSource,Files.end()) == false)
699 return false;
2d11135a
AL
700 }
701
b2e465d6
AL
702 if (_error->PendingError() == true)
703 return false;
704 if (OutMap != 0)
2d11135a 705 {
b2e465d6 706 if (CacheF != 0)
2d11135a 707 {
b2e465d6
AL
708 delete Map.UnGuard();
709 *OutMap = new MMap(*CacheF,MMap::Public | MMap::ReadOnly);
2d11135a 710 }
b2e465d6
AL
711 else
712 {
713 *OutMap = Map.UnGuard();
714 }
2d11135a
AL
715 }
716
b2e465d6
AL
717 return true;
718}
719 /*}}}*/
720// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/
721// ---------------------------------------------------------------------
722/* */
723bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap)
724{
725 unsigned long MapSize = _config->FindI("APT::Cache-Limit",4*1024*1024);
726 vector<pkgIndexFile *> Files;
727 unsigned long EndOfSource = Files.size();
728 if (_system->AddStatusFiles(Files) == false)
729 return false;
730
731 SPtr<DynamicMMap> Map;
732 Map = new DynamicMMap(MMap::Public,MapSize);
733 unsigned long CurrentSize = 0;
734 unsigned long TotalSize = 0;
735
736 TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
737
738 // Build the status cache
739 Progress.OverallProgress(0,1,1,_("Reading Package Lists"));
740 pkgCacheGenerator Gen(Map.Get(),&Progress);
2d11135a 741 if (_error->PendingError() == true)
b2e465d6
AL
742 return false;
743 if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
744 Files.begin()+EndOfSource,Files.end()) == false)
745 return false;
2d11135a 746
b2e465d6
AL
747 if (_error->PendingError() == true)
748 return false;
749 *OutMap = Map.UnGuard();
2d11135a 750
b2e465d6 751 return true;
2d11135a
AL
752}
753 /*}}}*/