HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Make Packages & Sources generation optional, during Generate call
[ntk/apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140
DK		 13#include <config.h>
14
b2e465d6
AL		 15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
453b82a3 20#include <apt-pkg/fileutl.h>
b2e465d6 21
453b82a3 22#include <algorithm>
152ab79e 23#include <climits>
b2e465d6 24#include <sys/time.h>
453b82a3
DK		 25#include <locale.h>
26#include <stdio.h>
27#include <sys/stat.h>
28#include <time.h>
29#include <functional>
30#include <iostream>
31#include <string>
32#include <vector>
b2e465d6 33
453b82a3
DK		 34#include "cachedb.h"
35#include "override.h"
ea542140 36#include "apt-ftparchive.h"
b2e465d6 37#include "multicompress.h"
ea542140
DK		 38#include "writer.h"
39
40#include <apti18n.h>
b2e465d6
AL		 41 /*}}}*/
42
d935bb17
AL		 43using namespace std;
44ostream c0out(0);
45ostream c1out(0);
46ostream c2out(0);
b2e465d6
AL		 47ofstream devnull("/dev/null");
48unsigned Quiet = 0;
49
50// struct PackageMap - List of all package files in the config file /*{{{*/
51// ---------------------------------------------------------------------
52/* */
53struct PackageMap
54{
55 // General Stuff
56 string BaseDir;
57 string InternalPrefix;
58 string FLFile;
59 string PkgExt;
60 string SrcExt;
61
62 // Stuff for the Package File
63 string PkgFile;
64 string BinCacheDB;
ce928105 65 string SrcCacheDB;
b2e465d6 66 string BinOverride;
64177f17 67 string ExtraOverride;
0b41e0e7
MV		 68
69 // We generate for this given arch
70 string Arch;
b2e465d6
AL		 71
72 // Stuff for the Source File
73 string SrcFile;
74 string SrcOverride;
64177f17 75 string SrcExtraOverride;
b2e465d6 76
66905344 77 // Translation master file
4e794c50 78 bool LongDesc;
66905344
DK		 79 TranslationWriter *TransWriter;
80
b2e465d6
AL		 81 // Contents
82 string Contents;
83 string ContentsHead;
84
85 // Random things
86 string Tag;
87 string PkgCompress;
88 string CntCompress;
89 string SrcCompress;
90 string PathPrefix;
91 unsigned int DeLinkLimit;
92 mode_t Permissions;
93
94 bool ContentsDone;
95 bool PkgDone;
96 bool SrcDone;
97 time_t ContentsMTime;
98
99 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
100 {
101 inline bool operator() (const PackageMap &x,const PackageMap &y)
102 {return x.ContentsMTime < y.ContentsMTime;};
103 };
104
105 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
106 {
107 inline bool operator() (const PackageMap &x,const PackageMap &y)
108 {return x.BinCacheDB < y.BinCacheDB;};
109 };
ce928105
MV		 110
111 struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
112 {
113 inline bool operator() (const PackageMap &x,const PackageMap &y)
114 {return x.SrcCacheDB < y.SrcCacheDB;};
115 };
b2e465d6
AL		 116
117 void GetGeneral(Configuration &Setup,Configuration &Block);
118 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
119 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
120 bool GenContents(Configuration &Setup,
d935bb17
AL		 121 vector<PackageMap>::iterator Begin,
122 vector<PackageMap>::iterator End,
b2e465d6
AL		 123 unsigned long &Left);
124
4e794c50 125 PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
66905344
DK		 126 ContentsDone(false), PkgDone(false), SrcDone(false),
127 ContentsMTime(0) {};
b2e465d6
AL		 128};
129 /*}}}*/
130
131// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
132// ---------------------------------------------------------------------
133/* */
134void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
135{
136 PathPrefix = Block.Find("PathPrefix");
137
138 if (Block.FindB("External-Links",true) == false)
139 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
140 else
141 DeLinkLimit = 0;
142
143 PkgCompress = Block.Find("Packages::Compress",
144 Setup.Find("Default::Packages::Compress",". gzip").c_str());
145 CntCompress = Block.Find("Contents::Compress",
146 Setup.Find("Default::Contents::Compress",". gzip").c_str());
147 SrcCompress = Block.Find("Sources::Compress",
148 Setup.Find("Default::Sources::Compress",". gzip").c_str());
149
150 SrcExt = Block.Find("Sources::Extensions",
151 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
152 PkgExt = Block.Find("Packages::Extensions",
153 Setup.Find("Default::Packages::Extensions",".deb").c_str());
154
2db5b414
DK		 155 Permissions = Setup.FindI("Default::FileMode",0644);
156
b2e465d6
AL		 157 if (FLFile.empty() == false)
158 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
159
160 if (Contents == " ")
161 Contents= string();
162}
163 /*}}}*/
164// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
165// ---------------------------------------------------------------------
166/* This generates the Package File described by this object. */
167bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
168{
169 if (PkgFile.empty() == true)
170 return true;
171
172 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
173 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
174 string CacheDir = Setup.FindDir("Dir::CacheDir");
175
176 struct timeval StartTime;
177 gettimeofday(&StartTime,0);
178
179 PkgDone = true;
180
181 // Create a package writer object.
182 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 183 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 184 flCombine(OverrideDir,ExtraOverride),
185 Arch);
b2e465d6 186 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 187 return _error->Error(_("Package extension list is too long"));
b2e465d6 188 if (_error->PendingError() == true)
db0db9fe 189 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 190
191 Packages.PathPrefix = PathPrefix;
192 Packages.DirStrip = ArchiveDir;
193 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
194
66905344 195 Packages.TransWriter = TransWriter;
4e794c50 196 Packages.LongDescription = LongDesc;
66905344 197
b2e465d6
AL		 198 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
199 Packages.DeLinkLimit = DeLinkLimit;
64177f17 200
b2e465d6
AL		 201 // Create a compressor object
202 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
203 PkgCompress,Permissions);
204 Packages.Output = Comp.Input;
205 if (_error->PendingError() == true)
db0db9fe 206 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 207
208 c0out << ' ' << BaseDir << ":" << flush;
209
210 // Do recursive directory searching
211 if (FLFile.empty() == true)
212 {
213 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
214 return false;
215 }
216 else
217 {
218 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
219 return false;
220 }
221
222 Packages.Output = 0; // Just in case
223
224 // Finish compressing
650faab0 225 unsigned long long Size;
b2e465d6
AL		 226 if (Comp.Finalize(Size) == false)
227 {
228 c0out << endl;
db0db9fe 229 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 230 }
231
232 if (Size != 0)
233 c0out << " New "
234 << SizeToStr(Size) << "B ";
235 else
236 c0out << ' ';
237
238 struct timeval NewTime;
239 gettimeofday(&NewTime,0);
240 double Delta = NewTime.tv_sec - StartTime.tv_sec +
241 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
ce928105 242
b2e465d6
AL		 243 c0out << Packages.Stats.Packages << " files " <<
244/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
245 SizeToStr(Packages.Stats.Bytes) << "B " <<
246 TimeToStr((long)Delta) << endl;
ce928105
MV		 247
248 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
249 c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
b2e465d6
AL		 250
251 Stats.Add(Packages.Stats);
252 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
253
254 return !_error->PendingError();
255}
98953965 256
b2e465d6 257 /*}}}*/
64177f17 258// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 259// ---------------------------------------------------------------------
260/* This generates the Sources File described by this object. */
261bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
262{
263 if (SrcFile.empty() == true)
264 return true;
265
266 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
267 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
268 string CacheDir = Setup.FindDir("Dir::CacheDir");
269
270 struct timeval StartTime;
271 gettimeofday(&StartTime,0);
272
273 SrcDone = true;
274
275 // Create a package writer object.
ce928105 276 SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
f6f06a8f 277 flCombine(OverrideDir,BinOverride),
64177f17
AL		 278 flCombine(OverrideDir,SrcOverride),
279 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 280 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 281 return _error->Error(_("Source extension list is too long"));
b2e465d6 282 if (_error->PendingError() == true)
db0db9fe 283 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 284
285 Sources.PathPrefix = PathPrefix;
286 Sources.DirStrip = ArchiveDir;
287 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
288
289 Sources.DeLinkLimit = DeLinkLimit;
290 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
291
292 // Create a compressor object
293 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
294 SrcCompress,Permissions);
295 Sources.Output = Comp.Input;
296 if (_error->PendingError() == true)
db0db9fe 297 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 298
299 c0out << ' ' << BaseDir << ":" << flush;
300
301 // Do recursive directory searching
302 if (FLFile.empty() == true)
303 {
304 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
305 return false;
306 }
307 else
308 {
309 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
310 return false;
311 }
312 Sources.Output = 0; // Just in case
313
314 // Finish compressing
650faab0 315 unsigned long long Size;
b2e465d6
AL		 316 if (Comp.Finalize(Size) == false)
317 {
318 c0out << endl;
db0db9fe 319 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 320 }
321
322 if (Size != 0)
323 c0out << " New "
324 << SizeToStr(Size) << "B ";
325 else
326 c0out << ' ';
327
328 struct timeval NewTime;
329 gettimeofday(&NewTime,0);
330 double Delta = NewTime.tv_sec - StartTime.tv_sec +
331 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
332
333 c0out << Sources.Stats.Packages << " pkgs in " <<
334 TimeToStr((long)Delta) << endl;
335
ce928105
MV		 336 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
337 c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
338
b2e465d6
AL		 339 Stats.Add(Sources.Stats);
340 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
341
342 return !_error->PendingError();
343}
344 /*}}}*/
345// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
346// ---------------------------------------------------------------------
347/* This generates the contents file partially described by this object.
348 It searches the given iterator range for other package files that map
349 into this contents file and includes their data as well when building. */
350bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 351 vector<PackageMap>::iterator Begin,
352 vector<PackageMap>::iterator End,
353 unsigned long &Left)
b2e465d6
AL		 354{
355 if (Contents.empty() == true)
356 return true;
357
358 if (Left == 0)
359 return true;
360
361 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
362 string CacheDir = Setup.FindDir("Dir::CacheDir");
363 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
364
365 struct timeval StartTime;
366 gettimeofday(&StartTime,0);
367
368 // Create a package writer object.
31981076 369 ContentsWriter Contents("", Arch);
b2e465d6 370 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 371 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 372 if (_error->PendingError() == true)
373 return false;
374
375 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
376 CntCompress,Permissions);
377 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
378 Contents.Output = Comp.Input;
379 if (_error->PendingError() == true)
380 return false;
381
382 // Write the header out.
383 if (ContentsHead.empty() == false)
384 {
385 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
386 if (_error->PendingError() == true)
387 return false;
388
650faab0 389 unsigned long long Size = Head.Size();
b2e465d6
AL		 390 unsigned char Buf[4096];
391 while (Size != 0)
392 {
650faab0 393 unsigned long long ToRead = Size;
b2e465d6
AL		 394 if (Size > sizeof(Buf))
395 ToRead = sizeof(Buf);
396
397 if (Head.Read(Buf,ToRead) == false)
398 return false;
399
400 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 401 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 402
403 Size -= ToRead;
404 }
405 }
406
407 /* Go over all the package file records and parse all the package
408 files associated with this contents file into one great big honking
409 memory structure, then dump the sorted version */
410 c0out << ' ' << this->Contents << ":" << flush;
f7f0d6c7 411 for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
b2e465d6
AL		 412 {
413 if (I->Contents != this->Contents)
414 continue;
415
416 Contents.Prefix = ArchiveDir;
417 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
418 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
419 I->PkgCompress);
420
421 I->ContentsDone = true;
422 }
423
424 Contents.Finish();
425
426 // Finish compressing
650faab0 427 unsigned long long Size;
b2e465d6
AL		 428 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
429 {
430 c0out << endl;
db0db9fe 431 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 432 this->Contents.c_str());
433 }
434
435 if (Size != 0)
436 {
437 c0out << " New " << SizeToStr(Size) << "B ";
438 if (Left > Size)
439 Left -= Size;
440 else
441 Left = 0;
442 }
443 else
444 c0out << ' ';
445
446 struct timeval NewTime;
447 gettimeofday(&NewTime,0);
448 double Delta = NewTime.tv_sec - StartTime.tv_sec +
449 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
450
ce928105
MV		 451 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
452 c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
453
b2e465d6
AL		 454 c0out << Contents.Stats.Packages << " files " <<
455 SizeToStr(Contents.Stats.Bytes) << "B " <<
456 TimeToStr((long)Delta) << endl;
457
458 return true;
459}
460 /*}}}*/
461
462// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
463// ---------------------------------------------------------------------
464/* This populates the PkgList with all the possible permutations of the
465 section/arch lists. */
c3ccac92 466static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
b2e465d6
AL		 467{
468 // Load the defaults
469 string DDir = Setup.Find("TreeDefault::Directory",
470 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
471 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
472 "$(DIST)/$(SECTION)/source/");
473 string DPkg = Setup.Find("TreeDefault::Packages",
474 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 475 string DTrans = Setup.Find("TreeDefault::Translation",
476 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 477 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
478 "$(DIST)/$(SECTION)/");
479 string DContents = Setup.Find("TreeDefault::Contents",
3adddfa8 480 "$(DIST)/$(SECTION)/Contents-$(ARCH)");
b2e465d6
AL		 481 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
482 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
483 "packages-$(ARCH).db");
ce928105
MV		 484 string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
485 "sources-$(SECTION).db");
b2e465d6
AL		 486 string DSources = Setup.Find("TreeDefault::Sources",
487 "$(DIST)/$(SECTION)/source/Sources");
488 string DFLFile = Setup.Find("TreeDefault::FileList", "");
489 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
490
2db5b414 491 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
34f1d96c
DK		 492
493 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 494 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 495 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
4e794c50 496
b2e465d6
AL		 497 // Process 'tree' type sections
498 const Configuration::Item *Top = Setup.Tree("tree");
499 for (Top = (Top == 0?0:Top->Child); Top != 0;)
500 {
501 Configuration Block(Top);
502 string Dist = Top->Tag;
503
504 // Parse the sections
d935bb17
AL		 505 string Tmp = Block.Find("Sections");
506 const char *Sections = Tmp.c_str();
b2e465d6
AL		 507 string Section;
508 while (ParseQuoteWord(Sections,Section) == true)
509 {
b2e465d6 510 string Arch;
66905344
DK		 511 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
512 {"$(SECTION)",&Section},
513 {"$(ARCH)",&Arch},
67c3067f 514 {NULL, NULL}};
34f1d96c 515 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 516 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
66905344 517 TranslationWriter *TransWriter;
4e794c50 518 if (DTrans.empty() == false && LongDesc == false)
66905344
DK		 519 {
520 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
521 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
34f1d96c
DK		 522 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
523 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
66905344
DK		 524 }
525 else
526 TransWriter = NULL;
527
528 string const Tmp2 = Block.Find("Architectures");
d935bb17 529 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 530 while (ParseQuoteWord(Archs,Arch) == true)
531 {
b2e465d6 532 PackageMap Itm;
34f1d96c 533 Itm.Permissions = Perms;
b2e465d6
AL		 534 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
535 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
536
537 if (stringcasecmp(Arch,"source") == 0)
538 {
539 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
540 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
541 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
542 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
543 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 544 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
ce928105 545 Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
b2e465d6
AL		 546 }
547 else
548 {
549 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
550 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
551 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
552 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 553 Itm.Arch = Arch;
4e794c50 554 Itm.LongDesc = LongDesc;
66905344
DK		 555 if (TransWriter != NULL)
556 {
557 TransWriter->IncreaseRefCounter();
558 Itm.TransWriter = TransWriter;
559 }
b2e465d6
AL		 560 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
561 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
562 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 563 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 564 }
565
566 Itm.GetGeneral(Setup,Block);
567 PkgList.push_back(Itm);
568 }
66905344
DK		 569 // we didn't use this TransWriter, so we can release it
570 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
571 delete TransWriter;
b2e465d6
AL		 572 }
573
574 Top = Top->Next;
575 }
576}
577 /*}}}*/
578// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
579// ---------------------------------------------------------------------
580/* */
c3ccac92 581static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
b2e465d6 582{
2db5b414
DK		 583 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
584
b2e465d6
AL		 585 // Process 'bindirectory' type sections
586 const Configuration::Item *Top = Setup.Tree("bindirectory");
587 for (Top = (Top == 0?0:Top->Child); Top != 0;)
588 {
589 Configuration Block(Top);
590
591 PackageMap Itm;
592 Itm.PkgFile = Block.Find("Packages");
593 Itm.SrcFile = Block.Find("Sources");
594 Itm.BinCacheDB = Block.Find("BinCacheDB");
ce928105 595 Itm.SrcCacheDB = Block.Find("SrcCacheDB");
b2e465d6 596 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 597 Itm.ExtraOverride = Block.Find("ExtraOverride");
598 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 599 Itm.SrcOverride = Block.Find("SrcOverride");
600 Itm.BaseDir = Top->Tag;
601 Itm.FLFile = Block.Find("FileList");
602 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
603 Itm.Contents = Block.Find("Contents");
604 Itm.ContentsHead = Block.Find("Contents::Header");
2db5b414 605 Itm.Permissions = Block.FindI("FileMode", Permissions);
b2e465d6
AL		 606
607 Itm.GetGeneral(Setup,Block);
608 PkgList.push_back(Itm);
609
610 Top = Top->Next;
611 }
612}
613 /*}}}*/
614
615// ShowHelp - Show the help text /*{{{*/
616// ---------------------------------------------------------------------
617/* */
65512241 618static bool ShowHelp(CommandLine &)
b2e465d6 619{
9179f697 620 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
5b28c804 621 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 622 if (_config->FindB("version") == true)
623 return true;
624
625 cout <<
dc738e7a 626 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 627 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 628 " sources srcpath [overridefile [pathprefix]]\n"
629 " contents path\n"
0dfd2728 630 " release path\n"
b2e465d6
AL		 631 " generate config [groups]\n"
632 " clean config\n"
633 "\n"
634 "apt-ftparchive generates index files for Debian archives. It supports\n"
635 "many styles of generation from fully automated to functional replacements\n"
636 "for dpkg-scanpackages and dpkg-scansources\n"
637 "\n"
638 "apt-ftparchive generates Package files from a tree of .debs. The\n"
639 "Package file contains the contents of all the control fields from\n"
640 "each package as well as the MD5 hash and filesize. An override file\n"
641 "is supported to force the value of Priority and Section.\n"
642 "\n"
643 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
644 "The --source-override option can be used to specify a src override file\n"
645 "\n"
646 "The 'packages' and 'sources' command should be run in the root of the\n"
647 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 648 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 649 "appended to the filename fields if present. Example usage from the \n"
75224826 650 "Debian archive:\n"
b2e465d6
AL		 651 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
652 " dists/potato/main/binary-i386/Packages\n"
653 "\n"
654 "Options:\n"
655 " -h This help text\n"
656 " --md5 Control MD5 generation\n"
657 " -s=? Source override file\n"
658 " -q Quiet\n"
659 " -d=? Select the optional caching database\n"
660 " --no-delink Enable delinking debug mode\n"
661 " --contents Control contents file generation\n"
662 " -c=? Read this configuration file\n"
8a2d7db6 663 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 664
665 return true;
666}
667 /*}}}*/
668// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
669// ---------------------------------------------------------------------
670/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
c3ccac92 671static bool SimpleGenPackages(CommandLine &CmdL)
b2e465d6
AL		 672{
673 if (CmdL.FileSize() < 2)
674 return ShowHelp(CmdL);
675
676 string Override;
677 if (CmdL.FileSize() >= 3)
678 Override = CmdL.FileList[2];
679
680 // Create a package writer object.
681 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 682 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 683 if (_error->PendingError() == true)
684 return false;
685
686 if (CmdL.FileSize() >= 4)
687 Packages.PathPrefix = CmdL.FileList[3];
688
689 // Do recursive directory searching
690 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
691 return false;
692
cf6bbca0
MV		 693 // Give some stats if asked for
694 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
695 c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
696
b2e465d6
AL		 697 return true;
698}
699 /*}}}*/
700// SimpleGenContents - Generate a Contents listing /*{{{*/
701// ---------------------------------------------------------------------
702/* */
c3ccac92 703static bool SimpleGenContents(CommandLine &CmdL)
b2e465d6
AL		 704{
705 if (CmdL.FileSize() < 2)
706 return ShowHelp(CmdL);
707
708 // Create a package writer object.
31981076 709 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 710 if (_error->PendingError() == true)
711 return false;
712
713 // Do recursive directory searching
714 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
715 return false;
716
717 Contents.Finish();
718
719 return true;
720}
721 /*}}}*/
722// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
723// ---------------------------------------------------------------------
724/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
c3ccac92 725static bool SimpleGenSources(CommandLine &CmdL)
b2e465d6
AL		 726{
727 if (CmdL.FileSize() < 2)
728 return ShowHelp(CmdL);
729
730 string Override;
731 if (CmdL.FileSize() >= 3)
732 Override = CmdL.FileList[2];
733
734 string SOverride;
735 if (Override.empty() == false)
736 SOverride = Override + ".src";
737
738 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
739 SOverride.c_str());
740
741 // Create a package writer object.
f6f06a8f 742 SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
b2e465d6
AL		 743 if (_error->PendingError() == true)
744 return false;
745
746 if (CmdL.FileSize() >= 4)
747 Sources.PathPrefix = CmdL.FileList[3];
748
749 // Do recursive directory searching
750 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
751 return false;
752
cf6bbca0
MV		 753 // Give some stats if asked for
754 if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
755 c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
756
b2e465d6
AL		 757 return true;
758}
98953965
AL		 759 /*}}}*/
760// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
761// ---------------------------------------------------------------------
c3ccac92 762static bool SimpleGenRelease(CommandLine &CmdL)
98953965 763{
0dfd2728
AL		 764 if (CmdL.FileSize() < 2)
765 return ShowHelp(CmdL);
766
c0eb6bc6
AL		 767 string Dir = CmdL.FileList[1];
768
98953965 769 ReleaseWriter Release("");
c0eb6bc6
AL		 770 Release.DirStrip = Dir;
771
98953965
AL		 772 if (_error->PendingError() == true)
773 return false;
774
c0eb6bc6 775 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 776 return false;
777
f7291f62
AL		 778 Release.Finish();
779
98953965
AL		 780 return true;
781}
782
b2e465d6 783 /*}}}*/
308b6fc9 784// DoGeneratePackagesAndSources - Helper for Generate /*{{{*/
b2e465d6 785// ---------------------------------------------------------------------
308b6fc9
DJL		 786static bool DoGeneratePackagesAndSources(Configuration &Setup,
787 vector<PackageMap> &PkgList,
788 struct CacheDB::Stats &SrcStats,
789 struct CacheDB::Stats &Stats,
790 CommandLine &CmdL)
b2e465d6 791{
b2e465d6
AL		 792 if (CmdL.FileSize() <= 2)
793 {
f7f0d6c7 794 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 795 if (I->GenPackages(Setup,Stats) == false)
796 _error->DumpErrors();
f7f0d6c7 797 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 798 if (I->GenSources(Setup,SrcStats) == false)
799 _error->DumpErrors();
800 }
801 else
802 {
803 // Make a choice list out of the package list..
804 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
805 RxChoiceList *End = List;
f7f0d6c7 806 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 807 {
d935bb17 808 End->UserData = &(*I);
b2e465d6
AL		 809 End->Str = I->BaseDir.c_str();
810 End++;
811
d935bb17 812 End->UserData = &(*I);
b2e465d6
AL		 813 End->Str = I->Tag.c_str();
814 End++;
815 }
816 End->Str = 0;
817
818 // Regex it
819 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
820 {
821 delete [] List;
dc738e7a 822 return _error->Error(_("No selections matched"));
b2e465d6
AL		 823 }
824 _error->DumpErrors();
825
826 // Do the generation for Packages
827 for (End = List; End->Str != 0; End++)
828 {
829 if (End->Hit == false)
830 continue;
831
832 PackageMap *I = (PackageMap *)End->UserData;
833 if (I->PkgDone == true)
834 continue;
835 if (I->GenPackages(Setup,Stats) == false)
836 _error->DumpErrors();
837 }
838
839 // Do the generation for Sources
840 for (End = List; End->Str != 0; End++)
841 {
842 if (End->Hit == false)
843 continue;
844
845 PackageMap *I = (PackageMap *)End->UserData;
846 if (I->SrcDone == true)
847 continue;
848 if (I->GenSources(Setup,SrcStats) == false)
849 _error->DumpErrors();
850 }
851
852 delete [] List;
853 }
66905344
DK		 854
855 // close the Translation master files
f7f0d6c7 856 for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
66905344
DK		 857 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
858 delete I->TransWriter;
859
308b6fc9
DJL		 860 return true;
861}
862
863 /*}}}*/
864// DoGenerateContents - Helper for Generate to generate the Contents /*{{{*/
865// ---------------------------------------------------------------------
866static bool DoGenerateContents(Configuration &Setup,
867 vector<PackageMap> &PkgList,
868 CommandLine &CmdL)
869{
c6474fb6 870 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 871
872 // Sort the contents file list by date
873 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
f7f0d6c7 874 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 875 {
876 struct stat A;
877 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
878 I->CntCompress,A) == false)
879 time(&I->ContentsMTime);
880 else
881 I->ContentsMTime = A.st_mtime;
882 }
883 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
884
885 /* Now for Contents.. The process here is to do a make-like dependency
886 check. Each contents file is verified to be newer than the package files
887 that describe the debs it indexes. Since the package files contain
888 hashes of the .debs this means they have not changed either so the
889 contents must be up to date. */
890 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
f7f0d6c7 891 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 892 {
1e3f4083 893 // This record is not relevant
b2e465d6
AL		 894 if (I->ContentsDone == true ||
895 I->Contents.empty() == true)
896 continue;
897
898 // Do not do everything if the user specified sections.
899 if (CmdL.FileSize() > 2 && I->PkgDone == false)
900 continue;
901
902 struct stat A,B;
903 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
904 {
905 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
906 {
dc738e7a 907 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 908 continue;
909 }
910
911 if (A.st_mtime > B.st_mtime)
912 continue;
913 }
914
915 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
916 MaxContentsChange) == false)
917 _error->DumpErrors();
918
919 // Hit the limit?
920 if (MaxContentsChange == 0)
921 {
922 c1out << "Hit contents update byte limit" << endl;
923 break;
924 }
925 }
308b6fc9
DJL		 926
927 return true;
928}
929
930 /*}}}*/
931// Generate - Full generate, using a config file /*{{{*/
932// ---------------------------------------------------------------------
933/* */
934static bool Generate(CommandLine &CmdL)
935{
936 struct CacheDB::Stats SrcStats;
937 if (CmdL.FileSize() < 2)
938 return ShowHelp(CmdL);
939
940 struct timeval StartTime;
941 gettimeofday(&StartTime,0);
942 struct CacheDB::Stats Stats;
b2e465d6 943
308b6fc9
DJL		 944 // Read the configuration file.
945 Configuration Setup;
946 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
947 return false;
948
949 vector<PackageMap> PkgList;
950 LoadTree(PkgList,Setup);
951 LoadBinDir(PkgList,Setup);
952
953 // Sort by cache DB to improve IO locality.
954 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
955 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
956
957 // Generate packages
958 if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
959 {
960 if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
961 return false;
962 } else {
963 c1out << "Skipping Packages/Sources generation" << endl;
964 }
965
966 // do Contents if needed
967 if (_config->FindB("APT::FTPArchive::Contents", true) == true)
968 if (DoGenerateContents(Setup, PkgList, CmdL) == false)
969 return false;
970
b2e465d6 971 struct timeval NewTime;
308b6fc9
DJL		 972 gettimeofday(&NewTime,0);
973 double Delta = NewTime.tv_sec - StartTime.tv_sec +
b2e465d6 974 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
308b6fc9 975 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
b2e465d6 976 << " archives. Took " << TimeToStr((long)Delta) << endl;
308b6fc9 977
b2e465d6
AL		 978 return true;
979}
308b6fc9
DJL		 980
981 /*}}}*/
b2e465d6
AL		 982// Clean - Clean out the databases /*{{{*/
983// ---------------------------------------------------------------------
984/* */
c3ccac92 985static bool Clean(CommandLine &CmdL)
b2e465d6
AL		 986{
987 if (CmdL.FileSize() != 2)
988 return ShowHelp(CmdL);
989
990 // Read the configuration file.
991 Configuration Setup;
992 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
993 return false;
994
995 vector<PackageMap> PkgList;
996 LoadTree(PkgList,Setup);
997 LoadBinDir(PkgList,Setup);
998
999 // Sort by cache DB to improve IO locality.
1000 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
53ba4e2c 1001 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
b2e465d6
AL		 1002
1003 string CacheDir = Setup.FindDir("Dir::CacheDir");
1004
d935bb17 1005 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6 1006 {
53ba4e2c
MV		 1007 if(I->BinCacheDB != "")
1008 c0out << I->BinCacheDB << endl;
1009 if(I->SrcCacheDB != "")
1010 c0out << I->SrcCacheDB << endl;
b2e465d6 1011 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
53ba4e2c 1012 CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
b2e465d6
AL		 1013 if (DB.Clean() == false)
1014 _error->DumpErrors();
53ba4e2c
MV		 1015 if (DB_SRC.Clean() == false)
1016 _error->DumpErrors();
b2e465d6
AL		 1017
1018 string CacheDB = I->BinCacheDB;
53ba4e2c
MV		 1019 string SrcCacheDB = I->SrcCacheDB;
1020 while(I != PkgList.end() &&
1021 I->BinCacheDB == CacheDB &&
1022 I->SrcCacheDB == SrcCacheDB)
1023 ++I;
1024
b2e465d6 1025 }
ce928105 1026
53ba4e2c 1027
b2e465d6
AL		 1028 return true;
1029}
1030 /*}}}*/
1031
1032int main(int argc, const char *argv[])
1033{
1692a2c7 1034 setlocale(LC_ALL, "");
b2e465d6
AL		 1035 CommandLine::Args Args[] = {
1036 {'h',"help","help",0},
1037 {0,"md5","APT::FTPArchive::MD5",0},
3c54407f
DK		 1038 {0,"sha1","APT::FTPArchive::SHA1",0},
1039 {0,"sha256","APT::FTPArchive::SHA256",0},
b2e465d6
AL		 1040 {'v',"version","version",0},
1041 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
1042 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
1043 {'q',"quiet","quiet",CommandLine::IntLevel},
1044 {'q',"silent","quiet",CommandLine::IntLevel},
1045 {0,"delink","APT::FTPArchive::DeLinkAct",0},
1046 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
1047 {0,"contents","APT::FTPArchive::Contents",0},
31981076 1048 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 1049 {'c',"config-file",0,CommandLine::ConfigFile},
1050 {'o',"option",0,CommandLine::ArbItem},
1051 {0,0,0,0}};
1052 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
1053 {"contents",&SimpleGenContents},
1054 {"sources",&SimpleGenSources},
98953965 1055 {"release",&SimpleGenRelease},
b2e465d6
AL		 1056 {"generate",&Generate},
1057 {"clean",&Clean},
1058 {"help",&ShowHelp},
1059 {0,0}};
1060
1061 // Parse the command line and initialize the package library
1062 CommandLine CmdL(Args,_config);
b34d4b47 1063 if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
b2e465d6
AL		 1064 {
1065 _error->DumpErrors();
1066 return 100;
1067 }
1068
1069 // See if the help should be shown
1070 if (_config->FindB("help") == true ||
1071 _config->FindB("version") == true ||
1072 CmdL.FileSize() == 0)
1073 {
1074 ShowHelp(CmdL);
1075 return 0;
1076 }
1077
1078 // Setup the output streams
1809f8d2
AL		 1079 c0out.rdbuf(clog.rdbuf());
1080 c1out.rdbuf(clog.rdbuf());
1081 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 1082 Quiet = _config->FindI("quiet",0);
1083 if (Quiet > 0)
1084 c0out.rdbuf(devnull.rdbuf());
1085 if (Quiet > 1)
1086 c1out.rdbuf(devnull.rdbuf());
1087
1088 // Match the operation
1089 CmdL.DispatchArg(Cmds);
1090
1091 if (_error->empty() == false)
1092 {
1093 bool Errors = _error->PendingError();
1094 _error->DumpErrors();
1095 return Errors == true?100:0;
1096 }
1097 return 0;
1098}
ntk's bugfix for apt