HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
* ftparchive/writer.cc:
[ntk/apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL		 13#include "apt-ftparchive.h"
14
15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
19#include <config.h>
20#include <apti18n.h>
21#include <algorithm>
22
152ab79e 23#include <climits>
b2e465d6
AL		 24#include <sys/time.h>
25#include <regex.h>
26
27#include "contents.h"
28#include "multicompress.h"
29#include "writer.h"
30 /*}}}*/
31
d935bb17
AL		 32using namespace std;
33ostream c0out(0);
34ostream c1out(0);
35ostream c2out(0);
b2e465d6
AL		 36ofstream devnull("/dev/null");
37unsigned Quiet = 0;
38
39// struct PackageMap - List of all package files in the config file /*{{{*/
40// ---------------------------------------------------------------------
41/* */
42struct PackageMap
43{
44 // General Stuff
45 string BaseDir;
46 string InternalPrefix;
47 string FLFile;
48 string PkgExt;
49 string SrcExt;
50
51 // Stuff for the Package File
52 string PkgFile;
53 string BinCacheDB;
54 string BinOverride;
64177f17 55 string ExtraOverride;
0b41e0e7
MV		 56
57 // We generate for this given arch
58 string Arch;
b2e465d6
AL		 59
60 // Stuff for the Source File
61 string SrcFile;
62 string SrcOverride;
64177f17 63 string SrcExtraOverride;
b2e465d6 64
66905344
DK		 65 // Translation master file
66 TranslationWriter *TransWriter;
67
b2e465d6
AL		 68 // Contents
69 string Contents;
70 string ContentsHead;
71
72 // Random things
73 string Tag;
74 string PkgCompress;
75 string CntCompress;
76 string SrcCompress;
77 string PathPrefix;
78 unsigned int DeLinkLimit;
79 mode_t Permissions;
80
81 bool ContentsDone;
82 bool PkgDone;
83 bool SrcDone;
84 time_t ContentsMTime;
85
86 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
87 {
88 inline bool operator() (const PackageMap &x,const PackageMap &y)
89 {return x.ContentsMTime < y.ContentsMTime;};
90 };
91
92 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
93 {
94 inline bool operator() (const PackageMap &x,const PackageMap &y)
95 {return x.BinCacheDB < y.BinCacheDB;};
96 };
97
98 void GetGeneral(Configuration &Setup,Configuration &Block);
99 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
100 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
101 bool GenContents(Configuration &Setup,
d935bb17
AL		 102 vector<PackageMap>::iterator Begin,
103 vector<PackageMap>::iterator End,
b2e465d6
AL		 104 unsigned long &Left);
105
66905344
DK		 106 PackageMap() : TransWriter(NULL), DeLinkLimit(0), Permissions(1),
107 ContentsDone(false), PkgDone(false), SrcDone(false),
108 ContentsMTime(0) {};
b2e465d6
AL		 109};
110 /*}}}*/
111
112// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
113// ---------------------------------------------------------------------
114/* */
115void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
116{
117 PathPrefix = Block.Find("PathPrefix");
118
119 if (Block.FindB("External-Links",true) == false)
120 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
121 else
122 DeLinkLimit = 0;
123
124 PkgCompress = Block.Find("Packages::Compress",
125 Setup.Find("Default::Packages::Compress",". gzip").c_str());
126 CntCompress = Block.Find("Contents::Compress",
127 Setup.Find("Default::Contents::Compress",". gzip").c_str());
128 SrcCompress = Block.Find("Sources::Compress",
129 Setup.Find("Default::Sources::Compress",". gzip").c_str());
130
131 SrcExt = Block.Find("Sources::Extensions",
132 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
133 PkgExt = Block.Find("Packages::Extensions",
134 Setup.Find("Default::Packages::Extensions",".deb").c_str());
135
136 Permissions = Setup.FindI("Default::FileMode",0644);
137
138 if (FLFile.empty() == false)
139 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
140
141 if (Contents == " ")
142 Contents= string();
143}
144 /*}}}*/
145// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
146// ---------------------------------------------------------------------
147/* This generates the Package File described by this object. */
148bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
149{
150 if (PkgFile.empty() == true)
151 return true;
152
153 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
154 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
155 string CacheDir = Setup.FindDir("Dir::CacheDir");
156
157 struct timeval StartTime;
158 gettimeofday(&StartTime,0);
159
160 PkgDone = true;
161
162 // Create a package writer object.
163 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 164 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 165 flCombine(OverrideDir,ExtraOverride),
166 Arch);
b2e465d6 167 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 168 return _error->Error(_("Package extension list is too long"));
b2e465d6 169 if (_error->PendingError() == true)
db0db9fe 170 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 171
172 Packages.PathPrefix = PathPrefix;
173 Packages.DirStrip = ArchiveDir;
174 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
175
66905344
DK		 176 Packages.TransWriter = TransWriter;
177
b2e465d6
AL		 178 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
179 Packages.DeLinkLimit = DeLinkLimit;
64177f17 180
b2e465d6
AL		 181 // Create a compressor object
182 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
183 PkgCompress,Permissions);
184 Packages.Output = Comp.Input;
185 if (_error->PendingError() == true)
db0db9fe 186 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 187
188 c0out << ' ' << BaseDir << ":" << flush;
189
190 // Do recursive directory searching
191 if (FLFile.empty() == true)
192 {
193 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
194 return false;
195 }
196 else
197 {
198 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
199 return false;
200 }
201
202 Packages.Output = 0; // Just in case
203
204 // Finish compressing
205 unsigned long Size;
206 if (Comp.Finalize(Size) == false)
207 {
208 c0out << endl;
db0db9fe 209 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 210 }
211
212 if (Size != 0)
213 c0out << " New "
214 << SizeToStr(Size) << "B ";
215 else
216 c0out << ' ';
217
218 struct timeval NewTime;
219 gettimeofday(&NewTime,0);
220 double Delta = NewTime.tv_sec - StartTime.tv_sec +
221 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
222
223 c0out << Packages.Stats.Packages << " files " <<
224/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
225 SizeToStr(Packages.Stats.Bytes) << "B " <<
226 TimeToStr((long)Delta) << endl;
227
228 Stats.Add(Packages.Stats);
229 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
230
231 return !_error->PendingError();
232}
98953965 233
b2e465d6 234 /*}}}*/
64177f17 235// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 236// ---------------------------------------------------------------------
237/* This generates the Sources File described by this object. */
238bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
239{
240 if (SrcFile.empty() == true)
241 return true;
242
243 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
244 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
245 string CacheDir = Setup.FindDir("Dir::CacheDir");
246
247 struct timeval StartTime;
248 gettimeofday(&StartTime,0);
249
250 SrcDone = true;
251
252 // Create a package writer object.
253 SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
64177f17
AL		 254 flCombine(OverrideDir,SrcOverride),
255 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 256 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 257 return _error->Error(_("Source extension list is too long"));
b2e465d6 258 if (_error->PendingError() == true)
db0db9fe 259 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 260
261 Sources.PathPrefix = PathPrefix;
262 Sources.DirStrip = ArchiveDir;
263 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
264
265 Sources.DeLinkLimit = DeLinkLimit;
266 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
267
268 // Create a compressor object
269 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
270 SrcCompress,Permissions);
271 Sources.Output = Comp.Input;
272 if (_error->PendingError() == true)
db0db9fe 273 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 274
275 c0out << ' ' << BaseDir << ":" << flush;
276
277 // Do recursive directory searching
278 if (FLFile.empty() == true)
279 {
280 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
281 return false;
282 }
283 else
284 {
285 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
286 return false;
287 }
288 Sources.Output = 0; // Just in case
289
290 // Finish compressing
291 unsigned long Size;
292 if (Comp.Finalize(Size) == false)
293 {
294 c0out << endl;
db0db9fe 295 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 296 }
297
298 if (Size != 0)
299 c0out << " New "
300 << SizeToStr(Size) << "B ";
301 else
302 c0out << ' ';
303
304 struct timeval NewTime;
305 gettimeofday(&NewTime,0);
306 double Delta = NewTime.tv_sec - StartTime.tv_sec +
307 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
308
309 c0out << Sources.Stats.Packages << " pkgs in " <<
310 TimeToStr((long)Delta) << endl;
311
312 Stats.Add(Sources.Stats);
313 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
314
315 return !_error->PendingError();
316}
317 /*}}}*/
318// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
319// ---------------------------------------------------------------------
320/* This generates the contents file partially described by this object.
321 It searches the given iterator range for other package files that map
322 into this contents file and includes their data as well when building. */
323bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 324 vector<PackageMap>::iterator Begin,
325 vector<PackageMap>::iterator End,
326 unsigned long &Left)
b2e465d6
AL		 327{
328 if (Contents.empty() == true)
329 return true;
330
331 if (Left == 0)
332 return true;
333
334 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
335 string CacheDir = Setup.FindDir("Dir::CacheDir");
336 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
337
338 struct timeval StartTime;
339 gettimeofday(&StartTime,0);
340
341 // Create a package writer object.
31981076 342 ContentsWriter Contents("", Arch);
b2e465d6 343 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 344 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 345 if (_error->PendingError() == true)
346 return false;
347
348 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
349 CntCompress,Permissions);
350 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
351 Contents.Output = Comp.Input;
352 if (_error->PendingError() == true)
353 return false;
354
355 // Write the header out.
356 if (ContentsHead.empty() == false)
357 {
358 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
359 if (_error->PendingError() == true)
360 return false;
361
362 unsigned long Size = Head.Size();
363 unsigned char Buf[4096];
364 while (Size != 0)
365 {
366 unsigned long ToRead = Size;
367 if (Size > sizeof(Buf))
368 ToRead = sizeof(Buf);
369
370 if (Head.Read(Buf,ToRead) == false)
371 return false;
372
373 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 374 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 375
376 Size -= ToRead;
377 }
378 }
379
380 /* Go over all the package file records and parse all the package
381 files associated with this contents file into one great big honking
382 memory structure, then dump the sorted version */
383 c0out << ' ' << this->Contents << ":" << flush;
d935bb17 384 for (vector<PackageMap>::iterator I = Begin; I != End; I++)
b2e465d6
AL		 385 {
386 if (I->Contents != this->Contents)
387 continue;
388
389 Contents.Prefix = ArchiveDir;
390 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
391 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
392 I->PkgCompress);
393
394 I->ContentsDone = true;
395 }
396
397 Contents.Finish();
398
399 // Finish compressing
400 unsigned long Size;
401 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
402 {
403 c0out << endl;
db0db9fe 404 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 405 this->Contents.c_str());
406 }
407
408 if (Size != 0)
409 {
410 c0out << " New " << SizeToStr(Size) << "B ";
411 if (Left > Size)
412 Left -= Size;
413 else
414 Left = 0;
415 }
416 else
417 c0out << ' ';
418
419 struct timeval NewTime;
420 gettimeofday(&NewTime,0);
421 double Delta = NewTime.tv_sec - StartTime.tv_sec +
422 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
423
424 c0out << Contents.Stats.Packages << " files " <<
425 SizeToStr(Contents.Stats.Bytes) << "B " <<
426 TimeToStr((long)Delta) << endl;
427
428 return true;
429}
430 /*}}}*/
431
432// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
433// ---------------------------------------------------------------------
434/* This populates the PkgList with all the possible permutations of the
435 section/arch lists. */
436void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
437{
438 // Load the defaults
439 string DDir = Setup.Find("TreeDefault::Directory",
440 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
441 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
442 "$(DIST)/$(SECTION)/source/");
443 string DPkg = Setup.Find("TreeDefault::Packages",
444 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 445 string DTrans = Setup.Find("TreeDefault::Translation",
446 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 447 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
448 "$(DIST)/$(SECTION)/");
449 string DContents = Setup.Find("TreeDefault::Contents",
450 "$(DIST)/Contents-$(ARCH)");
451 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
452 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
453 "packages-$(ARCH).db");
454 string DSources = Setup.Find("TreeDefault::Sources",
455 "$(DIST)/$(SECTION)/source/Sources");
456 string DFLFile = Setup.Find("TreeDefault::FileList", "");
457 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
458
459 // Process 'tree' type sections
460 const Configuration::Item *Top = Setup.Tree("tree");
461 for (Top = (Top == 0?0:Top->Child); Top != 0;)
462 {
463 Configuration Block(Top);
464 string Dist = Top->Tag;
465
466 // Parse the sections
d935bb17
AL		 467 string Tmp = Block.Find("Sections");
468 const char *Sections = Tmp.c_str();
b2e465d6
AL		 469 string Section;
470 while (ParseQuoteWord(Sections,Section) == true)
471 {
b2e465d6 472 string Arch;
66905344
DK		 473 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
474 {"$(SECTION)",&Section},
475 {"$(ARCH)",&Arch},
476 {}};
477 TranslationWriter *TransWriter;
478 if (DTrans.empty() == false)
479 {
480 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
481 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
482 TransWriter = new TranslationWriter(TranslationFile);
483 }
484 else
485 TransWriter = NULL;
486
487 string const Tmp2 = Block.Find("Architectures");
d935bb17 488 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 489 while (ParseQuoteWord(Archs,Arch) == true)
490 {
b2e465d6
AL		 491 PackageMap Itm;
492
493 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
494 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
495
496 if (stringcasecmp(Arch,"source") == 0)
497 {
498 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
499 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
500 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
501 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
502 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 503 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 504 }
505 else
506 {
507 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
508 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
509 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
510 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 511 Itm.Arch = Arch;
66905344
DK		 512 if (TransWriter != NULL)
513 {
514 TransWriter->IncreaseRefCounter();
515 Itm.TransWriter = TransWriter;
516 }
b2e465d6
AL		 517 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
518 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
519 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 520 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 521 }
522
523 Itm.GetGeneral(Setup,Block);
524 PkgList.push_back(Itm);
525 }
66905344
DK		 526 // we didn't use this TransWriter, so we can release it
527 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
528 delete TransWriter;
b2e465d6
AL		 529 }
530
531 Top = Top->Next;
532 }
533}
534 /*}}}*/
535// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
536// ---------------------------------------------------------------------
537/* */
538void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
539{
540 // Process 'bindirectory' type sections
541 const Configuration::Item *Top = Setup.Tree("bindirectory");
542 for (Top = (Top == 0?0:Top->Child); Top != 0;)
543 {
544 Configuration Block(Top);
545
546 PackageMap Itm;
547 Itm.PkgFile = Block.Find("Packages");
548 Itm.SrcFile = Block.Find("Sources");
549 Itm.BinCacheDB = Block.Find("BinCacheDB");
550 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 551 Itm.ExtraOverride = Block.Find("ExtraOverride");
552 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 553 Itm.SrcOverride = Block.Find("SrcOverride");
554 Itm.BaseDir = Top->Tag;
555 Itm.FLFile = Block.Find("FileList");
556 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
557 Itm.Contents = Block.Find("Contents");
558 Itm.ContentsHead = Block.Find("Contents::Header");
559
560 Itm.GetGeneral(Setup,Block);
561 PkgList.push_back(Itm);
562
563 Top = Top->Next;
564 }
565}
566 /*}}}*/
567
568// ShowHelp - Show the help text /*{{{*/
569// ---------------------------------------------------------------------
570/* */
571bool ShowHelp(CommandLine &CmdL)
572{
5b28c804
OS		 573 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
574 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 575 if (_config->FindB("version") == true)
576 return true;
577
578 cout <<
dc738e7a 579 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 580 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 581 " sources srcpath [overridefile [pathprefix]]\n"
582 " contents path\n"
0dfd2728 583 " release path\n"
b2e465d6
AL		 584 " generate config [groups]\n"
585 " clean config\n"
586 "\n"
587 "apt-ftparchive generates index files for Debian archives. It supports\n"
588 "many styles of generation from fully automated to functional replacements\n"
589 "for dpkg-scanpackages and dpkg-scansources\n"
590 "\n"
591 "apt-ftparchive generates Package files from a tree of .debs. The\n"
592 "Package file contains the contents of all the control fields from\n"
593 "each package as well as the MD5 hash and filesize. An override file\n"
594 "is supported to force the value of Priority and Section.\n"
595 "\n"
596 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
597 "The --source-override option can be used to specify a src override file\n"
598 "\n"
599 "The 'packages' and 'sources' command should be run in the root of the\n"
600 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 601 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 602 "appended to the filename fields if present. Example usage from the \n"
75224826 603 "Debian archive:\n"
b2e465d6
AL		 604 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
605 " dists/potato/main/binary-i386/Packages\n"
606 "\n"
607 "Options:\n"
608 " -h This help text\n"
609 " --md5 Control MD5 generation\n"
610 " -s=? Source override file\n"
611 " -q Quiet\n"
612 " -d=? Select the optional caching database\n"
613 " --no-delink Enable delinking debug mode\n"
614 " --contents Control contents file generation\n"
615 " -c=? Read this configuration file\n"
8a2d7db6 616 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 617
618 return true;
619}
620 /*}}}*/
621// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
622// ---------------------------------------------------------------------
623/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
624bool SimpleGenPackages(CommandLine &CmdL)
625{
626 if (CmdL.FileSize() < 2)
627 return ShowHelp(CmdL);
628
629 string Override;
630 if (CmdL.FileSize() >= 3)
631 Override = CmdL.FileList[2];
632
633 // Create a package writer object.
634 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 635 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 636 if (_error->PendingError() == true)
637 return false;
638
639 if (CmdL.FileSize() >= 4)
640 Packages.PathPrefix = CmdL.FileList[3];
641
642 // Do recursive directory searching
643 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
644 return false;
645
646 return true;
647}
648 /*}}}*/
649// SimpleGenContents - Generate a Contents listing /*{{{*/
650// ---------------------------------------------------------------------
651/* */
652bool SimpleGenContents(CommandLine &CmdL)
653{
654 if (CmdL.FileSize() < 2)
655 return ShowHelp(CmdL);
656
657 // Create a package writer object.
31981076 658 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 659 if (_error->PendingError() == true)
660 return false;
661
662 // Do recursive directory searching
663 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
664 return false;
665
666 Contents.Finish();
667
668 return true;
669}
670 /*}}}*/
671// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
672// ---------------------------------------------------------------------
673/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
674bool SimpleGenSources(CommandLine &CmdL)
675{
676 if (CmdL.FileSize() < 2)
677 return ShowHelp(CmdL);
678
679 string Override;
680 if (CmdL.FileSize() >= 3)
681 Override = CmdL.FileList[2];
682
683 string SOverride;
684 if (Override.empty() == false)
685 SOverride = Override + ".src";
686
687 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
688 SOverride.c_str());
689
690 // Create a package writer object.
691 SourcesWriter Sources(Override,SOverride);
692 if (_error->PendingError() == true)
693 return false;
694
695 if (CmdL.FileSize() >= 4)
696 Sources.PathPrefix = CmdL.FileList[3];
697
698 // Do recursive directory searching
699 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
700 return false;
701
702 return true;
703}
98953965
AL		 704 /*}}}*/
705// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
706// ---------------------------------------------------------------------
707bool SimpleGenRelease(CommandLine &CmdL)
708{
0dfd2728
AL		 709 if (CmdL.FileSize() < 2)
710 return ShowHelp(CmdL);
711
c0eb6bc6
AL		 712 string Dir = CmdL.FileList[1];
713
98953965 714 ReleaseWriter Release("");
c0eb6bc6
AL		 715 Release.DirStrip = Dir;
716
98953965
AL		 717 if (_error->PendingError() == true)
718 return false;
719
c0eb6bc6 720 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 721 return false;
722
f7291f62
AL		 723 Release.Finish();
724
98953965
AL		 725 return true;
726}
727
b2e465d6
AL		 728 /*}}}*/
729// Generate - Full generate, using a config file /*{{{*/
730// ---------------------------------------------------------------------
731/* */
732bool Generate(CommandLine &CmdL)
733{
734 struct CacheDB::Stats SrcStats;
735 if (CmdL.FileSize() < 2)
736 return ShowHelp(CmdL);
737
738 struct timeval StartTime;
739 gettimeofday(&StartTime,0);
740 struct CacheDB::Stats Stats;
741
742 // Read the configuration file.
743 Configuration Setup;
744 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
745 return false;
746
747 vector<PackageMap> PkgList;
748 LoadTree(PkgList,Setup);
749 LoadBinDir(PkgList,Setup);
750
751 // Sort by cache DB to improve IO locality.
752 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
753
754 // Generate packages
755 if (CmdL.FileSize() <= 2)
756 {
d935bb17 757 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 758 if (I->GenPackages(Setup,Stats) == false)
759 _error->DumpErrors();
d935bb17 760 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 761 if (I->GenSources(Setup,SrcStats) == false)
762 _error->DumpErrors();
763 }
764 else
765 {
766 // Make a choice list out of the package list..
767 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
768 RxChoiceList *End = List;
d935bb17 769 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6 770 {
d935bb17 771 End->UserData = &(*I);
b2e465d6
AL		 772 End->Str = I->BaseDir.c_str();
773 End++;
774
d935bb17 775 End->UserData = &(*I);
b2e465d6
AL		 776 End->Str = I->Tag.c_str();
777 End++;
778 }
779 End->Str = 0;
780
781 // Regex it
782 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
783 {
784 delete [] List;
dc738e7a 785 return _error->Error(_("No selections matched"));
b2e465d6
AL		 786 }
787 _error->DumpErrors();
788
789 // Do the generation for Packages
790 for (End = List; End->Str != 0; End++)
791 {
792 if (End->Hit == false)
793 continue;
794
795 PackageMap *I = (PackageMap *)End->UserData;
796 if (I->PkgDone == true)
797 continue;
798 if (I->GenPackages(Setup,Stats) == false)
799 _error->DumpErrors();
800 }
801
802 // Do the generation for Sources
803 for (End = List; End->Str != 0; End++)
804 {
805 if (End->Hit == false)
806 continue;
807
808 PackageMap *I = (PackageMap *)End->UserData;
809 if (I->SrcDone == true)
810 continue;
811 if (I->GenSources(Setup,SrcStats) == false)
812 _error->DumpErrors();
813 }
814
815 delete [] List;
816 }
66905344
DK		 817
818 // close the Translation master files
819 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
820 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
821 delete I->TransWriter;
822
b2e465d6
AL		 823 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
824 return true;
825
c6474fb6 826 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 827
828 // Sort the contents file list by date
829 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
d935bb17 830 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 831 {
832 struct stat A;
833 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
834 I->CntCompress,A) == false)
835 time(&I->ContentsMTime);
836 else
837 I->ContentsMTime = A.st_mtime;
838 }
839 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
840
841 /* Now for Contents.. The process here is to do a make-like dependency
842 check. Each contents file is verified to be newer than the package files
843 that describe the debs it indexes. Since the package files contain
844 hashes of the .debs this means they have not changed either so the
845 contents must be up to date. */
846 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
d935bb17 847 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 848 {
849 // This record is not relevent
850 if (I->ContentsDone == true ||
851 I->Contents.empty() == true)
852 continue;
853
854 // Do not do everything if the user specified sections.
855 if (CmdL.FileSize() > 2 && I->PkgDone == false)
856 continue;
857
858 struct stat A,B;
859 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
860 {
861 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
862 {
dc738e7a 863 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 864 continue;
865 }
866
867 if (A.st_mtime > B.st_mtime)
868 continue;
869 }
870
871 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
872 MaxContentsChange) == false)
873 _error->DumpErrors();
874
875 // Hit the limit?
876 if (MaxContentsChange == 0)
877 {
878 c1out << "Hit contents update byte limit" << endl;
879 break;
880 }
881 }
882
883 struct timeval NewTime;
884 gettimeofday(&NewTime,0);
885 double Delta = NewTime.tv_sec - StartTime.tv_sec +
886 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
887 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
888 << " archives. Took " << TimeToStr((long)Delta) << endl;
889
890 return true;
891}
892 /*}}}*/
893// Clean - Clean out the databases /*{{{*/
894// ---------------------------------------------------------------------
895/* */
896bool Clean(CommandLine &CmdL)
897{
898 if (CmdL.FileSize() != 2)
899 return ShowHelp(CmdL);
900
901 // Read the configuration file.
902 Configuration Setup;
903 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
904 return false;
905
906 vector<PackageMap> PkgList;
907 LoadTree(PkgList,Setup);
908 LoadBinDir(PkgList,Setup);
909
910 // Sort by cache DB to improve IO locality.
911 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
912
913 string CacheDir = Setup.FindDir("Dir::CacheDir");
914
d935bb17 915 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 916 {
917 c0out << I->BinCacheDB << endl;
918 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
919 if (DB.Clean() == false)
920 _error->DumpErrors();
921
922 string CacheDB = I->BinCacheDB;
923 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
924 }
925
926 return true;
927}
928 /*}}}*/
929
930int main(int argc, const char *argv[])
931{
1692a2c7 932 setlocale(LC_ALL, "");
b2e465d6
AL		 933 CommandLine::Args Args[] = {
934 {'h',"help","help",0},
935 {0,"md5","APT::FTPArchive::MD5",0},
936 {'v',"version","version",0},
937 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
938 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
939 {'q',"quiet","quiet",CommandLine::IntLevel},
940 {'q',"silent","quiet",CommandLine::IntLevel},
941 {0,"delink","APT::FTPArchive::DeLinkAct",0},
942 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
943 {0,"contents","APT::FTPArchive::Contents",0},
31981076 944 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 945 {'c',"config-file",0,CommandLine::ConfigFile},
946 {'o',"option",0,CommandLine::ArbItem},
947 {0,0,0,0}};
948 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
949 {"contents",&SimpleGenContents},
950 {"sources",&SimpleGenSources},
98953965 951 {"release",&SimpleGenRelease},
b2e465d6
AL		 952 {"generate",&Generate},
953 {"clean",&Clean},
954 {"help",&ShowHelp},
955 {0,0}};
956
957 // Parse the command line and initialize the package library
958 CommandLine CmdL(Args,_config);
959 if (CmdL.Parse(argc,argv) == false)
960 {
961 _error->DumpErrors();
962 return 100;
963 }
964
965 // See if the help should be shown
966 if (_config->FindB("help") == true ||
967 _config->FindB("version") == true ||
968 CmdL.FileSize() == 0)
969 {
970 ShowHelp(CmdL);
971 return 0;
972 }
973
974 // Setup the output streams
1809f8d2
AL		 975 c0out.rdbuf(clog.rdbuf());
976 c1out.rdbuf(clog.rdbuf());
977 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 978 Quiet = _config->FindI("quiet",0);
979 if (Quiet > 0)
980 c0out.rdbuf(devnull.rdbuf());
981 if (Quiet > 1)
982 c1out.rdbuf(devnull.rdbuf());
983
984 // Match the operation
985 CmdL.DispatchArg(Cmds);
986
987 if (_error->empty() == false)
988 {
989 bool Errors = _error->PendingError();
990 _error->DumpErrors();
991 return Errors == true?100:0;
992 }
993 return 0;
994}
ntk's bugfix for apt