HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
merged from debian-experimental-ma
[ntk/apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
b2e465d6
AL		 13#include "apt-ftparchive.h"
14
15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
b2e465d6
AL		 20#include <config.h>
21#include <apti18n.h>
22#include <algorithm>
23
152ab79e 24#include <climits>
b2e465d6
AL		 25#include <sys/time.h>
26#include <regex.h>
27
28#include "contents.h"
29#include "multicompress.h"
30#include "writer.h"
31 /*}}}*/
32
d935bb17
AL		 33using namespace std;
34ostream c0out(0);
35ostream c1out(0);
36ostream c2out(0);
b2e465d6
AL		 37ofstream devnull("/dev/null");
38unsigned Quiet = 0;
39
40// struct PackageMap - List of all package files in the config file /*{{{*/
41// ---------------------------------------------------------------------
42/* */
43struct PackageMap
44{
45 // General Stuff
46 string BaseDir;
47 string InternalPrefix;
48 string FLFile;
49 string PkgExt;
50 string SrcExt;
51
52 // Stuff for the Package File
53 string PkgFile;
54 string BinCacheDB;
55 string BinOverride;
64177f17 56 string ExtraOverride;
0b41e0e7
MV		 57
58 // We generate for this given arch
59 string Arch;
b2e465d6
AL		 60
61 // Stuff for the Source File
62 string SrcFile;
63 string SrcOverride;
64177f17 64 string SrcExtraOverride;
b2e465d6 65
66905344 66 // Translation master file
4e794c50 67 bool LongDesc;
66905344
DK		 68 TranslationWriter *TransWriter;
69
b2e465d6
AL		 70 // Contents
71 string Contents;
72 string ContentsHead;
73
74 // Random things
75 string Tag;
76 string PkgCompress;
77 string CntCompress;
78 string SrcCompress;
79 string PathPrefix;
80 unsigned int DeLinkLimit;
81 mode_t Permissions;
82
83 bool ContentsDone;
84 bool PkgDone;
85 bool SrcDone;
86 time_t ContentsMTime;
87
88 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
89 {
90 inline bool operator() (const PackageMap &x,const PackageMap &y)
91 {return x.ContentsMTime < y.ContentsMTime;};
92 };
93
94 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
95 {
96 inline bool operator() (const PackageMap &x,const PackageMap &y)
97 {return x.BinCacheDB < y.BinCacheDB;};
98 };
99
100 void GetGeneral(Configuration &Setup,Configuration &Block);
101 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
102 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
103 bool GenContents(Configuration &Setup,
d935bb17
AL		 104 vector<PackageMap>::iterator Begin,
105 vector<PackageMap>::iterator End,
b2e465d6
AL		 106 unsigned long &Left);
107
4e794c50 108 PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
66905344
DK		 109 ContentsDone(false), PkgDone(false), SrcDone(false),
110 ContentsMTime(0) {};
b2e465d6
AL		 111};
112 /*}}}*/
113
114// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
115// ---------------------------------------------------------------------
116/* */
117void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
118{
119 PathPrefix = Block.Find("PathPrefix");
120
121 if (Block.FindB("External-Links",true) == false)
122 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
123 else
124 DeLinkLimit = 0;
125
126 PkgCompress = Block.Find("Packages::Compress",
127 Setup.Find("Default::Packages::Compress",". gzip").c_str());
128 CntCompress = Block.Find("Contents::Compress",
129 Setup.Find("Default::Contents::Compress",". gzip").c_str());
130 SrcCompress = Block.Find("Sources::Compress",
131 Setup.Find("Default::Sources::Compress",". gzip").c_str());
132
133 SrcExt = Block.Find("Sources::Extensions",
134 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
135 PkgExt = Block.Find("Packages::Extensions",
136 Setup.Find("Default::Packages::Extensions",".deb").c_str());
137
b2e465d6
AL		 138 if (FLFile.empty() == false)
139 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
140
141 if (Contents == " ")
142 Contents= string();
143}
144 /*}}}*/
145// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
146// ---------------------------------------------------------------------
147/* This generates the Package File described by this object. */
148bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
149{
150 if (PkgFile.empty() == true)
151 return true;
152
153 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
154 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
155 string CacheDir = Setup.FindDir("Dir::CacheDir");
156
157 struct timeval StartTime;
158 gettimeofday(&StartTime,0);
159
160 PkgDone = true;
161
162 // Create a package writer object.
163 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 164 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 165 flCombine(OverrideDir,ExtraOverride),
166 Arch);
b2e465d6 167 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 168 return _error->Error(_("Package extension list is too long"));
b2e465d6 169 if (_error->PendingError() == true)
db0db9fe 170 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 171
172 Packages.PathPrefix = PathPrefix;
173 Packages.DirStrip = ArchiveDir;
174 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
175
66905344 176 Packages.TransWriter = TransWriter;
4e794c50 177 Packages.LongDescription = LongDesc;
66905344 178
b2e465d6
AL		 179 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
180 Packages.DeLinkLimit = DeLinkLimit;
64177f17 181
b2e465d6
AL		 182 // Create a compressor object
183 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
184 PkgCompress,Permissions);
185 Packages.Output = Comp.Input;
186 if (_error->PendingError() == true)
db0db9fe 187 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 188
189 c0out << ' ' << BaseDir << ":" << flush;
190
191 // Do recursive directory searching
192 if (FLFile.empty() == true)
193 {
194 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
195 return false;
196 }
197 else
198 {
199 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
200 return false;
201 }
202
203 Packages.Output = 0; // Just in case
204
205 // Finish compressing
206 unsigned long Size;
207 if (Comp.Finalize(Size) == false)
208 {
209 c0out << endl;
db0db9fe 210 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 211 }
212
213 if (Size != 0)
214 c0out << " New "
215 << SizeToStr(Size) << "B ";
216 else
217 c0out << ' ';
218
219 struct timeval NewTime;
220 gettimeofday(&NewTime,0);
221 double Delta = NewTime.tv_sec - StartTime.tv_sec +
222 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
223
224 c0out << Packages.Stats.Packages << " files " <<
225/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
226 SizeToStr(Packages.Stats.Bytes) << "B " <<
227 TimeToStr((long)Delta) << endl;
228
229 Stats.Add(Packages.Stats);
230 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
231
232 return !_error->PendingError();
233}
98953965 234
b2e465d6 235 /*}}}*/
64177f17 236// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 237// ---------------------------------------------------------------------
238/* This generates the Sources File described by this object. */
239bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
240{
241 if (SrcFile.empty() == true)
242 return true;
243
244 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
245 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
246 string CacheDir = Setup.FindDir("Dir::CacheDir");
247
248 struct timeval StartTime;
249 gettimeofday(&StartTime,0);
250
251 SrcDone = true;
252
253 // Create a package writer object.
254 SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
64177f17
AL		 255 flCombine(OverrideDir,SrcOverride),
256 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 257 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 258 return _error->Error(_("Source extension list is too long"));
b2e465d6 259 if (_error->PendingError() == true)
db0db9fe 260 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 261
262 Sources.PathPrefix = PathPrefix;
263 Sources.DirStrip = ArchiveDir;
264 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
265
266 Sources.DeLinkLimit = DeLinkLimit;
267 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
268
269 // Create a compressor object
270 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
271 SrcCompress,Permissions);
272 Sources.Output = Comp.Input;
273 if (_error->PendingError() == true)
db0db9fe 274 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 275
276 c0out << ' ' << BaseDir << ":" << flush;
277
278 // Do recursive directory searching
279 if (FLFile.empty() == true)
280 {
281 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
282 return false;
283 }
284 else
285 {
286 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
287 return false;
288 }
289 Sources.Output = 0; // Just in case
290
291 // Finish compressing
292 unsigned long Size;
293 if (Comp.Finalize(Size) == false)
294 {
295 c0out << endl;
db0db9fe 296 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 297 }
298
299 if (Size != 0)
300 c0out << " New "
301 << SizeToStr(Size) << "B ";
302 else
303 c0out << ' ';
304
305 struct timeval NewTime;
306 gettimeofday(&NewTime,0);
307 double Delta = NewTime.tv_sec - StartTime.tv_sec +
308 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
309
310 c0out << Sources.Stats.Packages << " pkgs in " <<
311 TimeToStr((long)Delta) << endl;
312
313 Stats.Add(Sources.Stats);
314 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
315
316 return !_error->PendingError();
317}
318 /*}}}*/
319// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
320// ---------------------------------------------------------------------
321/* This generates the contents file partially described by this object.
322 It searches the given iterator range for other package files that map
323 into this contents file and includes their data as well when building. */
324bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 325 vector<PackageMap>::iterator Begin,
326 vector<PackageMap>::iterator End,
327 unsigned long &Left)
b2e465d6
AL		 328{
329 if (Contents.empty() == true)
330 return true;
331
332 if (Left == 0)
333 return true;
334
335 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
336 string CacheDir = Setup.FindDir("Dir::CacheDir");
337 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
338
339 struct timeval StartTime;
340 gettimeofday(&StartTime,0);
341
342 // Create a package writer object.
31981076 343 ContentsWriter Contents("", Arch);
b2e465d6 344 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 345 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 346 if (_error->PendingError() == true)
347 return false;
348
349 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
350 CntCompress,Permissions);
351 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
352 Contents.Output = Comp.Input;
353 if (_error->PendingError() == true)
354 return false;
355
356 // Write the header out.
357 if (ContentsHead.empty() == false)
358 {
359 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
360 if (_error->PendingError() == true)
361 return false;
362
363 unsigned long Size = Head.Size();
364 unsigned char Buf[4096];
365 while (Size != 0)
366 {
367 unsigned long ToRead = Size;
368 if (Size > sizeof(Buf))
369 ToRead = sizeof(Buf);
370
371 if (Head.Read(Buf,ToRead) == false)
372 return false;
373
374 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 375 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 376
377 Size -= ToRead;
378 }
379 }
380
381 /* Go over all the package file records and parse all the package
382 files associated with this contents file into one great big honking
383 memory structure, then dump the sorted version */
384 c0out << ' ' << this->Contents << ":" << flush;
d935bb17 385 for (vector<PackageMap>::iterator I = Begin; I != End; I++)
b2e465d6
AL		 386 {
387 if (I->Contents != this->Contents)
388 continue;
389
390 Contents.Prefix = ArchiveDir;
391 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
392 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
393 I->PkgCompress);
394
395 I->ContentsDone = true;
396 }
397
398 Contents.Finish();
399
400 // Finish compressing
401 unsigned long Size;
402 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
403 {
404 c0out << endl;
db0db9fe 405 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 406 this->Contents.c_str());
407 }
408
409 if (Size != 0)
410 {
411 c0out << " New " << SizeToStr(Size) << "B ";
412 if (Left > Size)
413 Left -= Size;
414 else
415 Left = 0;
416 }
417 else
418 c0out << ' ';
419
420 struct timeval NewTime;
421 gettimeofday(&NewTime,0);
422 double Delta = NewTime.tv_sec - StartTime.tv_sec +
423 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
424
425 c0out << Contents.Stats.Packages << " files " <<
426 SizeToStr(Contents.Stats.Bytes) << "B " <<
427 TimeToStr((long)Delta) << endl;
428
429 return true;
430}
431 /*}}}*/
432
433// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
434// ---------------------------------------------------------------------
435/* This populates the PkgList with all the possible permutations of the
436 section/arch lists. */
437void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
438{
439 // Load the defaults
440 string DDir = Setup.Find("TreeDefault::Directory",
441 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
442 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
443 "$(DIST)/$(SECTION)/source/");
444 string DPkg = Setup.Find("TreeDefault::Packages",
445 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 446 string DTrans = Setup.Find("TreeDefault::Translation",
447 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 448 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
449 "$(DIST)/$(SECTION)/");
450 string DContents = Setup.Find("TreeDefault::Contents",
451 "$(DIST)/Contents-$(ARCH)");
452 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
453 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
454 "packages-$(ARCH).db");
455 string DSources = Setup.Find("TreeDefault::Sources",
456 "$(DIST)/$(SECTION)/source/Sources");
457 string DFLFile = Setup.Find("TreeDefault::FileList", "");
458 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
459
34f1d96c
DK		 460 int const Permissions = Setup.FindI("Default::FileMode",0644);
461
462 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 463 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 464 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
4e794c50 465
b2e465d6
AL		 466 // Process 'tree' type sections
467 const Configuration::Item *Top = Setup.Tree("tree");
468 for (Top = (Top == 0?0:Top->Child); Top != 0;)
469 {
470 Configuration Block(Top);
471 string Dist = Top->Tag;
472
473 // Parse the sections
d935bb17
AL		 474 string Tmp = Block.Find("Sections");
475 const char *Sections = Tmp.c_str();
b2e465d6
AL		 476 string Section;
477 while (ParseQuoteWord(Sections,Section) == true)
478 {
b2e465d6 479 string Arch;
66905344
DK		 480 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
481 {"$(SECTION)",&Section},
482 {"$(ARCH)",&Arch},
483 {}};
34f1d96c 484 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 485 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
66905344 486 TranslationWriter *TransWriter;
4e794c50 487 if (DTrans.empty() == false && LongDesc == false)
66905344
DK		 488 {
489 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
490 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
34f1d96c
DK		 491 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
492 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
66905344
DK		 493 }
494 else
495 TransWriter = NULL;
496
497 string const Tmp2 = Block.Find("Architectures");
d935bb17 498 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 499 while (ParseQuoteWord(Archs,Arch) == true)
500 {
b2e465d6 501 PackageMap Itm;
34f1d96c 502 Itm.Permissions = Perms;
b2e465d6
AL		 503 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
504 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
505
506 if (stringcasecmp(Arch,"source") == 0)
507 {
508 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
509 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
510 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
511 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
512 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 513 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 514 }
515 else
516 {
517 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
518 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
519 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
520 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 521 Itm.Arch = Arch;
4e794c50 522 Itm.LongDesc = LongDesc;
66905344
DK		 523 if (TransWriter != NULL)
524 {
525 TransWriter->IncreaseRefCounter();
526 Itm.TransWriter = TransWriter;
527 }
b2e465d6
AL		 528 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
529 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
530 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 531 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 532 }
533
534 Itm.GetGeneral(Setup,Block);
535 PkgList.push_back(Itm);
536 }
66905344
DK		 537 // we didn't use this TransWriter, so we can release it
538 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
539 delete TransWriter;
b2e465d6
AL		 540 }
541
542 Top = Top->Next;
543 }
544}
545 /*}}}*/
546// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
547// ---------------------------------------------------------------------
548/* */
549void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
550{
551 // Process 'bindirectory' type sections
552 const Configuration::Item *Top = Setup.Tree("bindirectory");
553 for (Top = (Top == 0?0:Top->Child); Top != 0;)
554 {
555 Configuration Block(Top);
556
557 PackageMap Itm;
558 Itm.PkgFile = Block.Find("Packages");
559 Itm.SrcFile = Block.Find("Sources");
560 Itm.BinCacheDB = Block.Find("BinCacheDB");
561 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 562 Itm.ExtraOverride = Block.Find("ExtraOverride");
563 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 564 Itm.SrcOverride = Block.Find("SrcOverride");
565 Itm.BaseDir = Top->Tag;
566 Itm.FLFile = Block.Find("FileList");
567 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
568 Itm.Contents = Block.Find("Contents");
569 Itm.ContentsHead = Block.Find("Contents::Header");
570
571 Itm.GetGeneral(Setup,Block);
572 PkgList.push_back(Itm);
573
574 Top = Top->Next;
575 }
576}
577 /*}}}*/
578
579// ShowHelp - Show the help text /*{{{*/
580// ---------------------------------------------------------------------
581/* */
582bool ShowHelp(CommandLine &CmdL)
583{
5b28c804
OS		 584 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
585 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 586 if (_config->FindB("version") == true)
587 return true;
588
589 cout <<
dc738e7a 590 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 591 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 592 " sources srcpath [overridefile [pathprefix]]\n"
593 " contents path\n"
0dfd2728 594 " release path\n"
b2e465d6
AL		 595 " generate config [groups]\n"
596 " clean config\n"
597 "\n"
598 "apt-ftparchive generates index files for Debian archives. It supports\n"
599 "many styles of generation from fully automated to functional replacements\n"
600 "for dpkg-scanpackages and dpkg-scansources\n"
601 "\n"
602 "apt-ftparchive generates Package files from a tree of .debs. The\n"
603 "Package file contains the contents of all the control fields from\n"
604 "each package as well as the MD5 hash and filesize. An override file\n"
605 "is supported to force the value of Priority and Section.\n"
606 "\n"
607 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
608 "The --source-override option can be used to specify a src override file\n"
609 "\n"
610 "The 'packages' and 'sources' command should be run in the root of the\n"
611 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 612 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 613 "appended to the filename fields if present. Example usage from the \n"
75224826 614 "Debian archive:\n"
b2e465d6
AL		 615 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
616 " dists/potato/main/binary-i386/Packages\n"
617 "\n"
618 "Options:\n"
619 " -h This help text\n"
620 " --md5 Control MD5 generation\n"
621 " -s=? Source override file\n"
622 " -q Quiet\n"
623 " -d=? Select the optional caching database\n"
624 " --no-delink Enable delinking debug mode\n"
625 " --contents Control contents file generation\n"
626 " -c=? Read this configuration file\n"
8a2d7db6 627 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 628
629 return true;
630}
631 /*}}}*/
632// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
633// ---------------------------------------------------------------------
634/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
635bool SimpleGenPackages(CommandLine &CmdL)
636{
637 if (CmdL.FileSize() < 2)
638 return ShowHelp(CmdL);
639
640 string Override;
641 if (CmdL.FileSize() >= 3)
642 Override = CmdL.FileList[2];
643
644 // Create a package writer object.
645 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 646 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 647 if (_error->PendingError() == true)
648 return false;
649
650 if (CmdL.FileSize() >= 4)
651 Packages.PathPrefix = CmdL.FileList[3];
652
653 // Do recursive directory searching
654 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
655 return false;
656
657 return true;
658}
659 /*}}}*/
660// SimpleGenContents - Generate a Contents listing /*{{{*/
661// ---------------------------------------------------------------------
662/* */
663bool SimpleGenContents(CommandLine &CmdL)
664{
665 if (CmdL.FileSize() < 2)
666 return ShowHelp(CmdL);
667
668 // Create a package writer object.
31981076 669 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 670 if (_error->PendingError() == true)
671 return false;
672
673 // Do recursive directory searching
674 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
675 return false;
676
677 Contents.Finish();
678
679 return true;
680}
681 /*}}}*/
682// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
683// ---------------------------------------------------------------------
684/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
685bool SimpleGenSources(CommandLine &CmdL)
686{
687 if (CmdL.FileSize() < 2)
688 return ShowHelp(CmdL);
689
690 string Override;
691 if (CmdL.FileSize() >= 3)
692 Override = CmdL.FileList[2];
693
694 string SOverride;
695 if (Override.empty() == false)
696 SOverride = Override + ".src";
697
698 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
699 SOverride.c_str());
700
701 // Create a package writer object.
702 SourcesWriter Sources(Override,SOverride);
703 if (_error->PendingError() == true)
704 return false;
705
706 if (CmdL.FileSize() >= 4)
707 Sources.PathPrefix = CmdL.FileList[3];
708
709 // Do recursive directory searching
710 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
711 return false;
712
713 return true;
714}
98953965
AL		 715 /*}}}*/
716// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
717// ---------------------------------------------------------------------
718bool SimpleGenRelease(CommandLine &CmdL)
719{
0dfd2728
AL		 720 if (CmdL.FileSize() < 2)
721 return ShowHelp(CmdL);
722
c0eb6bc6
AL		 723 string Dir = CmdL.FileList[1];
724
98953965 725 ReleaseWriter Release("");
c0eb6bc6
AL		 726 Release.DirStrip = Dir;
727
98953965
AL		 728 if (_error->PendingError() == true)
729 return false;
730
c0eb6bc6 731 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 732 return false;
733
f7291f62
AL		 734 Release.Finish();
735
98953965
AL		 736 return true;
737}
738
b2e465d6
AL		 739 /*}}}*/
740// Generate - Full generate, using a config file /*{{{*/
741// ---------------------------------------------------------------------
742/* */
743bool Generate(CommandLine &CmdL)
744{
745 struct CacheDB::Stats SrcStats;
746 if (CmdL.FileSize() < 2)
747 return ShowHelp(CmdL);
748
749 struct timeval StartTime;
750 gettimeofday(&StartTime,0);
751 struct CacheDB::Stats Stats;
752
753 // Read the configuration file.
754 Configuration Setup;
755 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
756 return false;
757
758 vector<PackageMap> PkgList;
759 LoadTree(PkgList,Setup);
760 LoadBinDir(PkgList,Setup);
761
762 // Sort by cache DB to improve IO locality.
763 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
764
765 // Generate packages
766 if (CmdL.FileSize() <= 2)
767 {
d935bb17 768 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 769 if (I->GenPackages(Setup,Stats) == false)
770 _error->DumpErrors();
d935bb17 771 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 772 if (I->GenSources(Setup,SrcStats) == false)
773 _error->DumpErrors();
774 }
775 else
776 {
777 // Make a choice list out of the package list..
778 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
779 RxChoiceList *End = List;
d935bb17 780 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6 781 {
d935bb17 782 End->UserData = &(*I);
b2e465d6
AL		 783 End->Str = I->BaseDir.c_str();
784 End++;
785
d935bb17 786 End->UserData = &(*I);
b2e465d6
AL		 787 End->Str = I->Tag.c_str();
788 End++;
789 }
790 End->Str = 0;
791
792 // Regex it
793 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
794 {
795 delete [] List;
dc738e7a 796 return _error->Error(_("No selections matched"));
b2e465d6
AL		 797 }
798 _error->DumpErrors();
799
800 // Do the generation for Packages
801 for (End = List; End->Str != 0; End++)
802 {
803 if (End->Hit == false)
804 continue;
805
806 PackageMap *I = (PackageMap *)End->UserData;
807 if (I->PkgDone == true)
808 continue;
809 if (I->GenPackages(Setup,Stats) == false)
810 _error->DumpErrors();
811 }
812
813 // Do the generation for Sources
814 for (End = List; End->Str != 0; End++)
815 {
816 if (End->Hit == false)
817 continue;
818
819 PackageMap *I = (PackageMap *)End->UserData;
820 if (I->SrcDone == true)
821 continue;
822 if (I->GenSources(Setup,SrcStats) == false)
823 _error->DumpErrors();
824 }
825
826 delete [] List;
827 }
66905344
DK		 828
829 // close the Translation master files
830 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
831 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
832 delete I->TransWriter;
833
b2e465d6
AL		 834 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
835 return true;
836
c6474fb6 837 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 838
839 // Sort the contents file list by date
840 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
d935bb17 841 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 842 {
843 struct stat A;
844 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
845 I->CntCompress,A) == false)
846 time(&I->ContentsMTime);
847 else
848 I->ContentsMTime = A.st_mtime;
849 }
850 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
851
852 /* Now for Contents.. The process here is to do a make-like dependency
853 check. Each contents file is verified to be newer than the package files
854 that describe the debs it indexes. Since the package files contain
855 hashes of the .debs this means they have not changed either so the
856 contents must be up to date. */
857 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
d935bb17 858 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 859 {
860 // This record is not relevent
861 if (I->ContentsDone == true ||
862 I->Contents.empty() == true)
863 continue;
864
865 // Do not do everything if the user specified sections.
866 if (CmdL.FileSize() > 2 && I->PkgDone == false)
867 continue;
868
869 struct stat A,B;
870 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
871 {
872 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
873 {
dc738e7a 874 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 875 continue;
876 }
877
878 if (A.st_mtime > B.st_mtime)
879 continue;
880 }
881
882 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
883 MaxContentsChange) == false)
884 _error->DumpErrors();
885
886 // Hit the limit?
887 if (MaxContentsChange == 0)
888 {
889 c1out << "Hit contents update byte limit" << endl;
890 break;
891 }
892 }
893
894 struct timeval NewTime;
895 gettimeofday(&NewTime,0);
896 double Delta = NewTime.tv_sec - StartTime.tv_sec +
897 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
898 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
899 << " archives. Took " << TimeToStr((long)Delta) << endl;
900
901 return true;
902}
903 /*}}}*/
904// Clean - Clean out the databases /*{{{*/
905// ---------------------------------------------------------------------
906/* */
907bool Clean(CommandLine &CmdL)
908{
909 if (CmdL.FileSize() != 2)
910 return ShowHelp(CmdL);
911
912 // Read the configuration file.
913 Configuration Setup;
914 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
915 return false;
916
917 vector<PackageMap> PkgList;
918 LoadTree(PkgList,Setup);
919 LoadBinDir(PkgList,Setup);
920
921 // Sort by cache DB to improve IO locality.
922 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
923
924 string CacheDir = Setup.FindDir("Dir::CacheDir");
925
d935bb17 926 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 927 {
928 c0out << I->BinCacheDB << endl;
929 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
930 if (DB.Clean() == false)
931 _error->DumpErrors();
932
933 string CacheDB = I->BinCacheDB;
934 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
935 }
936
937 return true;
938}
939 /*}}}*/
940
941int main(int argc, const char *argv[])
942{
1692a2c7 943 setlocale(LC_ALL, "");
b2e465d6
AL		 944 CommandLine::Args Args[] = {
945 {'h',"help","help",0},
946 {0,"md5","APT::FTPArchive::MD5",0},
947 {'v',"version","version",0},
948 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
949 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
950 {'q',"quiet","quiet",CommandLine::IntLevel},
951 {'q',"silent","quiet",CommandLine::IntLevel},
952 {0,"delink","APT::FTPArchive::DeLinkAct",0},
953 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
954 {0,"contents","APT::FTPArchive::Contents",0},
31981076 955 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 956 {'c',"config-file",0,CommandLine::ConfigFile},
957 {'o',"option",0,CommandLine::ArbItem},
958 {0,0,0,0}};
959 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
960 {"contents",&SimpleGenContents},
961 {"sources",&SimpleGenSources},
98953965 962 {"release",&SimpleGenRelease},
b2e465d6
AL		 963 {"generate",&Generate},
964 {"clean",&Clean},
965 {"help",&ShowHelp},
966 {0,0}};
967
968 // Parse the command line and initialize the package library
969 CommandLine CmdL(Args,_config);
b34d4b47 970 if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
b2e465d6
AL		 971 {
972 _error->DumpErrors();
973 return 100;
974 }
975
976 // See if the help should be shown
977 if (_config->FindB("help") == true ||
978 _config->FindB("version") == true ||
979 CmdL.FileSize() == 0)
980 {
981 ShowHelp(CmdL);
982 return 0;
983 }
984
985 // Setup the output streams
1809f8d2
AL		 986 c0out.rdbuf(clog.rdbuf());
987 c1out.rdbuf(clog.rdbuf());
988 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 989 Quiet = _config->FindI("quiet",0);
990 if (Quiet > 0)
991 c0out.rdbuf(devnull.rdbuf());
992 if (Quiet > 1)
993 c1out.rdbuf(devnull.rdbuf());
994
995 // Match the operation
996 CmdL.DispatchArg(Cmds);
997
998 if (_error->empty() == false)
999 {
1000 bool Errors = _error->PendingError();
1001 _error->DumpErrors();
1002 return Errors == true?100:0;
1003 }
1004 return 0;
1005}
ntk's bugfix for apt