HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
do not update po and pot files in the process of the build as this
[ntk/apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140
DK		 13#include <config.h>
14
b2e465d6
AL		 15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
b2e465d6
AL		 20#include <algorithm>
21
152ab79e 22#include <climits>
b2e465d6
AL		 23#include <sys/time.h>
24#include <regex.h>
25
ea542140 26#include "apt-ftparchive.h"
b2e465d6
AL		 27#include "contents.h"
28#include "multicompress.h"
ea542140
DK		 29#include "writer.h"
30
31#include <apti18n.h>
b2e465d6
AL		 32 /*}}}*/
33
d935bb17
AL		 34using namespace std;
35ostream c0out(0);
36ostream c1out(0);
37ostream c2out(0);
b2e465d6
AL		 38ofstream devnull("/dev/null");
39unsigned Quiet = 0;
40
41// struct PackageMap - List of all package files in the config file /*{{{*/
42// ---------------------------------------------------------------------
43/* */
44struct PackageMap
45{
46 // General Stuff
47 string BaseDir;
48 string InternalPrefix;
49 string FLFile;
50 string PkgExt;
51 string SrcExt;
52
53 // Stuff for the Package File
54 string PkgFile;
55 string BinCacheDB;
56 string BinOverride;
64177f17 57 string ExtraOverride;
0b41e0e7
MV		 58
59 // We generate for this given arch
60 string Arch;
b2e465d6
AL		 61
62 // Stuff for the Source File
63 string SrcFile;
64 string SrcOverride;
64177f17 65 string SrcExtraOverride;
b2e465d6 66
66905344 67 // Translation master file
4e794c50 68 bool LongDesc;
66905344
DK		 69 TranslationWriter *TransWriter;
70
b2e465d6
AL		 71 // Contents
72 string Contents;
73 string ContentsHead;
74
75 // Random things
76 string Tag;
77 string PkgCompress;
78 string CntCompress;
79 string SrcCompress;
80 string PathPrefix;
81 unsigned int DeLinkLimit;
82 mode_t Permissions;
83
84 bool ContentsDone;
85 bool PkgDone;
86 bool SrcDone;
87 time_t ContentsMTime;
88
89 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
90 {
91 inline bool operator() (const PackageMap &x,const PackageMap &y)
92 {return x.ContentsMTime < y.ContentsMTime;};
93 };
94
95 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
96 {
97 inline bool operator() (const PackageMap &x,const PackageMap &y)
98 {return x.BinCacheDB < y.BinCacheDB;};
99 };
100
101 void GetGeneral(Configuration &Setup,Configuration &Block);
102 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
103 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
104 bool GenContents(Configuration &Setup,
d935bb17
AL		 105 vector<PackageMap>::iterator Begin,
106 vector<PackageMap>::iterator End,
b2e465d6
AL		 107 unsigned long &Left);
108
4e794c50 109 PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
66905344
DK		 110 ContentsDone(false), PkgDone(false), SrcDone(false),
111 ContentsMTime(0) {};
b2e465d6
AL		 112};
113 /*}}}*/
114
115// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
116// ---------------------------------------------------------------------
117/* */
118void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
119{
120 PathPrefix = Block.Find("PathPrefix");
121
122 if (Block.FindB("External-Links",true) == false)
123 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
124 else
125 DeLinkLimit = 0;
126
127 PkgCompress = Block.Find("Packages::Compress",
128 Setup.Find("Default::Packages::Compress",". gzip").c_str());
129 CntCompress = Block.Find("Contents::Compress",
130 Setup.Find("Default::Contents::Compress",". gzip").c_str());
131 SrcCompress = Block.Find("Sources::Compress",
132 Setup.Find("Default::Sources::Compress",". gzip").c_str());
133
134 SrcExt = Block.Find("Sources::Extensions",
135 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
136 PkgExt = Block.Find("Packages::Extensions",
137 Setup.Find("Default::Packages::Extensions",".deb").c_str());
138
2db5b414
DK		 139 Permissions = Setup.FindI("Default::FileMode",0644);
140
b2e465d6
AL		 141 if (FLFile.empty() == false)
142 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
143
144 if (Contents == " ")
145 Contents= string();
146}
147 /*}}}*/
148// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
149// ---------------------------------------------------------------------
150/* This generates the Package File described by this object. */
151bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
152{
153 if (PkgFile.empty() == true)
154 return true;
155
156 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
157 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
158 string CacheDir = Setup.FindDir("Dir::CacheDir");
159
160 struct timeval StartTime;
161 gettimeofday(&StartTime,0);
162
163 PkgDone = true;
164
165 // Create a package writer object.
166 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 167 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 168 flCombine(OverrideDir,ExtraOverride),
169 Arch);
b2e465d6 170 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 171 return _error->Error(_("Package extension list is too long"));
b2e465d6 172 if (_error->PendingError() == true)
db0db9fe 173 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 174
175 Packages.PathPrefix = PathPrefix;
176 Packages.DirStrip = ArchiveDir;
177 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
178
66905344 179 Packages.TransWriter = TransWriter;
4e794c50 180 Packages.LongDescription = LongDesc;
66905344 181
b2e465d6
AL		 182 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
183 Packages.DeLinkLimit = DeLinkLimit;
64177f17 184
b2e465d6
AL		 185 // Create a compressor object
186 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
187 PkgCompress,Permissions);
188 Packages.Output = Comp.Input;
189 if (_error->PendingError() == true)
db0db9fe 190 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 191
192 c0out << ' ' << BaseDir << ":" << flush;
193
194 // Do recursive directory searching
195 if (FLFile.empty() == true)
196 {
197 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
198 return false;
199 }
200 else
201 {
202 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
203 return false;
204 }
205
206 Packages.Output = 0; // Just in case
207
208 // Finish compressing
650faab0 209 unsigned long long Size;
b2e465d6
AL		 210 if (Comp.Finalize(Size) == false)
211 {
212 c0out << endl;
db0db9fe 213 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 214 }
215
216 if (Size != 0)
217 c0out << " New "
218 << SizeToStr(Size) << "B ";
219 else
220 c0out << ' ';
221
222 struct timeval NewTime;
223 gettimeofday(&NewTime,0);
224 double Delta = NewTime.tv_sec - StartTime.tv_sec +
225 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
226
227 c0out << Packages.Stats.Packages << " files " <<
228/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
229 SizeToStr(Packages.Stats.Bytes) << "B " <<
230 TimeToStr((long)Delta) << endl;
231
232 Stats.Add(Packages.Stats);
233 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
234
235 return !_error->PendingError();
236}
98953965 237
b2e465d6 238 /*}}}*/
64177f17 239// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 240// ---------------------------------------------------------------------
241/* This generates the Sources File described by this object. */
242bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
243{
244 if (SrcFile.empty() == true)
245 return true;
246
247 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
248 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
249 string CacheDir = Setup.FindDir("Dir::CacheDir");
250
251 struct timeval StartTime;
252 gettimeofday(&StartTime,0);
253
254 SrcDone = true;
255
256 // Create a package writer object.
257 SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
64177f17
AL		 258 flCombine(OverrideDir,SrcOverride),
259 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 260 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 261 return _error->Error(_("Source extension list is too long"));
b2e465d6 262 if (_error->PendingError() == true)
db0db9fe 263 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 264
265 Sources.PathPrefix = PathPrefix;
266 Sources.DirStrip = ArchiveDir;
267 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
268
269 Sources.DeLinkLimit = DeLinkLimit;
270 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
271
272 // Create a compressor object
273 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
274 SrcCompress,Permissions);
275 Sources.Output = Comp.Input;
276 if (_error->PendingError() == true)
db0db9fe 277 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 278
279 c0out << ' ' << BaseDir << ":" << flush;
280
281 // Do recursive directory searching
282 if (FLFile.empty() == true)
283 {
284 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
285 return false;
286 }
287 else
288 {
289 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
290 return false;
291 }
292 Sources.Output = 0; // Just in case
293
294 // Finish compressing
650faab0 295 unsigned long long Size;
b2e465d6
AL		 296 if (Comp.Finalize(Size) == false)
297 {
298 c0out << endl;
db0db9fe 299 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 300 }
301
302 if (Size != 0)
303 c0out << " New "
304 << SizeToStr(Size) << "B ";
305 else
306 c0out << ' ';
307
308 struct timeval NewTime;
309 gettimeofday(&NewTime,0);
310 double Delta = NewTime.tv_sec - StartTime.tv_sec +
311 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
312
313 c0out << Sources.Stats.Packages << " pkgs in " <<
314 TimeToStr((long)Delta) << endl;
315
316 Stats.Add(Sources.Stats);
317 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
318
319 return !_error->PendingError();
320}
321 /*}}}*/
322// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
323// ---------------------------------------------------------------------
324/* This generates the contents file partially described by this object.
325 It searches the given iterator range for other package files that map
326 into this contents file and includes their data as well when building. */
327bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 328 vector<PackageMap>::iterator Begin,
329 vector<PackageMap>::iterator End,
330 unsigned long &Left)
b2e465d6
AL		 331{
332 if (Contents.empty() == true)
333 return true;
334
335 if (Left == 0)
336 return true;
337
338 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
339 string CacheDir = Setup.FindDir("Dir::CacheDir");
340 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
341
342 struct timeval StartTime;
343 gettimeofday(&StartTime,0);
344
345 // Create a package writer object.
31981076 346 ContentsWriter Contents("", Arch);
b2e465d6 347 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 348 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 349 if (_error->PendingError() == true)
350 return false;
351
352 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
353 CntCompress,Permissions);
354 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
355 Contents.Output = Comp.Input;
356 if (_error->PendingError() == true)
357 return false;
358
359 // Write the header out.
360 if (ContentsHead.empty() == false)
361 {
362 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
363 if (_error->PendingError() == true)
364 return false;
365
650faab0 366 unsigned long long Size = Head.Size();
b2e465d6
AL		 367 unsigned char Buf[4096];
368 while (Size != 0)
369 {
650faab0 370 unsigned long long ToRead = Size;
b2e465d6
AL		 371 if (Size > sizeof(Buf))
372 ToRead = sizeof(Buf);
373
374 if (Head.Read(Buf,ToRead) == false)
375 return false;
376
377 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 378 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 379
380 Size -= ToRead;
381 }
382 }
383
384 /* Go over all the package file records and parse all the package
385 files associated with this contents file into one great big honking
386 memory structure, then dump the sorted version */
387 c0out << ' ' << this->Contents << ":" << flush;
f7f0d6c7 388 for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
b2e465d6
AL		 389 {
390 if (I->Contents != this->Contents)
391 continue;
392
393 Contents.Prefix = ArchiveDir;
394 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
395 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
396 I->PkgCompress);
397
398 I->ContentsDone = true;
399 }
400
401 Contents.Finish();
402
403 // Finish compressing
650faab0 404 unsigned long long Size;
b2e465d6
AL		 405 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
406 {
407 c0out << endl;
db0db9fe 408 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 409 this->Contents.c_str());
410 }
411
412 if (Size != 0)
413 {
414 c0out << " New " << SizeToStr(Size) << "B ";
415 if (Left > Size)
416 Left -= Size;
417 else
418 Left = 0;
419 }
420 else
421 c0out << ' ';
422
423 struct timeval NewTime;
424 gettimeofday(&NewTime,0);
425 double Delta = NewTime.tv_sec - StartTime.tv_sec +
426 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
427
428 c0out << Contents.Stats.Packages << " files " <<
429 SizeToStr(Contents.Stats.Bytes) << "B " <<
430 TimeToStr((long)Delta) << endl;
431
432 return true;
433}
434 /*}}}*/
435
436// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
437// ---------------------------------------------------------------------
438/* This populates the PkgList with all the possible permutations of the
439 section/arch lists. */
440void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
441{
442 // Load the defaults
443 string DDir = Setup.Find("TreeDefault::Directory",
444 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
445 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
446 "$(DIST)/$(SECTION)/source/");
447 string DPkg = Setup.Find("TreeDefault::Packages",
448 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 449 string DTrans = Setup.Find("TreeDefault::Translation",
450 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 451 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
452 "$(DIST)/$(SECTION)/");
453 string DContents = Setup.Find("TreeDefault::Contents",
454 "$(DIST)/Contents-$(ARCH)");
455 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
456 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
457 "packages-$(ARCH).db");
458 string DSources = Setup.Find("TreeDefault::Sources",
459 "$(DIST)/$(SECTION)/source/Sources");
460 string DFLFile = Setup.Find("TreeDefault::FileList", "");
461 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
462
2db5b414 463 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
34f1d96c
DK		 464
465 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 466 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 467 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
4e794c50 468
b2e465d6
AL		 469 // Process 'tree' type sections
470 const Configuration::Item *Top = Setup.Tree("tree");
471 for (Top = (Top == 0?0:Top->Child); Top != 0;)
472 {
473 Configuration Block(Top);
474 string Dist = Top->Tag;
475
476 // Parse the sections
d935bb17
AL		 477 string Tmp = Block.Find("Sections");
478 const char *Sections = Tmp.c_str();
b2e465d6
AL		 479 string Section;
480 while (ParseQuoteWord(Sections,Section) == true)
481 {
b2e465d6 482 string Arch;
66905344
DK		 483 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
484 {"$(SECTION)",&Section},
485 {"$(ARCH)",&Arch},
486 {}};
34f1d96c 487 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 488 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
66905344 489 TranslationWriter *TransWriter;
4e794c50 490 if (DTrans.empty() == false && LongDesc == false)
66905344
DK		 491 {
492 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
493 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
34f1d96c
DK		 494 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
495 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
66905344
DK		 496 }
497 else
498 TransWriter = NULL;
499
500 string const Tmp2 = Block.Find("Architectures");
d935bb17 501 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 502 while (ParseQuoteWord(Archs,Arch) == true)
503 {
b2e465d6 504 PackageMap Itm;
34f1d96c 505 Itm.Permissions = Perms;
b2e465d6
AL		 506 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
507 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
508
509 if (stringcasecmp(Arch,"source") == 0)
510 {
511 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
512 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
513 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
514 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
515 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 516 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 517 }
518 else
519 {
520 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
521 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
522 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
523 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 524 Itm.Arch = Arch;
4e794c50 525 Itm.LongDesc = LongDesc;
66905344
DK		 526 if (TransWriter != NULL)
527 {
528 TransWriter->IncreaseRefCounter();
529 Itm.TransWriter = TransWriter;
530 }
b2e465d6
AL		 531 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
532 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
533 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 534 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 535 }
536
537 Itm.GetGeneral(Setup,Block);
538 PkgList.push_back(Itm);
539 }
66905344
DK		 540 // we didn't use this TransWriter, so we can release it
541 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
542 delete TransWriter;
b2e465d6
AL		 543 }
544
545 Top = Top->Next;
546 }
547}
548 /*}}}*/
549// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
550// ---------------------------------------------------------------------
551/* */
552void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
553{
2db5b414
DK		 554 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
555
b2e465d6
AL		 556 // Process 'bindirectory' type sections
557 const Configuration::Item *Top = Setup.Tree("bindirectory");
558 for (Top = (Top == 0?0:Top->Child); Top != 0;)
559 {
560 Configuration Block(Top);
561
562 PackageMap Itm;
563 Itm.PkgFile = Block.Find("Packages");
564 Itm.SrcFile = Block.Find("Sources");
565 Itm.BinCacheDB = Block.Find("BinCacheDB");
566 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 567 Itm.ExtraOverride = Block.Find("ExtraOverride");
568 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 569 Itm.SrcOverride = Block.Find("SrcOverride");
570 Itm.BaseDir = Top->Tag;
571 Itm.FLFile = Block.Find("FileList");
572 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
573 Itm.Contents = Block.Find("Contents");
574 Itm.ContentsHead = Block.Find("Contents::Header");
2db5b414 575 Itm.Permissions = Block.FindI("FileMode", Permissions);
b2e465d6
AL		 576
577 Itm.GetGeneral(Setup,Block);
578 PkgList.push_back(Itm);
579
580 Top = Top->Next;
581 }
582}
583 /*}}}*/
584
585// ShowHelp - Show the help text /*{{{*/
586// ---------------------------------------------------------------------
587/* */
588bool ShowHelp(CommandLine &CmdL)
589{
5b28c804
OS		 590 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
591 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 592 if (_config->FindB("version") == true)
593 return true;
594
595 cout <<
dc738e7a 596 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 597 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 598 " sources srcpath [overridefile [pathprefix]]\n"
599 " contents path\n"
0dfd2728 600 " release path\n"
b2e465d6
AL		 601 " generate config [groups]\n"
602 " clean config\n"
603 "\n"
604 "apt-ftparchive generates index files for Debian archives. It supports\n"
605 "many styles of generation from fully automated to functional replacements\n"
606 "for dpkg-scanpackages and dpkg-scansources\n"
607 "\n"
608 "apt-ftparchive generates Package files from a tree of .debs. The\n"
609 "Package file contains the contents of all the control fields from\n"
610 "each package as well as the MD5 hash and filesize. An override file\n"
611 "is supported to force the value of Priority and Section.\n"
612 "\n"
613 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
614 "The --source-override option can be used to specify a src override file\n"
615 "\n"
616 "The 'packages' and 'sources' command should be run in the root of the\n"
617 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 618 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 619 "appended to the filename fields if present. Example usage from the \n"
75224826 620 "Debian archive:\n"
b2e465d6
AL		 621 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
622 " dists/potato/main/binary-i386/Packages\n"
623 "\n"
624 "Options:\n"
625 " -h This help text\n"
626 " --md5 Control MD5 generation\n"
627 " -s=? Source override file\n"
628 " -q Quiet\n"
629 " -d=? Select the optional caching database\n"
630 " --no-delink Enable delinking debug mode\n"
631 " --contents Control contents file generation\n"
632 " -c=? Read this configuration file\n"
8a2d7db6 633 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 634
635 return true;
636}
637 /*}}}*/
638// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
639// ---------------------------------------------------------------------
640/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
641bool SimpleGenPackages(CommandLine &CmdL)
642{
643 if (CmdL.FileSize() < 2)
644 return ShowHelp(CmdL);
645
646 string Override;
647 if (CmdL.FileSize() >= 3)
648 Override = CmdL.FileList[2];
649
650 // Create a package writer object.
651 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 652 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 653 if (_error->PendingError() == true)
654 return false;
655
656 if (CmdL.FileSize() >= 4)
657 Packages.PathPrefix = CmdL.FileList[3];
658
659 // Do recursive directory searching
660 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
661 return false;
662
663 return true;
664}
665 /*}}}*/
666// SimpleGenContents - Generate a Contents listing /*{{{*/
667// ---------------------------------------------------------------------
668/* */
669bool SimpleGenContents(CommandLine &CmdL)
670{
671 if (CmdL.FileSize() < 2)
672 return ShowHelp(CmdL);
673
674 // Create a package writer object.
31981076 675 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 676 if (_error->PendingError() == true)
677 return false;
678
679 // Do recursive directory searching
680 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
681 return false;
682
683 Contents.Finish();
684
685 return true;
686}
687 /*}}}*/
688// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
689// ---------------------------------------------------------------------
690/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
691bool SimpleGenSources(CommandLine &CmdL)
692{
693 if (CmdL.FileSize() < 2)
694 return ShowHelp(CmdL);
695
696 string Override;
697 if (CmdL.FileSize() >= 3)
698 Override = CmdL.FileList[2];
699
700 string SOverride;
701 if (Override.empty() == false)
702 SOverride = Override + ".src";
703
704 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
705 SOverride.c_str());
706
707 // Create a package writer object.
708 SourcesWriter Sources(Override,SOverride);
709 if (_error->PendingError() == true)
710 return false;
711
712 if (CmdL.FileSize() >= 4)
713 Sources.PathPrefix = CmdL.FileList[3];
714
715 // Do recursive directory searching
716 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
717 return false;
718
719 return true;
720}
98953965
AL		 721 /*}}}*/
722// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
723// ---------------------------------------------------------------------
724bool SimpleGenRelease(CommandLine &CmdL)
725{
0dfd2728
AL		 726 if (CmdL.FileSize() < 2)
727 return ShowHelp(CmdL);
728
c0eb6bc6
AL		 729 string Dir = CmdL.FileList[1];
730
98953965 731 ReleaseWriter Release("");
c0eb6bc6
AL		 732 Release.DirStrip = Dir;
733
98953965
AL		 734 if (_error->PendingError() == true)
735 return false;
736
c0eb6bc6 737 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 738 return false;
739
f7291f62
AL		 740 Release.Finish();
741
98953965
AL		 742 return true;
743}
744
b2e465d6
AL		 745 /*}}}*/
746// Generate - Full generate, using a config file /*{{{*/
747// ---------------------------------------------------------------------
748/* */
749bool Generate(CommandLine &CmdL)
750{
751 struct CacheDB::Stats SrcStats;
752 if (CmdL.FileSize() < 2)
753 return ShowHelp(CmdL);
754
755 struct timeval StartTime;
756 gettimeofday(&StartTime,0);
757 struct CacheDB::Stats Stats;
758
759 // Read the configuration file.
760 Configuration Setup;
761 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
762 return false;
763
764 vector<PackageMap> PkgList;
765 LoadTree(PkgList,Setup);
766 LoadBinDir(PkgList,Setup);
767
768 // Sort by cache DB to improve IO locality.
769 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
770
771 // Generate packages
772 if (CmdL.FileSize() <= 2)
773 {
f7f0d6c7 774 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 775 if (I->GenPackages(Setup,Stats) == false)
776 _error->DumpErrors();
f7f0d6c7 777 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 778 if (I->GenSources(Setup,SrcStats) == false)
779 _error->DumpErrors();
780 }
781 else
782 {
783 // Make a choice list out of the package list..
784 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
785 RxChoiceList *End = List;
f7f0d6c7 786 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 787 {
d935bb17 788 End->UserData = &(*I);
b2e465d6
AL		 789 End->Str = I->BaseDir.c_str();
790 End++;
791
d935bb17 792 End->UserData = &(*I);
b2e465d6
AL		 793 End->Str = I->Tag.c_str();
794 End++;
795 }
796 End->Str = 0;
797
798 // Regex it
799 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
800 {
801 delete [] List;
dc738e7a 802 return _error->Error(_("No selections matched"));
b2e465d6
AL		 803 }
804 _error->DumpErrors();
805
806 // Do the generation for Packages
807 for (End = List; End->Str != 0; End++)
808 {
809 if (End->Hit == false)
810 continue;
811
812 PackageMap *I = (PackageMap *)End->UserData;
813 if (I->PkgDone == true)
814 continue;
815 if (I->GenPackages(Setup,Stats) == false)
816 _error->DumpErrors();
817 }
818
819 // Do the generation for Sources
820 for (End = List; End->Str != 0; End++)
821 {
822 if (End->Hit == false)
823 continue;
824
825 PackageMap *I = (PackageMap *)End->UserData;
826 if (I->SrcDone == true)
827 continue;
828 if (I->GenSources(Setup,SrcStats) == false)
829 _error->DumpErrors();
830 }
831
832 delete [] List;
833 }
66905344
DK		 834
835 // close the Translation master files
f7f0d6c7 836 for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
66905344
DK		 837 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
838 delete I->TransWriter;
839
b2e465d6
AL		 840 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
841 return true;
842
c6474fb6 843 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 844
845 // Sort the contents file list by date
846 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
f7f0d6c7 847 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 848 {
849 struct stat A;
850 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
851 I->CntCompress,A) == false)
852 time(&I->ContentsMTime);
853 else
854 I->ContentsMTime = A.st_mtime;
855 }
856 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
857
858 /* Now for Contents.. The process here is to do a make-like dependency
859 check. Each contents file is verified to be newer than the package files
860 that describe the debs it indexes. Since the package files contain
861 hashes of the .debs this means they have not changed either so the
862 contents must be up to date. */
863 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
f7f0d6c7 864 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 865 {
866 // This record is not relevent
867 if (I->ContentsDone == true ||
868 I->Contents.empty() == true)
869 continue;
870
871 // Do not do everything if the user specified sections.
872 if (CmdL.FileSize() > 2 && I->PkgDone == false)
873 continue;
874
875 struct stat A,B;
876 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
877 {
878 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
879 {
dc738e7a 880 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 881 continue;
882 }
883
884 if (A.st_mtime > B.st_mtime)
885 continue;
886 }
887
888 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
889 MaxContentsChange) == false)
890 _error->DumpErrors();
891
892 // Hit the limit?
893 if (MaxContentsChange == 0)
894 {
895 c1out << "Hit contents update byte limit" << endl;
896 break;
897 }
898 }
899
900 struct timeval NewTime;
901 gettimeofday(&NewTime,0);
902 double Delta = NewTime.tv_sec - StartTime.tv_sec +
903 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
904 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
905 << " archives. Took " << TimeToStr((long)Delta) << endl;
906
907 return true;
908}
909 /*}}}*/
910// Clean - Clean out the databases /*{{{*/
911// ---------------------------------------------------------------------
912/* */
913bool Clean(CommandLine &CmdL)
914{
915 if (CmdL.FileSize() != 2)
916 return ShowHelp(CmdL);
917
918 // Read the configuration file.
919 Configuration Setup;
920 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
921 return false;
922
923 vector<PackageMap> PkgList;
924 LoadTree(PkgList,Setup);
925 LoadBinDir(PkgList,Setup);
926
927 // Sort by cache DB to improve IO locality.
928 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
929
930 string CacheDir = Setup.FindDir("Dir::CacheDir");
931
d935bb17 932 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 933 {
934 c0out << I->BinCacheDB << endl;
935 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
936 if (DB.Clean() == false)
937 _error->DumpErrors();
938
939 string CacheDB = I->BinCacheDB;
f7f0d6c7 940 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I);
b2e465d6
AL		 941 }
942
943 return true;
944}
945 /*}}}*/
946
947int main(int argc, const char *argv[])
948{
1692a2c7 949 setlocale(LC_ALL, "");
b2e465d6
AL		 950 CommandLine::Args Args[] = {
951 {'h',"help","help",0},
952 {0,"md5","APT::FTPArchive::MD5",0},
3c54407f
DK		 953 {0,"sha1","APT::FTPArchive::SHA1",0},
954 {0,"sha256","APT::FTPArchive::SHA256",0},
b2e465d6
AL		 955 {'v',"version","version",0},
956 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
957 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
958 {'q',"quiet","quiet",CommandLine::IntLevel},
959 {'q',"silent","quiet",CommandLine::IntLevel},
960 {0,"delink","APT::FTPArchive::DeLinkAct",0},
961 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
962 {0,"contents","APT::FTPArchive::Contents",0},
31981076 963 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 964 {'c',"config-file",0,CommandLine::ConfigFile},
965 {'o',"option",0,CommandLine::ArbItem},
966 {0,0,0,0}};
967 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
968 {"contents",&SimpleGenContents},
969 {"sources",&SimpleGenSources},
98953965 970 {"release",&SimpleGenRelease},
b2e465d6
AL		 971 {"generate",&Generate},
972 {"clean",&Clean},
973 {"help",&ShowHelp},
974 {0,0}};
975
976 // Parse the command line and initialize the package library
977 CommandLine CmdL(Args,_config);
b34d4b47 978 if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
b2e465d6
AL		 979 {
980 _error->DumpErrors();
981 return 100;
982 }
983
984 // See if the help should be shown
985 if (_config->FindB("help") == true ||
986 _config->FindB("version") == true ||
987 CmdL.FileSize() == 0)
988 {
989 ShowHelp(CmdL);
990 return 0;
991 }
992
993 // Setup the output streams
1809f8d2
AL		 994 c0out.rdbuf(clog.rdbuf());
995 c1out.rdbuf(clog.rdbuf());
996 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 997 Quiet = _config->FindI("quiet",0);
998 if (Quiet > 0)
999 c0out.rdbuf(devnull.rdbuf());
1000 if (Quiet > 1)
1001 c1out.rdbuf(devnull.rdbuf());
1002
1003 // Match the operation
1004 CmdL.DispatchArg(Cmds);
1005
1006 if (_error->empty() == false)
1007 {
1008 bool Errors = _error->PendingError();
1009 _error->DumpErrors();
1010 return Errors == true?100:0;
1011 }
1012 return 0;
1013}
ntk's bugfix for apt