HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
use new Popen()
[ntk/apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
7db98ffc 3// $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
c6474fb6 6 apt-ftparchive - Efficient work-alike for dpkg-scanpackages
b2e465d6
AL		 7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
ea542140
DK		 13#include <config.h>
14
b2e465d6
AL		 15#include <apt-pkg/error.h>
16#include <apt-pkg/configuration.h>
17#include <apt-pkg/cmndline.h>
18#include <apt-pkg/strutl.h>
3b1fffc3 19#include <apt-pkg/init.h>
453b82a3 20#include <apt-pkg/fileutl.h>
b2e465d6 21
453b82a3 22#include <algorithm>
152ab79e 23#include <climits>
b2e465d6 24#include <sys/time.h>
453b82a3
DK		 25#include <locale.h>
26#include <stdio.h>
27#include <sys/stat.h>
28#include <time.h>
29#include <functional>
30#include <iostream>
31#include <string>
32#include <vector>
b2e465d6 33
453b82a3
DK		 34#include "cachedb.h"
35#include "override.h"
ea542140 36#include "apt-ftparchive.h"
b2e465d6 37#include "multicompress.h"
ea542140
DK		 38#include "writer.h"
39
40#include <apti18n.h>
b2e465d6
AL		 41 /*}}}*/
42
d935bb17
AL		 43using namespace std;
44ostream c0out(0);
45ostream c1out(0);
46ostream c2out(0);
b2e465d6
AL		 47ofstream devnull("/dev/null");
48unsigned Quiet = 0;
49
50// struct PackageMap - List of all package files in the config file /*{{{*/
51// ---------------------------------------------------------------------
52/* */
53struct PackageMap
54{
55 // General Stuff
56 string BaseDir;
57 string InternalPrefix;
58 string FLFile;
59 string PkgExt;
60 string SrcExt;
61
62 // Stuff for the Package File
63 string PkgFile;
64 string BinCacheDB;
65 string BinOverride;
64177f17 66 string ExtraOverride;
0b41e0e7
MV		 67
68 // We generate for this given arch
69 string Arch;
b2e465d6
AL		 70
71 // Stuff for the Source File
72 string SrcFile;
73 string SrcOverride;
64177f17 74 string SrcExtraOverride;
b2e465d6 75
66905344 76 // Translation master file
4e794c50 77 bool LongDesc;
66905344
DK		 78 TranslationWriter *TransWriter;
79
b2e465d6
AL		 80 // Contents
81 string Contents;
82 string ContentsHead;
83
84 // Random things
85 string Tag;
86 string PkgCompress;
87 string CntCompress;
88 string SrcCompress;
89 string PathPrefix;
90 unsigned int DeLinkLimit;
91 mode_t Permissions;
92
93 bool ContentsDone;
94 bool PkgDone;
95 bool SrcDone;
96 time_t ContentsMTime;
97
98 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
99 {
100 inline bool operator() (const PackageMap &x,const PackageMap &y)
101 {return x.ContentsMTime < y.ContentsMTime;};
102 };
103
104 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
105 {
106 inline bool operator() (const PackageMap &x,const PackageMap &y)
107 {return x.BinCacheDB < y.BinCacheDB;};
108 };
109
110 void GetGeneral(Configuration &Setup,Configuration &Block);
111 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
112 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
113 bool GenContents(Configuration &Setup,
d935bb17
AL		 114 vector<PackageMap>::iterator Begin,
115 vector<PackageMap>::iterator End,
b2e465d6
AL		 116 unsigned long &Left);
117
4e794c50 118 PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
66905344
DK		 119 ContentsDone(false), PkgDone(false), SrcDone(false),
120 ContentsMTime(0) {};
b2e465d6
AL		 121};
122 /*}}}*/
123
124// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
125// ---------------------------------------------------------------------
126/* */
127void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
128{
129 PathPrefix = Block.Find("PathPrefix");
130
131 if (Block.FindB("External-Links",true) == false)
132 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
133 else
134 DeLinkLimit = 0;
135
136 PkgCompress = Block.Find("Packages::Compress",
137 Setup.Find("Default::Packages::Compress",". gzip").c_str());
138 CntCompress = Block.Find("Contents::Compress",
139 Setup.Find("Default::Contents::Compress",". gzip").c_str());
140 SrcCompress = Block.Find("Sources::Compress",
141 Setup.Find("Default::Sources::Compress",". gzip").c_str());
142
143 SrcExt = Block.Find("Sources::Extensions",
144 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
145 PkgExt = Block.Find("Packages::Extensions",
146 Setup.Find("Default::Packages::Extensions",".deb").c_str());
147
2db5b414
DK		 148 Permissions = Setup.FindI("Default::FileMode",0644);
149
b2e465d6
AL		 150 if (FLFile.empty() == false)
151 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
152
153 if (Contents == " ")
154 Contents= string();
155}
156 /*}}}*/
157// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
158// ---------------------------------------------------------------------
159/* This generates the Package File described by this object. */
160bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
161{
162 if (PkgFile.empty() == true)
163 return true;
164
165 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
166 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
167 string CacheDir = Setup.FindDir("Dir::CacheDir");
168
169 struct timeval StartTime;
170 gettimeofday(&StartTime,0);
171
172 PkgDone = true;
173
174 // Create a package writer object.
175 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17 176 flCombine(OverrideDir,BinOverride),
0b41e0e7
MV		 177 flCombine(OverrideDir,ExtraOverride),
178 Arch);
b2e465d6 179 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 180 return _error->Error(_("Package extension list is too long"));
b2e465d6 181 if (_error->PendingError() == true)
db0db9fe 182 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 183
184 Packages.PathPrefix = PathPrefix;
185 Packages.DirStrip = ArchiveDir;
186 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
187
66905344 188 Packages.TransWriter = TransWriter;
4e794c50 189 Packages.LongDescription = LongDesc;
66905344 190
b2e465d6
AL		 191 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
192 Packages.DeLinkLimit = DeLinkLimit;
64177f17 193
b2e465d6
AL		 194 // Create a compressor object
195 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
196 PkgCompress,Permissions);
197 Packages.Output = Comp.Input;
198 if (_error->PendingError() == true)
db0db9fe 199 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 200
201 c0out << ' ' << BaseDir << ":" << flush;
202
203 // Do recursive directory searching
204 if (FLFile.empty() == true)
205 {
206 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
207 return false;
208 }
209 else
210 {
211 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
212 return false;
213 }
214
215 Packages.Output = 0; // Just in case
216
217 // Finish compressing
650faab0 218 unsigned long long Size;
b2e465d6
AL		 219 if (Comp.Finalize(Size) == false)
220 {
221 c0out << endl;
db0db9fe 222 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 223 }
224
225 if (Size != 0)
226 c0out << " New "
227 << SizeToStr(Size) << "B ";
228 else
229 c0out << ' ';
230
231 struct timeval NewTime;
232 gettimeofday(&NewTime,0);
233 double Delta = NewTime.tv_sec - StartTime.tv_sec +
234 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
235
236 c0out << Packages.Stats.Packages << " files " <<
237/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
238 SizeToStr(Packages.Stats.Bytes) << "B " <<
239 TimeToStr((long)Delta) << endl;
240
241 Stats.Add(Packages.Stats);
242 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
243
244 return !_error->PendingError();
245}
98953965 246
b2e465d6 247 /*}}}*/
64177f17 248// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 249// ---------------------------------------------------------------------
250/* This generates the Sources File described by this object. */
251bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
252{
253 if (SrcFile.empty() == true)
254 return true;
255
256 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
257 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
258 string CacheDir = Setup.FindDir("Dir::CacheDir");
259
260 struct timeval StartTime;
261 gettimeofday(&StartTime,0);
262
263 SrcDone = true;
264
265 // Create a package writer object.
f6f06a8f
MV		 266 SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),
267 flCombine(OverrideDir,BinOverride),
64177f17
AL		 268 flCombine(OverrideDir,SrcOverride),
269 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 270 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 271 return _error->Error(_("Source extension list is too long"));
b2e465d6 272 if (_error->PendingError() == true)
db0db9fe 273 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 274
275 Sources.PathPrefix = PathPrefix;
276 Sources.DirStrip = ArchiveDir;
277 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
278
279 Sources.DeLinkLimit = DeLinkLimit;
280 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
281
282 // Create a compressor object
283 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
284 SrcCompress,Permissions);
285 Sources.Output = Comp.Input;
286 if (_error->PendingError() == true)
db0db9fe 287 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 288
289 c0out << ' ' << BaseDir << ":" << flush;
290
291 // Do recursive directory searching
292 if (FLFile.empty() == true)
293 {
294 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
295 return false;
296 }
297 else
298 {
299 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
300 return false;
301 }
302 Sources.Output = 0; // Just in case
303
304 // Finish compressing
650faab0 305 unsigned long long Size;
b2e465d6
AL		 306 if (Comp.Finalize(Size) == false)
307 {
308 c0out << endl;
db0db9fe 309 return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 310 }
311
312 if (Size != 0)
313 c0out << " New "
314 << SizeToStr(Size) << "B ";
315 else
316 c0out << ' ';
317
318 struct timeval NewTime;
319 gettimeofday(&NewTime,0);
320 double Delta = NewTime.tv_sec - StartTime.tv_sec +
321 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
322
323 c0out << Sources.Stats.Packages << " pkgs in " <<
324 TimeToStr((long)Delta) << endl;
325
326 Stats.Add(Sources.Stats);
327 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
328
329 return !_error->PendingError();
330}
331 /*}}}*/
332// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
333// ---------------------------------------------------------------------
334/* This generates the contents file partially described by this object.
335 It searches the given iterator range for other package files that map
336 into this contents file and includes their data as well when building. */
337bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 338 vector<PackageMap>::iterator Begin,
339 vector<PackageMap>::iterator End,
340 unsigned long &Left)
b2e465d6
AL		 341{
342 if (Contents.empty() == true)
343 return true;
344
345 if (Left == 0)
346 return true;
347
348 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
349 string CacheDir = Setup.FindDir("Dir::CacheDir");
350 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
351
352 struct timeval StartTime;
353 gettimeofday(&StartTime,0);
354
355 // Create a package writer object.
31981076 356 ContentsWriter Contents("", Arch);
b2e465d6 357 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 358 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 359 if (_error->PendingError() == true)
360 return false;
361
362 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
363 CntCompress,Permissions);
364 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
365 Contents.Output = Comp.Input;
366 if (_error->PendingError() == true)
367 return false;
368
369 // Write the header out.
370 if (ContentsHead.empty() == false)
371 {
372 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
373 if (_error->PendingError() == true)
374 return false;
375
650faab0 376 unsigned long long Size = Head.Size();
b2e465d6
AL		 377 unsigned char Buf[4096];
378 while (Size != 0)
379 {
650faab0 380 unsigned long long ToRead = Size;
b2e465d6
AL		 381 if (Size > sizeof(Buf))
382 ToRead = sizeof(Buf);
383
384 if (Head.Read(Buf,ToRead) == false)
385 return false;
386
387 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 388 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 389
390 Size -= ToRead;
391 }
392 }
393
394 /* Go over all the package file records and parse all the package
395 files associated with this contents file into one great big honking
396 memory structure, then dump the sorted version */
397 c0out << ' ' << this->Contents << ":" << flush;
f7f0d6c7 398 for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
b2e465d6
AL		 399 {
400 if (I->Contents != this->Contents)
401 continue;
402
403 Contents.Prefix = ArchiveDir;
404 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
405 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
406 I->PkgCompress);
407
408 I->ContentsDone = true;
409 }
410
411 Contents.Finish();
412
413 // Finish compressing
650faab0 414 unsigned long long Size;
b2e465d6
AL		 415 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
416 {
417 c0out << endl;
db0db9fe 418 return _error->Error(_("Error processing contents %s"),
b2e465d6
AL		 419 this->Contents.c_str());
420 }
421
422 if (Size != 0)
423 {
424 c0out << " New " << SizeToStr(Size) << "B ";
425 if (Left > Size)
426 Left -= Size;
427 else
428 Left = 0;
429 }
430 else
431 c0out << ' ';
432
433 struct timeval NewTime;
434 gettimeofday(&NewTime,0);
435 double Delta = NewTime.tv_sec - StartTime.tv_sec +
436 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
437
438 c0out << Contents.Stats.Packages << " files " <<
439 SizeToStr(Contents.Stats.Bytes) << "B " <<
440 TimeToStr((long)Delta) << endl;
441
442 return true;
443}
444 /*}}}*/
445
446// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
447// ---------------------------------------------------------------------
448/* This populates the PkgList with all the possible permutations of the
449 section/arch lists. */
c3ccac92 450static void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
b2e465d6
AL		 451{
452 // Load the defaults
453 string DDir = Setup.Find("TreeDefault::Directory",
454 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
455 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
456 "$(DIST)/$(SECTION)/source/");
457 string DPkg = Setup.Find("TreeDefault::Packages",
458 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
66905344
DK		 459 string DTrans = Setup.Find("TreeDefault::Translation",
460 "$(DIST)/$(SECTION)/i18n/Translation-en");
b2e465d6
AL		 461 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
462 "$(DIST)/$(SECTION)/");
463 string DContents = Setup.Find("TreeDefault::Contents",
3adddfa8 464 "$(DIST)/$(SECTION)/Contents-$(ARCH)");
b2e465d6
AL		 465 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
466 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
467 "packages-$(ARCH).db");
468 string DSources = Setup.Find("TreeDefault::Sources",
469 "$(DIST)/$(SECTION)/source/Sources");
470 string DFLFile = Setup.Find("TreeDefault::FileList", "");
471 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
472
2db5b414 473 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
34f1d96c
DK		 474
475 bool const LongDescription = Setup.FindB("Default::LongDescription",
4e794c50 476 _config->FindB("APT::FTPArchive::LongDescription", true));
34f1d96c 477 string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
4e794c50 478
b2e465d6
AL		 479 // Process 'tree' type sections
480 const Configuration::Item *Top = Setup.Tree("tree");
481 for (Top = (Top == 0?0:Top->Child); Top != 0;)
482 {
483 Configuration Block(Top);
484 string Dist = Top->Tag;
485
486 // Parse the sections
d935bb17
AL		 487 string Tmp = Block.Find("Sections");
488 const char *Sections = Tmp.c_str();
b2e465d6
AL		 489 string Section;
490 while (ParseQuoteWord(Sections,Section) == true)
491 {
b2e465d6 492 string Arch;
66905344
DK		 493 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
494 {"$(SECTION)",&Section},
495 {"$(ARCH)",&Arch},
67c3067f 496 {NULL, NULL}};
34f1d96c 497 mode_t const Perms = Block.FindI("FileMode", Permissions);
4e794c50 498 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
66905344 499 TranslationWriter *TransWriter;
4e794c50 500 if (DTrans.empty() == false && LongDesc == false)
66905344
DK		 501 {
502 string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
503 SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
34f1d96c
DK		 504 string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
505 TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
66905344
DK		 506 }
507 else
508 TransWriter = NULL;
509
510 string const Tmp2 = Block.Find("Architectures");
d935bb17 511 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 512 while (ParseQuoteWord(Archs,Arch) == true)
513 {
b2e465d6 514 PackageMap Itm;
34f1d96c 515 Itm.Permissions = Perms;
b2e465d6
AL		 516 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
517 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
518
519 if (stringcasecmp(Arch,"source") == 0)
520 {
521 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
522 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
523 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
524 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
525 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 526 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 527 }
528 else
529 {
530 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
531 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
532 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
533 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
0b41e0e7 534 Itm.Arch = Arch;
4e794c50 535 Itm.LongDesc = LongDesc;
66905344
DK		 536 if (TransWriter != NULL)
537 {
538 TransWriter->IncreaseRefCounter();
539 Itm.TransWriter = TransWriter;
540 }
b2e465d6
AL		 541 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
542 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
543 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 544 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 545 }
546
547 Itm.GetGeneral(Setup,Block);
548 PkgList.push_back(Itm);
549 }
66905344
DK		 550 // we didn't use this TransWriter, so we can release it
551 if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
552 delete TransWriter;
b2e465d6
AL		 553 }
554
555 Top = Top->Next;
556 }
557}
558 /*}}}*/
559// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
560// ---------------------------------------------------------------------
561/* */
c3ccac92 562static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
b2e465d6 563{
2db5b414
DK		 564 mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
565
b2e465d6
AL		 566 // Process 'bindirectory' type sections
567 const Configuration::Item *Top = Setup.Tree("bindirectory");
568 for (Top = (Top == 0?0:Top->Child); Top != 0;)
569 {
570 Configuration Block(Top);
571
572 PackageMap Itm;
573 Itm.PkgFile = Block.Find("Packages");
574 Itm.SrcFile = Block.Find("Sources");
575 Itm.BinCacheDB = Block.Find("BinCacheDB");
576 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 577 Itm.ExtraOverride = Block.Find("ExtraOverride");
578 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 579 Itm.SrcOverride = Block.Find("SrcOverride");
580 Itm.BaseDir = Top->Tag;
581 Itm.FLFile = Block.Find("FileList");
582 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
583 Itm.Contents = Block.Find("Contents");
584 Itm.ContentsHead = Block.Find("Contents::Header");
2db5b414 585 Itm.Permissions = Block.FindI("FileMode", Permissions);
b2e465d6
AL		 586
587 Itm.GetGeneral(Setup,Block);
588 PkgList.push_back(Itm);
589
590 Top = Top->Next;
591 }
592}
593 /*}}}*/
594
595// ShowHelp - Show the help text /*{{{*/
596// ---------------------------------------------------------------------
597/* */
65512241 598static bool ShowHelp(CommandLine &)
b2e465d6 599{
9179f697 600 ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
5b28c804 601 COMMON_ARCH,__DATE__,__TIME__);
b2e465d6
AL		 602 if (_config->FindB("version") == true)
603 return true;
604
605 cout <<
dc738e7a 606 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 607 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 608 " sources srcpath [overridefile [pathprefix]]\n"
609 " contents path\n"
0dfd2728 610 " release path\n"
b2e465d6
AL		 611 " generate config [groups]\n"
612 " clean config\n"
613 "\n"
614 "apt-ftparchive generates index files for Debian archives. It supports\n"
615 "many styles of generation from fully automated to functional replacements\n"
616 "for dpkg-scanpackages and dpkg-scansources\n"
617 "\n"
618 "apt-ftparchive generates Package files from a tree of .debs. The\n"
619 "Package file contains the contents of all the control fields from\n"
620 "each package as well as the MD5 hash and filesize. An override file\n"
621 "is supported to force the value of Priority and Section.\n"
622 "\n"
623 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
624 "The --source-override option can be used to specify a src override file\n"
625 "\n"
626 "The 'packages' and 'sources' command should be run in the root of the\n"
627 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 628 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 629 "appended to the filename fields if present. Example usage from the \n"
75224826 630 "Debian archive:\n"
b2e465d6
AL		 631 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
632 " dists/potato/main/binary-i386/Packages\n"
633 "\n"
634 "Options:\n"
635 " -h This help text\n"
636 " --md5 Control MD5 generation\n"
637 " -s=? Source override file\n"
638 " -q Quiet\n"
639 " -d=? Select the optional caching database\n"
640 " --no-delink Enable delinking debug mode\n"
641 " --contents Control contents file generation\n"
642 " -c=? Read this configuration file\n"
8a2d7db6 643 " -o=? Set an arbitrary configuration option") << endl;
b2e465d6
AL		 644
645 return true;
646}
647 /*}}}*/
648// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
649// ---------------------------------------------------------------------
650/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
c3ccac92 651static bool SimpleGenPackages(CommandLine &CmdL)
b2e465d6
AL		 652{
653 if (CmdL.FileSize() < 2)
654 return ShowHelp(CmdL);
655
656 string Override;
657 if (CmdL.FileSize() >= 3)
658 Override = CmdL.FileList[2];
659
660 // Create a package writer object.
661 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
31981076 662 Override, "", _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 663 if (_error->PendingError() == true)
664 return false;
665
666 if (CmdL.FileSize() >= 4)
667 Packages.PathPrefix = CmdL.FileList[3];
668
669 // Do recursive directory searching
670 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
671 return false;
672
673 return true;
674}
675 /*}}}*/
676// SimpleGenContents - Generate a Contents listing /*{{{*/
677// ---------------------------------------------------------------------
678/* */
c3ccac92 679static bool SimpleGenContents(CommandLine &CmdL)
b2e465d6
AL		 680{
681 if (CmdL.FileSize() < 2)
682 return ShowHelp(CmdL);
683
684 // Create a package writer object.
31981076 685 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
b2e465d6
AL		 686 if (_error->PendingError() == true)
687 return false;
688
689 // Do recursive directory searching
690 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
691 return false;
692
693 Contents.Finish();
694
695 return true;
696}
697 /*}}}*/
698// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
699// ---------------------------------------------------------------------
700/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
c3ccac92 701static bool SimpleGenSources(CommandLine &CmdL)
b2e465d6
AL		 702{
703 if (CmdL.FileSize() < 2)
704 return ShowHelp(CmdL);
705
706 string Override;
707 if (CmdL.FileSize() >= 3)
708 Override = CmdL.FileList[2];
709
710 string SOverride;
711 if (Override.empty() == false)
712 SOverride = Override + ".src";
713
714 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
715 SOverride.c_str());
716
717 // Create a package writer object.
f6f06a8f 718 SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
b2e465d6
AL		 719 if (_error->PendingError() == true)
720 return false;
721
722 if (CmdL.FileSize() >= 4)
723 Sources.PathPrefix = CmdL.FileList[3];
724
725 // Do recursive directory searching
726 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
727 return false;
728
729 return true;
730}
98953965
AL		 731 /*}}}*/
732// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
733// ---------------------------------------------------------------------
c3ccac92 734static bool SimpleGenRelease(CommandLine &CmdL)
98953965 735{
0dfd2728
AL		 736 if (CmdL.FileSize() < 2)
737 return ShowHelp(CmdL);
738
c0eb6bc6
AL		 739 string Dir = CmdL.FileList[1];
740
98953965 741 ReleaseWriter Release("");
c0eb6bc6
AL		 742 Release.DirStrip = Dir;
743
98953965
AL		 744 if (_error->PendingError() == true)
745 return false;
746
c0eb6bc6 747 if (Release.RecursiveScan(Dir) == false)
98953965
AL		 748 return false;
749
f7291f62
AL		 750 Release.Finish();
751
98953965
AL		 752 return true;
753}
754
b2e465d6
AL		 755 /*}}}*/
756// Generate - Full generate, using a config file /*{{{*/
757// ---------------------------------------------------------------------
758/* */
c3ccac92 759static bool Generate(CommandLine &CmdL)
b2e465d6
AL		 760{
761 struct CacheDB::Stats SrcStats;
762 if (CmdL.FileSize() < 2)
763 return ShowHelp(CmdL);
764
765 struct timeval StartTime;
766 gettimeofday(&StartTime,0);
767 struct CacheDB::Stats Stats;
768
769 // Read the configuration file.
770 Configuration Setup;
771 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
772 return false;
773
774 vector<PackageMap> PkgList;
775 LoadTree(PkgList,Setup);
776 LoadBinDir(PkgList,Setup);
777
778 // Sort by cache DB to improve IO locality.
779 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
780
781 // Generate packages
782 if (CmdL.FileSize() <= 2)
783 {
f7f0d6c7 784 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 785 if (I->GenPackages(Setup,Stats) == false)
786 _error->DumpErrors();
f7f0d6c7 787 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 788 if (I->GenSources(Setup,SrcStats) == false)
789 _error->DumpErrors();
790 }
791 else
792 {
793 // Make a choice list out of the package list..
794 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
795 RxChoiceList *End = List;
f7f0d6c7 796 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 797 {
d935bb17 798 End->UserData = &(*I);
b2e465d6
AL		 799 End->Str = I->BaseDir.c_str();
800 End++;
801
d935bb17 802 End->UserData = &(*I);
b2e465d6
AL		 803 End->Str = I->Tag.c_str();
804 End++;
805 }
806 End->Str = 0;
807
808 // Regex it
809 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
810 {
811 delete [] List;
dc738e7a 812 return _error->Error(_("No selections matched"));
b2e465d6
AL		 813 }
814 _error->DumpErrors();
815
816 // Do the generation for Packages
817 for (End = List; End->Str != 0; End++)
818 {
819 if (End->Hit == false)
820 continue;
821
822 PackageMap *I = (PackageMap *)End->UserData;
823 if (I->PkgDone == true)
824 continue;
825 if (I->GenPackages(Setup,Stats) == false)
826 _error->DumpErrors();
827 }
828
829 // Do the generation for Sources
830 for (End = List; End->Str != 0; End++)
831 {
832 if (End->Hit == false)
833 continue;
834
835 PackageMap *I = (PackageMap *)End->UserData;
836 if (I->SrcDone == true)
837 continue;
838 if (I->GenSources(Setup,SrcStats) == false)
839 _error->DumpErrors();
840 }
841
842 delete [] List;
843 }
66905344
DK		 844
845 // close the Translation master files
f7f0d6c7 846 for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
66905344
DK		 847 if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
848 delete I->TransWriter;
849
b2e465d6
AL		 850 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
851 return true;
852
c6474fb6 853 c1out << "Packages done, Starting contents." << endl;
b2e465d6
AL		 854
855 // Sort the contents file list by date
856 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
f7f0d6c7 857 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6
AL		 858 {
859 struct stat A;
860 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
861 I->CntCompress,A) == false)
862 time(&I->ContentsMTime);
863 else
864 I->ContentsMTime = A.st_mtime;
865 }
866 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
867
868 /* Now for Contents.. The process here is to do a make-like dependency
869 check. Each contents file is verified to be newer than the package files
870 that describe the debs it indexes. Since the package files contain
871 hashes of the .debs this means they have not changed either so the
872 contents must be up to date. */
873 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
f7f0d6c7 874 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
b2e465d6 875 {
1e3f4083 876 // This record is not relevant
b2e465d6
AL		 877 if (I->ContentsDone == true ||
878 I->Contents.empty() == true)
879 continue;
880
881 // Do not do everything if the user specified sections.
882 if (CmdL.FileSize() > 2 && I->PkgDone == false)
883 continue;
884
885 struct stat A,B;
886 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
887 {
888 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
889 {
dc738e7a 890 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 891 continue;
892 }
893
894 if (A.st_mtime > B.st_mtime)
895 continue;
896 }
897
898 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
899 MaxContentsChange) == false)
900 _error->DumpErrors();
901
902 // Hit the limit?
903 if (MaxContentsChange == 0)
904 {
905 c1out << "Hit contents update byte limit" << endl;
906 break;
907 }
908 }
909
910 struct timeval NewTime;
911 gettimeofday(&NewTime,0);
912 double Delta = NewTime.tv_sec - StartTime.tv_sec +
913 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
914 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
915 << " archives. Took " << TimeToStr((long)Delta) << endl;
916
917 return true;
918}
919 /*}}}*/
920// Clean - Clean out the databases /*{{{*/
921// ---------------------------------------------------------------------
922/* */
c3ccac92 923static bool Clean(CommandLine &CmdL)
b2e465d6
AL		 924{
925 if (CmdL.FileSize() != 2)
926 return ShowHelp(CmdL);
927
928 // Read the configuration file.
929 Configuration Setup;
930 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
931 return false;
932
933 vector<PackageMap> PkgList;
934 LoadTree(PkgList,Setup);
935 LoadBinDir(PkgList,Setup);
936
937 // Sort by cache DB to improve IO locality.
938 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
939
940 string CacheDir = Setup.FindDir("Dir::CacheDir");
941
d935bb17 942 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 943 {
944 c0out << I->BinCacheDB << endl;
945 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
946 if (DB.Clean() == false)
947 _error->DumpErrors();
948
949 string CacheDB = I->BinCacheDB;
f7f0d6c7 950 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I);
b2e465d6
AL		 951 }
952
953 return true;
954}
955 /*}}}*/
956
957int main(int argc, const char *argv[])
958{
1692a2c7 959 setlocale(LC_ALL, "");
b2e465d6
AL		 960 CommandLine::Args Args[] = {
961 {'h',"help","help",0},
962 {0,"md5","APT::FTPArchive::MD5",0},
3c54407f
DK		 963 {0,"sha1","APT::FTPArchive::SHA1",0},
964 {0,"sha256","APT::FTPArchive::SHA256",0},
b2e465d6
AL		 965 {'v',"version","version",0},
966 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
967 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
968 {'q',"quiet","quiet",CommandLine::IntLevel},
969 {'q',"silent","quiet",CommandLine::IntLevel},
970 {0,"delink","APT::FTPArchive::DeLinkAct",0},
971 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
972 {0,"contents","APT::FTPArchive::Contents",0},
31981076 973 {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
b2e465d6
AL		 974 {'c',"config-file",0,CommandLine::ConfigFile},
975 {'o',"option",0,CommandLine::ArbItem},
976 {0,0,0,0}};
977 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
978 {"contents",&SimpleGenContents},
979 {"sources",&SimpleGenSources},
98953965 980 {"release",&SimpleGenRelease},
b2e465d6
AL		 981 {"generate",&Generate},
982 {"clean",&Clean},
983 {"help",&ShowHelp},
984 {0,0}};
985
986 // Parse the command line and initialize the package library
987 CommandLine CmdL(Args,_config);
b34d4b47 988 if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
b2e465d6
AL		 989 {
990 _error->DumpErrors();
991 return 100;
992 }
993
994 // See if the help should be shown
995 if (_config->FindB("help") == true ||
996 _config->FindB("version") == true ||
997 CmdL.FileSize() == 0)
998 {
999 ShowHelp(CmdL);
1000 return 0;
1001 }
1002
1003 // Setup the output streams
1809f8d2
AL		 1004 c0out.rdbuf(clog.rdbuf());
1005 c1out.rdbuf(clog.rdbuf());
1006 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 1007 Quiet = _config->FindI("quiet",0);
1008 if (Quiet > 0)
1009 c0out.rdbuf(devnull.rdbuf());
1010 if (Quiet > 1)
1011 c1out.rdbuf(devnull.rdbuf());
1012
1013 // Match the operation
1014 CmdL.DispatchArg(Cmds);
1015
1016 if (_error->empty() == false)
1017 {
1018 bool Errors = _error->PendingError();
1019 _error->DumpErrors();
1020 return Errors == true?100:0;
1021 }
1022 return 0;
1023}
ntk's bugfix for apt