HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Add SHA1 support to "apt-ftparchive release"
[ntk/apt.git] / ftparchive / apt-ftparchive.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
f7291f62 3// $Id: apt-ftparchive.cc,v 1.10 2003/12/26 20:50:01 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
6 apt-scanpackages - Efficient work-alike for dpkg-scanpackages
7
8 Let contents be disabled from the conf
9
10 ##################################################################### */
11 /*}}}*/
12// Include Files /*{{{*/
13#ifdef __GNUG__
14#pragma implementation "apt-ftparchive.h"
15#endif
16
17#include "apt-ftparchive.h"
18
19#include <apt-pkg/error.h>
20#include <apt-pkg/configuration.h>
21#include <apt-pkg/cmndline.h>
22#include <apt-pkg/strutl.h>
23#include <config.h>
24#include <apti18n.h>
25#include <algorithm>
26
27#include <sys/time.h>
28#include <regex.h>
29
30#include "contents.h"
31#include "multicompress.h"
32#include "writer.h"
33 /*}}}*/
34
d935bb17
AL		 35using namespace std;
36ostream c0out(0);
37ostream c1out(0);
38ostream c2out(0);
b2e465d6
AL		 39ofstream devnull("/dev/null");
40unsigned Quiet = 0;
41
42// struct PackageMap - List of all package files in the config file /*{{{*/
43// ---------------------------------------------------------------------
44/* */
45struct PackageMap
46{
47 // General Stuff
48 string BaseDir;
49 string InternalPrefix;
50 string FLFile;
51 string PkgExt;
52 string SrcExt;
53
54 // Stuff for the Package File
55 string PkgFile;
56 string BinCacheDB;
57 string BinOverride;
64177f17 58 string ExtraOverride;
b2e465d6
AL		 59
60 // Stuff for the Source File
61 string SrcFile;
62 string SrcOverride;
64177f17 63 string SrcExtraOverride;
b2e465d6
AL		 64
65 // Contents
66 string Contents;
67 string ContentsHead;
68
69 // Random things
70 string Tag;
71 string PkgCompress;
72 string CntCompress;
73 string SrcCompress;
74 string PathPrefix;
75 unsigned int DeLinkLimit;
76 mode_t Permissions;
77
78 bool ContentsDone;
79 bool PkgDone;
80 bool SrcDone;
81 time_t ContentsMTime;
82
83 struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
84 {
85 inline bool operator() (const PackageMap &x,const PackageMap &y)
86 {return x.ContentsMTime < y.ContentsMTime;};
87 };
88
89 struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
90 {
91 inline bool operator() (const PackageMap &x,const PackageMap &y)
92 {return x.BinCacheDB < y.BinCacheDB;};
93 };
94
95 void GetGeneral(Configuration &Setup,Configuration &Block);
96 bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
97 bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
98 bool GenContents(Configuration &Setup,
d935bb17
AL		 99 vector<PackageMap>::iterator Begin,
100 vector<PackageMap>::iterator End,
b2e465d6
AL		 101 unsigned long &Left);
102
103 PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
104 PkgDone(false), SrcDone(false), ContentsMTime(0) {};
105};
106 /*}}}*/
107
108// PackageMap::GetGeneral - Common per-section definitions /*{{{*/
109// ---------------------------------------------------------------------
110/* */
111void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
112{
113 PathPrefix = Block.Find("PathPrefix");
114
115 if (Block.FindB("External-Links",true) == false)
116 DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
117 else
118 DeLinkLimit = 0;
119
120 PkgCompress = Block.Find("Packages::Compress",
121 Setup.Find("Default::Packages::Compress",". gzip").c_str());
122 CntCompress = Block.Find("Contents::Compress",
123 Setup.Find("Default::Contents::Compress",". gzip").c_str());
124 SrcCompress = Block.Find("Sources::Compress",
125 Setup.Find("Default::Sources::Compress",". gzip").c_str());
126
127 SrcExt = Block.Find("Sources::Extensions",
128 Setup.Find("Default::Sources::Extensions",".dsc").c_str());
129 PkgExt = Block.Find("Packages::Extensions",
130 Setup.Find("Default::Packages::Extensions",".deb").c_str());
131
132 Permissions = Setup.FindI("Default::FileMode",0644);
133
134 if (FLFile.empty() == false)
135 FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
136
137 if (Contents == " ")
138 Contents= string();
139}
140 /*}}}*/
141// PackageMap::GenPackages - Actually generate a Package file /*{{{*/
142// ---------------------------------------------------------------------
143/* This generates the Package File described by this object. */
144bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
145{
146 if (PkgFile.empty() == true)
147 return true;
148
149 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
150 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
151 string CacheDir = Setup.FindDir("Dir::CacheDir");
152
153 struct timeval StartTime;
154 gettimeofday(&StartTime,0);
155
156 PkgDone = true;
157
158 // Create a package writer object.
159 PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
64177f17
AL		 160 flCombine(OverrideDir,BinOverride),
161 flCombine(OverrideDir,ExtraOverride));
b2e465d6 162 if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
dc738e7a 163 return _error->Error(_("Package extension list is too long"));
b2e465d6 164 if (_error->PendingError() == true)
dc738e7a 165 return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 166
167 Packages.PathPrefix = PathPrefix;
168 Packages.DirStrip = ArchiveDir;
169 Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
170
171 Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
172 Packages.DeLinkLimit = DeLinkLimit;
64177f17 173
b2e465d6
AL		 174 // Create a compressor object
175 MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
176 PkgCompress,Permissions);
177 Packages.Output = Comp.Input;
178 if (_error->PendingError() == true)
dc738e7a 179 return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 180
181 c0out << ' ' << BaseDir << ":" << flush;
182
183 // Do recursive directory searching
184 if (FLFile.empty() == true)
185 {
186 if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
187 return false;
188 }
189 else
190 {
191 if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
192 return false;
193 }
194
195 Packages.Output = 0; // Just in case
196
197 // Finish compressing
198 unsigned long Size;
199 if (Comp.Finalize(Size) == false)
200 {
201 c0out << endl;
dc738e7a 202 return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 203 }
204
205 if (Size != 0)
206 c0out << " New "
207 << SizeToStr(Size) << "B ";
208 else
209 c0out << ' ';
210
211 struct timeval NewTime;
212 gettimeofday(&NewTime,0);
213 double Delta = NewTime.tv_sec - StartTime.tv_sec +
214 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
215
216 c0out << Packages.Stats.Packages << " files " <<
217/* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
218 SizeToStr(Packages.Stats.Bytes) << "B " <<
219 TimeToStr((long)Delta) << endl;
220
221 Stats.Add(Packages.Stats);
222 Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
223
224 return !_error->PendingError();
225}
98953965 226
b2e465d6 227 /*}}}*/
64177f17 228// PackageMap::GenSources - Actually generate a Source file /*{{{*/
b2e465d6
AL		 229// ---------------------------------------------------------------------
230/* This generates the Sources File described by this object. */
231bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
232{
233 if (SrcFile.empty() == true)
234 return true;
235
236 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
237 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
238 string CacheDir = Setup.FindDir("Dir::CacheDir");
239
240 struct timeval StartTime;
241 gettimeofday(&StartTime,0);
242
243 SrcDone = true;
244
245 // Create a package writer object.
246 SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
64177f17
AL		 247 flCombine(OverrideDir,SrcOverride),
248 flCombine(OverrideDir,SrcExtraOverride));
b2e465d6 249 if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
dc738e7a 250 return _error->Error(_("Source extension list is too long"));
b2e465d6 251 if (_error->PendingError() == true)
dc738e7a 252 return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 253
254 Sources.PathPrefix = PathPrefix;
255 Sources.DirStrip = ArchiveDir;
256 Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
257
258 Sources.DeLinkLimit = DeLinkLimit;
259 Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
260
261 // Create a compressor object
262 MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
263 SrcCompress,Permissions);
264 Sources.Output = Comp.Input;
265 if (_error->PendingError() == true)
dc738e7a 266 return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 267
268 c0out << ' ' << BaseDir << ":" << flush;
269
270 // Do recursive directory searching
271 if (FLFile.empty() == true)
272 {
273 if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
274 return false;
275 }
276 else
277 {
278 if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
279 return false;
280 }
281 Sources.Output = 0; // Just in case
282
283 // Finish compressing
284 unsigned long Size;
285 if (Comp.Finalize(Size) == false)
286 {
287 c0out << endl;
dc738e7a 288 return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
b2e465d6
AL		 289 }
290
291 if (Size != 0)
292 c0out << " New "
293 << SizeToStr(Size) << "B ";
294 else
295 c0out << ' ';
296
297 struct timeval NewTime;
298 gettimeofday(&NewTime,0);
299 double Delta = NewTime.tv_sec - StartTime.tv_sec +
300 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
301
302 c0out << Sources.Stats.Packages << " pkgs in " <<
303 TimeToStr((long)Delta) << endl;
304
305 Stats.Add(Sources.Stats);
306 Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
307
308 return !_error->PendingError();
309}
310 /*}}}*/
311// PackageMap::GenContents - Actually generate a Contents file /*{{{*/
312// ---------------------------------------------------------------------
313/* This generates the contents file partially described by this object.
314 It searches the given iterator range for other package files that map
315 into this contents file and includes their data as well when building. */
316bool PackageMap::GenContents(Configuration &Setup,
d935bb17
AL		 317 vector<PackageMap>::iterator Begin,
318 vector<PackageMap>::iterator End,
319 unsigned long &Left)
b2e465d6
AL		 320{
321 if (Contents.empty() == true)
322 return true;
323
324 if (Left == 0)
325 return true;
326
327 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
328 string CacheDir = Setup.FindDir("Dir::CacheDir");
329 string OverrideDir = Setup.FindDir("Dir::OverrideDir");
330
331 struct timeval StartTime;
332 gettimeofday(&StartTime,0);
333
334 // Create a package writer object.
335 ContentsWriter Contents("");
336 if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
dc738e7a 337 return _error->Error(_("Package extension list is too long"));
b2e465d6
AL		 338 if (_error->PendingError() == true)
339 return false;
340
341 MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
342 CntCompress,Permissions);
343 Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
344 Contents.Output = Comp.Input;
345 if (_error->PendingError() == true)
346 return false;
347
348 // Write the header out.
349 if (ContentsHead.empty() == false)
350 {
351 FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
352 if (_error->PendingError() == true)
353 return false;
354
355 unsigned long Size = Head.Size();
356 unsigned char Buf[4096];
357 while (Size != 0)
358 {
359 unsigned long ToRead = Size;
360 if (Size > sizeof(Buf))
361 ToRead = sizeof(Buf);
362
363 if (Head.Read(Buf,ToRead) == false)
364 return false;
365
366 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
dc738e7a 367 return _error->Errno("fwrite",_("Error writing header to contents file"));
b2e465d6
AL		 368
369 Size -= ToRead;
370 }
371 }
372
373 /* Go over all the package file records and parse all the package
374 files associated with this contents file into one great big honking
375 memory structure, then dump the sorted version */
376 c0out << ' ' << this->Contents << ":" << flush;
d935bb17 377 for (vector<PackageMap>::iterator I = Begin; I != End; I++)
b2e465d6
AL		 378 {
379 if (I->Contents != this->Contents)
380 continue;
381
382 Contents.Prefix = ArchiveDir;
383 Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
384 Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
385 I->PkgCompress);
386
387 I->ContentsDone = true;
388 }
389
390 Contents.Finish();
391
392 // Finish compressing
393 unsigned long Size;
394 if (Comp.Finalize(Size) == false || _error->PendingError() == true)
395 {
396 c0out << endl;
dc738e7a 397 return _error->Error(_("Error Processing Contents %s"),
b2e465d6
AL		 398 this->Contents.c_str());
399 }
400
401 if (Size != 0)
402 {
403 c0out << " New " << SizeToStr(Size) << "B ";
404 if (Left > Size)
405 Left -= Size;
406 else
407 Left = 0;
408 }
409 else
410 c0out << ' ';
411
412 struct timeval NewTime;
413 gettimeofday(&NewTime,0);
414 double Delta = NewTime.tv_sec - StartTime.tv_sec +
415 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
416
417 c0out << Contents.Stats.Packages << " files " <<
418 SizeToStr(Contents.Stats.Bytes) << "B " <<
419 TimeToStr((long)Delta) << endl;
420
421 return true;
422}
423 /*}}}*/
424
425// LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
426// ---------------------------------------------------------------------
427/* This populates the PkgList with all the possible permutations of the
428 section/arch lists. */
429void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
430{
431 // Load the defaults
432 string DDir = Setup.Find("TreeDefault::Directory",
433 "$(DIST)/$(SECTION)/binary-$(ARCH)/");
434 string DSDir = Setup.Find("TreeDefault::SrcDirectory",
435 "$(DIST)/$(SECTION)/source/");
436 string DPkg = Setup.Find("TreeDefault::Packages",
437 "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
438 string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
439 "$(DIST)/$(SECTION)/");
440 string DContents = Setup.Find("TreeDefault::Contents",
441 "$(DIST)/Contents-$(ARCH)");
442 string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
443 string DBCache = Setup.Find("TreeDefault::BinCacheDB",
444 "packages-$(ARCH).db");
445 string DSources = Setup.Find("TreeDefault::Sources",
446 "$(DIST)/$(SECTION)/source/Sources");
447 string DFLFile = Setup.Find("TreeDefault::FileList", "");
448 string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
449
450 // Process 'tree' type sections
451 const Configuration::Item *Top = Setup.Tree("tree");
452 for (Top = (Top == 0?0:Top->Child); Top != 0;)
453 {
454 Configuration Block(Top);
455 string Dist = Top->Tag;
456
457 // Parse the sections
d935bb17
AL		 458 string Tmp = Block.Find("Sections");
459 const char *Sections = Tmp.c_str();
b2e465d6
AL		 460 string Section;
461 while (ParseQuoteWord(Sections,Section) == true)
462 {
d935bb17 463 string Tmp2 = Block.Find("Architectures");
b2e465d6 464 string Arch;
d935bb17 465 const char *Archs = Tmp2.c_str();
b2e465d6
AL		 466 while (ParseQuoteWord(Archs,Arch) == true)
467 {
468 struct SubstVar Vars[] = {{"$(DIST)",&Dist},
469 {"$(SECTION)",&Section},
470 {"$(ARCH)",&Arch},
471 {}};
472 PackageMap Itm;
473
474 Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
475 Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
476
477 if (stringcasecmp(Arch,"source") == 0)
478 {
479 Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
480 Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
481 Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
482 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
483 Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
64177f17 484 Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
b2e465d6
AL		 485 }
486 else
487 {
488 Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
489 Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
490 Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
491 Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
492 Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
493 Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
494 Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
64177f17 495 Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
b2e465d6
AL		 496 }
497
498 Itm.GetGeneral(Setup,Block);
499 PkgList.push_back(Itm);
500 }
501 }
502
503 Top = Top->Next;
504 }
505}
506 /*}}}*/
507// LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
508// ---------------------------------------------------------------------
509/* */
510void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
511{
512 // Process 'bindirectory' type sections
513 const Configuration::Item *Top = Setup.Tree("bindirectory");
514 for (Top = (Top == 0?0:Top->Child); Top != 0;)
515 {
516 Configuration Block(Top);
517
518 PackageMap Itm;
519 Itm.PkgFile = Block.Find("Packages");
520 Itm.SrcFile = Block.Find("Sources");
521 Itm.BinCacheDB = Block.Find("BinCacheDB");
522 Itm.BinOverride = Block.Find("BinOverride");
64177f17
AL		 523 Itm.ExtraOverride = Block.Find("ExtraOverride");
524 Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
b2e465d6
AL		 525 Itm.SrcOverride = Block.Find("SrcOverride");
526 Itm.BaseDir = Top->Tag;
527 Itm.FLFile = Block.Find("FileList");
528 Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
529 Itm.Contents = Block.Find("Contents");
530 Itm.ContentsHead = Block.Find("Contents::Header");
531
532 Itm.GetGeneral(Setup,Block);
533 PkgList.push_back(Itm);
534
535 Top = Top->Next;
536 }
537}
538 /*}}}*/
539
540// ShowHelp - Show the help text /*{{{*/
541// ---------------------------------------------------------------------
542/* */
543bool ShowHelp(CommandLine &CmdL)
544{
545 ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
546 COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
547 if (_config->FindB("version") == true)
548 return true;
549
550 cout <<
dc738e7a 551 _("Usage: apt-ftparchive [options] command\n"
7a3e9e0a 552 "Commands: packages binarypath [overridefile [pathprefix]]\n"
b2e465d6
AL		 553 " sources srcpath [overridefile [pathprefix]]\n"
554 " contents path\n"
555 " generate config [groups]\n"
556 " clean config\n"
557 "\n"
558 "apt-ftparchive generates index files for Debian archives. It supports\n"
559 "many styles of generation from fully automated to functional replacements\n"
560 "for dpkg-scanpackages and dpkg-scansources\n"
561 "\n"
562 "apt-ftparchive generates Package files from a tree of .debs. The\n"
563 "Package file contains the contents of all the control fields from\n"
564 "each package as well as the MD5 hash and filesize. An override file\n"
565 "is supported to force the value of Priority and Section.\n"
566 "\n"
567 "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
568 "The --source-override option can be used to specify a src override file\n"
569 "\n"
570 "The 'packages' and 'sources' command should be run in the root of the\n"
571 "tree. BinaryPath should point to the base of the recursive search and \n"
75224826 572 "override file should contain the override flags. Pathprefix is\n"
b2e465d6 573 "appended to the filename fields if present. Example usage from the \n"
75224826 574 "Debian archive:\n"
b2e465d6
AL		 575 " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
576 " dists/potato/main/binary-i386/Packages\n"
577 "\n"
578 "Options:\n"
579 " -h This help text\n"
580 " --md5 Control MD5 generation\n"
581 " -s=? Source override file\n"
582 " -q Quiet\n"
583 " -d=? Select the optional caching database\n"
584 " --no-delink Enable delinking debug mode\n"
585 " --contents Control contents file generation\n"
586 " -c=? Read this configuration file\n"
dc738e7a 587 " -o=? Set an arbitary configuration option") << endl;
b2e465d6
AL		 588
589 return true;
590}
591 /*}}}*/
592// SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
593// ---------------------------------------------------------------------
594/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
595bool SimpleGenPackages(CommandLine &CmdL)
596{
597 if (CmdL.FileSize() < 2)
598 return ShowHelp(CmdL);
599
600 string Override;
601 if (CmdL.FileSize() >= 3)
602 Override = CmdL.FileList[2];
603
604 // Create a package writer object.
605 PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
64177f17 606 Override, "");
b2e465d6
AL		 607 if (_error->PendingError() == true)
608 return false;
609
610 if (CmdL.FileSize() >= 4)
611 Packages.PathPrefix = CmdL.FileList[3];
612
613 // Do recursive directory searching
614 if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
615 return false;
616
617 return true;
618}
619 /*}}}*/
620// SimpleGenContents - Generate a Contents listing /*{{{*/
621// ---------------------------------------------------------------------
622/* */
623bool SimpleGenContents(CommandLine &CmdL)
624{
625 if (CmdL.FileSize() < 2)
626 return ShowHelp(CmdL);
627
628 // Create a package writer object.
629 ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
630 if (_error->PendingError() == true)
631 return false;
632
633 // Do recursive directory searching
634 if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
635 return false;
636
637 Contents.Finish();
638
639 return true;
640}
641 /*}}}*/
642// SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
643// ---------------------------------------------------------------------
644/* This emulates dpkg-scanpackages's command line interface. 'mostly' */
645bool SimpleGenSources(CommandLine &CmdL)
646{
647 if (CmdL.FileSize() < 2)
648 return ShowHelp(CmdL);
649
650 string Override;
651 if (CmdL.FileSize() >= 3)
652 Override = CmdL.FileList[2];
653
654 string SOverride;
655 if (Override.empty() == false)
656 SOverride = Override + ".src";
657
658 SOverride = _config->Find("APT::FTPArchive::SourceOverride",
659 SOverride.c_str());
660
661 // Create a package writer object.
662 SourcesWriter Sources(Override,SOverride);
663 if (_error->PendingError() == true)
664 return false;
665
666 if (CmdL.FileSize() >= 4)
667 Sources.PathPrefix = CmdL.FileList[3];
668
669 // Do recursive directory searching
670 if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
671 return false;
672
673 return true;
674}
98953965
AL		 675 /*}}}*/
676// SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
677// ---------------------------------------------------------------------
678bool SimpleGenRelease(CommandLine &CmdL)
679{
680 ReleaseWriter Release("");
681 if (_error->PendingError() == true)
682 return false;
683
684 if (Release.RecursiveScan(CmdL.FileList[1]) == false)
685 return false;
686
f7291f62
AL		 687 Release.Finish();
688
98953965
AL		 689 return true;
690}
691
b2e465d6
AL		 692 /*}}}*/
693// Generate - Full generate, using a config file /*{{{*/
694// ---------------------------------------------------------------------
695/* */
696bool Generate(CommandLine &CmdL)
697{
698 struct CacheDB::Stats SrcStats;
699 if (CmdL.FileSize() < 2)
700 return ShowHelp(CmdL);
701
702 struct timeval StartTime;
703 gettimeofday(&StartTime,0);
704 struct CacheDB::Stats Stats;
705
706 // Read the configuration file.
707 Configuration Setup;
708 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
709 return false;
710
711 vector<PackageMap> PkgList;
712 LoadTree(PkgList,Setup);
713 LoadBinDir(PkgList,Setup);
714
715 // Sort by cache DB to improve IO locality.
716 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
717
718 // Generate packages
719 if (CmdL.FileSize() <= 2)
720 {
d935bb17 721 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 722 if (I->GenPackages(Setup,Stats) == false)
723 _error->DumpErrors();
d935bb17 724 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 725 if (I->GenSources(Setup,SrcStats) == false)
726 _error->DumpErrors();
727 }
728 else
729 {
730 // Make a choice list out of the package list..
731 RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
732 RxChoiceList *End = List;
d935bb17 733 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6 734 {
d935bb17 735 End->UserData = &(*I);
b2e465d6
AL		 736 End->Str = I->BaseDir.c_str();
737 End++;
738
d935bb17 739 End->UserData = &(*I);
b2e465d6
AL		 740 End->Str = I->Tag.c_str();
741 End++;
742 }
743 End->Str = 0;
744
745 // Regex it
746 if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
747 {
748 delete [] List;
dc738e7a 749 return _error->Error(_("No selections matched"));
b2e465d6
AL		 750 }
751 _error->DumpErrors();
752
753 // Do the generation for Packages
754 for (End = List; End->Str != 0; End++)
755 {
756 if (End->Hit == false)
757 continue;
758
759 PackageMap *I = (PackageMap *)End->UserData;
760 if (I->PkgDone == true)
761 continue;
762 if (I->GenPackages(Setup,Stats) == false)
763 _error->DumpErrors();
764 }
765
766 // Do the generation for Sources
767 for (End = List; End->Str != 0; End++)
768 {
769 if (End->Hit == false)
770 continue;
771
772 PackageMap *I = (PackageMap *)End->UserData;
773 if (I->SrcDone == true)
774 continue;
775 if (I->GenSources(Setup,SrcStats) == false)
776 _error->DumpErrors();
777 }
778
779 delete [] List;
780 }
781
782 if (_config->FindB("APT::FTPArchive::Contents",true) == false)
783 return true;
784
785 c1out << "Done Packages, Starting contents." << endl;
786
787 // Sort the contents file list by date
788 string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
d935bb17 789 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 790 {
791 struct stat A;
792 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
793 I->CntCompress,A) == false)
794 time(&I->ContentsMTime);
795 else
796 I->ContentsMTime = A.st_mtime;
797 }
798 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
799
800 /* Now for Contents.. The process here is to do a make-like dependency
801 check. Each contents file is verified to be newer than the package files
802 that describe the debs it indexes. Since the package files contain
803 hashes of the .debs this means they have not changed either so the
804 contents must be up to date. */
805 unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
d935bb17 806 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
b2e465d6
AL		 807 {
808 // This record is not relevent
809 if (I->ContentsDone == true ||
810 I->Contents.empty() == true)
811 continue;
812
813 // Do not do everything if the user specified sections.
814 if (CmdL.FileSize() > 2 && I->PkgDone == false)
815 continue;
816
817 struct stat A,B;
818 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
819 {
820 if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
821 {
dc738e7a 822 _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
b2e465d6
AL		 823 continue;
824 }
825
826 if (A.st_mtime > B.st_mtime)
827 continue;
828 }
829
830 if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
831 MaxContentsChange) == false)
832 _error->DumpErrors();
833
834 // Hit the limit?
835 if (MaxContentsChange == 0)
836 {
837 c1out << "Hit contents update byte limit" << endl;
838 break;
839 }
840 }
841
842 struct timeval NewTime;
843 gettimeofday(&NewTime,0);
844 double Delta = NewTime.tv_sec - StartTime.tv_sec +
845 (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
846 c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
847 << " archives. Took " << TimeToStr((long)Delta) << endl;
848
849 return true;
850}
851 /*}}}*/
852// Clean - Clean out the databases /*{{{*/
853// ---------------------------------------------------------------------
854/* */
855bool Clean(CommandLine &CmdL)
856{
857 if (CmdL.FileSize() != 2)
858 return ShowHelp(CmdL);
859
860 // Read the configuration file.
861 Configuration Setup;
862 if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
863 return false;
864
865 vector<PackageMap> PkgList;
866 LoadTree(PkgList,Setup);
867 LoadBinDir(PkgList,Setup);
868
869 // Sort by cache DB to improve IO locality.
870 stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
871
872 string CacheDir = Setup.FindDir("Dir::CacheDir");
873
d935bb17 874 for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
b2e465d6
AL		 875 {
876 c0out << I->BinCacheDB << endl;
877 CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
878 if (DB.Clean() == false)
879 _error->DumpErrors();
880
881 string CacheDB = I->BinCacheDB;
882 for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
883 }
884
885 return true;
886}
887 /*}}}*/
888
889int main(int argc, const char *argv[])
890{
891 CommandLine::Args Args[] = {
892 {'h',"help","help",0},
893 {0,"md5","APT::FTPArchive::MD5",0},
894 {'v',"version","version",0},
895 {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
896 {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
897 {'q',"quiet","quiet",CommandLine::IntLevel},
898 {'q',"silent","quiet",CommandLine::IntLevel},
899 {0,"delink","APT::FTPArchive::DeLinkAct",0},
900 {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
901 {0,"contents","APT::FTPArchive::Contents",0},
902 {'c',"config-file",0,CommandLine::ConfigFile},
903 {'o',"option",0,CommandLine::ArbItem},
904 {0,0,0,0}};
905 CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
906 {"contents",&SimpleGenContents},
907 {"sources",&SimpleGenSources},
98953965 908 {"release",&SimpleGenRelease},
b2e465d6
AL		 909 {"generate",&Generate},
910 {"clean",&Clean},
911 {"help",&ShowHelp},
912 {0,0}};
913
914 // Parse the command line and initialize the package library
915 CommandLine CmdL(Args,_config);
916 if (CmdL.Parse(argc,argv) == false)
917 {
918 _error->DumpErrors();
919 return 100;
920 }
921
922 // See if the help should be shown
923 if (_config->FindB("help") == true ||
924 _config->FindB("version") == true ||
925 CmdL.FileSize() == 0)
926 {
927 ShowHelp(CmdL);
928 return 0;
929 }
930
931 // Setup the output streams
1809f8d2
AL		 932 c0out.rdbuf(clog.rdbuf());
933 c1out.rdbuf(clog.rdbuf());
934 c2out.rdbuf(clog.rdbuf());
b2e465d6
AL		 935 Quiet = _config->FindI("quiet",0);
936 if (Quiet > 0)
937 c0out.rdbuf(devnull.rdbuf());
938 if (Quiet > 1)
939 c1out.rdbuf(devnull.rdbuf());
940
941 // Match the operation
942 CmdL.DispatchArg(Cmds);
943
944 if (_error->empty() == false)
945 {
946 bool Errors = _error->PendingError();
947 _error->DumpErrors();
948 return Errors == true?100:0;
949 }
950 return 0;
951}
ntk's bugfix for apt