ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <config.h>
  20 #include <apti18n.h>
  21 #include <algorithm>
  22
  23 #include <climits>
  24 #include <sys/time.h>
  25 #include <regex.h>
  26
  27 #include "contents.h"
  28 #include "multicompress.h"
  29 #include "writer.h"
  30                                                                         /*}}}*/
  31
  32 using namespace std;
  33 ostream c0out(0);
  34 ostream c1out(0);
  35 ostream c2out(0);
  36 ofstream devnull("/dev/null");
  37 unsigned Quiet = 0;
  38
  39 // struct PackageMap - List of all package files in the config file     /*{{{*/
  40 // ---------------------------------------------------------------------
  41 /* */
  42 struct PackageMap
  43 {
  44    // General Stuff
  45    string BaseDir;
  46    string InternalPrefix;
  47    string FLFile;
  48    string PkgExt;
  49    string SrcExt;
  50
  51    // Stuff for the Package File
  52    string PkgFile;
  53    string BinCacheDB;
  54    string BinOverride;
  55    string ExtraOverride;
  56
  57    // We generate for this given arch
  58    string Arch;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Translation master file
  66    TranslationWriter *TransWriter;
  67
  68    // Contents
  69    string Contents;
  70    string ContentsHead;
  71
  72    // Random things
  73    string Tag;
  74    string PkgCompress;
  75    string CntCompress;
  76    string SrcCompress;
  77    string PathPrefix;
  78    unsigned int DeLinkLimit;
  79    mode_t Permissions;
  80
  81    bool ContentsDone;
  82    bool PkgDone;
  83    bool SrcDone;
  84    time_t ContentsMTime;
  85
  86    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  87    {
  88       inline bool operator() (const PackageMap &x,const PackageMap &y)
  89       {return x.ContentsMTime < y.ContentsMTime;};
  90    };
  91
  92    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  93    {
  94       inline bool operator() (const PackageMap &x,const PackageMap &y)
  95       {return x.BinCacheDB < y.BinCacheDB;};
  96    };
  97
  98    void GetGeneral(Configuration &Setup,Configuration &Block);
  99    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 100    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 101    bool GenContents(Configuration &Setup,
 102                     vector<PackageMap>::iterator Begin,
 103                     vector<PackageMap>::iterator End,
 104                     unsigned long &Left);
 105
 106    PackageMap() : TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 107                   ContentsDone(false), PkgDone(false), SrcDone(false),
 108                   ContentsMTime(0) {};
 109 };
 110                                                                         /*}}}*/
 111
 112 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* */
 115 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 116 {
 117    PathPrefix = Block.Find("PathPrefix");
 118
 119    if (Block.FindB("External-Links",true) == false)
 120       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 121    else
 122       DeLinkLimit = 0;
 123
 124    PkgCompress = Block.Find("Packages::Compress",
 125                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 126    CntCompress = Block.Find("Contents::Compress",
 127                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 128    SrcCompress = Block.Find("Sources::Compress",
 129                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 130
 131    SrcExt = Block.Find("Sources::Extensions",
 132                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 133    PkgExt = Block.Find("Packages::Extensions",
 134                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 135
 136    Permissions = Setup.FindI("Default::FileMode",0644);
 137
 138    if (FLFile.empty() == false)
 139       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 140
 141    if (Contents == " ")
 142       Contents= string();
 143 }
 144                                                                         /*}}}*/
 145 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 146 // ---------------------------------------------------------------------
 147 /* This generates the Package File described by this object. */
 148 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 149 {
 150    if (PkgFile.empty() == true)
 151       return true;
 152
 153    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 154    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 155    string CacheDir = Setup.FindDir("Dir::CacheDir");
 156
 157    struct timeval StartTime;
 158    gettimeofday(&StartTime,0);
 159
 160    PkgDone = true;
 161
 162    // Create a package writer object.
 163    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 164                            flCombine(OverrideDir,BinOverride),
 165                            flCombine(OverrideDir,ExtraOverride),
 166                            Arch);
 167    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 168       return _error->Error(_("Package extension list is too long"));
 169    if (_error->PendingError() == true)
 170       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 171
 172    Packages.PathPrefix = PathPrefix;
 173    Packages.DirStrip = ArchiveDir;
 174    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 175
 176    Packages.TransWriter = TransWriter;
 177
 178    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 179    Packages.DeLinkLimit = DeLinkLimit;
 180
 181    // Create a compressor object
 182    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 183                       PkgCompress,Permissions);
 184    Packages.Output = Comp.Input;
 185    if (_error->PendingError() == true)
 186       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 187
 188    c0out << ' ' << BaseDir << ":" << flush;
 189
 190    // Do recursive directory searching
 191    if (FLFile.empty() == true)
 192    {
 193       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 194          return false;
 195    }
 196    else
 197    {
 198       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 199          return false;
 200    }
 201
 202    Packages.Output = 0;      // Just in case
 203
 204    // Finish compressing
 205    unsigned long Size;
 206    if (Comp.Finalize(Size) == false)
 207    {
 208       c0out << endl;
 209       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 210    }
 211
 212    if (Size != 0)
 213       c0out << " New "
 214              << SizeToStr(Size) << "B ";
 215    else
 216       c0out << ' ';
 217
 218    struct timeval NewTime;
 219    gettimeofday(&NewTime,0);
 220    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 221                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 222
 223    c0out << Packages.Stats.Packages << " files " <<
 224 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 225       SizeToStr(Packages.Stats.Bytes) << "B " <<
 226       TimeToStr((long)Delta) << endl;
 227
 228    Stats.Add(Packages.Stats);
 229    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 230
 231    return !_error->PendingError();
 232 }
 233
 234                                                                         /*}}}*/
 235 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 236 // ---------------------------------------------------------------------
 237 /* This generates the Sources File described by this object. */
 238 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 239 {
 240    if (SrcFile.empty() == true)
 241       return true;
 242
 243    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 244    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 245    string CacheDir = Setup.FindDir("Dir::CacheDir");
 246
 247    struct timeval StartTime;
 248    gettimeofday(&StartTime,0);
 249
 250    SrcDone = true;
 251
 252    // Create a package writer object.
 253    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 254                          flCombine(OverrideDir,SrcOverride),
 255                          flCombine(OverrideDir,SrcExtraOverride));
 256    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 257       return _error->Error(_("Source extension list is too long"));
 258    if (_error->PendingError() == true)
 259       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 260
 261    Sources.PathPrefix = PathPrefix;
 262    Sources.DirStrip = ArchiveDir;
 263    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 264
 265    Sources.DeLinkLimit = DeLinkLimit;
 266    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 267
 268    // Create a compressor object
 269    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 270                       SrcCompress,Permissions);
 271    Sources.Output = Comp.Input;
 272    if (_error->PendingError() == true)
 273       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 274
 275    c0out << ' ' << BaseDir << ":" << flush;
 276
 277    // Do recursive directory searching
 278    if (FLFile.empty() == true)
 279    {
 280       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 281          return false;
 282    }
 283    else
 284    {
 285       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 286          return false;
 287    }
 288    Sources.Output = 0;      // Just in case
 289
 290    // Finish compressing
 291    unsigned long Size;
 292    if (Comp.Finalize(Size) == false)
 293    {
 294       c0out << endl;
 295       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 296    }
 297
 298    if (Size != 0)
 299       c0out << " New "
 300              << SizeToStr(Size) << "B ";
 301    else
 302       c0out << ' ';
 303
 304    struct timeval NewTime;
 305    gettimeofday(&NewTime,0);
 306    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 307                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 308
 309    c0out << Sources.Stats.Packages << " pkgs in " <<
 310       TimeToStr((long)Delta) << endl;
 311
 312    Stats.Add(Sources.Stats);
 313    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 314
 315    return !_error->PendingError();
 316 }
 317                                                                         /*}}}*/
 318 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 319 // ---------------------------------------------------------------------
 320 /* This generates the contents file partially described by this object.
 321    It searches the given iterator range for other package files that map
 322    into this contents file and includes their data as well when building. */
 323 bool PackageMap::GenContents(Configuration &Setup,
 324                              vector<PackageMap>::iterator Begin,
 325                              vector<PackageMap>::iterator End,
 326                              unsigned long &Left)
 327 {
 328    if (Contents.empty() == true)
 329       return true;
 330
 331    if (Left == 0)
 332       return true;
 333
 334    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 335    string CacheDir = Setup.FindDir("Dir::CacheDir");
 336    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 337
 338    struct timeval StartTime;
 339    gettimeofday(&StartTime,0);
 340
 341    // Create a package writer object.
 342    ContentsWriter Contents("", Arch);
 343    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 344       return _error->Error(_("Package extension list is too long"));
 345    if (_error->PendingError() == true)
 346       return false;
 347
 348    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 349                       CntCompress,Permissions);
 350    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 351    Contents.Output = Comp.Input;
 352    if (_error->PendingError() == true)
 353       return false;
 354
 355    // Write the header out.
 356    if (ContentsHead.empty() == false)
 357    {
 358       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 359       if (_error->PendingError() == true)
 360          return false;
 361
 362       unsigned long Size = Head.Size();
 363       unsigned char Buf[4096];
 364       while (Size != 0)
 365       {
 366          unsigned long ToRead = Size;
 367          if (Size > sizeof(Buf))
 368             ToRead = sizeof(Buf);
 369
 370          if (Head.Read(Buf,ToRead) == false)
 371             return false;
 372
 373          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 374             return _error->Errno("fwrite",_("Error writing header to contents file"));
 375
 376          Size -= ToRead;
 377       }
 378    }
 379
 380    /* Go over all the package file records and parse all the package
 381       files associated with this contents file into one great big honking
 382       memory structure, then dump the sorted version */
 383    c0out << ' ' << this->Contents << ":" << flush;
 384    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 385    {
 386       if (I->Contents != this->Contents)
 387          continue;
 388
 389       Contents.Prefix = ArchiveDir;
 390       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 391       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 392                             I->PkgCompress);
 393
 394       I->ContentsDone = true;
 395    }
 396
 397    Contents.Finish();
 398
 399    // Finish compressing
 400    unsigned long Size;
 401    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 402    {
 403       c0out << endl;
 404       return _error->Error(_("Error processing contents %s"),
 405                            this->Contents.c_str());
 406    }
 407
 408    if (Size != 0)
 409    {
 410       c0out << " New " << SizeToStr(Size) << "B ";
 411       if (Left > Size)
 412          Left -= Size;
 413       else
 414          Left = 0;
 415    }
 416    else
 417       c0out << ' ';
 418
 419    struct timeval NewTime;
 420    gettimeofday(&NewTime,0);
 421    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 422                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 423
 424    c0out << Contents.Stats.Packages << " files " <<
 425       SizeToStr(Contents.Stats.Bytes) << "B " <<
 426       TimeToStr((long)Delta) << endl;
 427
 428    return true;
 429 }
 430                                                                         /*}}}*/
 431
 432 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 433 // ---------------------------------------------------------------------
 434 /* This populates the PkgList with all the possible permutations of the
 435    section/arch lists. */
 436 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 437 {
 438    // Load the defaults
 439    string DDir = Setup.Find("TreeDefault::Directory",
 440                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 441    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 442                             "$(DIST)/$(SECTION)/source/");
 443    string DPkg = Setup.Find("TreeDefault::Packages",
 444                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 445    string DTrans = Setup.Find("TreeDefault::Translation",
 446                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 447    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 448                             "$(DIST)/$(SECTION)/");
 449    string DContents = Setup.Find("TreeDefault::Contents",
 450                             "$(DIST)/Contents-$(ARCH)");
 451    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 452    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 453                                "packages-$(ARCH).db");
 454    string DSources = Setup.Find("TreeDefault::Sources",
 455                                 "$(DIST)/$(SECTION)/source/Sources");
 456    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 457    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 458
 459    // Process 'tree' type sections
 460    const Configuration::Item *Top = Setup.Tree("tree");
 461    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 462    {
 463       Configuration Block(Top);
 464       string Dist = Top->Tag;
 465
 466       // Parse the sections
 467       string Tmp = Block.Find("Sections");
 468       const char *Sections = Tmp.c_str();
 469       string Section;
 470       while (ParseQuoteWord(Sections,Section) == true)
 471       {
 472          string Arch;
 473          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 474                                          {"$(SECTION)",&Section},
 475                                          {"$(ARCH)",&Arch},
 476                                          {}};
 477          TranslationWriter *TransWriter;
 478          if (DTrans.empty() == false)
 479          {
 480             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 481                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 482             TransWriter = new TranslationWriter(TranslationFile);
 483          }
 484          else
 485             TransWriter = NULL;
 486
 487          string const Tmp2 = Block.Find("Architectures");
 488          const char *Archs = Tmp2.c_str();
 489          while (ParseQuoteWord(Archs,Arch) == true)
 490          {
 491             PackageMap Itm;
 492
 493             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 494             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 495
 496             if (stringcasecmp(Arch,"source") == 0)
 497             {
 498                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 499                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 500                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 501                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 502                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 503                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 504             }
 505             else
 506             {
 507                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 508                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 509                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 510                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 511                Itm.Arch = Arch;
 512                if (TransWriter != NULL)
 513                {
 514                   TransWriter->IncreaseRefCounter();
 515                   Itm.TransWriter = TransWriter;
 516                }
 517                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 518                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 519                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 520                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 521             }
 522
 523             Itm.GetGeneral(Setup,Block);
 524             PkgList.push_back(Itm);
 525          }
 526          // we didn't use this TransWriter, so we can release it
 527          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 528             delete TransWriter;
 529       }
 530
 531       Top = Top->Next;
 532    }
 533 }
 534                                                                         /*}}}*/
 535 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 536 // ---------------------------------------------------------------------
 537 /* */
 538 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 539 {
 540    // Process 'bindirectory' type sections
 541    const Configuration::Item *Top = Setup.Tree("bindirectory");
 542    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 543    {
 544       Configuration Block(Top);
 545
 546       PackageMap Itm;
 547       Itm.PkgFile = Block.Find("Packages");
 548       Itm.SrcFile = Block.Find("Sources");
 549       Itm.BinCacheDB = Block.Find("BinCacheDB");
 550       Itm.BinOverride = Block.Find("BinOverride");
 551       Itm.ExtraOverride = Block.Find("ExtraOverride");
 552       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 553       Itm.SrcOverride = Block.Find("SrcOverride");
 554       Itm.BaseDir = Top->Tag;
 555       Itm.FLFile = Block.Find("FileList");
 556       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 557       Itm.Contents = Block.Find("Contents");
 558       Itm.ContentsHead = Block.Find("Contents::Header");
 559
 560       Itm.GetGeneral(Setup,Block);
 561       PkgList.push_back(Itm);
 562
 563       Top = Top->Next;
 564    }
 565 }
 566                                                                         /*}}}*/
 567
 568 // ShowHelp - Show the help text                                        /*{{{*/
 569 // ---------------------------------------------------------------------
 570 /* */
 571 bool ShowHelp(CommandLine &CmdL)
 572 {
 573    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,VERSION,
 574             COMMON_ARCH,__DATE__,__TIME__);
 575    if (_config->FindB("version") == true)
 576       return true;
 577
 578    cout <<
 579     _("Usage: apt-ftparchive [options] command\n"
 580       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 581       "          sources srcpath [overridefile [pathprefix]]\n"
 582       "          contents path\n"
 583       "          release path\n"
 584       "          generate config [groups]\n"
 585       "          clean config\n"
 586       "\n"
 587       "apt-ftparchive generates index files for Debian archives. It supports\n"
 588       "many styles of generation from fully automated to functional replacements\n"
 589       "for dpkg-scanpackages and dpkg-scansources\n"
 590       "\n"
 591       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 592       "Package file contains the contents of all the control fields from\n"
 593       "each package as well as the MD5 hash and filesize. An override file\n"
 594       "is supported to force the value of Priority and Section.\n"
 595       "\n"
 596       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 597       "The --source-override option can be used to specify a src override file\n"
 598       "\n"
 599       "The 'packages' and 'sources' command should be run in the root of the\n"
 600       "tree. BinaryPath should point to the base of the recursive search and \n"
 601       "override file should contain the override flags. Pathprefix is\n"
 602       "appended to the filename fields if present. Example usage from the \n"
 603       "Debian archive:\n"
 604       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 605       "               dists/potato/main/binary-i386/Packages\n"
 606       "\n"
 607       "Options:\n"
 608       "  -h    This help text\n"
 609       "  --md5 Control MD5 generation\n"
 610       "  -s=?  Source override file\n"
 611       "  -q    Quiet\n"
 612       "  -d=?  Select the optional caching database\n"
 613       "  --no-delink Enable delinking debug mode\n"
 614       "  --contents  Control contents file generation\n"
 615       "  -c=?  Read this configuration file\n"
 616       "  -o=?  Set an arbitrary configuration option") << endl;
 617
 618    return true;
 619 }
 620                                                                         /*}}}*/
 621 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 624 bool SimpleGenPackages(CommandLine &CmdL)
 625 {
 626    if (CmdL.FileSize() < 2)
 627       return ShowHelp(CmdL);
 628
 629    string Override;
 630    if (CmdL.FileSize() >= 3)
 631       Override = CmdL.FileList[2];
 632
 633    // Create a package writer object.
 634    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 635                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 636    if (_error->PendingError() == true)
 637       return false;
 638
 639    if (CmdL.FileSize() >= 4)
 640       Packages.PathPrefix = CmdL.FileList[3];
 641
 642    // Do recursive directory searching
 643    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 644       return false;
 645
 646    return true;
 647 }
 648                                                                         /*}}}*/
 649 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 650 // ---------------------------------------------------------------------
 651 /* */
 652 bool SimpleGenContents(CommandLine &CmdL)
 653 {
 654    if (CmdL.FileSize() < 2)
 655       return ShowHelp(CmdL);
 656
 657    // Create a package writer object.
 658    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 659    if (_error->PendingError() == true)
 660       return false;
 661
 662    // Do recursive directory searching
 663    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 664       return false;
 665
 666    Contents.Finish();
 667
 668    return true;
 669 }
 670                                                                         /*}}}*/
 671 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 672 // ---------------------------------------------------------------------
 673 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 674 bool SimpleGenSources(CommandLine &CmdL)
 675 {
 676    if (CmdL.FileSize() < 2)
 677       return ShowHelp(CmdL);
 678
 679    string Override;
 680    if (CmdL.FileSize() >= 3)
 681       Override = CmdL.FileList[2];
 682
 683    string SOverride;
 684    if (Override.empty() == false)
 685       SOverride = Override + ".src";
 686
 687    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 688                              SOverride.c_str());
 689
 690    // Create a package writer object.
 691    SourcesWriter Sources(Override,SOverride);
 692    if (_error->PendingError() == true)
 693       return false;
 694
 695    if (CmdL.FileSize() >= 4)
 696       Sources.PathPrefix = CmdL.FileList[3];
 697
 698    // Do recursive directory searching
 699    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 700       return false;
 701
 702    return true;
 703 }
 704                                                                         /*}}}*/
 705 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 706 // ---------------------------------------------------------------------
 707 bool SimpleGenRelease(CommandLine &CmdL)
 708 {
 709    if (CmdL.FileSize() < 2)
 710       return ShowHelp(CmdL);
 711
 712    string Dir = CmdL.FileList[1];
 713
 714    ReleaseWriter Release("");
 715    Release.DirStrip = Dir;
 716
 717    if (_error->PendingError() == true)
 718       return false;
 719
 720    if (Release.RecursiveScan(Dir) == false)
 721       return false;
 722
 723    Release.Finish();
 724
 725    return true;
 726 }
 727
 728                                                                         /*}}}*/
 729 // Generate - Full generate, using a config file                        /*{{{*/
 730 // ---------------------------------------------------------------------
 731 /* */
 732 bool Generate(CommandLine &CmdL)
 733 {
 734    struct CacheDB::Stats SrcStats;
 735    if (CmdL.FileSize() < 2)
 736       return ShowHelp(CmdL);
 737
 738    struct timeval StartTime;
 739    gettimeofday(&StartTime,0);
 740    struct CacheDB::Stats Stats;
 741
 742    // Read the configuration file.
 743    Configuration Setup;
 744    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 745       return false;
 746
 747    vector<PackageMap> PkgList;
 748    LoadTree(PkgList,Setup);
 749    LoadBinDir(PkgList,Setup);
 750
 751    // Sort by cache DB to improve IO locality.
 752    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 753
 754    // Generate packages
 755    if (CmdL.FileSize() <= 2)
 756    {
 757       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 758          if (I->GenPackages(Setup,Stats) == false)
 759             _error->DumpErrors();
 760       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 761          if (I->GenSources(Setup,SrcStats) == false)
 762             _error->DumpErrors();
 763    }
 764    else
 765    {
 766       // Make a choice list out of the package list..
 767       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 768       RxChoiceList *End = List;
 769       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 770       {
 771          End->UserData = &(*I);
 772          End->Str = I->BaseDir.c_str();
 773          End++;
 774
 775          End->UserData = &(*I);
 776          End->Str = I->Tag.c_str();
 777          End++;
 778       }
 779       End->Str = 0;
 780
 781       // Regex it
 782       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 783       {
 784          delete [] List;
 785          return _error->Error(_("No selections matched"));
 786       }
 787       _error->DumpErrors();
 788
 789       // Do the generation for Packages
 790       for (End = List; End->Str != 0; End++)
 791       {
 792          if (End->Hit == false)
 793             continue;
 794
 795          PackageMap *I = (PackageMap *)End->UserData;
 796          if (I->PkgDone == true)
 797             continue;
 798          if (I->GenPackages(Setup,Stats) == false)
 799             _error->DumpErrors();
 800       }
 801
 802       // Do the generation for Sources
 803       for (End = List; End->Str != 0; End++)
 804       {
 805          if (End->Hit == false)
 806             continue;
 807
 808          PackageMap *I = (PackageMap *)End->UserData;
 809          if (I->SrcDone == true)
 810             continue;
 811          if (I->GenSources(Setup,SrcStats) == false)
 812             _error->DumpErrors();
 813       }
 814
 815       delete [] List;
 816    }
 817
 818    // close the Translation master files
 819    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 820       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 821          delete I->TransWriter;
 822
 823    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 824       return true;
 825
 826    c1out << "Packages done, Starting contents." << endl;
 827
 828    // Sort the contents file list by date
 829    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 830    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 831    {
 832       struct stat A;
 833       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 834                                  I->CntCompress,A) == false)
 835          time(&I->ContentsMTime);
 836       else
 837          I->ContentsMTime = A.st_mtime;
 838    }
 839    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 840
 841    /* Now for Contents.. The process here is to do a make-like dependency
 842       check. Each contents file is verified to be newer than the package files
 843       that describe the debs it indexes. Since the package files contain
 844       hashes of the .debs this means they have not changed either so the
 845       contents must be up to date. */
 846    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 847    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 848    {
 849       // This record is not relevent
 850       if (I->ContentsDone == true ||
 851           I->Contents.empty() == true)
 852          continue;
 853
 854       // Do not do everything if the user specified sections.
 855       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 856          continue;
 857
 858       struct stat A,B;
 859       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 860       {
 861          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 862          {
 863             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 864             continue;
 865          }
 866
 867          if (A.st_mtime > B.st_mtime)
 868             continue;
 869       }
 870
 871       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 872                          MaxContentsChange) == false)
 873          _error->DumpErrors();
 874
 875       // Hit the limit?
 876       if (MaxContentsChange == 0)
 877       {
 878          c1out << "Hit contents update byte limit" << endl;
 879          break;
 880       }
 881    }
 882
 883    struct timeval NewTime;
 884    gettimeofday(&NewTime,0);
 885    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 886                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 887    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 888          << " archives. Took " << TimeToStr((long)Delta) << endl;
 889
 890    return true;
 891 }
 892                                                                         /*}}}*/
 893 // Clean - Clean out the databases                                      /*{{{*/
 894 // ---------------------------------------------------------------------
 895 /* */
 896 bool Clean(CommandLine &CmdL)
 897 {
 898    if (CmdL.FileSize() != 2)
 899       return ShowHelp(CmdL);
 900
 901    // Read the configuration file.
 902    Configuration Setup;
 903    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 904       return false;
 905
 906    vector<PackageMap> PkgList;
 907    LoadTree(PkgList,Setup);
 908    LoadBinDir(PkgList,Setup);
 909
 910    // Sort by cache DB to improve IO locality.
 911    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 912
 913    string CacheDir = Setup.FindDir("Dir::CacheDir");
 914
 915    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 916    {
 917       c0out << I->BinCacheDB << endl;
 918       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 919       if (DB.Clean() == false)
 920          _error->DumpErrors();
 921
 922       string CacheDB = I->BinCacheDB;
 923       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 924    }
 925
 926    return true;
 927 }
 928                                                                         /*}}}*/
 929
 930 int main(int argc, const char *argv[])
 931 {
 932    setlocale(LC_ALL, "");
 933    CommandLine::Args Args[] = {
 934       {'h',"help","help",0},
 935       {0,"md5","APT::FTPArchive::MD5",0},
 936       {'v',"version","version",0},
 937       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 938       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 939       {'q',"quiet","quiet",CommandLine::IntLevel},
 940       {'q',"silent","quiet",CommandLine::IntLevel},
 941       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 942       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 943       {0,"contents","APT::FTPArchive::Contents",0},
 944       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 945       {'c',"config-file",0,CommandLine::ConfigFile},
 946       {'o',"option",0,CommandLine::ArbItem},
 947       {0,0,0,0}};
 948    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 949                                    {"contents",&SimpleGenContents},
 950                                    {"sources",&SimpleGenSources},
 951                                    {"release",&SimpleGenRelease},
 952                                    {"generate",&Generate},
 953                                    {"clean",&Clean},
 954                                    {"help",&ShowHelp},
 955                                    {0,0}};
 956
 957    // Parse the command line and initialize the package library
 958    CommandLine CmdL(Args,_config);
 959    if (CmdL.Parse(argc,argv) == false)
 960    {
 961       _error->DumpErrors();
 962       return 100;
 963    }
 964
 965    // See if the help should be shown
 966    if (_config->FindB("help") == true ||
 967        _config->FindB("version") == true ||
 968        CmdL.FileSize() == 0)
 969    {
 970       ShowHelp(CmdL);
 971       return 0;
 972    }
 973
 974    // Setup the output streams
 975    c0out.rdbuf(clog.rdbuf());
 976    c1out.rdbuf(clog.rdbuf());
 977    c2out.rdbuf(clog.rdbuf());
 978    Quiet = _config->FindI("quiet",0);
 979    if (Quiet > 0)
 980       c0out.rdbuf(devnull.rdbuf());
 981    if (Quiet > 1)
 982       c1out.rdbuf(devnull.rdbuf());
 983
 984    // Match the operation
 985    CmdL.DispatchArg(Cmds);
 986
 987    if (_error->empty() == false)
 988    {
 989       bool Errors = _error->PendingError();
 990       _error->DumpErrors();
 991       return Errors == true?100:0;
 992    }
 993    return 0;
 994 }