ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-ftparchive - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <apt-pkg/init.h>
  20 #include <algorithm>
  21
  22 #include <climits>
  23 #include <sys/time.h>
  24 #include <regex.h>
  25
  26 #include "apt-ftparchive.h"
  27 #include "contents.h"
  28 #include "multicompress.h"
  29 #include "writer.h"
  30
  31 #include <apti18n.h>
  32                                                                         /*}}}*/
  33
  34 using namespace std;
  35 ostream c0out(0);
  36 ostream c1out(0);
  37 ostream c2out(0);
  38 ofstream devnull("/dev/null");
  39 unsigned Quiet = 0;
  40
  41 // struct PackageMap - List of all package files in the config file     /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 struct PackageMap
  45 {
  46    // General Stuff
  47    string BaseDir;
  48    string InternalPrefix;
  49    string FLFile;
  50    string PkgExt;
  51    string SrcExt;
  52
  53    // Stuff for the Package File
  54    string PkgFile;
  55    string BinCacheDB;
  56    string BinOverride;
  57    string ExtraOverride;
  58
  59    // We generate for this given arch
  60    string Arch;
  61
  62    // Stuff for the Source File
  63    string SrcFile;
  64    string SrcOverride;
  65    string SrcExtraOverride;
  66
  67    // Translation master file
  68    bool LongDesc;
  69    TranslationWriter *TransWriter;
  70
  71    // Contents
  72    string Contents;
  73    string ContentsHead;
  74
  75    // Random things
  76    string Tag;
  77    string PkgCompress;
  78    string CntCompress;
  79    string SrcCompress;
  80    string PathPrefix;
  81    unsigned int DeLinkLimit;
  82    mode_t Permissions;
  83
  84    bool ContentsDone;
  85    bool PkgDone;
  86    bool SrcDone;
  87    time_t ContentsMTime;
  88
  89    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.ContentsMTime < y.ContentsMTime;};
  93    };
  94
  95    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  96    {
  97       inline bool operator() (const PackageMap &x,const PackageMap &y)
  98       {return x.BinCacheDB < y.BinCacheDB;};
  99    };
 100
 101    void GetGeneral(Configuration &Setup,Configuration &Block);
 102    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
 103    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
 104    bool GenContents(Configuration &Setup,
 105                     vector<PackageMap>::iterator Begin,
 106                     vector<PackageMap>::iterator End,
 107                     unsigned long &Left);
 108
 109    PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
 110                   ContentsDone(false), PkgDone(false), SrcDone(false),
 111                   ContentsMTime(0) {};
 112 };
 113                                                                         /*}}}*/
 114
 115 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* */
 118 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 119 {
 120    PathPrefix = Block.Find("PathPrefix");
 121
 122    if (Block.FindB("External-Links",true) == false)
 123       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 124    else
 125       DeLinkLimit = 0;
 126
 127    PkgCompress = Block.Find("Packages::Compress",
 128                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 129    CntCompress = Block.Find("Contents::Compress",
 130                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 131    SrcCompress = Block.Find("Sources::Compress",
 132                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 133
 134    SrcExt = Block.Find("Sources::Extensions",
 135                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 136    PkgExt = Block.Find("Packages::Extensions",
 137                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 138
 139    Permissions = Setup.FindI("Default::FileMode",0644);
 140
 141    if (FLFile.empty() == false)
 142       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 143
 144    if (Contents == " ")
 145       Contents= string();
 146 }
 147                                                                         /*}}}*/
 148 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 149 // ---------------------------------------------------------------------
 150 /* This generates the Package File described by this object. */
 151 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 152 {
 153    if (PkgFile.empty() == true)
 154       return true;
 155
 156    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 157    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 158    string CacheDir = Setup.FindDir("Dir::CacheDir");
 159
 160    struct timeval StartTime;
 161    gettimeofday(&StartTime,0);
 162
 163    PkgDone = true;
 164
 165    // Create a package writer object.
 166    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 167                            flCombine(OverrideDir,BinOverride),
 168                            flCombine(OverrideDir,ExtraOverride),
 169                            Arch);
 170    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 171       return _error->Error(_("Package extension list is too long"));
 172    if (_error->PendingError() == true)
 173       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 174
 175    Packages.PathPrefix = PathPrefix;
 176    Packages.DirStrip = ArchiveDir;
 177    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 178
 179    Packages.TransWriter = TransWriter;
 180    Packages.LongDescription = LongDesc;
 181
 182    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 183    Packages.DeLinkLimit = DeLinkLimit;
 184
 185    // Create a compressor object
 186    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 187                       PkgCompress,Permissions);
 188    Packages.Output = Comp.Input;
 189    if (_error->PendingError() == true)
 190       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 191
 192    c0out << ' ' << BaseDir << ":" << flush;
 193
 194    // Do recursive directory searching
 195    if (FLFile.empty() == true)
 196    {
 197       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 198          return false;
 199    }
 200    else
 201    {
 202       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 203          return false;
 204    }
 205
 206    Packages.Output = 0;      // Just in case
 207
 208    // Finish compressing
 209    unsigned long long Size;
 210    if (Comp.Finalize(Size) == false)
 211    {
 212       c0out << endl;
 213       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 214    }
 215
 216    if (Size != 0)
 217       c0out << " New "
 218              << SizeToStr(Size) << "B ";
 219    else
 220       c0out << ' ';
 221
 222    struct timeval NewTime;
 223    gettimeofday(&NewTime,0);
 224    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 225                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 226
 227    c0out << Packages.Stats.Packages << " files " <<
 228 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 229       SizeToStr(Packages.Stats.Bytes) << "B " <<
 230       TimeToStr((long)Delta) << endl;
 231
 232    Stats.Add(Packages.Stats);
 233    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 234
 235    return !_error->PendingError();
 236 }
 237
 238                                                                         /*}}}*/
 239 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This generates the Sources File described by this object. */
 242 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 243 {
 244    if (SrcFile.empty() == true)
 245       return true;
 246
 247    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 248    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 249    string CacheDir = Setup.FindDir("Dir::CacheDir");
 250
 251    struct timeval StartTime;
 252    gettimeofday(&StartTime,0);
 253
 254    SrcDone = true;
 255
 256    // Create a package writer object.
 257    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),
 258                          flCombine(OverrideDir,BinOverride),
 259                          flCombine(OverrideDir,SrcOverride),
 260                          flCombine(OverrideDir,SrcExtraOverride));
 261    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 262       return _error->Error(_("Source extension list is too long"));
 263    if (_error->PendingError() == true)
 264       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 265
 266    Sources.PathPrefix = PathPrefix;
 267    Sources.DirStrip = ArchiveDir;
 268    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 269
 270    Sources.DeLinkLimit = DeLinkLimit;
 271    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 272
 273    // Create a compressor object
 274    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 275                       SrcCompress,Permissions);
 276    Sources.Output = Comp.Input;
 277    if (_error->PendingError() == true)
 278       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 279
 280    c0out << ' ' << BaseDir << ":" << flush;
 281
 282    // Do recursive directory searching
 283    if (FLFile.empty() == true)
 284    {
 285       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 286          return false;
 287    }
 288    else
 289    {
 290       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 291          return false;
 292    }
 293    Sources.Output = 0;      // Just in case
 294
 295    // Finish compressing
 296    unsigned long long Size;
 297    if (Comp.Finalize(Size) == false)
 298    {
 299       c0out << endl;
 300       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 301    }
 302
 303    if (Size != 0)
 304       c0out << " New "
 305              << SizeToStr(Size) << "B ";
 306    else
 307       c0out << ' ';
 308
 309    struct timeval NewTime;
 310    gettimeofday(&NewTime,0);
 311    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 312                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 313
 314    c0out << Sources.Stats.Packages << " pkgs in " <<
 315       TimeToStr((long)Delta) << endl;
 316
 317    Stats.Add(Sources.Stats);
 318    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 319
 320    return !_error->PendingError();
 321 }
 322                                                                         /*}}}*/
 323 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 324 // ---------------------------------------------------------------------
 325 /* This generates the contents file partially described by this object.
 326    It searches the given iterator range for other package files that map
 327    into this contents file and includes their data as well when building. */
 328 bool PackageMap::GenContents(Configuration &Setup,
 329                              vector<PackageMap>::iterator Begin,
 330                              vector<PackageMap>::iterator End,
 331                              unsigned long &Left)
 332 {
 333    if (Contents.empty() == true)
 334       return true;
 335
 336    if (Left == 0)
 337       return true;
 338
 339    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 340    string CacheDir = Setup.FindDir("Dir::CacheDir");
 341    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 342
 343    struct timeval StartTime;
 344    gettimeofday(&StartTime,0);
 345
 346    // Create a package writer object.
 347    ContentsWriter Contents("", Arch);
 348    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 349       return _error->Error(_("Package extension list is too long"));
 350    if (_error->PendingError() == true)
 351       return false;
 352
 353    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 354                       CntCompress,Permissions);
 355    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 356    Contents.Output = Comp.Input;
 357    if (_error->PendingError() == true)
 358       return false;
 359
 360    // Write the header out.
 361    if (ContentsHead.empty() == false)
 362    {
 363       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 364       if (_error->PendingError() == true)
 365          return false;
 366
 367       unsigned long long Size = Head.Size();
 368       unsigned char Buf[4096];
 369       while (Size != 0)
 370       {
 371          unsigned long long ToRead = Size;
 372          if (Size > sizeof(Buf))
 373             ToRead = sizeof(Buf);
 374
 375          if (Head.Read(Buf,ToRead) == false)
 376             return false;
 377
 378          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 379             return _error->Errno("fwrite",_("Error writing header to contents file"));
 380
 381          Size -= ToRead;
 382       }
 383    }
 384
 385    /* Go over all the package file records and parse all the package
 386       files associated with this contents file into one great big honking
 387       memory structure, then dump the sorted version */
 388    c0out << ' ' << this->Contents << ":" << flush;
 389    for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
 390    {
 391       if (I->Contents != this->Contents)
 392          continue;
 393
 394       Contents.Prefix = ArchiveDir;
 395       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 396       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 397                             I->PkgCompress);
 398
 399       I->ContentsDone = true;
 400    }
 401
 402    Contents.Finish();
 403
 404    // Finish compressing
 405    unsigned long long Size;
 406    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 407    {
 408       c0out << endl;
 409       return _error->Error(_("Error processing contents %s"),
 410                            this->Contents.c_str());
 411    }
 412
 413    if (Size != 0)
 414    {
 415       c0out << " New " << SizeToStr(Size) << "B ";
 416       if (Left > Size)
 417          Left -= Size;
 418       else
 419          Left = 0;
 420    }
 421    else
 422       c0out << ' ';
 423
 424    struct timeval NewTime;
 425    gettimeofday(&NewTime,0);
 426    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 427                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 428
 429    c0out << Contents.Stats.Packages << " files " <<
 430       SizeToStr(Contents.Stats.Bytes) << "B " <<
 431       TimeToStr((long)Delta) << endl;
 432
 433    return true;
 434 }
 435                                                                         /*}}}*/
 436
 437 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 438 // ---------------------------------------------------------------------
 439 /* This populates the PkgList with all the possible permutations of the
 440    section/arch lists. */
 441 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 442 {
 443    // Load the defaults
 444    string DDir = Setup.Find("TreeDefault::Directory",
 445                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 446    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 447                             "$(DIST)/$(SECTION)/source/");
 448    string DPkg = Setup.Find("TreeDefault::Packages",
 449                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 450    string DTrans = Setup.Find("TreeDefault::Translation",
 451                             "$(DIST)/$(SECTION)/i18n/Translation-en");
 452    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 453                             "$(DIST)/$(SECTION)/");
 454    string DContents = Setup.Find("TreeDefault::Contents",
 455                             "$(DIST)/$(SECTION)/Contents-$(ARCH)");
 456    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 457    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 458                                "packages-$(ARCH).db");
 459    string DSources = Setup.Find("TreeDefault::Sources",
 460                                 "$(DIST)/$(SECTION)/source/Sources");
 461    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 462    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 463
 464    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 465
 466    bool const LongDescription = Setup.FindB("Default::LongDescription",
 467                                         _config->FindB("APT::FTPArchive::LongDescription", true));
 468    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
 469
 470    // Process 'tree' type sections
 471    const Configuration::Item *Top = Setup.Tree("tree");
 472    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 473    {
 474       Configuration Block(Top);
 475       string Dist = Top->Tag;
 476
 477       // Parse the sections
 478       string Tmp = Block.Find("Sections");
 479       const char *Sections = Tmp.c_str();
 480       string Section;
 481       while (ParseQuoteWord(Sections,Section) == true)
 482       {
 483          string Arch;
 484          struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
 485                                          {"$(SECTION)",&Section},
 486                                          {"$(ARCH)",&Arch},
 487                                          {}};
 488          mode_t const Perms = Block.FindI("FileMode", Permissions);
 489          bool const LongDesc = Block.FindB("LongDescription", LongDescription);
 490          TranslationWriter *TransWriter;
 491          if (DTrans.empty() == false && LongDesc == false)
 492          {
 493             string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
 494                         SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
 495             string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
 496             TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
 497          }
 498          else
 499             TransWriter = NULL;
 500
 501          string const Tmp2 = Block.Find("Architectures");
 502          const char *Archs = Tmp2.c_str();
 503          while (ParseQuoteWord(Archs,Arch) == true)
 504          {
 505             PackageMap Itm;
 506             Itm.Permissions = Perms;
 507             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 508             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 509
 510             if (stringcasecmp(Arch,"source") == 0)
 511             {
 512                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 513                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 514                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 515                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 516                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 517                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 518             }
 519             else
 520             {
 521                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 522                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 523                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 524                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 525                Itm.Arch = Arch;
 526                Itm.LongDesc = LongDesc;
 527                if (TransWriter != NULL)
 528                {
 529                   TransWriter->IncreaseRefCounter();
 530                   Itm.TransWriter = TransWriter;
 531                }
 532                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 533                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 534                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 535                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 536             }
 537
 538             Itm.GetGeneral(Setup,Block);
 539             PkgList.push_back(Itm);
 540          }
 541          // we didn't use this TransWriter, so we can release it
 542          if (TransWriter != NULL && TransWriter->GetRefCounter() == 0)
 543             delete TransWriter;
 544       }
 545
 546       Top = Top->Next;
 547    }
 548 }
 549                                                                         /*}}}*/
 550 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* */
 553 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 554 {
 555    mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
 556
 557    // Process 'bindirectory' type sections
 558    const Configuration::Item *Top = Setup.Tree("bindirectory");
 559    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 560    {
 561       Configuration Block(Top);
 562
 563       PackageMap Itm;
 564       Itm.PkgFile = Block.Find("Packages");
 565       Itm.SrcFile = Block.Find("Sources");
 566       Itm.BinCacheDB = Block.Find("BinCacheDB");
 567       Itm.BinOverride = Block.Find("BinOverride");
 568       Itm.ExtraOverride = Block.Find("ExtraOverride");
 569       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 570       Itm.SrcOverride = Block.Find("SrcOverride");
 571       Itm.BaseDir = Top->Tag;
 572       Itm.FLFile = Block.Find("FileList");
 573       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 574       Itm.Contents = Block.Find("Contents");
 575       Itm.ContentsHead = Block.Find("Contents::Header");
 576       Itm.Permissions = Block.FindI("FileMode", Permissions);
 577
 578       Itm.GetGeneral(Setup,Block);
 579       PkgList.push_back(Itm);
 580
 581       Top = Top->Next;
 582    }
 583 }
 584                                                                         /*}}}*/
 585
 586 // ShowHelp - Show the help text                                        /*{{{*/
 587 // ---------------------------------------------------------------------
 588 /* */
 589 bool ShowHelp(CommandLine &CmdL)
 590 {
 591    ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
 592             COMMON_ARCH,__DATE__,__TIME__);
 593    if (_config->FindB("version") == true)
 594       return true;
 595
 596    cout <<
 597     _("Usage: apt-ftparchive [options] command\n"
 598       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 599       "          sources srcpath [overridefile [pathprefix]]\n"
 600       "          contents path\n"
 601       "          release path\n"
 602       "          generate config [groups]\n"
 603       "          clean config\n"
 604       "\n"
 605       "apt-ftparchive generates index files for Debian archives. It supports\n"
 606       "many styles of generation from fully automated to functional replacements\n"
 607       "for dpkg-scanpackages and dpkg-scansources\n"
 608       "\n"
 609       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 610       "Package file contains the contents of all the control fields from\n"
 611       "each package as well as the MD5 hash and filesize. An override file\n"
 612       "is supported to force the value of Priority and Section.\n"
 613       "\n"
 614       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 615       "The --source-override option can be used to specify a src override file\n"
 616       "\n"
 617       "The 'packages' and 'sources' command should be run in the root of the\n"
 618       "tree. BinaryPath should point to the base of the recursive search and \n"
 619       "override file should contain the override flags. Pathprefix is\n"
 620       "appended to the filename fields if present. Example usage from the \n"
 621       "Debian archive:\n"
 622       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 623       "               dists/potato/main/binary-i386/Packages\n"
 624       "\n"
 625       "Options:\n"
 626       "  -h    This help text\n"
 627       "  --md5 Control MD5 generation\n"
 628       "  -s=?  Source override file\n"
 629       "  -q    Quiet\n"
 630       "  -d=?  Select the optional caching database\n"
 631       "  --no-delink Enable delinking debug mode\n"
 632       "  --contents  Control contents file generation\n"
 633       "  -c=?  Read this configuration file\n"
 634       "  -o=?  Set an arbitrary configuration option") << endl;
 635
 636    return true;
 637 }
 638                                                                         /*}}}*/
 639 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 642 bool SimpleGenPackages(CommandLine &CmdL)
 643 {
 644    if (CmdL.FileSize() < 2)
 645       return ShowHelp(CmdL);
 646
 647    string Override;
 648    if (CmdL.FileSize() >= 3)
 649       Override = CmdL.FileList[2];
 650
 651    // Create a package writer object.
 652    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 653                            Override, "", _config->Find("APT::FTPArchive::Architecture"));
 654    if (_error->PendingError() == true)
 655       return false;
 656
 657    if (CmdL.FileSize() >= 4)
 658       Packages.PathPrefix = CmdL.FileList[3];
 659
 660    // Do recursive directory searching
 661    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 662       return false;
 663
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 668 // ---------------------------------------------------------------------
 669 /* */
 670 bool SimpleGenContents(CommandLine &CmdL)
 671 {
 672    if (CmdL.FileSize() < 2)
 673       return ShowHelp(CmdL);
 674
 675    // Create a package writer object.
 676    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
 677    if (_error->PendingError() == true)
 678       return false;
 679
 680    // Do recursive directory searching
 681    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 682       return false;
 683
 684    Contents.Finish();
 685
 686    return true;
 687 }
 688                                                                         /*}}}*/
 689 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 690 // ---------------------------------------------------------------------
 691 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 692 bool SimpleGenSources(CommandLine &CmdL)
 693 {
 694    if (CmdL.FileSize() < 2)
 695       return ShowHelp(CmdL);
 696
 697    string Override;
 698    if (CmdL.FileSize() >= 3)
 699       Override = CmdL.FileList[2];
 700
 701    string SOverride;
 702    if (Override.empty() == false)
 703       SOverride = Override + ".src";
 704
 705    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 706                              SOverride.c_str());
 707
 708    // Create a package writer object.
 709    SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
 710    if (_error->PendingError() == true)
 711       return false;
 712
 713    if (CmdL.FileSize() >= 4)
 714       Sources.PathPrefix = CmdL.FileList[3];
 715
 716    // Do recursive directory searching
 717    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 718       return false;
 719
 720    return true;
 721 }
 722                                                                         /*}}}*/
 723 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 724 // ---------------------------------------------------------------------
 725 bool SimpleGenRelease(CommandLine &CmdL)
 726 {
 727    if (CmdL.FileSize() < 2)
 728       return ShowHelp(CmdL);
 729
 730    string Dir = CmdL.FileList[1];
 731
 732    ReleaseWriter Release("");
 733    Release.DirStrip = Dir;
 734
 735    if (_error->PendingError() == true)
 736       return false;
 737
 738    if (Release.RecursiveScan(Dir) == false)
 739       return false;
 740
 741    Release.Finish();
 742
 743    return true;
 744 }
 745
 746                                                                         /*}}}*/
 747 // Generate - Full generate, using a config file                        /*{{{*/
 748 // ---------------------------------------------------------------------
 749 /* */
 750 bool Generate(CommandLine &CmdL)
 751 {
 752    struct CacheDB::Stats SrcStats;
 753    if (CmdL.FileSize() < 2)
 754       return ShowHelp(CmdL);
 755
 756    struct timeval StartTime;
 757    gettimeofday(&StartTime,0);
 758    struct CacheDB::Stats Stats;
 759
 760    // Read the configuration file.
 761    Configuration Setup;
 762    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 763       return false;
 764
 765    vector<PackageMap> PkgList;
 766    LoadTree(PkgList,Setup);
 767    LoadBinDir(PkgList,Setup);
 768
 769    // Sort by cache DB to improve IO locality.
 770    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 771
 772    // Generate packages
 773    if (CmdL.FileSize() <= 2)
 774    {
 775       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 776          if (I->GenPackages(Setup,Stats) == false)
 777             _error->DumpErrors();
 778       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 779          if (I->GenSources(Setup,SrcStats) == false)
 780             _error->DumpErrors();
 781    }
 782    else
 783    {
 784       // Make a choice list out of the package list..
 785       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 786       RxChoiceList *End = List;
 787       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 788       {
 789          End->UserData = &(*I);
 790          End->Str = I->BaseDir.c_str();
 791          End++;
 792
 793          End->UserData = &(*I);
 794          End->Str = I->Tag.c_str();
 795          End++;
 796       }
 797       End->Str = 0;
 798
 799       // Regex it
 800       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 801       {
 802          delete [] List;
 803          return _error->Error(_("No selections matched"));
 804       }
 805       _error->DumpErrors();
 806
 807       // Do the generation for Packages
 808       for (End = List; End->Str != 0; End++)
 809       {
 810          if (End->Hit == false)
 811             continue;
 812
 813          PackageMap *I = (PackageMap *)End->UserData;
 814          if (I->PkgDone == true)
 815             continue;
 816          if (I->GenPackages(Setup,Stats) == false)
 817             _error->DumpErrors();
 818       }
 819
 820       // Do the generation for Sources
 821       for (End = List; End->Str != 0; End++)
 822       {
 823          if (End->Hit == false)
 824             continue;
 825
 826          PackageMap *I = (PackageMap *)End->UserData;
 827          if (I->SrcDone == true)
 828             continue;
 829          if (I->GenSources(Setup,SrcStats) == false)
 830             _error->DumpErrors();
 831       }
 832
 833       delete [] List;
 834    }
 835
 836    // close the Translation master files
 837    for (vector<PackageMap>::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I)
 838       if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0)
 839          delete I->TransWriter;
 840
 841    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 842       return true;
 843
 844    c1out << "Packages done, Starting contents." << endl;
 845
 846    // Sort the contents file list by date
 847    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 848    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 849    {
 850       struct stat A;
 851       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 852                                  I->CntCompress,A) == false)
 853          time(&I->ContentsMTime);
 854       else
 855          I->ContentsMTime = A.st_mtime;
 856    }
 857    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 858
 859    /* Now for Contents.. The process here is to do a make-like dependency
 860       check. Each contents file is verified to be newer than the package files
 861       that describe the debs it indexes. Since the package files contain
 862       hashes of the .debs this means they have not changed either so the
 863       contents must be up to date. */
 864    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 865    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
 866    {
 867       // This record is not relevant
 868       if (I->ContentsDone == true ||
 869           I->Contents.empty() == true)
 870          continue;
 871
 872       // Do not do everything if the user specified sections.
 873       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 874          continue;
 875
 876       struct stat A,B;
 877       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 878       {
 879          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 880          {
 881             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 882             continue;
 883          }
 884
 885          if (A.st_mtime > B.st_mtime)
 886             continue;
 887       }
 888
 889       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 890                          MaxContentsChange) == false)
 891          _error->DumpErrors();
 892
 893       // Hit the limit?
 894       if (MaxContentsChange == 0)
 895       {
 896          c1out << "Hit contents update byte limit" << endl;
 897          break;
 898       }
 899    }
 900
 901    struct timeval NewTime;
 902    gettimeofday(&NewTime,0);
 903    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 904                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 905    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 906          << " archives. Took " << TimeToStr((long)Delta) << endl;
 907
 908    return true;
 909 }
 910                                                                         /*}}}*/
 911 // Clean - Clean out the databases                                      /*{{{*/
 912 // ---------------------------------------------------------------------
 913 /* */
 914 bool Clean(CommandLine &CmdL)
 915 {
 916    if (CmdL.FileSize() != 2)
 917       return ShowHelp(CmdL);
 918
 919    // Read the configuration file.
 920    Configuration Setup;
 921    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 922       return false;
 923
 924    vector<PackageMap> PkgList;
 925    LoadTree(PkgList,Setup);
 926    LoadBinDir(PkgList,Setup);
 927
 928    // Sort by cache DB to improve IO locality.
 929    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 930
 931    string CacheDir = Setup.FindDir("Dir::CacheDir");
 932
 933    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 934    {
 935       c0out << I->BinCacheDB << endl;
 936       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 937       if (DB.Clean() == false)
 938          _error->DumpErrors();
 939
 940       string CacheDB = I->BinCacheDB;
 941       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; ++I);
 942    }
 943
 944    return true;
 945 }
 946                                                                         /*}}}*/
 947
 948 int main(int argc, const char *argv[])
 949 {
 950    setlocale(LC_ALL, "");
 951    CommandLine::Args Args[] = {
 952       {'h',"help","help",0},
 953       {0,"md5","APT::FTPArchive::MD5",0},
 954       {0,"sha1","APT::FTPArchive::SHA1",0},
 955       {0,"sha256","APT::FTPArchive::SHA256",0},
 956       {'v',"version","version",0},
 957       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 958       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 959       {'q',"quiet","quiet",CommandLine::IntLevel},
 960       {'q',"silent","quiet",CommandLine::IntLevel},
 961       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 962       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 963       {0,"contents","APT::FTPArchive::Contents",0},
 964       {'a',"arch","APT::FTPArchive::Architecture",CommandLine::HasArg},
 965       {'c',"config-file",0,CommandLine::ConfigFile},
 966       {'o',"option",0,CommandLine::ArbItem},
 967       {0,0,0,0}};
 968    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 969                                    {"contents",&SimpleGenContents},
 970                                    {"sources",&SimpleGenSources},
 971                                    {"release",&SimpleGenRelease},
 972                                    {"generate",&Generate},
 973                                    {"clean",&Clean},
 974                                    {"help",&ShowHelp},
 975                                    {0,0}};
 976
 977    // Parse the command line and initialize the package library
 978    CommandLine CmdL(Args,_config);
 979    if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false)
 980    {
 981       _error->DumpErrors();
 982       return 100;
 983    }
 984
 985    // See if the help should be shown
 986    if (_config->FindB("help") == true ||
 987        _config->FindB("version") == true ||
 988        CmdL.FileSize() == 0)
 989    {
 990       ShowHelp(CmdL);
 991       return 0;
 992    }
 993
 994    // Setup the output streams
 995    c0out.rdbuf(clog.rdbuf());
 996    c1out.rdbuf(clog.rdbuf());
 997    c2out.rdbuf(clog.rdbuf());
 998    Quiet = _config->FindI("quiet",0);
 999    if (Quiet > 0)
1000       c0out.rdbuf(devnull.rdbuf());
1001    if (Quiet > 1)
1002       c1out.rdbuf(devnull.rdbuf());
1003
1004    // Match the operation
1005    CmdL.DispatchArg(Cmds);
1006
1007    if (_error->empty() == false)
1008    {
1009       bool Errors = _error->PendingError();
1010       _error->DumpErrors();
1011       return Errors == true?100:0;
1012    }
1013    return 0;
1014 }