ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.9 2003/12/26 20:08:56 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #ifdef __GNUG__
  14 #pragma implementation "apt-ftparchive.h"
  15 #endif
  16
  17 #include "apt-ftparchive.h"
  18
  19 #include <apt-pkg/error.h>
  20 #include <apt-pkg/configuration.h>
  21 #include <apt-pkg/cmndline.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <config.h>
  24 #include <apti18n.h>
  25 #include <algorithm>
  26
  27 #include <sys/time.h>
  28 #include <regex.h>
  29
  30 #include "contents.h"
  31 #include "multicompress.h"
  32 #include "writer.h"
  33                                                                         /*}}}*/
  34
  35 using namespace std;
  36 ostream c0out(0);
  37 ostream c1out(0);
  38 ostream c2out(0);
  39 ofstream devnull("/dev/null");
  40 unsigned Quiet = 0;
  41
  42 // struct PackageMap - List of all package files in the config file     /*{{{*/
  43 // ---------------------------------------------------------------------
  44 /* */
  45 struct PackageMap
  46 {
  47    // General Stuff
  48    string BaseDir;
  49    string InternalPrefix;
  50    string FLFile;
  51    string PkgExt;
  52    string SrcExt;
  53
  54    // Stuff for the Package File
  55    string PkgFile;
  56    string BinCacheDB;
  57    string BinOverride;
  58    string ExtraOverride;
  59
  60    // Stuff for the Source File
  61    string SrcFile;
  62    string SrcOverride;
  63    string SrcExtraOverride;
  64
  65    // Contents
  66    string Contents;
  67    string ContentsHead;
  68
  69    // Random things
  70    string Tag;
  71    string PkgCompress;
  72    string CntCompress;
  73    string SrcCompress;
  74    string PathPrefix;
  75    unsigned int DeLinkLimit;
  76    mode_t Permissions;
  77
  78    bool ContentsDone;
  79    bool PkgDone;
  80    bool SrcDone;
  81    time_t ContentsMTime;
  82
  83    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  84    {
  85       inline bool operator() (const PackageMap &x,const PackageMap &y)
  86       {return x.ContentsMTime < y.ContentsMTime;};
  87    };
  88
  89    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  90    {
  91       inline bool operator() (const PackageMap &x,const PackageMap &y)
  92       {return x.BinCacheDB < y.BinCacheDB;};
  93    };
  94
  95    void GetGeneral(Configuration &Setup,Configuration &Block);
  96    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  97    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  98    bool GenContents(Configuration &Setup,
  99                     vector<PackageMap>::iterator Begin,
 100                     vector<PackageMap>::iterator End,
 101                     unsigned long &Left);
 102
 103    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 104         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 105 };
 106                                                                         /*}}}*/
 107
 108 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* */
 111 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 112 {
 113    PathPrefix = Block.Find("PathPrefix");
 114
 115    if (Block.FindB("External-Links",true) == false)
 116       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 117    else
 118       DeLinkLimit = 0;
 119
 120    PkgCompress = Block.Find("Packages::Compress",
 121                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 122    CntCompress = Block.Find("Contents::Compress",
 123                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 124    SrcCompress = Block.Find("Sources::Compress",
 125                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 126
 127    SrcExt = Block.Find("Sources::Extensions",
 128                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 129    PkgExt = Block.Find("Packages::Extensions",
 130                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 131
 132    Permissions = Setup.FindI("Default::FileMode",0644);
 133
 134    if (FLFile.empty() == false)
 135       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 136
 137    if (Contents == " ")
 138       Contents= string();
 139 }
 140                                                                         /*}}}*/
 141 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 142 // ---------------------------------------------------------------------
 143 /* This generates the Package File described by this object. */
 144 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 145 {
 146    if (PkgFile.empty() == true)
 147       return true;
 148
 149    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 150    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 151    string CacheDir = Setup.FindDir("Dir::CacheDir");
 152
 153    struct timeval StartTime;
 154    gettimeofday(&StartTime,0);
 155
 156    PkgDone = true;
 157
 158    // Create a package writer object.
 159    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 160                            flCombine(OverrideDir,BinOverride),
 161                            flCombine(OverrideDir,ExtraOverride));
 162    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 163       return _error->Error(_("Package extension list is too long"));
 164    if (_error->PendingError() == true)
 165       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 166
 167    Packages.PathPrefix = PathPrefix;
 168    Packages.DirStrip = ArchiveDir;
 169    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 170
 171    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 172    Packages.DeLinkLimit = DeLinkLimit;
 173
 174    // Create a compressor object
 175    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 176                       PkgCompress,Permissions);
 177    Packages.Output = Comp.Input;
 178    if (_error->PendingError() == true)
 179       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 180
 181    c0out << ' ' << BaseDir << ":" << flush;
 182
 183    // Do recursive directory searching
 184    if (FLFile.empty() == true)
 185    {
 186       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 187          return false;
 188    }
 189    else
 190    {
 191       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 192          return false;
 193    }
 194
 195    Packages.Output = 0;      // Just in case
 196
 197    // Finish compressing
 198    unsigned long Size;
 199    if (Comp.Finalize(Size) == false)
 200    {
 201       c0out << endl;
 202       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 203    }
 204
 205    if (Size != 0)
 206       c0out << " New "
 207              << SizeToStr(Size) << "B ";
 208    else
 209       c0out << ' ';
 210
 211    struct timeval NewTime;
 212    gettimeofday(&NewTime,0);
 213    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 214                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 215
 216    c0out << Packages.Stats.Packages << " files " <<
 217 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 218       SizeToStr(Packages.Stats.Bytes) << "B " <<
 219       TimeToStr((long)Delta) << endl;
 220
 221    Stats.Add(Packages.Stats);
 222    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 223
 224    return !_error->PendingError();
 225 }
 226
 227                                                                         /*}}}*/
 228 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* This generates the Sources File described by this object. */
 231 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 232 {
 233    if (SrcFile.empty() == true)
 234       return true;
 235
 236    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 237    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 238    string CacheDir = Setup.FindDir("Dir::CacheDir");
 239
 240    struct timeval StartTime;
 241    gettimeofday(&StartTime,0);
 242
 243    SrcDone = true;
 244
 245    // Create a package writer object.
 246    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 247                          flCombine(OverrideDir,SrcOverride),
 248                          flCombine(OverrideDir,SrcExtraOverride));
 249    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 250       return _error->Error(_("Source extension list is too long"));
 251    if (_error->PendingError() == true)
 252       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 253
 254    Sources.PathPrefix = PathPrefix;
 255    Sources.DirStrip = ArchiveDir;
 256    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 257
 258    Sources.DeLinkLimit = DeLinkLimit;
 259    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 260
 261    // Create a compressor object
 262    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 263                       SrcCompress,Permissions);
 264    Sources.Output = Comp.Input;
 265    if (_error->PendingError() == true)
 266       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 267
 268    c0out << ' ' << BaseDir << ":" << flush;
 269
 270    // Do recursive directory searching
 271    if (FLFile.empty() == true)
 272    {
 273       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 274          return false;
 275    }
 276    else
 277    {
 278       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 279          return false;
 280    }
 281    Sources.Output = 0;      // Just in case
 282
 283    // Finish compressing
 284    unsigned long Size;
 285    if (Comp.Finalize(Size) == false)
 286    {
 287       c0out << endl;
 288       return _error->Error(_("Error Processing directory %s"),BaseDir.c_str());
 289    }
 290
 291    if (Size != 0)
 292       c0out << " New "
 293              << SizeToStr(Size) << "B ";
 294    else
 295       c0out << ' ';
 296
 297    struct timeval NewTime;
 298    gettimeofday(&NewTime,0);
 299    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 300                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 301
 302    c0out << Sources.Stats.Packages << " pkgs in " <<
 303       TimeToStr((long)Delta) << endl;
 304
 305    Stats.Add(Sources.Stats);
 306    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 307
 308    return !_error->PendingError();
 309 }
 310                                                                         /*}}}*/
 311 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* This generates the contents file partially described by this object.
 314    It searches the given iterator range for other package files that map
 315    into this contents file and includes their data as well when building. */
 316 bool PackageMap::GenContents(Configuration &Setup,
 317                              vector<PackageMap>::iterator Begin,
 318                              vector<PackageMap>::iterator End,
 319                              unsigned long &Left)
 320 {
 321    if (Contents.empty() == true)
 322       return true;
 323
 324    if (Left == 0)
 325       return true;
 326
 327    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 328    string CacheDir = Setup.FindDir("Dir::CacheDir");
 329    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 330
 331    struct timeval StartTime;
 332    gettimeofday(&StartTime,0);
 333
 334    // Create a package writer object.
 335    ContentsWriter Contents("");
 336    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 337       return _error->Error(_("Package extension list is too long"));
 338    if (_error->PendingError() == true)
 339       return false;
 340
 341    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 342                       CntCompress,Permissions);
 343    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 344    Contents.Output = Comp.Input;
 345    if (_error->PendingError() == true)
 346       return false;
 347
 348    // Write the header out.
 349    if (ContentsHead.empty() == false)
 350    {
 351       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 352       if (_error->PendingError() == true)
 353          return false;
 354
 355       unsigned long Size = Head.Size();
 356       unsigned char Buf[4096];
 357       while (Size != 0)
 358       {
 359          unsigned long ToRead = Size;
 360          if (Size > sizeof(Buf))
 361             ToRead = sizeof(Buf);
 362
 363          if (Head.Read(Buf,ToRead) == false)
 364             return false;
 365
 366          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 367             return _error->Errno("fwrite",_("Error writing header to contents file"));
 368
 369          Size -= ToRead;
 370       }
 371    }
 372
 373    /* Go over all the package file records and parse all the package
 374       files associated with this contents file into one great big honking
 375       memory structure, then dump the sorted version */
 376    c0out << ' ' << this->Contents << ":" << flush;
 377    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 378    {
 379       if (I->Contents != this->Contents)
 380          continue;
 381
 382       Contents.Prefix = ArchiveDir;
 383       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 384       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 385                             I->PkgCompress);
 386
 387       I->ContentsDone = true;
 388    }
 389
 390    Contents.Finish();
 391
 392    // Finish compressing
 393    unsigned long Size;
 394    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 395    {
 396       c0out << endl;
 397       return _error->Error(_("Error Processing Contents %s"),
 398                            this->Contents.c_str());
 399    }
 400
 401    if (Size != 0)
 402    {
 403       c0out << " New " << SizeToStr(Size) << "B ";
 404       if (Left > Size)
 405          Left -= Size;
 406       else
 407          Left = 0;
 408    }
 409    else
 410       c0out << ' ';
 411
 412    struct timeval NewTime;
 413    gettimeofday(&NewTime,0);
 414    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 415                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 416
 417    c0out << Contents.Stats.Packages << " files " <<
 418       SizeToStr(Contents.Stats.Bytes) << "B " <<
 419       TimeToStr((long)Delta) << endl;
 420
 421    return true;
 422 }
 423                                                                         /*}}}*/
 424
 425 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* This populates the PkgList with all the possible permutations of the
 428    section/arch lists. */
 429 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 430 {
 431    // Load the defaults
 432    string DDir = Setup.Find("TreeDefault::Directory",
 433                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 434    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 435                             "$(DIST)/$(SECTION)/source/");
 436    string DPkg = Setup.Find("TreeDefault::Packages",
 437                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 438    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 439                             "$(DIST)/$(SECTION)/");
 440    string DContents = Setup.Find("TreeDefault::Contents",
 441                             "$(DIST)/Contents-$(ARCH)");
 442    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 443    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 444                                "packages-$(ARCH).db");
 445    string DSources = Setup.Find("TreeDefault::Sources",
 446                                 "$(DIST)/$(SECTION)/source/Sources");
 447    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 448    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 449
 450    // Process 'tree' type sections
 451    const Configuration::Item *Top = Setup.Tree("tree");
 452    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 453    {
 454       Configuration Block(Top);
 455       string Dist = Top->Tag;
 456
 457       // Parse the sections
 458       string Tmp = Block.Find("Sections");
 459       const char *Sections = Tmp.c_str();
 460       string Section;
 461       while (ParseQuoteWord(Sections,Section) == true)
 462       {
 463          string Tmp2 = Block.Find("Architectures");
 464          string Arch;
 465          const char *Archs = Tmp2.c_str();
 466          while (ParseQuoteWord(Archs,Arch) == true)
 467          {
 468             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 469                                       {"$(SECTION)",&Section},
 470                                       {"$(ARCH)",&Arch},
 471                                       {}};
 472             PackageMap Itm;
 473
 474             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 475             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 476
 477             if (stringcasecmp(Arch,"source") == 0)
 478             {
 479                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 480                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 481                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 482                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 483                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 484                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 485             }
 486             else
 487             {
 488                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 489                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 490                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 491                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 492                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 493                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 494                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 495                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 496             }
 497
 498             Itm.GetGeneral(Setup,Block);
 499             PkgList.push_back(Itm);
 500          }
 501       }
 502
 503       Top = Top->Next;
 504    }
 505 }
 506                                                                         /*}}}*/
 507 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 508 // ---------------------------------------------------------------------
 509 /* */
 510 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 511 {
 512    // Process 'bindirectory' type sections
 513    const Configuration::Item *Top = Setup.Tree("bindirectory");
 514    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 515    {
 516       Configuration Block(Top);
 517
 518       PackageMap Itm;
 519       Itm.PkgFile = Block.Find("Packages");
 520       Itm.SrcFile = Block.Find("Sources");
 521       Itm.BinCacheDB = Block.Find("BinCacheDB");
 522       Itm.BinOverride = Block.Find("BinOverride");
 523       Itm.ExtraOverride = Block.Find("ExtraOverride");
 524       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 525       Itm.SrcOverride = Block.Find("SrcOverride");
 526       Itm.BaseDir = Top->Tag;
 527       Itm.FLFile = Block.Find("FileList");
 528       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 529       Itm.Contents = Block.Find("Contents");
 530       Itm.ContentsHead = Block.Find("Contents::Header");
 531
 532       Itm.GetGeneral(Setup,Block);
 533       PkgList.push_back(Itm);
 534
 535       Top = Top->Next;
 536    }
 537 }
 538                                                                         /*}}}*/
 539
 540 // ShowHelp - Show the help text                                        /*{{{*/
 541 // ---------------------------------------------------------------------
 542 /* */
 543 bool ShowHelp(CommandLine &CmdL)
 544 {
 545    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 546             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 547    if (_config->FindB("version") == true)
 548       return true;
 549
 550    cout <<
 551     _("Usage: apt-ftparchive [options] command\n"
 552       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 553       "          sources srcpath [overridefile [pathprefix]]\n"
 554       "          contents path\n"
 555       "          generate config [groups]\n"
 556       "          clean config\n"
 557       "\n"
 558       "apt-ftparchive generates index files for Debian archives. It supports\n"
 559       "many styles of generation from fully automated to functional replacements\n"
 560       "for dpkg-scanpackages and dpkg-scansources\n"
 561       "\n"
 562       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 563       "Package file contains the contents of all the control fields from\n"
 564       "each package as well as the MD5 hash and filesize. An override file\n"
 565       "is supported to force the value of Priority and Section.\n"
 566       "\n"
 567       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 568       "The --source-override option can be used to specify a src override file\n"
 569       "\n"
 570       "The 'packages' and 'sources' command should be run in the root of the\n"
 571       "tree. BinaryPath should point to the base of the recursive search and \n"
 572       "override file should contain the override flags. Pathprefix is\n"
 573       "appended to the filename fields if present. Example usage from the \n"
 574       "Debian archive:\n"
 575       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 576       "               dists/potato/main/binary-i386/Packages\n"
 577       "\n"
 578       "Options:\n"
 579       "  -h    This help text\n"
 580       "  --md5 Control MD5 generation\n"
 581       "  -s=?  Source override file\n"
 582       "  -q    Quiet\n"
 583       "  -d=?  Select the optional caching database\n"
 584       "  --no-delink Enable delinking debug mode\n"
 585       "  --contents  Control contents file generation\n"
 586       "  -c=?  Read this configuration file\n"
 587       "  -o=?  Set an arbitary configuration option") << endl;
 588
 589    return true;
 590 }
 591                                                                         /*}}}*/
 592 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 593 // ---------------------------------------------------------------------
 594 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 595 bool SimpleGenPackages(CommandLine &CmdL)
 596 {
 597    if (CmdL.FileSize() < 2)
 598       return ShowHelp(CmdL);
 599
 600    string Override;
 601    if (CmdL.FileSize() >= 3)
 602       Override = CmdL.FileList[2];
 603
 604    // Create a package writer object.
 605    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 606                            Override, "");
 607    if (_error->PendingError() == true)
 608       return false;
 609
 610    if (CmdL.FileSize() >= 4)
 611       Packages.PathPrefix = CmdL.FileList[3];
 612
 613    // Do recursive directory searching
 614    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 615       return false;
 616
 617    return true;
 618 }
 619                                                                         /*}}}*/
 620 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 621 // ---------------------------------------------------------------------
 622 /* */
 623 bool SimpleGenContents(CommandLine &CmdL)
 624 {
 625    if (CmdL.FileSize() < 2)
 626       return ShowHelp(CmdL);
 627
 628    // Create a package writer object.
 629    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 630    if (_error->PendingError() == true)
 631       return false;
 632
 633    // Do recursive directory searching
 634    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 635       return false;
 636
 637    Contents.Finish();
 638
 639    return true;
 640 }
 641                                                                         /*}}}*/
 642 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 643 // ---------------------------------------------------------------------
 644 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 645 bool SimpleGenSources(CommandLine &CmdL)
 646 {
 647    if (CmdL.FileSize() < 2)
 648       return ShowHelp(CmdL);
 649
 650    string Override;
 651    if (CmdL.FileSize() >= 3)
 652       Override = CmdL.FileList[2];
 653
 654    string SOverride;
 655    if (Override.empty() == false)
 656       SOverride = Override + ".src";
 657
 658    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 659                              SOverride.c_str());
 660
 661    // Create a package writer object.
 662    SourcesWriter Sources(Override,SOverride);
 663    if (_error->PendingError() == true)
 664       return false;
 665
 666    if (CmdL.FileSize() >= 4)
 667       Sources.PathPrefix = CmdL.FileList[3];
 668
 669    // Do recursive directory searching
 670    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 671       return false;
 672
 673    return true;
 674 }
 675                                                                         /*}}}*/
 676 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 677 // ---------------------------------------------------------------------
 678 bool SimpleGenRelease(CommandLine &CmdL)
 679 {
 680    ReleaseWriter Release("");
 681    if (_error->PendingError() == true)
 682       return false;
 683
 684    if (Release.RecursiveScan(CmdL.FileList[1]) == false)
 685       return false;
 686
 687    return true;
 688 }
 689
 690                                                                         /*}}}*/
 691 // Generate - Full generate, using a config file                        /*{{{*/
 692 // ---------------------------------------------------------------------
 693 /* */
 694 bool Generate(CommandLine &CmdL)
 695 {
 696    struct CacheDB::Stats SrcStats;
 697    if (CmdL.FileSize() < 2)
 698       return ShowHelp(CmdL);
 699
 700    struct timeval StartTime;
 701    gettimeofday(&StartTime,0);
 702    struct CacheDB::Stats Stats;
 703
 704    // Read the configuration file.
 705    Configuration Setup;
 706    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 707       return false;
 708
 709    vector<PackageMap> PkgList;
 710    LoadTree(PkgList,Setup);
 711    LoadBinDir(PkgList,Setup);
 712
 713    // Sort by cache DB to improve IO locality.
 714    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 715
 716    // Generate packages
 717    if (CmdL.FileSize() <= 2)
 718    {
 719       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 720          if (I->GenPackages(Setup,Stats) == false)
 721             _error->DumpErrors();
 722       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 723          if (I->GenSources(Setup,SrcStats) == false)
 724             _error->DumpErrors();
 725    }
 726    else
 727    {
 728       // Make a choice list out of the package list..
 729       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 730       RxChoiceList *End = List;
 731       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 732       {
 733          End->UserData = &(*I);
 734          End->Str = I->BaseDir.c_str();
 735          End++;
 736
 737          End->UserData = &(*I);
 738          End->Str = I->Tag.c_str();
 739          End++;
 740       }
 741       End->Str = 0;
 742
 743       // Regex it
 744       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 745       {
 746          delete [] List;
 747          return _error->Error(_("No selections matched"));
 748       }
 749       _error->DumpErrors();
 750
 751       // Do the generation for Packages
 752       for (End = List; End->Str != 0; End++)
 753       {
 754          if (End->Hit == false)
 755             continue;
 756
 757          PackageMap *I = (PackageMap *)End->UserData;
 758          if (I->PkgDone == true)
 759             continue;
 760          if (I->GenPackages(Setup,Stats) == false)
 761             _error->DumpErrors();
 762       }
 763
 764       // Do the generation for Sources
 765       for (End = List; End->Str != 0; End++)
 766       {
 767          if (End->Hit == false)
 768             continue;
 769
 770          PackageMap *I = (PackageMap *)End->UserData;
 771          if (I->SrcDone == true)
 772             continue;
 773          if (I->GenSources(Setup,SrcStats) == false)
 774             _error->DumpErrors();
 775       }
 776
 777       delete [] List;
 778    }
 779
 780    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 781       return true;
 782
 783    c1out << "Done Packages, Starting contents." << endl;
 784
 785    // Sort the contents file list by date
 786    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 787    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 788    {
 789       struct stat A;
 790       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 791                                  I->CntCompress,A) == false)
 792          time(&I->ContentsMTime);
 793       else
 794          I->ContentsMTime = A.st_mtime;
 795    }
 796    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 797
 798    /* Now for Contents.. The process here is to do a make-like dependency
 799       check. Each contents file is verified to be newer than the package files
 800       that describe the debs it indexes. Since the package files contain
 801       hashes of the .debs this means they have not changed either so the
 802       contents must be up to date. */
 803    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 804    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 805    {
 806       // This record is not relevent
 807       if (I->ContentsDone == true ||
 808           I->Contents.empty() == true)
 809          continue;
 810
 811       // Do not do everything if the user specified sections.
 812       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 813          continue;
 814
 815       struct stat A,B;
 816       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 817       {
 818          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 819          {
 820             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 821             continue;
 822          }
 823
 824          if (A.st_mtime > B.st_mtime)
 825             continue;
 826       }
 827
 828       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 829                          MaxContentsChange) == false)
 830          _error->DumpErrors();
 831
 832       // Hit the limit?
 833       if (MaxContentsChange == 0)
 834       {
 835          c1out << "Hit contents update byte limit" << endl;
 836          break;
 837       }
 838    }
 839
 840    struct timeval NewTime;
 841    gettimeofday(&NewTime,0);
 842    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 843                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 844    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 845          << " archives. Took " << TimeToStr((long)Delta) << endl;
 846
 847    return true;
 848 }
 849                                                                         /*}}}*/
 850 // Clean - Clean out the databases                                      /*{{{*/
 851 // ---------------------------------------------------------------------
 852 /* */
 853 bool Clean(CommandLine &CmdL)
 854 {
 855    if (CmdL.FileSize() != 2)
 856       return ShowHelp(CmdL);
 857
 858    // Read the configuration file.
 859    Configuration Setup;
 860    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 861       return false;
 862
 863    vector<PackageMap> PkgList;
 864    LoadTree(PkgList,Setup);
 865    LoadBinDir(PkgList,Setup);
 866
 867    // Sort by cache DB to improve IO locality.
 868    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 869
 870    string CacheDir = Setup.FindDir("Dir::CacheDir");
 871
 872    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 873    {
 874       c0out << I->BinCacheDB << endl;
 875       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 876       if (DB.Clean() == false)
 877          _error->DumpErrors();
 878
 879       string CacheDB = I->BinCacheDB;
 880       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 881    }
 882
 883    return true;
 884 }
 885                                                                         /*}}}*/
 886
 887 int main(int argc, const char *argv[])
 888 {
 889    CommandLine::Args Args[] = {
 890       {'h',"help","help",0},
 891       {0,"md5","APT::FTPArchive::MD5",0},
 892       {'v',"version","version",0},
 893       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 894       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 895       {'q',"quiet","quiet",CommandLine::IntLevel},
 896       {'q',"silent","quiet",CommandLine::IntLevel},
 897       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 898       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 899       {0,"contents","APT::FTPArchive::Contents",0},
 900       {'c',"config-file",0,CommandLine::ConfigFile},
 901       {'o',"option",0,CommandLine::ArbItem},
 902       {0,0,0,0}};
 903    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 904                                    {"contents",&SimpleGenContents},
 905                                    {"sources",&SimpleGenSources},
 906                                    {"release",&SimpleGenRelease},
 907                                    {"generate",&Generate},
 908                                    {"clean",&Clean},
 909                                    {"help",&ShowHelp},
 910                                    {0,0}};
 911
 912    // Parse the command line and initialize the package library
 913    CommandLine CmdL(Args,_config);
 914    if (CmdL.Parse(argc,argv) == false)
 915    {
 916       _error->DumpErrors();
 917       return 100;
 918    }
 919
 920    // See if the help should be shown
 921    if (_config->FindB("help") == true ||
 922        _config->FindB("version") == true ||
 923        CmdL.FileSize() == 0)
 924    {
 925       ShowHelp(CmdL);
 926       return 0;
 927    }
 928
 929    // Setup the output streams
 930    c0out.rdbuf(clog.rdbuf());
 931    c1out.rdbuf(clog.rdbuf());
 932    c2out.rdbuf(clog.rdbuf());
 933    Quiet = _config->FindI("quiet",0);
 934    if (Quiet > 0)
 935       c0out.rdbuf(devnull.rdbuf());
 936    if (Quiet > 1)
 937       c1out.rdbuf(devnull.rdbuf());
 938
 939    // Match the operation
 940    CmdL.DispatchArg(Cmds);
 941
 942    if (_error->empty() == false)
 943    {
 944       bool Errors = _error->PendingError();
 945       _error->DumpErrors();
 946       return Errors == true?100:0;
 947    }
 948    return 0;
 949 }