ftparchive/apt-ftparchive.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: apt-ftparchive.cc,v 1.8.2.3 2004/01/02 22:01:48 mdz Exp $
   4 /* ######################################################################
   5
   6    apt-scanpackages - Efficient work-alike for dpkg-scanpackages
   7
   8    Let contents be disabled from the conf
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "apt-ftparchive.h"
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/configuration.h>
  17 #include <apt-pkg/cmndline.h>
  18 #include <apt-pkg/strutl.h>
  19 #include <config.h>
  20 #include <apti18n.h>
  21 #include <algorithm>
  22
  23 #include <sys/time.h>
  24 #include <regex.h>
  25
  26 #include "contents.h"
  27 #include "multicompress.h"
  28 #include "writer.h"
  29                                                                         /*}}}*/
  30
  31 using namespace std;
  32 ostream c0out(0);
  33 ostream c1out(0);
  34 ostream c2out(0);
  35 ofstream devnull("/dev/null");
  36 unsigned Quiet = 0;
  37
  38 // struct PackageMap - List of all package files in the config file     /*{{{*/
  39 // ---------------------------------------------------------------------
  40 /* */
  41 struct PackageMap
  42 {
  43    // General Stuff
  44    string BaseDir;
  45    string InternalPrefix;
  46    string FLFile;
  47    string PkgExt;
  48    string SrcExt;
  49
  50    // Stuff for the Package File
  51    string PkgFile;
  52    string BinCacheDB;
  53    string BinOverride;
  54    string ExtraOverride;
  55
  56    // We generate for this given arch
  57    string Arch;
  58
  59    // Stuff for the Source File
  60    string SrcFile;
  61    string SrcOverride;
  62    string SrcExtraOverride;
  63
  64    // Contents
  65    string Contents;
  66    string ContentsHead;
  67
  68    // Random things
  69    string Tag;
  70    string PkgCompress;
  71    string CntCompress;
  72    string SrcCompress;
  73    string PathPrefix;
  74    unsigned int DeLinkLimit;
  75    mode_t Permissions;
  76
  77    bool ContentsDone;
  78    bool PkgDone;
  79    bool SrcDone;
  80    time_t ContentsMTime;
  81
  82    struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  83    {
  84       inline bool operator() (const PackageMap &x,const PackageMap &y)
  85       {return x.ContentsMTime < y.ContentsMTime;};
  86    };
  87
  88    struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  89    {
  90       inline bool operator() (const PackageMap &x,const PackageMap &y)
  91       {return x.BinCacheDB < y.BinCacheDB;};
  92    };
  93
  94    void GetGeneral(Configuration &Setup,Configuration &Block);
  95    bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  96    bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  97    bool GenContents(Configuration &Setup,
  98                     vector<PackageMap>::iterator Begin,
  99                     vector<PackageMap>::iterator End,
 100                     unsigned long &Left);
 101
 102    PackageMap() : DeLinkLimit(0), Permissions(1), ContentsDone(false),
 103         PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 104 };
 105                                                                         /*}}}*/
 106
 107 // PackageMap::GetGeneral - Common per-section definitions              /*{{{*/
 108 // ---------------------------------------------------------------------
 109 /* */
 110 void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
 111 {
 112    PathPrefix = Block.Find("PathPrefix");
 113
 114    if (Block.FindB("External-Links",true) == false)
 115       DeLinkLimit = Setup.FindI("Default::DeLinkLimit",UINT_MAX);
 116    else
 117       DeLinkLimit = 0;
 118
 119    PkgCompress = Block.Find("Packages::Compress",
 120                             Setup.Find("Default::Packages::Compress",". gzip").c_str());
 121    CntCompress = Block.Find("Contents::Compress",
 122                             Setup.Find("Default::Contents::Compress",". gzip").c_str());
 123    SrcCompress = Block.Find("Sources::Compress",
 124                             Setup.Find("Default::Sources::Compress",". gzip").c_str());
 125
 126    SrcExt = Block.Find("Sources::Extensions",
 127                        Setup.Find("Default::Sources::Extensions",".dsc").c_str());
 128    PkgExt = Block.Find("Packages::Extensions",
 129                        Setup.Find("Default::Packages::Extensions",".deb").c_str());
 130
 131    Permissions = Setup.FindI("Default::FileMode",0644);
 132
 133    if (FLFile.empty() == false)
 134       FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
 135
 136    if (Contents == " ")
 137       Contents= string();
 138 }
 139                                                                         /*}}}*/
 140 // PackageMap::GenPackages - Actually generate a Package file           /*{{{*/
 141 // ---------------------------------------------------------------------
 142 /* This generates the Package File described by this object. */
 143 bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
 144 {
 145    if (PkgFile.empty() == true)
 146       return true;
 147
 148    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 149    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 150    string CacheDir = Setup.FindDir("Dir::CacheDir");
 151
 152    struct timeval StartTime;
 153    gettimeofday(&StartTime,0);
 154
 155    PkgDone = true;
 156
 157    // Create a package writer object.
 158    PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
 159                            flCombine(OverrideDir,BinOverride),
 160                            flCombine(OverrideDir,ExtraOverride),
 161                            Arch);
 162    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
 163       return _error->Error(_("Package extension list is too long"));
 164    if (_error->PendingError() == true)
 165       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 166
 167    Packages.PathPrefix = PathPrefix;
 168    Packages.DirStrip = ArchiveDir;
 169    Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 170
 171    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
 172    Packages.DeLinkLimit = DeLinkLimit;
 173
 174    // Create a compressor object
 175    MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
 176                       PkgCompress,Permissions);
 177    Packages.Output = Comp.Input;
 178    if (_error->PendingError() == true)
 179       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 180
 181    c0out << ' ' << BaseDir << ":" << flush;
 182
 183    // Do recursive directory searching
 184    if (FLFile.empty() == true)
 185    {
 186       if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
 187          return false;
 188    }
 189    else
 190    {
 191       if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
 192          return false;
 193    }
 194
 195    Packages.Output = 0;      // Just in case
 196
 197    // Finish compressing
 198    unsigned long Size;
 199    if (Comp.Finalize(Size) == false)
 200    {
 201       c0out << endl;
 202       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 203    }
 204
 205    if (Size != 0)
 206       c0out << " New "
 207              << SizeToStr(Size) << "B ";
 208    else
 209       c0out << ' ';
 210
 211    struct timeval NewTime;
 212    gettimeofday(&NewTime,0);
 213    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 214                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 215
 216    c0out << Packages.Stats.Packages << " files " <<
 217 /*      SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
 218       SizeToStr(Packages.Stats.Bytes) << "B " <<
 219       TimeToStr((long)Delta) << endl;
 220
 221    Stats.Add(Packages.Stats);
 222    Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
 223
 224    return !_error->PendingError();
 225 }
 226
 227                                                                         /*}}}*/
 228 // PackageMap::GenSources - Actually generate a Source file             /*{{{*/
 229 // ---------------------------------------------------------------------
 230 /* This generates the Sources File described by this object. */
 231 bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 232 {
 233    if (SrcFile.empty() == true)
 234       return true;
 235
 236    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 237    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 238    string CacheDir = Setup.FindDir("Dir::CacheDir");
 239
 240    struct timeval StartTime;
 241    gettimeofday(&StartTime,0);
 242
 243    SrcDone = true;
 244
 245    // Create a package writer object.
 246    SourcesWriter Sources(flCombine(OverrideDir,BinOverride),
 247                          flCombine(OverrideDir,SrcOverride),
 248                          flCombine(OverrideDir,SrcExtraOverride));
 249    if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
 250       return _error->Error(_("Source extension list is too long"));
 251    if (_error->PendingError() == true)
 252       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 253
 254    Sources.PathPrefix = PathPrefix;
 255    Sources.DirStrip = ArchiveDir;
 256    Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
 257
 258    Sources.DeLinkLimit = DeLinkLimit;
 259    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
 260
 261    // Create a compressor object
 262    MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
 263                       SrcCompress,Permissions);
 264    Sources.Output = Comp.Input;
 265    if (_error->PendingError() == true)
 266       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 267
 268    c0out << ' ' << BaseDir << ":" << flush;
 269
 270    // Do recursive directory searching
 271    if (FLFile.empty() == true)
 272    {
 273       if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
 274          return false;
 275    }
 276    else
 277    {
 278       if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
 279          return false;
 280    }
 281    Sources.Output = 0;      // Just in case
 282
 283    // Finish compressing
 284    unsigned long Size;
 285    if (Comp.Finalize(Size) == false)
 286    {
 287       c0out << endl;
 288       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 289    }
 290
 291    if (Size != 0)
 292       c0out << " New "
 293              << SizeToStr(Size) << "B ";
 294    else
 295       c0out << ' ';
 296
 297    struct timeval NewTime;
 298    gettimeofday(&NewTime,0);
 299    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 300                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 301
 302    c0out << Sources.Stats.Packages << " pkgs in " <<
 303       TimeToStr((long)Delta) << endl;
 304
 305    Stats.Add(Sources.Stats);
 306    Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
 307
 308    return !_error->PendingError();
 309 }
 310                                                                         /*}}}*/
 311 // PackageMap::GenContents - Actually generate a Contents file          /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* This generates the contents file partially described by this object.
 314    It searches the given iterator range for other package files that map
 315    into this contents file and includes their data as well when building. */
 316 bool PackageMap::GenContents(Configuration &Setup,
 317                              vector<PackageMap>::iterator Begin,
 318                              vector<PackageMap>::iterator End,
 319                              unsigned long &Left)
 320 {
 321    if (Contents.empty() == true)
 322       return true;
 323
 324    if (Left == 0)
 325       return true;
 326
 327    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 328    string CacheDir = Setup.FindDir("Dir::CacheDir");
 329    string OverrideDir = Setup.FindDir("Dir::OverrideDir");
 330
 331    struct timeval StartTime;
 332    gettimeofday(&StartTime,0);
 333
 334    // Create a package writer object.
 335    ContentsWriter Contents("");
 336    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
 337       return _error->Error(_("Package extension list is too long"));
 338    if (_error->PendingError() == true)
 339       return false;
 340
 341    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 342                       CntCompress,Permissions);
 343    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
 344    Contents.Output = Comp.Input;
 345    if (_error->PendingError() == true)
 346       return false;
 347
 348    // Write the header out.
 349    if (ContentsHead.empty() == false)
 350    {
 351       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
 352       if (_error->PendingError() == true)
 353          return false;
 354
 355       unsigned long Size = Head.Size();
 356       unsigned char Buf[4096];
 357       while (Size != 0)
 358       {
 359          unsigned long ToRead = Size;
 360          if (Size > sizeof(Buf))
 361             ToRead = sizeof(Buf);
 362
 363          if (Head.Read(Buf,ToRead) == false)
 364             return false;
 365
 366          if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
 367             return _error->Errno("fwrite",_("Error writing header to contents file"));
 368
 369          Size -= ToRead;
 370       }
 371    }
 372
 373    /* Go over all the package file records and parse all the package
 374       files associated with this contents file into one great big honking
 375       memory structure, then dump the sorted version */
 376    c0out << ' ' << this->Contents << ":" << flush;
 377    for (vector<PackageMap>::iterator I = Begin; I != End; I++)
 378    {
 379       if (I->Contents != this->Contents)
 380          continue;
 381
 382       Contents.Prefix = ArchiveDir;
 383       Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
 384       Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
 385                             I->PkgCompress);
 386
 387       I->ContentsDone = true;
 388    }
 389
 390    Contents.Finish();
 391
 392    // Finish compressing
 393    unsigned long Size;
 394    if (Comp.Finalize(Size) == false || _error->PendingError() == true)
 395    {
 396       c0out << endl;
 397       return _error->Error(_("Error processing contents %s"),
 398                            this->Contents.c_str());
 399    }
 400
 401    if (Size != 0)
 402    {
 403       c0out << " New " << SizeToStr(Size) << "B ";
 404       if (Left > Size)
 405          Left -= Size;
 406       else
 407          Left = 0;
 408    }
 409    else
 410       c0out << ' ';
 411
 412    struct timeval NewTime;
 413    gettimeofday(&NewTime,0);
 414    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 415                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 416
 417    c0out << Contents.Stats.Packages << " files " <<
 418       SizeToStr(Contents.Stats.Bytes) << "B " <<
 419       TimeToStr((long)Delta) << endl;
 420
 421    return true;
 422 }
 423                                                                         /*}}}*/
 424
 425 // LoadTree - Load a 'tree' section from the Generate Config            /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* This populates the PkgList with all the possible permutations of the
 428    section/arch lists. */
 429 void LoadTree(vector<PackageMap> &PkgList,Configuration &Setup)
 430 {
 431    // Load the defaults
 432    string DDir = Setup.Find("TreeDefault::Directory",
 433                             "$(DIST)/$(SECTION)/binary-$(ARCH)/");
 434    string DSDir = Setup.Find("TreeDefault::SrcDirectory",
 435                             "$(DIST)/$(SECTION)/source/");
 436    string DPkg = Setup.Find("TreeDefault::Packages",
 437                             "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
 438    string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
 439                             "$(DIST)/$(SECTION)/");
 440    string DContents = Setup.Find("TreeDefault::Contents",
 441                             "$(DIST)/Contents-$(ARCH)");
 442    string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
 443    string DBCache = Setup.Find("TreeDefault::BinCacheDB",
 444                                "packages-$(ARCH).db");
 445    string DSources = Setup.Find("TreeDefault::Sources",
 446                                 "$(DIST)/$(SECTION)/source/Sources");
 447    string DFLFile = Setup.Find("TreeDefault::FileList", "");
 448    string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
 449
 450    // Process 'tree' type sections
 451    const Configuration::Item *Top = Setup.Tree("tree");
 452    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 453    {
 454       Configuration Block(Top);
 455       string Dist = Top->Tag;
 456
 457       // Parse the sections
 458       string Tmp = Block.Find("Sections");
 459       const char *Sections = Tmp.c_str();
 460       string Section;
 461       while (ParseQuoteWord(Sections,Section) == true)
 462       {
 463          string Tmp2 = Block.Find("Architectures");
 464          string Arch;
 465          const char *Archs = Tmp2.c_str();
 466          while (ParseQuoteWord(Archs,Arch) == true)
 467          {
 468             struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 469                                       {"$(SECTION)",&Section},
 470                                       {"$(ARCH)",&Arch},
 471                                       {}};
 472             PackageMap Itm;
 473
 474             Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 475             Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 476
 477             if (stringcasecmp(Arch,"source") == 0)
 478             {
 479                Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 480                Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
 481                Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
 482                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
 483                Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
 484                Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
 485             }
 486             else
 487             {
 488                Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
 489                Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
 490                Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 491                Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 492                Itm.Arch = Arch;
 493                Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
 494                Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
 495                Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
 496                Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
 497             }
 498
 499             Itm.GetGeneral(Setup,Block);
 500             PkgList.push_back(Itm);
 501          }
 502       }
 503
 504       Top = Top->Next;
 505    }
 506 }
 507                                                                         /*}}}*/
 508 // LoadBinDir - Load a 'bindirectory' section from the Generate Config  /*{{{*/
 509 // ---------------------------------------------------------------------
 510 /* */
 511 void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
 512 {
 513    // Process 'bindirectory' type sections
 514    const Configuration::Item *Top = Setup.Tree("bindirectory");
 515    for (Top = (Top == 0?0:Top->Child); Top != 0;)
 516    {
 517       Configuration Block(Top);
 518
 519       PackageMap Itm;
 520       Itm.PkgFile = Block.Find("Packages");
 521       Itm.SrcFile = Block.Find("Sources");
 522       Itm.BinCacheDB = Block.Find("BinCacheDB");
 523       Itm.BinOverride = Block.Find("BinOverride");
 524       Itm.ExtraOverride = Block.Find("ExtraOverride");
 525       Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
 526       Itm.SrcOverride = Block.Find("SrcOverride");
 527       Itm.BaseDir = Top->Tag;
 528       Itm.FLFile = Block.Find("FileList");
 529       Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
 530       Itm.Contents = Block.Find("Contents");
 531       Itm.ContentsHead = Block.Find("Contents::Header");
 532
 533       Itm.GetGeneral(Setup,Block);
 534       PkgList.push_back(Itm);
 535
 536       Top = Top->Next;
 537    }
 538 }
 539                                                                         /*}}}*/
 540
 541 // ShowHelp - Show the help text                                        /*{{{*/
 542 // ---------------------------------------------------------------------
 543 /* */
 544 bool ShowHelp(CommandLine &CmdL)
 545 {
 546    ioprintf(cout,_("%s %s for %s %s compiled on %s %s\n"),PACKAGE,VERSION,
 547             COMMON_OS,COMMON_CPU,__DATE__,__TIME__);
 548    if (_config->FindB("version") == true)
 549       return true;
 550
 551    cout <<
 552     _("Usage: apt-ftparchive [options] command\n"
 553       "Commands: packages binarypath [overridefile [pathprefix]]\n"
 554       "          sources srcpath [overridefile [pathprefix]]\n"
 555       "          contents path\n"
 556       "          release path\n"
 557       "          generate config [groups]\n"
 558       "          clean config\n"
 559       "\n"
 560       "apt-ftparchive generates index files for Debian archives. It supports\n"
 561       "many styles of generation from fully automated to functional replacements\n"
 562       "for dpkg-scanpackages and dpkg-scansources\n"
 563       "\n"
 564       "apt-ftparchive generates Package files from a tree of .debs. The\n"
 565       "Package file contains the contents of all the control fields from\n"
 566       "each package as well as the MD5 hash and filesize. An override file\n"
 567       "is supported to force the value of Priority and Section.\n"
 568       "\n"
 569       "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
 570       "The --source-override option can be used to specify a src override file\n"
 571       "\n"
 572       "The 'packages' and 'sources' command should be run in the root of the\n"
 573       "tree. BinaryPath should point to the base of the recursive search and \n"
 574       "override file should contain the override flags. Pathprefix is\n"
 575       "appended to the filename fields if present. Example usage from the \n"
 576       "Debian archive:\n"
 577       "   apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
 578       "               dists/potato/main/binary-i386/Packages\n"
 579       "\n"
 580       "Options:\n"
 581       "  -h    This help text\n"
 582       "  --md5 Control MD5 generation\n"
 583       "  -s=?  Source override file\n"
 584       "  -q    Quiet\n"
 585       "  -d=?  Select the optional caching database\n"
 586       "  --no-delink Enable delinking debug mode\n"
 587       "  --contents  Control contents file generation\n"
 588       "  -c=?  Read this configuration file\n"
 589       "  -o=?  Set an arbitrary configuration option") << endl;
 590
 591    return true;
 592 }
 593                                                                         /*}}}*/
 594 // SimpleGenPackages - Generate a Packages file for a directory tree    /*{{{*/
 595 // ---------------------------------------------------------------------
 596 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 597 bool SimpleGenPackages(CommandLine &CmdL)
 598 {
 599    if (CmdL.FileSize() < 2)
 600       return ShowHelp(CmdL);
 601
 602    string Override;
 603    if (CmdL.FileSize() >= 3)
 604       Override = CmdL.FileList[2];
 605
 606    // Create a package writer object.
 607    PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
 608                            Override, "");
 609    if (_error->PendingError() == true)
 610       return false;
 611
 612    if (CmdL.FileSize() >= 4)
 613       Packages.PathPrefix = CmdL.FileList[3];
 614
 615    // Do recursive directory searching
 616    if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
 617       return false;
 618
 619    return true;
 620 }
 621                                                                         /*}}}*/
 622 // SimpleGenContents - Generate a Contents listing                      /*{{{*/
 623 // ---------------------------------------------------------------------
 624 /* */
 625 bool SimpleGenContents(CommandLine &CmdL)
 626 {
 627    if (CmdL.FileSize() < 2)
 628       return ShowHelp(CmdL);
 629
 630    // Create a package writer object.
 631    ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"));
 632    if (_error->PendingError() == true)
 633       return false;
 634
 635    // Do recursive directory searching
 636    if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
 637       return false;
 638
 639    Contents.Finish();
 640
 641    return true;
 642 }
 643                                                                         /*}}}*/
 644 // SimpleGenSources - Generate a Sources file for a directory tree      /*{{{*/
 645 // ---------------------------------------------------------------------
 646 /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
 647 bool SimpleGenSources(CommandLine &CmdL)
 648 {
 649    if (CmdL.FileSize() < 2)
 650       return ShowHelp(CmdL);
 651
 652    string Override;
 653    if (CmdL.FileSize() >= 3)
 654       Override = CmdL.FileList[2];
 655
 656    string SOverride;
 657    if (Override.empty() == false)
 658       SOverride = Override + ".src";
 659
 660    SOverride = _config->Find("APT::FTPArchive::SourceOverride",
 661                              SOverride.c_str());
 662
 663    // Create a package writer object.
 664    SourcesWriter Sources(Override,SOverride);
 665    if (_error->PendingError() == true)
 666       return false;
 667
 668    if (CmdL.FileSize() >= 4)
 669       Sources.PathPrefix = CmdL.FileList[3];
 670
 671    // Do recursive directory searching
 672    if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
 673       return false;
 674
 675    return true;
 676 }
 677                                                                         /*}}}*/
 678 // SimpleGenRelease - Generate a Release file for a directory tree      /*{{{*/
 679 // ---------------------------------------------------------------------
 680 bool SimpleGenRelease(CommandLine &CmdL)
 681 {
 682    if (CmdL.FileSize() < 2)
 683       return ShowHelp(CmdL);
 684
 685    string Dir = CmdL.FileList[1];
 686
 687    ReleaseWriter Release("");
 688    Release.DirStrip = Dir;
 689
 690    if (_error->PendingError() == true)
 691       return false;
 692
 693    if (Release.RecursiveScan(Dir) == false)
 694       return false;
 695
 696    Release.Finish();
 697
 698    return true;
 699 }
 700
 701                                                                         /*}}}*/
 702 // Generate - Full generate, using a config file                        /*{{{*/
 703 // ---------------------------------------------------------------------
 704 /* */
 705 bool Generate(CommandLine &CmdL)
 706 {
 707    struct CacheDB::Stats SrcStats;
 708    if (CmdL.FileSize() < 2)
 709       return ShowHelp(CmdL);
 710
 711    struct timeval StartTime;
 712    gettimeofday(&StartTime,0);
 713    struct CacheDB::Stats Stats;
 714
 715    // Read the configuration file.
 716    Configuration Setup;
 717    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 718       return false;
 719
 720    vector<PackageMap> PkgList;
 721    LoadTree(PkgList,Setup);
 722    LoadBinDir(PkgList,Setup);
 723
 724    // Sort by cache DB to improve IO locality.
 725    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 726
 727    // Generate packages
 728    if (CmdL.FileSize() <= 2)
 729    {
 730       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 731          if (I->GenPackages(Setup,Stats) == false)
 732             _error->DumpErrors();
 733       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 734          if (I->GenSources(Setup,SrcStats) == false)
 735             _error->DumpErrors();
 736    }
 737    else
 738    {
 739       // Make a choice list out of the package list..
 740       RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
 741       RxChoiceList *End = List;
 742       for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 743       {
 744          End->UserData = &(*I);
 745          End->Str = I->BaseDir.c_str();
 746          End++;
 747
 748          End->UserData = &(*I);
 749          End->Str = I->Tag.c_str();
 750          End++;
 751       }
 752       End->Str = 0;
 753
 754       // Regex it
 755       if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
 756       {
 757          delete [] List;
 758          return _error->Error(_("No selections matched"));
 759       }
 760       _error->DumpErrors();
 761
 762       // Do the generation for Packages
 763       for (End = List; End->Str != 0; End++)
 764       {
 765          if (End->Hit == false)
 766             continue;
 767
 768          PackageMap *I = (PackageMap *)End->UserData;
 769          if (I->PkgDone == true)
 770             continue;
 771          if (I->GenPackages(Setup,Stats) == false)
 772             _error->DumpErrors();
 773       }
 774
 775       // Do the generation for Sources
 776       for (End = List; End->Str != 0; End++)
 777       {
 778          if (End->Hit == false)
 779             continue;
 780
 781          PackageMap *I = (PackageMap *)End->UserData;
 782          if (I->SrcDone == true)
 783             continue;
 784          if (I->GenSources(Setup,SrcStats) == false)
 785             _error->DumpErrors();
 786       }
 787
 788       delete [] List;
 789    }
 790
 791    if (_config->FindB("APT::FTPArchive::Contents",true) == false)
 792       return true;
 793
 794    c1out << "Done Packages, Starting contents." << endl;
 795
 796    // Sort the contents file list by date
 797    string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
 798    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 799    {
 800       struct stat A;
 801       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
 802                                  I->CntCompress,A) == false)
 803          time(&I->ContentsMTime);
 804       else
 805          I->ContentsMTime = A.st_mtime;
 806    }
 807    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
 808
 809    /* Now for Contents.. The process here is to do a make-like dependency
 810       check. Each contents file is verified to be newer than the package files
 811       that describe the debs it indexes. Since the package files contain
 812       hashes of the .debs this means they have not changed either so the
 813       contents must be up to date. */
 814    unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",UINT_MAX)*1024;
 815    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); I++)
 816    {
 817       // This record is not relevent
 818       if (I->ContentsDone == true ||
 819           I->Contents.empty() == true)
 820          continue;
 821
 822       // Do not do everything if the user specified sections.
 823       if (CmdL.FileSize() > 2 && I->PkgDone == false)
 824          continue;
 825
 826       struct stat A,B;
 827       if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
 828       {
 829          if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
 830          {
 831             _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
 832             continue;
 833          }
 834
 835          if (A.st_mtime > B.st_mtime)
 836             continue;
 837       }
 838
 839       if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
 840                          MaxContentsChange) == false)
 841          _error->DumpErrors();
 842
 843       // Hit the limit?
 844       if (MaxContentsChange == 0)
 845       {
 846          c1out << "Hit contents update byte limit" << endl;
 847          break;
 848       }
 849    }
 850
 851    struct timeval NewTime;
 852    gettimeofday(&NewTime,0);
 853    double Delta = NewTime.tv_sec - StartTime.tv_sec +
 854                   (NewTime.tv_usec - StartTime.tv_usec)/1000000.0;
 855    c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
 856          << " archives. Took " << TimeToStr((long)Delta) << endl;
 857
 858    return true;
 859 }
 860                                                                         /*}}}*/
 861 // Clean - Clean out the databases                                      /*{{{*/
 862 // ---------------------------------------------------------------------
 863 /* */
 864 bool Clean(CommandLine &CmdL)
 865 {
 866    if (CmdL.FileSize() != 2)
 867       return ShowHelp(CmdL);
 868
 869    // Read the configuration file.
 870    Configuration Setup;
 871    if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
 872       return false;
 873
 874    vector<PackageMap> PkgList;
 875    LoadTree(PkgList,Setup);
 876    LoadBinDir(PkgList,Setup);
 877
 878    // Sort by cache DB to improve IO locality.
 879    stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
 880
 881    string CacheDir = Setup.FindDir("Dir::CacheDir");
 882
 883    for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
 884    {
 885       c0out << I->BinCacheDB << endl;
 886       CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
 887       if (DB.Clean() == false)
 888          _error->DumpErrors();
 889
 890       string CacheDB = I->BinCacheDB;
 891       for (; I != PkgList.end() && I->BinCacheDB == CacheDB; I++);
 892    }
 893
 894    return true;
 895 }
 896                                                                         /*}}}*/
 897
 898 int main(int argc, const char *argv[])
 899 {
 900    CommandLine::Args Args[] = {
 901       {'h',"help","help",0},
 902       {0,"md5","APT::FTPArchive::MD5",0},
 903       {'v',"version","version",0},
 904       {'d',"db","APT::FTPArchive::DB",CommandLine::HasArg},
 905       {'s',"source-override","APT::FTPArchive::SourceOverride",CommandLine::HasArg},
 906       {'q',"quiet","quiet",CommandLine::IntLevel},
 907       {'q',"silent","quiet",CommandLine::IntLevel},
 908       {0,"delink","APT::FTPArchive::DeLinkAct",0},
 909       {0,"readonly","APT::FTPArchive::ReadOnlyDB",0},
 910       {0,"contents","APT::FTPArchive::Contents",0},
 911       {'c',"config-file",0,CommandLine::ConfigFile},
 912       {'o',"option",0,CommandLine::ArbItem},
 913       {0,0,0,0}};
 914    CommandLine::Dispatch Cmds[] = {{"packages",&SimpleGenPackages},
 915                                    {"contents",&SimpleGenContents},
 916                                    {"sources",&SimpleGenSources},
 917                                    {"release",&SimpleGenRelease},
 918                                    {"generate",&Generate},
 919                                    {"clean",&Clean},
 920                                    {"help",&ShowHelp},
 921                                    {0,0}};
 922
 923    // Parse the command line and initialize the package library
 924    CommandLine CmdL(Args,_config);
 925    if (CmdL.Parse(argc,argv) == false)
 926    {
 927       _error->DumpErrors();
 928       return 100;
 929    }
 930
 931    // See if the help should be shown
 932    if (_config->FindB("help") == true ||
 933        _config->FindB("version") == true ||
 934        CmdL.FileSize() == 0)
 935    {
 936       ShowHelp(CmdL);
 937       return 0;
 938    }
 939
 940    // Setup the output streams
 941    c0out.rdbuf(clog.rdbuf());
 942    c1out.rdbuf(clog.rdbuf());
 943    c2out.rdbuf(clog.rdbuf());
 944    Quiet = _config->FindI("quiet",0);
 945    if (Quiet > 0)
 946       c0out.rdbuf(devnull.rdbuf());
 947    if (Quiet > 1)
 948       c1out.rdbuf(devnull.rdbuf());
 949
 950    // Match the operation
 951    CmdL.DispatchArg(Cmds);
 952
 953    if (_error->empty() == false)
 954    {
 955       bool Errors = _error->PendingError();
 956       _error->DumpErrors();
 957       return Errors == true?100:0;
 958    }
 959    return 0;
 960 }