ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include <config.h>
  15
  16 #include <apt-pkg/strutl.h>
  17 #include <apt-pkg/error.h>
  18 #include <apt-pkg/configuration.h>
  19 #include <apt-pkg/aptconfiguration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/deblistparser.h>
  23
  24 #include <sys/types.h>
  25 #include <unistd.h>
  26 #include <ctime>
  27 #include <ftw.h>
  28 #include <fnmatch.h>
  29 #include <iostream>
  30 #include <sstream>
  31 #include <memory>
  32
  33 #include "writer.h"
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37
  38 #include <apti18n.h>
  39                                                                         /*}}}*/
  40 using namespace std;
  41 FTWScanner *FTWScanner::Owner;
  42
  43 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* */
  46 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  47                              const char *tag,
  48                              const char *rewrite,
  49                              const char *newtag = 0)
  50 {
  51     tfrd.Tag = tag;
  52     tfrd.Rewrite = rewrite;
  53     tfrd.NewTag = newtag;
  54 }
  55                                                                         /*}}}*/
  56
  57 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  58 // ---------------------------------------------------------------------
  59 /* */
  60 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  61 {
  62    ErrorPrinted = false;
  63    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  64
  65    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
  66    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
  67    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
  68    DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
  69 }
  70                                                                         /*}}}*/
  71 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  72 // ---------------------------------------------------------------------
  73 /* This is the FTW scanner, it processes each directory element in the
  74    directory tree. */
  75 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  76 {
  77    if (Flag == FTW_DNR)
  78    {
  79       Owner->NewLine(1);
  80       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  81    }
  82    if (Flag == FTW_NS)
  83    {
  84       Owner->NewLine(1);
  85       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  86    }
  87    if (Flag != FTW_F)
  88       return 0;
  89
  90    return ScannerFile(File, true);
  91 }
  92                                                                         /*}}}*/
  93 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  94 // ---------------------------------------------------------------------
  95 /* */
  96 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  97 {
  98    const char *LastComponent = strrchr(File, '/');
  99    char *RealPath = NULL;
 100
 101    if (LastComponent == NULL)
 102       LastComponent = File;
 103    else
 104       LastComponent++;
 105
 106    vector<string>::const_iterator I;
 107    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 108    {
 109       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 110          break;
 111    }
 112    if (I == Owner->Patterns.end())
 113       return 0;
 114
 115    /* Process it. If the file is a link then resolve it into an absolute
 116       name.. This works best if the directory components the scanner are
 117       given are not links themselves. */
 118    char Jnk[2];
 119    Owner->OriginalPath = File;
 120    if (ReadLink &&
 121        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 122        (RealPath = realpath(File,NULL)) != 0)
 123    {
 124       Owner->DoPackage(RealPath);
 125       free(RealPath);
 126    }
 127    else
 128       Owner->DoPackage(File);
 129
 130    if (_error->empty() == false)
 131    {
 132       // Print any errors or warnings found
 133       string Err;
 134       bool SeenPath = false;
 135       while (_error->empty() == false)
 136       {
 137          Owner->NewLine(1);
 138
 139          bool const Type = _error->PopMessage(Err);
 140          if (Type == true)
 141             cerr << _("E: ") << Err << endl;
 142          else
 143             cerr << _("W: ") << Err << endl;
 144
 145          if (Err.find(File) != string::npos)
 146             SeenPath = true;
 147       }
 148
 149       if (SeenPath == false)
 150          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 151       return 0;
 152    }
 153
 154    return 0;
 155 }
 156                                                                         /*}}}*/
 157 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* */
 160 bool FTWScanner::RecursiveScan(string const &Dir)
 161 {
 162    char *RealPath = NULL;
 163    /* If noprefix is set then jam the scan root in, so we don't generate
 164       link followed paths out of control */
 165    if (InternalPrefix.empty() == true)
 166    {
 167       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 168          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 169       InternalPrefix = RealPath;
 170       free(RealPath);
 171    }
 172
 173    // Do recursive directory searching
 174    Owner = this;
 175    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 176
 177    // Error treewalking?
 178    if (Res != 0)
 179    {
 180       if (_error->PendingError() == false)
 181          _error->Errno("ftw",_("Tree walking failed"));
 182       return false;
 183    }
 184
 185    return true;
 186 }
 187                                                                         /*}}}*/
 188 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 189 // ---------------------------------------------------------------------
 190 /* This is an alternative to using FTW to locate files, it reads the list
 191    of files from another file. */
 192 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 193 {
 194    char *RealPath = NULL;
 195    /* If noprefix is set then jam the scan root in, so we don't generate
 196       link followed paths out of control */
 197    if (InternalPrefix.empty() == true)
 198    {
 199       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 200          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 201       InternalPrefix = RealPath;
 202       free(RealPath);
 203    }
 204
 205    Owner = this;
 206    FILE *List = fopen(File.c_str(),"r");
 207    if (List == 0)
 208       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 209
 210    /* We are a tad tricky here.. We prefix the buffer with the directory
 211       name, that way if we need a full path with just use line.. Sneaky and
 212       fully evil. */
 213    char Line[1000];
 214    char *FileStart;
 215    if (Dir.empty() == true || Dir.end()[-1] != '/')
 216       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 217    else
 218       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 219    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 220    {
 221       char *FileName = _strstrip(FileStart);
 222       if (FileName[0] == 0)
 223          continue;
 224
 225       if (FileName[0] != '/')
 226       {
 227          if (FileName != FileStart)
 228             memmove(FileStart,FileName,strlen(FileStart));
 229          FileName = Line;
 230       }
 231
 232 #if 0
 233       struct stat St;
 234       int Flag = FTW_F;
 235       if (stat(FileName,&St) != 0)
 236          Flag = FTW_NS;
 237 #endif
 238
 239       if (ScannerFile(FileName, false) != 0)
 240          break;
 241    }
 242
 243    fclose(List);
 244    return true;
 245 }
 246                                                                         /*}}}*/
 247 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 251                         unsigned long long &DeLinkBytes,
 252                         unsigned long long const &FileSize)
 253 {
 254    // See if this isn't an internaly prefix'd file name.
 255    if (InternalPrefix.empty() == false &&
 256        InternalPrefix.length() < FileName.length() &&
 257        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 258                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 259    {
 260       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 261       {
 262          // Tidy up the display
 263          if (DeLinkBytes == 0)
 264             cout << endl;
 265
 266          NewLine(1);
 267          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 268                     SizeToStr(FileSize).c_str());
 269          c1out << flush;
 270
 271          if (NoLinkAct == false)
 272          {
 273             char OldLink[400];
 274             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 275                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 276             else
 277             {
 278                if (unlink(OriginalPath) != 0)
 279                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 280                else
 281                {
 282                   if (link(FileName.c_str(),OriginalPath) != 0)
 283                   {
 284                      // Panic! Restore the symlink
 285                      symlink(OldLink,OriginalPath);
 286                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 287                                           FileName.c_str(),
 288                                           OriginalPath);
 289                   }
 290                }
 291             }
 292          }
 293
 294          DeLinkBytes += FileSize;
 295          if (DeLinkBytes/1024 >= DeLinkLimit)
 296             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 297       }
 298
 299       FileName = OriginalPath;
 300    }
 301
 302    return true;
 303 }
 304                                                                         /*}}}*/
 305
 306 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 307 // ---------------------------------------------------------------------
 308 /* */
 309 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 310                                string const &Arch) :
 311    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 312 {
 313    Output = stdout;
 314    SetExts(".deb .udeb");
 315    DeLinkLimit = 0;
 316
 317    // Process the command line options
 318    DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
 319    DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
 320    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
 321    DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA512",true);
 322    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 323    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 324    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 325    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 326
 327    if (Db.Loaded() == false)
 328       DoContents = false;
 329
 330    // Read the override file
 331    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 332       return;
 333    else
 334       NoOverride = true;
 335
 336    if (ExtOverrides.empty() == false)
 337       Over.ReadExtraOverride(ExtOverrides);
 338
 339    _error->DumpErrors();
 340 }
 341                                                                         /*}}}*/
 342 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 343 // ---------------------------------------------------------------------
 344 /* */
 345 bool FTWScanner::SetExts(string const &Vals)
 346 {
 347    ClearPatterns();
 348    string::size_type Start = 0;
 349    while (Start <= Vals.length()-1)
 350    {
 351       string::size_type const Space = Vals.find(' ',Start);
 352       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 353       if ( Arch.empty() == false )
 354       {
 355          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 356          AddPattern(string("*_all") + Vals.substr(Start, Length));
 357       }
 358       else
 359          AddPattern(string("*") + Vals.substr(Start, Length));
 360
 361       Start += Length + 1;
 362    }
 363
 364    return true;
 365 }
 366
 367                                                                         /*}}}*/
 368 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 369 // ---------------------------------------------------------------------
 370 /* This method takes a package and gets its control information and
 371    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 372    rewritten and the path/size/hash appended. */
 373 bool PackagesWriter::DoPackage(string FileName)
 374 {
 375    // Pull all the data we need form the DB
 376    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
 377                   == false)
 378    {
 379       return false;
 380    }
 381
 382    unsigned long long FileSize = Db.GetFileSize();
 383    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 384       return false;
 385
 386    // Lookup the overide information
 387    pkgTagSection &Tags = Db.Control.Section;
 388    string Package = Tags.FindS("Package");
 389    string Architecture;
 390    // if we generate a Packages file for a given arch, we use it to
 391    // look for overrides. if we run in "simple" mode without the
 392    // "Architecures" variable in the config we use the architecure value
 393    // from the deb file
 394    if(Arch != "")
 395       Architecture = Arch;
 396    else
 397       Architecture = Tags.FindS("Architecture");
 398    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 399
 400    if (Package.empty() == true)
 401       return _error->Error(_("Archive had no package field"));
 402
 403    // If we need to do any rewriting of the header do it now..
 404    if (OverItem.get() == 0)
 405    {
 406       if (NoOverride == false)
 407       {
 408          NewLine(1);
 409          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 410       }
 411
 412       OverItem = auto_ptr<Override::Item>(new Override::Item);
 413       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 414       OverItem->Priority = Tags.FindS("Priority");
 415    }
 416
 417    char Size[40];
 418    sprintf(Size,"%llu", (unsigned long long) FileSize);
 419
 420    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 421    string NewFileName;
 422    if (DirStrip.empty() == false &&
 423        FileName.length() > DirStrip.length() &&
 424        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 425                  DirStrip.begin(),DirStrip.end()) == 0)
 426       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 427    else
 428       NewFileName = FileName;
 429    if (PathPrefix.empty() == false)
 430       NewFileName = flCombine(PathPrefix,NewFileName);
 431
 432    /* Configuration says we don't want to include the long Description
 433       in the package file - instead we want to ship a separated file */
 434    string desc;
 435    if (LongDescription == false) {
 436       desc = Tags.FindS("Description").append("\n");
 437       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 438    }
 439
 440    // This lists all the changes to the fields we are going to make.
 441    // (7 hardcoded + maintainer + suggests + end marker)
 442    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 443
 444    unsigned int End = 0;
 445    SetTFRewriteData(Changes[End++], "Size", Size);
 446    if (DoMD5 == true)
 447       SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 448    if (DoSHA1 == true)
 449       SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 450    if (DoSHA256 == true)
 451       SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 452    if (DoSHA512 == true)
 453       SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
 454    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 455    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 456    SetTFRewriteData(Changes[End++], "Status", 0);
 457    SetTFRewriteData(Changes[End++], "Optional", 0);
 458
 459    string DescriptionMd5;
 460    if (LongDescription == false) {
 461       MD5Summation descmd5;
 462       descmd5.Add(desc.c_str());
 463       DescriptionMd5 = descmd5.Result().Value();
 464       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 465       if (TransWriter != NULL)
 466          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 467    }
 468
 469    // Rewrite the maintainer field if necessary
 470    bool MaintFailed;
 471    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 472    if (MaintFailed == true)
 473    {
 474       if (NoOverride == false)
 475       {
 476          NewLine(1);
 477          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 478                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 479       }
 480    }
 481
 482    if (NewMaint.empty() == false)
 483       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 484
 485    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 486       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 487       but dpkg does this append bit. So we do the append bit, at least that way the
 488       status file and package file will remain similar. There are other transforms
 489       but optional is the only legacy one still in use for some lazy reason. */
 490    string OptionalStr = Tags.FindS("Optional");
 491    if (OptionalStr.empty() == false)
 492    {
 493       if (Tags.FindS("Suggests").empty() == false)
 494          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 495       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 496    }
 497
 498    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 499         I != OverItem->FieldOverride.end(); I++)
 500       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 501
 502    SetTFRewriteData(Changes[End++], 0, 0);
 503
 504    // Rewrite and store the fields.
 505    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 506       return false;
 507    fprintf(Output,"\n");
 508
 509    return Db.Finish();
 510 }
 511                                                                         /*}}}*/
 512
 513 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 514 // ---------------------------------------------------------------------
 515 /* Create a Translation-Master file for this Packages file */
 516 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 517                                         mode_t const &Permissions) : Output(NULL),
 518                                                         RefCounter(0)
 519 {
 520    if (File.empty() == true)
 521       return;
 522
 523    Comp = new MultiCompress(File, TransCompress, Permissions);
 524    Output = Comp->Input;
 525 }
 526                                                                         /*}}}*/
 527 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 528 // ---------------------------------------------------------------------
 529 /* Create a Translation-Master file for this Packages file */
 530 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 531                                   string const &MD5)
 532 {
 533    if (Output == NULL)
 534       return true;
 535
 536    // Different archs can include different versions and therefore
 537    // different descriptions - so we need to check for both name and md5.
 538    string const Record = Pkg + ":" + MD5;
 539
 540    if (Included.find(Record) != Included.end())
 541       return true;
 542
 543    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 544            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 545
 546    Included.insert(Record);
 547    return true;
 548 }
 549                                                                         /*}}}*/
 550 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* */
 553 TranslationWriter::~TranslationWriter()
 554 {
 555    if (Comp == NULL)
 556       return;
 557
 558    delete Comp;
 559 }
 560                                                                         /*}}}*/
 561
 562 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 563 // ---------------------------------------------------------------------
 564 /* */
 565 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 566                              string const &ExtOverrides)
 567 {
 568    Output = stdout;
 569    AddPattern("*.dsc");
 570    DeLinkLimit = 0;
 571    Buffer = 0;
 572    BufSize = 0;
 573
 574    // Process the command line options
 575    DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
 576    DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
 577    DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
 578    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 579
 580    // Read the override file
 581    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 582       return;
 583    else
 584       NoOverride = true;
 585
 586    // WTF?? The logic above: if we can't read binary overrides, don't even try
 587    // reading source overrides. if we can read binary overrides, then say there
 588    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 589
 590    if (ExtOverrides.empty() == false)
 591       SOver.ReadExtraOverride(ExtOverrides);
 592
 593    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 594       SOver.ReadOverride(SOverrides,true);
 595 }
 596                                                                         /*}}}*/
 597 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* */
 600 bool SourcesWriter::DoPackage(string FileName)
 601 {
 602    // Open the archive
 603    FileFd F(FileName,FileFd::ReadOnly);
 604    if (_error->PendingError() == true)
 605       return false;
 606
 607    // Stat the file for later
 608    struct stat St;
 609    if (fstat(F.Fd(),&St) != 0)
 610       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 611
 612    if (St.st_size > 128*1024)
 613       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 614
 615    if (BufSize < (unsigned long long)St.st_size+1)
 616    {
 617       BufSize = St.st_size+1;
 618       Buffer = (char *)realloc(Buffer,St.st_size+1);
 619    }
 620
 621    if (F.Read(Buffer,St.st_size) == false)
 622       return false;
 623
 624    // Hash the file
 625    char *Start = Buffer;
 626    char *BlkEnd = Buffer + St.st_size;
 627
 628    MD5Summation MD5;
 629    SHA1Summation SHA1;
 630    SHA256Summation SHA256;
 631    SHA256Summation SHA512;
 632
 633    if (DoMD5 == true)
 634       MD5.Add((unsigned char *)Start,BlkEnd - Start);
 635    if (DoSHA1 == true)
 636       SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 637    if (DoSHA256 == true)
 638       SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 639    if (DoSHA512 == true)
 640       SHA512.Add((unsigned char *)Start,BlkEnd - Start);
 641
 642    // Add an extra \n to the end, just in case
 643    *BlkEnd++ = '\n';
 644
 645    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 646       before */
 647    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 648    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 649    {
 650       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 651       {
 652          MsgEnd[1] = '\n';
 653          break;
 654       }
 655    }
 656
 657    /* Read records until we locate the Source record. This neatly skips the
 658       GPG header (which is RFC822 formed) without any trouble. */
 659    pkgTagSection Tags;
 660    do
 661    {
 662       unsigned Pos;
 663       if (Tags.Scan(Start,BlkEnd - Start) == false)
 664          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 665       if (Tags.Find("Source",Pos) == true)
 666          break;
 667       Start += Tags.size();
 668    }
 669    while (1);
 670    Tags.Trim();
 671
 672    // Lookup the overide information, finding first the best priority.
 673    string BestPrio;
 674    string Bins = Tags.FindS("Binary");
 675    char Buffer[Bins.length() + 1];
 676    auto_ptr<Override::Item> OverItem(0);
 677    if (Bins.empty() == false)
 678    {
 679       strcpy(Buffer,Bins.c_str());
 680
 681       // Ignore too-long errors.
 682       char *BinList[400];
 683       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 684
 685       // Look at all the binaries
 686       unsigned char BestPrioV = pkgCache::State::Extra;
 687       for (unsigned I = 0; BinList[I] != 0; I++)
 688       {
 689          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 690          if (Itm.get() == 0)
 691             continue;
 692
 693          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 694          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 695          {
 696             BestPrioV = NewPrioV;
 697             BestPrio = Itm->Priority;
 698          }
 699
 700          if (OverItem.get() == 0)
 701             OverItem = Itm;
 702       }
 703    }
 704
 705    // If we need to do any rewriting of the header do it now..
 706    if (OverItem.get() == 0)
 707    {
 708       if (NoOverride == false)
 709       {
 710          NewLine(1);
 711          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 712       }
 713
 714       OverItem = auto_ptr<Override::Item>(new Override::Item);
 715    }
 716
 717    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 718    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 719    if (SOverItem.get() == 0)
 720    {
 721       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 722       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 723       if (SOverItem.get() == 0)
 724       {
 725         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 726          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 727          *SOverItem = *OverItem;
 728       }
 729    }
 730
 731    // Add the dsc to the files hash list
 732    string const strippedName = flNotDir(FileName);
 733    std::ostringstream ostreamFiles;
 734    if (DoMD5 == true && Tags.Exists("Files"))
 735       ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 736                    << strippedName << "\n " << Tags.FindS("Files");
 737    string const Files = ostreamFiles.str();
 738
 739    std::ostringstream ostreamSha1;
 740    if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
 741       ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 742                    << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 743    string const ChecksumsSha1 = ostreamSha1.str();
 744
 745    std::ostringstream ostreamSha256;
 746    if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
 747       ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 748                    << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 749    string const ChecksumsSha256 = ostreamSha256.str();
 750
 751    std::ostringstream ostreamSha512;
 752    if (Tags.Exists("Checksums-Sha512"))
 753       ostreamSha512 << "\n " << string(SHA512.Result()) << " " << St.st_size << " "
 754                    << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
 755    string const ChecksumsSha512 = ostreamSha512.str();
 756
 757    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 758    string NewFileName;
 759    if (DirStrip.empty() == false &&
 760        FileName.length() > DirStrip.length() &&
 761        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 762       NewFileName = string(OriginalPath + DirStrip.length());
 763    else
 764       NewFileName = OriginalPath;
 765    if (PathPrefix.empty() == false)
 766       NewFileName = flCombine(PathPrefix,NewFileName);
 767
 768    string Directory = flNotFile(OriginalPath);
 769    string Package = Tags.FindS("Source");
 770
 771    // Perform the delinking operation over all of the files
 772    string ParseJnk;
 773    const char *C = Files.c_str();
 774    char *RealPath = NULL;
 775    for (;isspace(*C); C++);
 776    while (*C != 0)
 777    {
 778       // Parse each of the elements
 779       if (ParseQuoteWord(C,ParseJnk) == false ||
 780           ParseQuoteWord(C,ParseJnk) == false ||
 781           ParseQuoteWord(C,ParseJnk) == false)
 782          return _error->Error("Error parsing file record");
 783
 784       char Jnk[2];
 785       string OriginalPath = Directory + ParseJnk;
 786       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 787           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 788       {
 789          string RP = RealPath;
 790          free(RealPath);
 791          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 792             return false;
 793       }
 794    }
 795
 796    Directory = flNotFile(NewFileName);
 797    if (Directory.length() > 2)
 798       Directory.erase(Directory.end()-1);
 799
 800    // This lists all the changes to the fields we are going to make.
 801    // (5 hardcoded + checksums + maintainer + end marker)
 802    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 803
 804    unsigned int End = 0;
 805    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 806    if (Files.empty() == false)
 807       SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 808    if (ChecksumsSha1.empty() == false)
 809       SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 810    if (ChecksumsSha256.empty() == false)
 811       SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 812    if (ChecksumsSha512.empty() == false)
 813       SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
 814    if (Directory != "./")
 815       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 816    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 817    SetTFRewriteData(Changes[End++],"Status",0);
 818
 819    // Rewrite the maintainer field if necessary
 820    bool MaintFailed;
 821    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 822    if (MaintFailed == true)
 823    {
 824       if (NoOverride == false)
 825       {
 826          NewLine(1);
 827          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 828                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 829       }
 830    }
 831    if (NewMaint.empty() == false)
 832       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 833
 834    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 835         I != SOverItem->FieldOverride.end(); I++)
 836       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 837
 838    SetTFRewriteData(Changes[End++], 0, 0);
 839
 840    // Rewrite and store the fields.
 841    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 842       return false;
 843    fprintf(Output,"\n");
 844
 845    Stats.Packages++;
 846
 847    return true;
 848 }
 849                                                                         /*}}}*/
 850
 851 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 852 // ---------------------------------------------------------------------
 853 /* */
 854 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 855                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 856
 857 {
 858    SetExts(".deb");
 859    Output = stdout;
 860 }
 861                                                                         /*}}}*/
 862 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 863 // ---------------------------------------------------------------------
 864 /* If Package is the empty string the control record will be parsed to
 865    determine what the package name is. */
 866 bool ContentsWriter::DoPackage(string FileName, string Package)
 867 {
 868    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 869    {
 870       return false;
 871    }
 872
 873    // Parse the package name
 874    if (Package.empty() == true)
 875    {
 876       Package = Db.Control.Section.FindS("Package");
 877    }
 878
 879    Db.Contents.Add(Gen,Package);
 880
 881    return Db.Finish();
 882 }
 883                                                                         /*}}}*/
 884 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 885 // ---------------------------------------------------------------------
 886 /* */
 887 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 888 {
 889    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 890    if (_error->PendingError() == true)
 891       return false;
 892
 893    // Open the package file
 894    int CompFd = -1;
 895    pid_t Proc = -1;
 896    if (Pkgs.OpenOld(CompFd,Proc) == false)
 897       return false;
 898
 899    // No auto-close FD
 900    FileFd Fd(CompFd,false);
 901    pkgTagFile Tags(&Fd);
 902    if (_error->PendingError() == true)
 903    {
 904       Pkgs.CloseOld(CompFd,Proc);
 905       return false;
 906    }
 907
 908    // Parse.
 909    pkgTagSection Section;
 910    while (Tags.Step(Section) == true)
 911    {
 912       string File = flCombine(Prefix,Section.FindS("FileName"));
 913       string Package = Section.FindS("Section");
 914       if (Package.empty() == false && Package.end()[-1] != '/')
 915       {
 916          Package += '/';
 917          Package += Section.FindS("Package");
 918       }
 919       else
 920          Package += Section.FindS("Package");
 921
 922       DoPackage(File,Package);
 923       if (_error->empty() == false)
 924       {
 925          _error->Error("Errors apply to file '%s'",File.c_str());
 926          _error->DumpErrors();
 927       }
 928    }
 929
 930    // Tidy the compressor
 931    if (Pkgs.CloseOld(CompFd,Proc) == false)
 932       return false;
 933
 934    return true;
 935 }
 936
 937                                                                         /*}}}*/
 938
 939 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* */
 942 ReleaseWriter::ReleaseWriter(string const &DB)
 943 {
 944    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
 945    {
 946       AddPattern("Packages");
 947       AddPattern("Packages.gz");
 948       AddPattern("Packages.bz2");
 949       AddPattern("Packages.lzma");
 950       AddPattern("Packages.xz");
 951       AddPattern("Sources");
 952       AddPattern("Sources.gz");
 953       AddPattern("Sources.bz2");
 954       AddPattern("Sources.lzma");
 955       AddPattern("Sources.xz");
 956       AddPattern("Release");
 957       AddPattern("Index");
 958       AddPattern("md5sum.txt");
 959    }
 960    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 961
 962    Output = stdout;
 963    time_t const now = time(NULL);
 964
 965    setlocale(LC_TIME, "C");
 966
 967    char datestr[128];
 968    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 969                 gmtime(&now)) == 0)
 970    {
 971       datestr[0] = '\0';
 972    }
 973
 974    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 975    char validstr[128];
 976    if (now == validuntil ||
 977        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 978                 gmtime(&validuntil)) == 0)
 979    {
 980       validstr[0] = '\0';
 981    }
 982
 983    setlocale(LC_TIME, "");
 984
 985    map<string,string> Fields;
 986    Fields["Origin"] = "";
 987    Fields["Label"] = "";
 988    Fields["Suite"] = "";
 989    Fields["Version"] = "";
 990    Fields["Codename"] = "";
 991    Fields["Date"] = datestr;
 992    Fields["Valid-Until"] = validstr;
 993    Fields["Architectures"] = "";
 994    Fields["Components"] = "";
 995    Fields["Description"] = "";
 996
 997    for(map<string,string>::const_iterator I = Fields.begin();
 998        I != Fields.end();
 999        ++I)
1000    {
1001       string Config = string("APT::FTPArchive::Release::") + (*I).first;
1002       string Value = _config->Find(Config, (*I).second.c_str());
1003       if (Value == "")
1004          continue;
1005
1006       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1007    }
1008
1009    DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1010    DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1011    DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
1012 }
1013                                                                         /*}}}*/
1014 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
1015 // ---------------------------------------------------------------------
1016 bool ReleaseWriter::DoPackage(string FileName)
1017 {
1018    // Strip the DirStrip prefix from the FileName and add the PathPrefix
1019    string NewFileName;
1020    if (DirStrip.empty() == false &&
1021        FileName.length() > DirStrip.length() &&
1022        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1023                  DirStrip.begin(),DirStrip.end()) == 0)
1024    {
1025       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
1026       while (NewFileName[0] == '/')
1027          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
1028    }
1029    else
1030       NewFileName = FileName;
1031
1032    if (PathPrefix.empty() == false)
1033       NewFileName = flCombine(PathPrefix,NewFileName);
1034
1035    FileFd fd(FileName, FileFd::ReadOnly);
1036
1037    if (!fd.IsOpen())
1038    {
1039       return false;
1040    }
1041
1042    CheckSums[NewFileName].size = fd.Size();
1043
1044    Hashes hs;
1045    hs.AddFD(fd.Fd(), 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
1046    if (DoMD5 == true)
1047       CheckSums[NewFileName].MD5 = hs.MD5.Result();
1048    if (DoSHA1 == true)
1049       CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
1050    if (DoSHA256 == true)
1051       CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
1052    if (DoSHA512 == true)
1053       CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
1054    fd.Close();
1055
1056    return true;
1057 }
1058
1059                                                                         /*}}}*/
1060 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1061 // ---------------------------------------------------------------------
1062 void ReleaseWriter::Finish()
1063 {
1064    if (DoMD5 == true)
1065    {
1066       fprintf(Output, "MD5Sum:\n");
1067       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1068           I != CheckSums.end(); ++I)
1069       {
1070          fprintf(Output, " %s %16llu %s\n",
1071                  (*I).second.MD5.c_str(),
1072                  (*I).second.size,
1073                  (*I).first.c_str());
1074       }
1075    }
1076    if (DoSHA1 == true)
1077    {
1078       fprintf(Output, "SHA1:\n");
1079       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1080           I != CheckSums.end(); ++I)
1081       {
1082          fprintf(Output, " %s %16llu %s\n",
1083                  (*I).second.SHA1.c_str(),
1084                  (*I).second.size,
1085                  (*I).first.c_str());
1086       }
1087    }
1088    if (DoSHA256 == true)
1089    {
1090       fprintf(Output, "SHA256:\n");
1091       for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1092           I != CheckSums.end(); ++I)
1093       {
1094          fprintf(Output, " %s %16llu %s\n",
1095                  (*I).second.SHA256.c_str(),
1096                  (*I).second.size,
1097                  (*I).first.c_str());
1098       }
1099    }
1100
1101    fprintf(Output, "SHA512:\n");
1102    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1103        I != CheckSums.end();
1104        ++I)
1105    {
1106       fprintf(Output, " %s %16llu %s\n",
1107               (*I).second.SHA512.c_str(),
1108               (*I).second.size,
1109               (*I).first.c_str());
1110    }
1111
1112 }