ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/sha1.h>
  22 #include <apt-pkg/sha256.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <ctime>
  28 #include <ftw.h>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <sstream>
  32 #include <memory>
  33
  34 #include "cachedb.h"
  35 #include "apt-ftparchive.h"
  36 #include "multicompress.h"
  37                                                                         /*}}}*/
  38 using namespace std;
  39 FTWScanner *FTWScanner::Owner;
  40
  41 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  45                              const char *tag,
  46                              const char *rewrite,
  47                              const char *newtag = 0)
  48 {
  49     tfrd.Tag = tag;
  50     tfrd.Rewrite = rewrite;
  51     tfrd.NewTag = newtag;
  52 }
  53                                                                         /*}}}*/
  54
  55 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  56 // ---------------------------------------------------------------------
  57 /* */
  58 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  59 {
  60    ErrorPrinted = false;
  61    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  62 }
  63                                                                         /*}}}*/
  64 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* This is the FTW scanner, it processes each directory element in the
  67    directory tree. */
  68 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  69 {
  70    if (Flag == FTW_DNR)
  71    {
  72       Owner->NewLine(1);
  73       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  74    }
  75    if (Flag == FTW_NS)
  76    {
  77       Owner->NewLine(1);
  78       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  79    }
  80    if (Flag != FTW_F)
  81       return 0;
  82
  83    return ScannerFile(File, true);
  84 }
  85                                                                         /*}}}*/
  86 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* */
  89 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  90 {
  91    const char *LastComponent = strrchr(File, '/');
  92    char *RealPath = NULL;
  93
  94    if (LastComponent == NULL)
  95       LastComponent = File;
  96    else
  97       LastComponent++;
  98
  99    vector<string>::const_iterator I;
 100    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 101    {
 102       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 103          break;
 104    }
 105    if (I == Owner->Patterns.end())
 106       return 0;
 107
 108    /* Process it. If the file is a link then resolve it into an absolute
 109       name.. This works best if the directory components the scanner are
 110       given are not links themselves. */
 111    char Jnk[2];
 112    Owner->OriginalPath = File;
 113    if (ReadLink &&
 114        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 115        (RealPath = realpath(File,NULL)) != 0)
 116    {
 117       Owner->DoPackage(RealPath);
 118       free(RealPath);
 119    }
 120    else
 121       Owner->DoPackage(File);
 122
 123    if (_error->empty() == false)
 124    {
 125       // Print any errors or warnings found
 126       string Err;
 127       bool SeenPath = false;
 128       while (_error->empty() == false)
 129       {
 130          Owner->NewLine(1);
 131
 132          bool const Type = _error->PopMessage(Err);
 133          if (Type == true)
 134             cerr << _("E: ") << Err << endl;
 135          else
 136             cerr << _("W: ") << Err << endl;
 137
 138          if (Err.find(File) != string::npos)
 139             SeenPath = true;
 140       }
 141
 142       if (SeenPath == false)
 143          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 144       return 0;
 145    }
 146
 147    return 0;
 148 }
 149                                                                         /*}}}*/
 150 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 151 // ---------------------------------------------------------------------
 152 /* */
 153 bool FTWScanner::RecursiveScan(string const &Dir)
 154 {
 155    char *RealPath = NULL;
 156    /* If noprefix is set then jam the scan root in, so we don't generate
 157       link followed paths out of control */
 158    if (InternalPrefix.empty() == true)
 159    {
 160       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 161          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 162       InternalPrefix = RealPath;
 163       free(RealPath);
 164    }
 165
 166    // Do recursive directory searching
 167    Owner = this;
 168    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 169
 170    // Error treewalking?
 171    if (Res != 0)
 172    {
 173       if (_error->PendingError() == false)
 174          _error->Errno("ftw",_("Tree walking failed"));
 175       return false;
 176    }
 177
 178    return true;
 179 }
 180                                                                         /*}}}*/
 181 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 182 // ---------------------------------------------------------------------
 183 /* This is an alternative to using FTW to locate files, it reads the list
 184    of files from another file. */
 185 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 186 {
 187    char *RealPath = NULL;
 188    /* If noprefix is set then jam the scan root in, so we don't generate
 189       link followed paths out of control */
 190    if (InternalPrefix.empty() == true)
 191    {
 192       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 193          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 194       InternalPrefix = RealPath;
 195       free(RealPath);
 196    }
 197
 198    Owner = this;
 199    FILE *List = fopen(File.c_str(),"r");
 200    if (List == 0)
 201       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 202
 203    /* We are a tad tricky here.. We prefix the buffer with the directory
 204       name, that way if we need a full path with just use line.. Sneaky and
 205       fully evil. */
 206    char Line[1000];
 207    char *FileStart;
 208    if (Dir.empty() == true || Dir.end()[-1] != '/')
 209       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 210    else
 211       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 212    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 213    {
 214       char *FileName = _strstrip(FileStart);
 215       if (FileName[0] == 0)
 216          continue;
 217
 218       if (FileName[0] != '/')
 219       {
 220          if (FileName != FileStart)
 221             memmove(FileStart,FileName,strlen(FileStart));
 222          FileName = Line;
 223       }
 224
 225 #if 0
 226       struct stat St;
 227       int Flag = FTW_F;
 228       if (stat(FileName,&St) != 0)
 229          Flag = FTW_NS;
 230 #endif
 231
 232       if (ScannerFile(FileName, false) != 0)
 233          break;
 234    }
 235
 236    fclose(List);
 237    return true;
 238 }
 239                                                                         /*}}}*/
 240 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 241 // ---------------------------------------------------------------------
 242 /* */
 243 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 244                         unsigned long &DeLinkBytes,
 245                         off_t const &FileSize)
 246 {
 247    // See if this isn't an internaly prefix'd file name.
 248    if (InternalPrefix.empty() == false &&
 249        InternalPrefix.length() < FileName.length() &&
 250        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 251                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 252    {
 253       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 254       {
 255          // Tidy up the display
 256          if (DeLinkBytes == 0)
 257             cout << endl;
 258
 259          NewLine(1);
 260          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 261                     SizeToStr(FileSize).c_str());
 262          c1out << flush;
 263
 264          if (NoLinkAct == false)
 265          {
 266             char OldLink[400];
 267             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 268                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 269             else
 270             {
 271                if (unlink(OriginalPath) != 0)
 272                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 273                else
 274                {
 275                   if (link(FileName.c_str(),OriginalPath) != 0)
 276                   {
 277                      // Panic! Restore the symlink
 278                      symlink(OldLink,OriginalPath);
 279                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 280                                           FileName.c_str(),
 281                                           OriginalPath);
 282                   }
 283                }
 284             }
 285          }
 286
 287          DeLinkBytes += FileSize;
 288          if (DeLinkBytes/1024 >= DeLinkLimit)
 289             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 290       }
 291
 292       FileName = OriginalPath;
 293    }
 294
 295    return true;
 296 }
 297                                                                         /*}}}*/
 298
 299 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 300 // ---------------------------------------------------------------------
 301 /* */
 302 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 303                                string const &Arch) :
 304    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 305 {
 306    Output = stdout;
 307    SetExts(".deb .udeb");
 308    DeLinkLimit = 0;
 309
 310    // Process the command line options
 311    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
 312    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
 313    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
 314    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 315    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 316    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 317    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 318
 319    if (Db.Loaded() == false)
 320       DoContents = false;
 321
 322    // Read the override file
 323    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 324       return;
 325    else
 326       NoOverride = true;
 327
 328    if (ExtOverrides.empty() == false)
 329       Over.ReadExtraOverride(ExtOverrides);
 330
 331    _error->DumpErrors();
 332 }
 333                                                                         /*}}}*/
 334 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 335 // ---------------------------------------------------------------------
 336 /* */
 337 bool FTWScanner::SetExts(string const &Vals)
 338 {
 339    ClearPatterns();
 340    string::size_type Start = 0;
 341    while (Start <= Vals.length()-1)
 342    {
 343       string::size_type const Space = Vals.find(' ',Start);
 344       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 345       if ( Arch.empty() == false )
 346       {
 347          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 348          AddPattern(string("*_all") + Vals.substr(Start, Length));
 349       }
 350       else
 351          AddPattern(string("*") + Vals.substr(Start, Length));
 352
 353       Start += Length + 1;
 354    }
 355
 356    return true;
 357 }
 358
 359                                                                         /*}}}*/
 360 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 361 // ---------------------------------------------------------------------
 362 /* This method takes a package and gets its control information and
 363    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 364    rewritten and the path/size/hash appended. */
 365 bool PackagesWriter::DoPackage(string FileName)
 366 {
 367    // Pull all the data we need form the DB
 368    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoAlwaysStat)
 369                   == false)
 370    {
 371       return false;
 372    }
 373
 374    off_t FileSize = Db.GetFileSize();
 375    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 376       return false;
 377
 378    // Lookup the overide information
 379    pkgTagSection &Tags = Db.Control.Section;
 380    string Package = Tags.FindS("Package");
 381    string Architecture;
 382    // if we generate a Packages file for a given arch, we use it to
 383    // look for overrides. if we run in "simple" mode without the
 384    // "Architecures" variable in the config we use the architecure value
 385    // from the deb file
 386    if(Arch != "")
 387       Architecture = Arch;
 388    else
 389       Architecture = Tags.FindS("Architecture");
 390    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 391
 392    if (Package.empty() == true)
 393       return _error->Error(_("Archive had no package field"));
 394
 395    // If we need to do any rewriting of the header do it now..
 396    if (OverItem.get() == 0)
 397    {
 398       if (NoOverride == false)
 399       {
 400          NewLine(1);
 401          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 402       }
 403
 404       OverItem = auto_ptr<Override::Item>(new Override::Item);
 405       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 406       OverItem->Priority = Tags.FindS("Priority");
 407    }
 408
 409    char Size[40];
 410    sprintf(Size,"%lu", (unsigned long) FileSize);
 411
 412    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 413    string NewFileName;
 414    if (DirStrip.empty() == false &&
 415        FileName.length() > DirStrip.length() &&
 416        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 417                  DirStrip.begin(),DirStrip.end()) == 0)
 418       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 419    else
 420       NewFileName = FileName;
 421    if (PathPrefix.empty() == false)
 422       NewFileName = flCombine(PathPrefix,NewFileName);
 423
 424    /* Configuration says we don't want to include the long Description
 425       in the package file - instead we want to ship a separated file */
 426    string desc;
 427    if (LongDescription == false) {
 428       desc = Tags.FindS("Description").append("\n");
 429       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 430    }
 431
 432    // This lists all the changes to the fields we are going to make.
 433    // (7 hardcoded + maintainer + suggests + end marker)
 434    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 435
 436    unsigned int End = 0;
 437    SetTFRewriteData(Changes[End++], "Size", Size);
 438    SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 439    SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 440    SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 441    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 442    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 443    SetTFRewriteData(Changes[End++], "Status", 0);
 444    SetTFRewriteData(Changes[End++], "Optional", 0);
 445
 446    string DescriptionMd5;
 447    if (LongDescription == false) {
 448       MD5Summation descmd5;
 449       descmd5.Add(desc.c_str());
 450       DescriptionMd5 = descmd5.Result().Value();
 451       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 452       if (TransWriter != NULL)
 453          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 454    }
 455
 456    // Rewrite the maintainer field if necessary
 457    bool MaintFailed;
 458    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 459    if (MaintFailed == true)
 460    {
 461       if (NoOverride == false)
 462       {
 463          NewLine(1);
 464          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 465                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 466       }
 467    }
 468
 469    if (NewMaint.empty() == false)
 470       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 471
 472    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 473       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 474       but dpkg does this append bit. So we do the append bit, at least that way the
 475       status file and package file will remain similar. There are other transforms
 476       but optional is the only legacy one still in use for some lazy reason. */
 477    string OptionalStr = Tags.FindS("Optional");
 478    if (OptionalStr.empty() == false)
 479    {
 480       if (Tags.FindS("Suggests").empty() == false)
 481          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 482       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 483    }
 484
 485    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 486         I != OverItem->FieldOverride.end(); I++)
 487       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 488
 489    SetTFRewriteData(Changes[End++], 0, 0);
 490
 491    // Rewrite and store the fields.
 492    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 493       return false;
 494    fprintf(Output,"\n");
 495
 496    return Db.Finish();
 497 }
 498                                                                         /*}}}*/
 499
 500 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 501 // ---------------------------------------------------------------------
 502 /* Create a Translation-Master file for this Packages file */
 503 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 504                                         mode_t const &Permissions) : Output(NULL),
 505                                                         RefCounter(0)
 506 {
 507    if (File.empty() == true)
 508       return;
 509
 510    Comp = new MultiCompress(File, TransCompress, Permissions);
 511    Output = Comp->Input;
 512 }
 513                                                                         /*}}}*/
 514 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 515 // ---------------------------------------------------------------------
 516 /* Create a Translation-Master file for this Packages file */
 517 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 518                                   string const &MD5)
 519 {
 520    if (Output == NULL)
 521       return true;
 522
 523    // Different archs can include different versions and therefore
 524    // different descriptions - so we need to check for both name and md5.
 525    string const Record = Pkg + ":" + MD5;
 526
 527    if (Included.find(Record) != Included.end())
 528       return true;
 529
 530    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 531            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 532
 533    Included.insert(Record);
 534    return true;
 535 }
 536                                                                         /*}}}*/
 537 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 538 // ---------------------------------------------------------------------
 539 /* */
 540 TranslationWriter::~TranslationWriter()
 541 {
 542    if (Comp == NULL)
 543       return;
 544
 545    delete Comp;
 546 }
 547                                                                         /*}}}*/
 548
 549 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* */
 552 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 553                              string const &ExtOverrides)
 554 {
 555    Output = stdout;
 556    AddPattern("*.dsc");
 557    DeLinkLimit = 0;
 558    Buffer = 0;
 559    BufSize = 0;
 560
 561    // Process the command line options
 562    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 563
 564    // Read the override file
 565    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 566       return;
 567    else
 568       NoOverride = true;
 569
 570    // WTF?? The logic above: if we can't read binary overrides, don't even try
 571    // reading source overrides. if we can read binary overrides, then say there
 572    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 573
 574    if (ExtOverrides.empty() == false)
 575       SOver.ReadExtraOverride(ExtOverrides);
 576
 577    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 578       SOver.ReadOverride(SOverrides,true);
 579 }
 580                                                                         /*}}}*/
 581 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* */
 584 bool SourcesWriter::DoPackage(string FileName)
 585 {
 586    // Open the archive
 587    FileFd F(FileName,FileFd::ReadOnly);
 588    if (_error->PendingError() == true)
 589       return false;
 590
 591    // Stat the file for later
 592    struct stat St;
 593    if (fstat(F.Fd(),&St) != 0)
 594       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 595
 596    if (St.st_size > 128*1024)
 597       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 598
 599    if (BufSize < (unsigned)St.st_size+1)
 600    {
 601       BufSize = St.st_size+1;
 602       Buffer = (char *)realloc(Buffer,St.st_size+1);
 603    }
 604
 605    if (F.Read(Buffer,St.st_size) == false)
 606       return false;
 607
 608    // Hash the file
 609    char *Start = Buffer;
 610    char *BlkEnd = Buffer + St.st_size;
 611    MD5Summation MD5;
 612    MD5.Add((unsigned char *)Start,BlkEnd - Start);
 613
 614    SHA1Summation SHA1;
 615    SHA256Summation SHA256;
 616    SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 617    SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 618
 619    // Add an extra \n to the end, just in case
 620    *BlkEnd++ = '\n';
 621
 622    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 623       before */
 624    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 625    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 626    {
 627       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 628       {
 629          MsgEnd[1] = '\n';
 630          break;
 631       }
 632    }
 633
 634    /* Read records until we locate the Source record. This neatly skips the
 635       GPG header (which is RFC822 formed) without any trouble. */
 636    pkgTagSection Tags;
 637    do
 638    {
 639       unsigned Pos;
 640       if (Tags.Scan(Start,BlkEnd - Start) == false)
 641          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 642       if (Tags.Find("Source",Pos) == true)
 643          break;
 644       Start += Tags.size();
 645    }
 646    while (1);
 647    Tags.Trim();
 648
 649    // Lookup the overide information, finding first the best priority.
 650    string BestPrio;
 651    string Bins = Tags.FindS("Binary");
 652    char Buffer[Bins.length() + 1];
 653    auto_ptr<Override::Item> OverItem(0);
 654    if (Bins.empty() == false)
 655    {
 656       strcpy(Buffer,Bins.c_str());
 657
 658       // Ignore too-long errors.
 659       char *BinList[400];
 660       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 661
 662       // Look at all the binaries
 663       unsigned char BestPrioV = pkgCache::State::Extra;
 664       for (unsigned I = 0; BinList[I] != 0; I++)
 665       {
 666          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 667          if (Itm.get() == 0)
 668             continue;
 669
 670          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 671          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 672          {
 673             BestPrioV = NewPrioV;
 674             BestPrio = Itm->Priority;
 675          }
 676
 677          if (OverItem.get() == 0)
 678             OverItem = Itm;
 679       }
 680    }
 681
 682    // If we need to do any rewriting of the header do it now..
 683    if (OverItem.get() == 0)
 684    {
 685       if (NoOverride == false)
 686       {
 687          NewLine(1);
 688          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 689       }
 690
 691       OverItem = auto_ptr<Override::Item>(new Override::Item);
 692    }
 693
 694    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 695    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 696    if (SOverItem.get() == 0)
 697    {
 698       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 699       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 700       if (SOverItem.get() == 0)
 701       {
 702         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 703          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 704          *SOverItem = *OverItem;
 705       }
 706    }
 707
 708    // Add the dsc to the files hash list
 709    string const strippedName = flNotDir(FileName);
 710    std::ostringstream ostreamFiles;
 711    ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
 712                 << strippedName << "\n " << Tags.FindS("Files");
 713    string const Files = ostreamFiles.str();
 714
 715    std::ostringstream ostreamSha1;
 716    ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
 717                 << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
 718    string const ChecksumsSha1 = ostreamSha1.str();
 719
 720    std::ostringstream ostreamSha256;
 721    ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
 722                 << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
 723    string const ChecksumsSha256 = ostreamSha256.str();
 724
 725    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 726    string NewFileName;
 727    if (DirStrip.empty() == false &&
 728        FileName.length() > DirStrip.length() &&
 729        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 730       NewFileName = string(OriginalPath + DirStrip.length());
 731    else
 732       NewFileName = OriginalPath;
 733    if (PathPrefix.empty() == false)
 734       NewFileName = flCombine(PathPrefix,NewFileName);
 735
 736    string Directory = flNotFile(OriginalPath);
 737    string Package = Tags.FindS("Source");
 738
 739    // Perform the delinking operation over all of the files
 740    string ParseJnk;
 741    const char *C = Files.c_str();
 742    char *RealPath = NULL;
 743    for (;isspace(*C); C++);
 744    while (*C != 0)
 745    {
 746       // Parse each of the elements
 747       if (ParseQuoteWord(C,ParseJnk) == false ||
 748           ParseQuoteWord(C,ParseJnk) == false ||
 749           ParseQuoteWord(C,ParseJnk) == false)
 750          return _error->Error("Error parsing file record");
 751
 752       char Jnk[2];
 753       string OriginalPath = Directory + ParseJnk;
 754       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 755           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 756       {
 757          string RP = RealPath;
 758          free(RealPath);
 759          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 760             return false;
 761       }
 762    }
 763
 764    Directory = flNotFile(NewFileName);
 765    if (Directory.length() > 2)
 766       Directory.erase(Directory.end()-1);
 767
 768    // This lists all the changes to the fields we are going to make.
 769    // (5 hardcoded + checksums + maintainer + end marker)
 770    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 771
 772    unsigned int End = 0;
 773    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 774    SetTFRewriteData(Changes[End++],"Files",Files.c_str());
 775    SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
 776    SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
 777    if (Directory != "./")
 778       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 779    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 780    SetTFRewriteData(Changes[End++],"Status",0);
 781
 782    // Rewrite the maintainer field if necessary
 783    bool MaintFailed;
 784    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 785    if (MaintFailed == true)
 786    {
 787       if (NoOverride == false)
 788       {
 789          NewLine(1);
 790          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 791                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 792       }
 793    }
 794    if (NewMaint.empty() == false)
 795       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 796
 797    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 798         I != SOverItem->FieldOverride.end(); I++)
 799       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 800
 801    SetTFRewriteData(Changes[End++], 0, 0);
 802
 803    // Rewrite and store the fields.
 804    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 805       return false;
 806    fprintf(Output,"\n");
 807
 808    Stats.Packages++;
 809
 810    return true;
 811 }
 812                                                                         /*}}}*/
 813
 814 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 815 // ---------------------------------------------------------------------
 816 /* */
 817 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 818                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 819
 820 {
 821    SetExts(".deb");
 822    Output = stdout;
 823 }
 824                                                                         /*}}}*/
 825 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 826 // ---------------------------------------------------------------------
 827 /* If Package is the empty string the control record will be parsed to
 828    determine what the package name is. */
 829 bool ContentsWriter::DoPackage(string FileName, string Package)
 830 {
 831    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 832    {
 833       return false;
 834    }
 835
 836    // Parse the package name
 837    if (Package.empty() == true)
 838    {
 839       Package = Db.Control.Section.FindS("Package");
 840    }
 841
 842    Db.Contents.Add(Gen,Package);
 843
 844    return Db.Finish();
 845 }
 846                                                                         /*}}}*/
 847 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* */
 850 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 851 {
 852    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 853    if (_error->PendingError() == true)
 854       return false;
 855
 856    // Open the package file
 857    int CompFd = -1;
 858    pid_t Proc = -1;
 859    if (Pkgs.OpenOld(CompFd,Proc) == false)
 860       return false;
 861
 862    // No auto-close FD
 863    FileFd Fd(CompFd,false);
 864    pkgTagFile Tags(&Fd);
 865    if (_error->PendingError() == true)
 866    {
 867       Pkgs.CloseOld(CompFd,Proc);
 868       return false;
 869    }
 870
 871    // Parse.
 872    pkgTagSection Section;
 873    while (Tags.Step(Section) == true)
 874    {
 875       string File = flCombine(Prefix,Section.FindS("FileName"));
 876       string Package = Section.FindS("Section");
 877       if (Package.empty() == false && Package.end()[-1] != '/')
 878       {
 879          Package += '/';
 880          Package += Section.FindS("Package");
 881       }
 882       else
 883          Package += Section.FindS("Package");
 884
 885       DoPackage(File,Package);
 886       if (_error->empty() == false)
 887       {
 888          _error->Error("Errors apply to file '%s'",File.c_str());
 889          _error->DumpErrors();
 890       }
 891    }
 892
 893    // Tidy the compressor
 894    if (Pkgs.CloseOld(CompFd,Proc) == false)
 895       return false;
 896
 897    return true;
 898 }
 899
 900                                                                         /*}}}*/
 901
 902 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 903 // ---------------------------------------------------------------------
 904 /* */
 905 ReleaseWriter::ReleaseWriter(string const &DB)
 906 {
 907    AddPattern("Packages");
 908    AddPattern("Packages.gz");
 909    AddPattern("Packages.bz2");
 910    AddPattern("Packages.lzma");
 911    AddPattern("Sources");
 912    AddPattern("Sources.gz");
 913    AddPattern("Sources.bz2");
 914    AddPattern("Sources.lzma");
 915    AddPattern("Release");
 916    AddPattern("md5sum.txt");
 917
 918    Output = stdout;
 919    time_t const now = time(NULL);
 920    char datestr[128];
 921    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 922                 gmtime(&now)) == 0)
 923    {
 924       datestr[0] = '\0';
 925    }
 926
 927    time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
 928    char validstr[128];
 929    if (now == validuntil ||
 930        strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
 931                 gmtime(&validuntil)) == 0)
 932    {
 933       validstr[0] = '\0';
 934    }
 935
 936    map<string,string> Fields;
 937    Fields["Origin"] = "";
 938    Fields["Label"] = "";
 939    Fields["Suite"] = "";
 940    Fields["Version"] = "";
 941    Fields["Codename"] = "";
 942    Fields["Date"] = datestr;
 943    Fields["Valid-Until"] = validstr;
 944    Fields["Architectures"] = "";
 945    Fields["Components"] = "";
 946    Fields["Description"] = "";
 947
 948    for(map<string,string>::const_iterator I = Fields.begin();
 949        I != Fields.end();
 950        ++I)
 951    {
 952       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 953       string Value = _config->Find(Config, (*I).second.c_str());
 954       if (Value == "")
 955          continue;
 956
 957       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 958    }
 959 }
 960                                                                         /*}}}*/
 961 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 962 // ---------------------------------------------------------------------
 963 bool ReleaseWriter::DoPackage(string FileName)
 964 {
 965    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 966    string NewFileName;
 967    if (DirStrip.empty() == false &&
 968        FileName.length() > DirStrip.length() &&
 969        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 970                  DirStrip.begin(),DirStrip.end()) == 0)
 971    {
 972       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 973       while (NewFileName[0] == '/')
 974          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
 975    }
 976    else
 977       NewFileName = FileName;
 978
 979    if (PathPrefix.empty() == false)
 980       NewFileName = flCombine(PathPrefix,NewFileName);
 981
 982    FileFd fd(FileName, FileFd::ReadOnly);
 983
 984    if (!fd.IsOpen())
 985    {
 986       return false;
 987    }
 988
 989    CheckSums[NewFileName].size = fd.Size();
 990
 991    MD5Summation MD5;
 992    MD5.AddFD(fd.Fd(), fd.Size());
 993    CheckSums[NewFileName].MD5 = MD5.Result();
 994
 995    fd.Seek(0);
 996    SHA1Summation SHA1;
 997    SHA1.AddFD(fd.Fd(), fd.Size());
 998    CheckSums[NewFileName].SHA1 = SHA1.Result();
 999
1000    fd.Seek(0);
1001    SHA256Summation SHA256;
1002    SHA256.AddFD(fd.Fd(), fd.Size());
1003    CheckSums[NewFileName].SHA256 = SHA256.Result();
1004
1005    fd.Close();
1006
1007    return true;
1008 }
1009
1010                                                                         /*}}}*/
1011 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1012 // ---------------------------------------------------------------------
1013 void ReleaseWriter::Finish()
1014 {
1015    fprintf(Output, "MD5Sum:\n");
1016    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1017        I != CheckSums.end();
1018        ++I)
1019    {
1020       fprintf(Output, " %s %16ld %s\n",
1021               (*I).second.MD5.c_str(),
1022               (*I).second.size,
1023               (*I).first.c_str());
1024    }
1025
1026    fprintf(Output, "SHA1:\n");
1027    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1028        I != CheckSums.end();
1029        ++I)
1030    {
1031       fprintf(Output, " %s %16ld %s\n",
1032               (*I).second.SHA1.c_str(),
1033               (*I).second.size,
1034               (*I).first.c_str());
1035    }
1036
1037    fprintf(Output, "SHA256:\n");
1038    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1039        I != CheckSums.end();
1040        ++I)
1041    {
1042       fprintf(Output, " %s %16ld %s\n",
1043               (*I).second.SHA256.c_str(),
1044               (*I).second.size,
1045               (*I).first.c_str());
1046    }
1047 }
1048