ftparchive/writer.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
   4 /* ######################################################################
   5
   6    Writer
   7
   8    The file writer classes. These write various types of output, sources,
   9    packages and contents.
  10
  11    ##################################################################### */
  12                                                                         /*}}}*/
  13 // Include Files                                                        /*{{{*/
  14 #include "writer.h"
  15
  16 #include <apti18n.h>
  17 #include <apt-pkg/strutl.h>
  18 #include <apt-pkg/error.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/md5.h>
  21 #include <apt-pkg/sha1.h>
  22 #include <apt-pkg/sha256.h>
  23 #include <apt-pkg/deblistparser.h>
  24
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <ctime>
  28 #include <ftw.h>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <memory>
  32
  33 #include "cachedb.h"
  34 #include "apt-ftparchive.h"
  35 #include "multicompress.h"
  36                                                                         /*}}}*/
  37 using namespace std;
  38 FTWScanner *FTWScanner::Owner;
  39
  40 // SetTFRewriteData - Helper for setting rewrite lists                  /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 inline void SetTFRewriteData(struct TFRewriteData &tfrd,
  44                              const char *tag,
  45                              const char *rewrite,
  46                              const char *newtag = 0)
  47 {
  48     tfrd.Tag = tag;
  49     tfrd.Rewrite = rewrite;
  50     tfrd.NewTag = newtag;
  51 }
  52                                                                         /*}}}*/
  53
  54 // FTWScanner::FTWScanner - Constructor                                 /*{{{*/
  55 // ---------------------------------------------------------------------
  56 /* */
  57 FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
  58 {
  59    ErrorPrinted = false;
  60    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  61 }
  62                                                                         /*}}}*/
  63 // FTWScanner::Scanner - FTW Scanner                                    /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* This is the FTW scanner, it processes each directory element in the
  66    directory tree. */
  67 int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
  68 {
  69    if (Flag == FTW_DNR)
  70    {
  71       Owner->NewLine(1);
  72       ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  73    }
  74    if (Flag == FTW_NS)
  75    {
  76       Owner->NewLine(1);
  77       ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  78    }
  79    if (Flag != FTW_F)
  80       return 0;
  81
  82    return ScannerFile(File, true);
  83 }
  84                                                                         /*}}}*/
  85 // FTWScanner::ScannerFile - File Scanner                               /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* */
  88 int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
  89 {
  90    const char *LastComponent = strrchr(File, '/');
  91    char *RealPath = NULL;
  92
  93    if (LastComponent == NULL)
  94       LastComponent = File;
  95    else
  96       LastComponent++;
  97
  98    vector<string>::const_iterator I;
  99    for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
 100    {
 101       if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
 102          break;
 103    }
 104    if (I == Owner->Patterns.end())
 105       return 0;
 106
 107    /* Process it. If the file is a link then resolve it into an absolute
 108       name.. This works best if the directory components the scanner are
 109       given are not links themselves. */
 110    char Jnk[2];
 111    Owner->OriginalPath = File;
 112    if (ReadLink &&
 113        readlink(File,Jnk,sizeof(Jnk)) != -1 &&
 114        (RealPath = realpath(File,NULL)) != 0)
 115    {
 116       Owner->DoPackage(RealPath);
 117       free(RealPath);
 118    }
 119    else
 120       Owner->DoPackage(File);
 121
 122    if (_error->empty() == false)
 123    {
 124       // Print any errors or warnings found
 125       string Err;
 126       bool SeenPath = false;
 127       while (_error->empty() == false)
 128       {
 129          Owner->NewLine(1);
 130
 131          bool const Type = _error->PopMessage(Err);
 132          if (Type == true)
 133             cerr << _("E: ") << Err << endl;
 134          else
 135             cerr << _("W: ") << Err << endl;
 136
 137          if (Err.find(File) != string::npos)
 138             SeenPath = true;
 139       }
 140
 141       if (SeenPath == false)
 142          cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
 143       return 0;
 144    }
 145
 146    return 0;
 147 }
 148                                                                         /*}}}*/
 149 // FTWScanner::RecursiveScan - Just scan a directory tree               /*{{{*/
 150 // ---------------------------------------------------------------------
 151 /* */
 152 bool FTWScanner::RecursiveScan(string const &Dir)
 153 {
 154    char *RealPath = NULL;
 155    /* If noprefix is set then jam the scan root in, so we don't generate
 156       link followed paths out of control */
 157    if (InternalPrefix.empty() == true)
 158    {
 159       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 160          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 161       InternalPrefix = RealPath;
 162       free(RealPath);
 163    }
 164
 165    // Do recursive directory searching
 166    Owner = this;
 167    int const Res = ftw(Dir.c_str(),ScannerFTW,30);
 168
 169    // Error treewalking?
 170    if (Res != 0)
 171    {
 172       if (_error->PendingError() == false)
 173          _error->Errno("ftw",_("Tree walking failed"));
 174       return false;
 175    }
 176
 177    return true;
 178 }
 179                                                                         /*}}}*/
 180 // FTWScanner::LoadFileList - Load the file list from a file            /*{{{*/
 181 // ---------------------------------------------------------------------
 182 /* This is an alternative to using FTW to locate files, it reads the list
 183    of files from another file. */
 184 bool FTWScanner::LoadFileList(string const &Dir, string const &File)
 185 {
 186    char *RealPath = NULL;
 187    /* If noprefix is set then jam the scan root in, so we don't generate
 188       link followed paths out of control */
 189    if (InternalPrefix.empty() == true)
 190    {
 191       if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
 192          return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
 193       InternalPrefix = RealPath;
 194       free(RealPath);
 195    }
 196
 197    Owner = this;
 198    FILE *List = fopen(File.c_str(),"r");
 199    if (List == 0)
 200       return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
 201
 202    /* We are a tad tricky here.. We prefix the buffer with the directory
 203       name, that way if we need a full path with just use line.. Sneaky and
 204       fully evil. */
 205    char Line[1000];
 206    char *FileStart;
 207    if (Dir.empty() == true || Dir.end()[-1] != '/')
 208       FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
 209    else
 210       FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
 211    while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
 212    {
 213       char *FileName = _strstrip(FileStart);
 214       if (FileName[0] == 0)
 215          continue;
 216
 217       if (FileName[0] != '/')
 218       {
 219          if (FileName != FileStart)
 220             memmove(FileStart,FileName,strlen(FileStart));
 221          FileName = Line;
 222       }
 223
 224 #if 0
 225       struct stat St;
 226       int Flag = FTW_F;
 227       if (stat(FileName,&St) != 0)
 228          Flag = FTW_NS;
 229 #endif
 230
 231       if (ScannerFile(FileName, false) != 0)
 232          break;
 233    }
 234
 235    fclose(List);
 236    return true;
 237 }
 238                                                                         /*}}}*/
 239 // FTWScanner::Delink - Delink symlinks                                 /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* */
 242 bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 243                         unsigned long &DeLinkBytes,
 244                         off_t const &FileSize)
 245 {
 246    // See if this isn't an internaly prefix'd file name.
 247    if (InternalPrefix.empty() == false &&
 248        InternalPrefix.length() < FileName.length() &&
 249        stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
 250                  InternalPrefix.begin(),InternalPrefix.end()) != 0)
 251    {
 252       if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
 253       {
 254          // Tidy up the display
 255          if (DeLinkBytes == 0)
 256             cout << endl;
 257
 258          NewLine(1);
 259          ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
 260                     SizeToStr(FileSize).c_str());
 261          c1out << flush;
 262
 263          if (NoLinkAct == false)
 264          {
 265             char OldLink[400];
 266             if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
 267                _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
 268             else
 269             {
 270                if (unlink(OriginalPath) != 0)
 271                   _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
 272                else
 273                {
 274                   if (link(FileName.c_str(),OriginalPath) != 0)
 275                   {
 276                      // Panic! Restore the symlink
 277                      symlink(OldLink,OriginalPath);
 278                      return _error->Errno("link",_("*** Failed to link %s to %s"),
 279                                           FileName.c_str(),
 280                                           OriginalPath);
 281                   }
 282                }
 283             }
 284          }
 285
 286          DeLinkBytes += FileSize;
 287          if (DeLinkBytes/1024 >= DeLinkLimit)
 288             ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
 289       }
 290
 291       FileName = OriginalPath;
 292    }
 293
 294    return true;
 295 }
 296                                                                         /*}}}*/
 297
 298 // PackagesWriter::PackagesWriter - Constructor                         /*{{{*/
 299 // ---------------------------------------------------------------------
 300 /* */
 301 PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
 302                                string const &Arch) :
 303    FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 304 {
 305    Output = stdout;
 306    SetExts(".deb .udeb");
 307    DeLinkLimit = 0;
 308
 309    // Process the command line options
 310    DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
 311    DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
 312    DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
 313    DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
 314    DoContents = _config->FindB("APT::FTPArchive::Contents",true);
 315    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 316    LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
 317
 318    if (Db.Loaded() == false)
 319       DoContents = false;
 320
 321    // Read the override file
 322    if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
 323       return;
 324    else
 325       NoOverride = true;
 326
 327    if (ExtOverrides.empty() == false)
 328       Over.ReadExtraOverride(ExtOverrides);
 329
 330    _error->DumpErrors();
 331 }
 332                                                                         /*}}}*/
 333 // FTWScanner::SetExts - Set extensions to support                      /*{{{*/
 334 // ---------------------------------------------------------------------
 335 /* */
 336 bool FTWScanner::SetExts(string const &Vals)
 337 {
 338    ClearPatterns();
 339    string::size_type Start = 0;
 340    while (Start <= Vals.length()-1)
 341    {
 342       string::size_type const Space = Vals.find(' ',Start);
 343       string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
 344       if ( Arch.empty() == false )
 345       {
 346          AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
 347          AddPattern(string("*_all") + Vals.substr(Start, Length));
 348       }
 349       else
 350          AddPattern(string("*") + Vals.substr(Start, Length));
 351
 352       Start += Length + 1;
 353    }
 354
 355    return true;
 356 }
 357
 358                                                                         /*}}}*/
 359 // PackagesWriter::DoPackage - Process a single package                 /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* This method takes a package and gets its control information and
 362    MD5, SHA1 and SHA256 then writes out a control record with the proper fields
 363    rewritten and the path/size/hash appended. */
 364 bool PackagesWriter::DoPackage(string FileName)
 365 {
 366    // Pull all the data we need form the DB
 367    if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoAlwaysStat)
 368                   == false)
 369    {
 370       return false;
 371    }
 372
 373    off_t FileSize = Db.GetFileSize();
 374    if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
 375       return false;
 376
 377    // Lookup the overide information
 378    pkgTagSection &Tags = Db.Control.Section;
 379    string Package = Tags.FindS("Package");
 380    string Architecture;
 381    // if we generate a Packages file for a given arch, we use it to
 382    // look for overrides. if we run in "simple" mode without the
 383    // "Architecures" variable in the config we use the architecure value
 384    // from the deb file
 385    if(Arch != "")
 386       Architecture = Arch;
 387    else
 388       Architecture = Tags.FindS("Architecture");
 389    auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
 390
 391    if (Package.empty() == true)
 392       return _error->Error(_("Archive had no package field"));
 393
 394    // If we need to do any rewriting of the header do it now..
 395    if (OverItem.get() == 0)
 396    {
 397       if (NoOverride == false)
 398       {
 399          NewLine(1);
 400          ioprintf(c1out, _("  %s has no override entry\n"), Package.c_str());
 401       }
 402
 403       OverItem = auto_ptr<Override::Item>(new Override::Item);
 404       OverItem->FieldOverride["Section"] = Tags.FindS("Section");
 405       OverItem->Priority = Tags.FindS("Priority");
 406    }
 407
 408    char Size[40];
 409    sprintf(Size,"%lu", (unsigned long) FileSize);
 410
 411    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 412    string NewFileName;
 413    if (DirStrip.empty() == false &&
 414        FileName.length() > DirStrip.length() &&
 415        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 416                  DirStrip.begin(),DirStrip.end()) == 0)
 417       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 418    else
 419       NewFileName = FileName;
 420    if (PathPrefix.empty() == false)
 421       NewFileName = flCombine(PathPrefix,NewFileName);
 422
 423    /* Configuration says we don't want to include the long Description
 424       in the package file - instead we want to ship a separated file */
 425    string desc;
 426    if (LongDescription == false) {
 427       desc = Tags.FindS("Description").append("\n");
 428       OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
 429    }
 430
 431    // This lists all the changes to the fields we are going to make.
 432    // (7 hardcoded + maintainer + suggests + end marker)
 433    TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
 434
 435    unsigned int End = 0;
 436    SetTFRewriteData(Changes[End++], "Size", Size);
 437    SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
 438    SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
 439    SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
 440    SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
 441    SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
 442    SetTFRewriteData(Changes[End++], "Status", 0);
 443    SetTFRewriteData(Changes[End++], "Optional", 0);
 444
 445    string DescriptionMd5;
 446    if (LongDescription == false) {
 447       MD5Summation descmd5;
 448       descmd5.Add(desc.c_str());
 449       DescriptionMd5 = descmd5.Result().Value();
 450       SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
 451       if (TransWriter != NULL)
 452          TransWriter->DoPackage(Package, desc, DescriptionMd5);
 453    }
 454
 455    // Rewrite the maintainer field if necessary
 456    bool MaintFailed;
 457    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 458    if (MaintFailed == true)
 459    {
 460       if (NoOverride == false)
 461       {
 462          NewLine(1);
 463          ioprintf(c1out, _("  %s maintainer is %s not %s\n"),
 464                Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 465       }
 466    }
 467
 468    if (NewMaint.empty() == false)
 469       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 470
 471    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
 472       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
 473       but dpkg does this append bit. So we do the append bit, at least that way the
 474       status file and package file will remain similar. There are other transforms
 475       but optional is the only legacy one still in use for some lazy reason. */
 476    string OptionalStr = Tags.FindS("Optional");
 477    if (OptionalStr.empty() == false)
 478    {
 479       if (Tags.FindS("Suggests").empty() == false)
 480          OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
 481       SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
 482    }
 483
 484    for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
 485         I != OverItem->FieldOverride.end(); I++)
 486       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 487
 488    SetTFRewriteData(Changes[End++], 0, 0);
 489
 490    // Rewrite and store the fields.
 491    if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
 492       return false;
 493    fprintf(Output,"\n");
 494
 495    return Db.Finish();
 496 }
 497                                                                         /*}}}*/
 498
 499 // TranslationWriter::TranslationWriter - Constructor                   /*{{{*/
 500 // ---------------------------------------------------------------------
 501 /* Create a Translation-Master file for this Packages file */
 502 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
 503                                         mode_t const &Permissions) : Output(NULL),
 504                                                         RefCounter(0)
 505 {
 506    if (File.empty() == true)
 507       return;
 508
 509    Comp = new MultiCompress(File, TransCompress, Permissions);
 510    Output = Comp->Input;
 511 }
 512                                                                         /*}}}*/
 513 // TranslationWriter::DoPackage - Process a single package              /*{{{*/
 514 // ---------------------------------------------------------------------
 515 /* Create a Translation-Master file for this Packages file */
 516 bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
 517                                   string const &MD5)
 518 {
 519    if (Output == NULL)
 520       return true;
 521
 522    // Different archs can include different versions and therefore
 523    // different descriptions - so we need to check for both name and md5.
 524    string const Record = Pkg + ":" + MD5;
 525
 526    if (Included.find(Record) != Included.end())
 527       return true;
 528
 529    fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 530            Pkg.c_str(), MD5.c_str(), Desc.c_str());
 531
 532    Included.insert(Record);
 533    return true;
 534 }
 535                                                                         /*}}}*/
 536 // TranslationWriter::~TranslationWriter - Destructor                   /*{{{*/
 537 // ---------------------------------------------------------------------
 538 /* */
 539 TranslationWriter::~TranslationWriter()
 540 {
 541    if (Comp == NULL)
 542       return;
 543
 544    delete Comp;
 545 }
 546                                                                         /*}}}*/
 547
 548 // SourcesWriter::SourcesWriter - Constructor                           /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* */
 551 SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
 552                              string const &ExtOverrides)
 553 {
 554    Output = stdout;
 555    AddPattern("*.dsc");
 556    DeLinkLimit = 0;
 557    Buffer = 0;
 558    BufSize = 0;
 559
 560    // Process the command line options
 561    NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
 562
 563    // Read the override file
 564    if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
 565       return;
 566    else
 567       NoOverride = true;
 568
 569    // WTF?? The logic above: if we can't read binary overrides, don't even try
 570    // reading source overrides. if we can read binary overrides, then say there
 571    // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
 572
 573    if (ExtOverrides.empty() == false)
 574       SOver.ReadExtraOverride(ExtOverrides);
 575
 576    if (SOverrides.empty() == false && FileExists(SOverrides) == true)
 577       SOver.ReadOverride(SOverrides,true);
 578 }
 579                                                                         /*}}}*/
 580 // SourcesWriter::DoPackage - Process a single package                  /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* */
 583 bool SourcesWriter::DoPackage(string FileName)
 584 {
 585    // Open the archive
 586    FileFd F(FileName,FileFd::ReadOnly);
 587    if (_error->PendingError() == true)
 588       return false;
 589
 590    // Stat the file for later
 591    struct stat St;
 592    if (fstat(F.Fd(),&St) != 0)
 593       return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
 594
 595    if (St.st_size > 128*1024)
 596       return _error->Error("DSC file '%s' is too large!",FileName.c_str());
 597
 598    if (BufSize < (unsigned)St.st_size+1)
 599    {
 600       BufSize = St.st_size+1;
 601       Buffer = (char *)realloc(Buffer,St.st_size+1);
 602    }
 603
 604    if (F.Read(Buffer,St.st_size) == false)
 605       return false;
 606
 607    // Hash the file
 608    char *Start = Buffer;
 609    char *BlkEnd = Buffer + St.st_size;
 610    MD5Summation MD5;
 611    MD5.Add((unsigned char *)Start,BlkEnd - Start);
 612
 613    SHA1Summation SHA1;
 614    SHA256Summation SHA256;
 615    SHA1.Add((unsigned char *)Start,BlkEnd - Start);
 616    SHA256.Add((unsigned char *)Start,BlkEnd - Start);
 617
 618    // Add an extra \n to the end, just in case
 619    *BlkEnd++ = '\n';
 620
 621    /* Remove the PGP trailer. Some .dsc's have this without a blank line
 622       before */
 623    const char *Key = "-----BEGIN PGP SIGNATURE-----";
 624    for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
 625    {
 626       if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
 627       {
 628          MsgEnd[1] = '\n';
 629          break;
 630       }
 631    }
 632
 633    /* Read records until we locate the Source record. This neatly skips the
 634       GPG header (which is RFC822 formed) without any trouble. */
 635    pkgTagSection Tags;
 636    do
 637    {
 638       unsigned Pos;
 639       if (Tags.Scan(Start,BlkEnd - Start) == false)
 640          return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
 641       if (Tags.Find("Source",Pos) == true)
 642          break;
 643       Start += Tags.size();
 644    }
 645    while (1);
 646    Tags.Trim();
 647
 648    // Lookup the overide information, finding first the best priority.
 649    string BestPrio;
 650    string Bins = Tags.FindS("Binary");
 651    char Buffer[Bins.length() + 1];
 652    auto_ptr<Override::Item> OverItem(0);
 653    if (Bins.empty() == false)
 654    {
 655       strcpy(Buffer,Bins.c_str());
 656
 657       // Ignore too-long errors.
 658       char *BinList[400];
 659       TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
 660
 661       // Look at all the binaries
 662       unsigned char BestPrioV = pkgCache::State::Extra;
 663       for (unsigned I = 0; BinList[I] != 0; I++)
 664       {
 665          auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
 666          if (Itm.get() == 0)
 667             continue;
 668
 669          unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
 670          if (NewPrioV < BestPrioV || BestPrio.empty() == true)
 671          {
 672             BestPrioV = NewPrioV;
 673             BestPrio = Itm->Priority;
 674          }
 675
 676          if (OverItem.get() == 0)
 677             OverItem = Itm;
 678       }
 679    }
 680
 681    // If we need to do any rewriting of the header do it now..
 682    if (OverItem.get() == 0)
 683    {
 684       if (NoOverride == false)
 685       {
 686          NewLine(1);
 687          ioprintf(c1out, _("  %s has no override entry\n"), Tags.FindS("Source").c_str());
 688       }
 689
 690       OverItem = auto_ptr<Override::Item>(new Override::Item);
 691    }
 692
 693    auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
 694    // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
 695    if (SOverItem.get() == 0)
 696    {
 697       ioprintf(c1out, _("  %s has no source override entry\n"), Tags.FindS("Source").c_str());
 698       SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
 699       if (SOverItem.get() == 0)
 700       {
 701         ioprintf(c1out, _("  %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
 702          SOverItem = auto_ptr<Override::Item>(new Override::Item);
 703          *SOverItem = *OverItem;
 704       }
 705    }
 706
 707    // Add the dsc to the files hash list
 708    string const strippedName = flNotDir(FileName);
 709    char Files[1000];
 710    snprintf(Files,sizeof(Files),"\n %s %lu %s\n %s",
 711             string(MD5.Result()).c_str(),St.st_size,
 712             strippedName.c_str(),
 713             Tags.FindS("Files").c_str());
 714
 715    char ChecksumsSha1[1000];
 716    snprintf(ChecksumsSha1,sizeof(ChecksumsSha1),"\n %s %lu %s\n %s",
 717             string(SHA1.Result()).c_str(),St.st_size,
 718             strippedName.c_str(),
 719             Tags.FindS("Checksums-Sha1").c_str());
 720
 721    char ChecksumsSha256[1000];
 722    snprintf(ChecksumsSha256,sizeof(ChecksumsSha256),"\n %s %lu %s\n %s",
 723             string(SHA256.Result()).c_str(),St.st_size,
 724             strippedName.c_str(),
 725             Tags.FindS("Checksums-Sha256").c_str());
 726
 727    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 728    string NewFileName;
 729    if (DirStrip.empty() == false &&
 730        FileName.length() > DirStrip.length() &&
 731        stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
 732       NewFileName = string(OriginalPath + DirStrip.length());
 733    else
 734       NewFileName = OriginalPath;
 735    if (PathPrefix.empty() == false)
 736       NewFileName = flCombine(PathPrefix,NewFileName);
 737
 738    string Directory = flNotFile(OriginalPath);
 739    string Package = Tags.FindS("Source");
 740
 741    // Perform the delinking operation over all of the files
 742    string ParseJnk;
 743    const char *C = Files;
 744    char *RealPath = NULL;
 745    for (;isspace(*C); C++);
 746    while (*C != 0)
 747    {
 748       // Parse each of the elements
 749       if (ParseQuoteWord(C,ParseJnk) == false ||
 750           ParseQuoteWord(C,ParseJnk) == false ||
 751           ParseQuoteWord(C,ParseJnk) == false)
 752          return _error->Error("Error parsing file record");
 753
 754       char Jnk[2];
 755       string OriginalPath = Directory + ParseJnk;
 756       if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
 757           (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
 758       {
 759          string RP = RealPath;
 760          free(RealPath);
 761          if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
 762             return false;
 763       }
 764    }
 765
 766    Directory = flNotFile(NewFileName);
 767    if (Directory.length() > 2)
 768       Directory.erase(Directory.end()-1);
 769
 770    // This lists all the changes to the fields we are going to make.
 771    // (5 hardcoded + checksums + maintainer + end marker)
 772    TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
 773
 774    unsigned int End = 0;
 775    SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
 776    SetTFRewriteData(Changes[End++],"Files",Files);
 777    SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1);
 778    SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256);
 779    if (Directory != "./")
 780       SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
 781    SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
 782    SetTFRewriteData(Changes[End++],"Status",0);
 783
 784    // Rewrite the maintainer field if necessary
 785    bool MaintFailed;
 786    string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
 787    if (MaintFailed == true)
 788    {
 789       if (NoOverride == false)
 790       {
 791          NewLine(1);
 792          ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 793                Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
 794       }
 795    }
 796    if (NewMaint.empty() == false)
 797       SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
 798
 799    for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
 800         I != SOverItem->FieldOverride.end(); I++)
 801       SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
 802
 803    SetTFRewriteData(Changes[End++], 0, 0);
 804
 805    // Rewrite and store the fields.
 806    if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
 807       return false;
 808    fprintf(Output,"\n");
 809
 810    Stats.Packages++;
 811
 812    return true;
 813 }
 814                                                                         /*}}}*/
 815
 816 // ContentsWriter::ContentsWriter - Constructor                         /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* */
 819 ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
 820                     FTWScanner(Arch), Db(DB), Stats(Db.Stats)
 821
 822 {
 823    SetExts(".deb");
 824    Output = stdout;
 825 }
 826                                                                         /*}}}*/
 827 // ContentsWriter::DoPackage - Process a single package                 /*{{{*/
 828 // ---------------------------------------------------------------------
 829 /* If Package is the empty string the control record will be parsed to
 830    determine what the package name is. */
 831 bool ContentsWriter::DoPackage(string FileName, string Package)
 832 {
 833    if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
 834    {
 835       return false;
 836    }
 837
 838    // Parse the package name
 839    if (Package.empty() == true)
 840    {
 841       Package = Db.Control.Section.FindS("Package");
 842    }
 843
 844    Db.Contents.Add(Gen,Package);
 845
 846    return Db.Finish();
 847 }
 848                                                                         /*}}}*/
 849 // ContentsWriter::ReadFromPkgs - Read from a packages file             /*{{{*/
 850 // ---------------------------------------------------------------------
 851 /* */
 852 bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
 853 {
 854    MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
 855    if (_error->PendingError() == true)
 856       return false;
 857
 858    // Open the package file
 859    int CompFd = -1;
 860    pid_t Proc = -1;
 861    if (Pkgs.OpenOld(CompFd,Proc) == false)
 862       return false;
 863
 864    // No auto-close FD
 865    FileFd Fd(CompFd,false);
 866    pkgTagFile Tags(&Fd);
 867    if (_error->PendingError() == true)
 868    {
 869       Pkgs.CloseOld(CompFd,Proc);
 870       return false;
 871    }
 872
 873    // Parse.
 874    pkgTagSection Section;
 875    while (Tags.Step(Section) == true)
 876    {
 877       string File = flCombine(Prefix,Section.FindS("FileName"));
 878       string Package = Section.FindS("Section");
 879       if (Package.empty() == false && Package.end()[-1] != '/')
 880       {
 881          Package += '/';
 882          Package += Section.FindS("Package");
 883       }
 884       else
 885          Package += Section.FindS("Package");
 886
 887       DoPackage(File,Package);
 888       if (_error->empty() == false)
 889       {
 890          _error->Error("Errors apply to file '%s'",File.c_str());
 891          _error->DumpErrors();
 892       }
 893    }
 894
 895    // Tidy the compressor
 896    if (Pkgs.CloseOld(CompFd,Proc) == false)
 897       return false;
 898
 899    return true;
 900 }
 901
 902                                                                         /*}}}*/
 903
 904 // ReleaseWriter::ReleaseWriter - Constructor                           /*{{{*/
 905 // ---------------------------------------------------------------------
 906 /* */
 907 ReleaseWriter::ReleaseWriter(string const &DB)
 908 {
 909    AddPattern("Packages");
 910    AddPattern("Packages.gz");
 911    AddPattern("Packages.bz2");
 912    AddPattern("Packages.lzma");
 913    AddPattern("Sources");
 914    AddPattern("Sources.gz");
 915    AddPattern("Sources.bz2");
 916    AddPattern("Sources.lzma");
 917    AddPattern("Release");
 918    AddPattern("md5sum.txt");
 919
 920    Output = stdout;
 921    time_t const now = time(NULL);
 922    char datestr[128];
 923    if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
 924                 gmtime(&now)) == 0)
 925    {
 926       datestr[0] = '\0';
 927    }
 928
 929    map<string,string> Fields;
 930    Fields["Origin"] = "";
 931    Fields["Label"] = "";
 932    Fields["Suite"] = "";
 933    Fields["Version"] = "";
 934    Fields["Codename"] = "";
 935    Fields["Date"] = datestr;
 936    Fields["Architectures"] = "";
 937    Fields["Components"] = "";
 938    Fields["Description"] = "";
 939
 940    for(map<string,string>::const_iterator I = Fields.begin();
 941        I != Fields.end();
 942        ++I)
 943    {
 944       string Config = string("APT::FTPArchive::Release::") + (*I).first;
 945       string Value = _config->Find(Config, (*I).second.c_str());
 946       if (Value == "")
 947          continue;
 948
 949       fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
 950    }
 951 }
 952                                                                         /*}}}*/
 953 // ReleaseWriter::DoPackage - Process a single package                  /*{{{*/
 954 // ---------------------------------------------------------------------
 955 bool ReleaseWriter::DoPackage(string FileName)
 956 {
 957    // Strip the DirStrip prefix from the FileName and add the PathPrefix
 958    string NewFileName;
 959    if (DirStrip.empty() == false &&
 960        FileName.length() > DirStrip.length() &&
 961        stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
 962                  DirStrip.begin(),DirStrip.end()) == 0)
 963    {
 964       NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
 965       while (NewFileName[0] == '/')
 966          NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
 967    }
 968    else
 969       NewFileName = FileName;
 970
 971    if (PathPrefix.empty() == false)
 972       NewFileName = flCombine(PathPrefix,NewFileName);
 973
 974    FileFd fd(FileName, FileFd::ReadOnly);
 975
 976    if (!fd.IsOpen())
 977    {
 978       return false;
 979    }
 980
 981    CheckSums[NewFileName].size = fd.Size();
 982
 983    MD5Summation MD5;
 984    MD5.AddFD(fd.Fd(), fd.Size());
 985    CheckSums[NewFileName].MD5 = MD5.Result();
 986
 987    fd.Seek(0);
 988    SHA1Summation SHA1;
 989    SHA1.AddFD(fd.Fd(), fd.Size());
 990    CheckSums[NewFileName].SHA1 = SHA1.Result();
 991
 992    fd.Seek(0);
 993    SHA256Summation SHA256;
 994    SHA256.AddFD(fd.Fd(), fd.Size());
 995    CheckSums[NewFileName].SHA256 = SHA256.Result();
 996
 997    fd.Close();
 998
 999    return true;
1000 }
1001
1002                                                                         /*}}}*/
1003 // ReleaseWriter::Finish - Output the checksums                         /*{{{*/
1004 // ---------------------------------------------------------------------
1005 void ReleaseWriter::Finish()
1006 {
1007    fprintf(Output, "MD5Sum:\n");
1008    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1009        I != CheckSums.end();
1010        ++I)
1011    {
1012       fprintf(Output, " %s %16ld %s\n",
1013               (*I).second.MD5.c_str(),
1014               (*I).second.size,
1015               (*I).first.c_str());
1016    }
1017
1018    fprintf(Output, "SHA1:\n");
1019    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1020        I != CheckSums.end();
1021        ++I)
1022    {
1023       fprintf(Output, " %s %16ld %s\n",
1024               (*I).second.SHA1.c_str(),
1025               (*I).second.size,
1026               (*I).first.c_str());
1027    }
1028
1029    fprintf(Output, "SHA256:\n");
1030    for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1031        I != CheckSums.end();
1032        ++I)
1033    {
1034       fprintf(Output, " %s %16ld %s\n",
1035               (*I).second.SHA256.c_str(),
1036               (*I).second.size,
1037               (*I).first.c_str());
1038    }
1039 }
1040