apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
  54 {
  55    if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
  56       return;
  57    std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
  58    for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
  59       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  60    std::cerr << " Actual Hash: " << std::endl;
  61    for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
  62       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  63 }
  64                                                                         /*}}}*/
  65
  66 // Acquire::Item::Item - Constructor                                    /*{{{*/
  67 pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) :
  68    Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
  69    Local(false), QueueCounter(0), ExpectedAdditionalItems(0),
  70    ExpectedHashes(ExpectedHashes)
  71 {
  72    Owner->Add(this);
  73    Status = StatIdle;
  74 }
  75                                                                         /*}}}*/
  76 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* */
  79 pkgAcquire::Item::~Item()
  80 {
  81    Owner->Remove(this);
  82 }
  83                                                                         /*}}}*/
  84 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* We return to an idle state if there are still other queues that could
  87    fetch this object */
  88 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  89 {
  90    Status = StatIdle;
  91    ErrorText = LookupTag(Message,"Message");
  92    UsedMirror =  LookupTag(Message,"UsedMirror");
  93    if (QueueCounter <= 1)
  94    {
  95       /* This indicates that the file is not available right now but might
  96          be sometime later. If we do a retry cycle then this should be
  97          retried [CDROMs] */
  98       if (Cnf->LocalOnly == true &&
  99           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 100       {
 101          Status = StatIdle;
 102          Dequeue();
 103          return;
 104       }
 105
 106       Status = StatError;
 107       Dequeue();
 108    }
 109
 110    // report mirror failure back to LP if we actually use a mirror
 111    string FailReason = LookupTag(Message, "FailReason");
 112    if(FailReason.size() != 0)
 113       ReportMirrorFailure(FailReason);
 114    else
 115       ReportMirrorFailure(ErrorText);
 116 }
 117                                                                         /*}}}*/
 118 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* Stash status and the file size. Note that setting Complete means
 121    sub-phases of the acquire process such as decompresion are operating */
 122 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 123 {
 124    Status = StatFetching;
 125    if (FileSize == 0 && Complete == false)
 126       FileSize = Size;
 127 }
 128                                                                         /*}}}*/
 129 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 130 // ---------------------------------------------------------------------
 131 /* */
 132 void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/,
 133                             pkgAcquire::MethodConfig * /*Cnf*/)
 134 {
 135    // We just downloaded something..
 136    string FileName = LookupTag(Message,"Filename");
 137    UsedMirror =  LookupTag(Message,"UsedMirror");
 138    if (Complete == false && !Local && FileName == DestFile)
 139    {
 140       if (Owner->Log != 0)
 141          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 142    }
 143
 144    if (FileSize == 0)
 145       FileSize= Size;
 146    Status = StatDone;
 147    ErrorText = string();
 148    Owner->Dequeue(this);
 149 }
 150                                                                         /*}}}*/
 151 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 152 // ---------------------------------------------------------------------
 153 /* This helper function is used by a lot of item methods as their final
 154    step */
 155 void pkgAcquire::Item::Rename(string From,string To)
 156 {
 157    if (rename(From.c_str(),To.c_str()) != 0)
 158    {
 159       char S[300];
 160       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 161               From.c_str(),To.c_str());
 162       Status = StatError;
 163       ErrorText = S;
 164    }
 165 }
 166                                                                         /*}}}*/
 167 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 168 {
 169    if(FileExists(DestFile))
 170       Rename(DestFile, DestFile + ".FAILED");
 171
 172    switch (error)
 173    {
 174       case HashSumMismatch:
 175          ErrorText = _("Hash Sum mismatch");
 176          Status = StatAuthError;
 177          ReportMirrorFailure("HashChecksumFailure");
 178          break;
 179       case SizeMismatch:
 180          ErrorText = _("Size mismatch");
 181          Status = StatAuthError;
 182          ReportMirrorFailure("SizeFailure");
 183          break;
 184       case InvalidFormat:
 185          ErrorText = _("Invalid file format");
 186          Status = StatError;
 187          // do not report as usually its not the mirrors fault, but Portal/Proxy
 188          break;
 189    }
 190    return false;
 191 }
 192                                                                         /*}}}*/
 193 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 194 // ---------------------------------------------------------------------
 195 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 196 {
 197    // we only act if a mirror was used at all
 198    if(UsedMirror.empty())
 199       return;
 200 #if 0
 201    std::cerr << "\nReportMirrorFailure: "
 202              << UsedMirror
 203              << " Uri: " << DescURI()
 204              << " FailCode: "
 205              << FailCode << std::endl;
 206 #endif
 207    const char *Args[40];
 208    unsigned int i = 0;
 209    string report = _config->Find("Methods::Mirror::ProblemReporting",
 210                                  "/usr/lib/apt/apt-report-mirror-failure");
 211    if(!FileExists(report))
 212       return;
 213    Args[i++] = report.c_str();
 214    Args[i++] = UsedMirror.c_str();
 215    Args[i++] = DescURI().c_str();
 216    Args[i++] = FailCode.c_str();
 217    Args[i++] = NULL;
 218    pid_t pid = ExecFork();
 219    if(pid < 0)
 220    {
 221       _error->Error("ReportMirrorFailure Fork failed");
 222       return;
 223    }
 224    else if(pid == 0)
 225    {
 226       execvp(Args[0], (char**)Args);
 227       std::cerr << "Could not exec " << Args[0] << std::endl;
 228       _exit(100);
 229    }
 230    if(!ExecWait(pid, "report-mirror-failure"))
 231    {
 232       _error->Warning("Couldn't report problem to '%s'",
 233                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 234    }
 235 }
 236                                                                         /*}}}*/
 237 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 238 // ---------------------------------------------------------------------
 239 /* Get a sub-index file based on checksums from a 'master' file and
 240    possibly query additional files */
 241 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 242                                  string const &URIDesc, string const &ShortDesc,
 243                                  HashStringList const &ExpectedHashes)
 244    : Item(Owner, ExpectedHashes)
 245 {
 246    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 247    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 248
 249    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 250    DestFile += URItoFileName(URI);
 251
 252    Desc.URI = URI;
 253    Desc.Description = URIDesc;
 254    Desc.Owner = this;
 255    Desc.ShortDesc = ShortDesc;
 256
 257    QueueURI(Desc);
 258
 259    if(Debug)
 260       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 261 }
 262                                                                         /*}}}*/
 263 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 264 // ---------------------------------------------------------------------
 265 /* The only header we use is the last-modified header. */
 266 string pkgAcqSubIndex::Custom600Headers() const
 267 {
 268    string Final = _config->FindDir("Dir::State::lists");
 269    Final += URItoFileName(Desc.URI);
 270
 271    struct stat Buf;
 272    if (stat(Final.c_str(),&Buf) != 0)
 273       return "\nIndex-File: true\nFail-Ignore: true\n";
 274    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 278 {
 279    if(Debug)
 280       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 281
 282    Complete = false;
 283    Status = StatDone;
 284    Dequeue();
 285
 286    // No good Index is provided
 287 }
 288                                                                         /*}}}*/
 289 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,  /*{{{*/
 290                            pkgAcquire::MethodConfig *Cnf)
 291 {
 292    if(Debug)
 293       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 294
 295    string FileName = LookupTag(Message,"Filename");
 296    if (FileName.empty() == true)
 297    {
 298       Status = StatError;
 299       ErrorText = "Method gave a blank filename";
 300       return;
 301    }
 302
 303    if (FileName != DestFile)
 304    {
 305       Local = true;
 306       Desc.URI = "copy:" + FileName;
 307       QueueURI(Desc);
 308       return;
 309    }
 310
 311    Item::Done(Message, Size, Hashes, Cnf);
 312
 313    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 314
 315    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 316    indexRecords SubIndexParser;
 317    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 318       Status = StatError;
 319       ErrorText = SubIndexParser.ErrorText;
 320       return;
 321    }
 322
 323    // success in downloading the index
 324    // rename the index
 325    if(Debug)
 326       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 327    Rename(DestFile,FinalFile);
 328    chmod(FinalFile.c_str(),0644);
 329    DestFile = FinalFile;
 330
 331    if(ParseIndex(DestFile) == false)
 332       return Failed("", NULL);
 333
 334    Complete = true;
 335    Status = StatDone;
 336    Dequeue();
 337    return;
 338 }
 339                                                                         /*}}}*/
 340 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 341 {
 342    indexRecords SubIndexParser;
 343    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 344       return false;
 345    // so something with the downloaded index
 346    return true;
 347 }
 348                                                                         /*}}}*/
 349 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 350 // ---------------------------------------------------------------------
 351 /* Get the DiffIndex file first and see if there are patches available
 352  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 353  * patches. If anything goes wrong in that process, it will fall back to
 354  * the original packages file
 355  */
 356 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 357                                  IndexTarget const * const Target,
 358                                  HashStringList const &ExpectedHashes,
 359                                  indexRecords *MetaIndexParser)
 360    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser)
 361 {
 362
 363    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 364
 365    RealURI = Target->URI;
 366    Desc.Owner = this;
 367    Desc.Description = Target->Description + "/DiffIndex";
 368    Desc.ShortDesc = Target->ShortDesc;
 369    Desc.URI = Target->URI + ".diff/Index";
 370
 371    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 372    DestFile += URItoFileName(Target->URI) + string(".DiffIndex");
 373
 374    if(Debug)
 375       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 376
 377    // look for the current package file
 378    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 379    CurrentPackagesFile += URItoFileName(RealURI);
 380
 381    // FIXME: this file:/ check is a hack to prevent fetching
 382    //        from local sources. this is really silly, and
 383    //        should be fixed cleanly as soon as possible
 384    if(!FileExists(CurrentPackagesFile) ||
 385       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 386    {
 387       // we don't have a pkg file or we don't want to queue
 388       if(Debug)
 389          std::clog << "No index file, local or canceld by user" << std::endl;
 390       Failed("", NULL);
 391       return;
 392    }
 393
 394    if(Debug)
 395       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 396          << CurrentPackagesFile << std::endl;
 397
 398    QueueURI(Desc);
 399
 400 }
 401                                                                         /*}}}*/
 402 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 403 // ---------------------------------------------------------------------
 404 /* The only header we use is the last-modified header. */
 405 string pkgAcqDiffIndex::Custom600Headers() const
 406 {
 407    string Final = _config->FindDir("Dir::State::lists");
 408    Final += URItoFileName(RealURI) + string(".IndexDiff");
 409
 410    if(Debug)
 411       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 412
 413    struct stat Buf;
 414    if (stat(Final.c_str(),&Buf) != 0)
 415       return "\nIndex-File: true";
 416
 417    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 418 }
 419                                                                         /*}}}*/
 420 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 421 {
 422    if(Debug)
 423       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 424          << std::endl;
 425
 426    pkgTagSection Tags;
 427    string ServerSha1;
 428    vector<DiffInfo> available_patches;
 429
 430    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 431    pkgTagFile TF(&Fd);
 432    if (_error->PendingError() == true)
 433       return false;
 434
 435    if(TF.Step(Tags) == true)
 436    {
 437       bool found = false;
 438       DiffInfo d;
 439       string size;
 440
 441       string const tmp = Tags.FindS("SHA1-Current");
 442       std::stringstream ss(tmp);
 443       ss >> ServerSha1 >> size;
 444       unsigned long const ServerSize = atol(size.c_str());
 445
 446       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 447       SHA1Summation SHA1;
 448       SHA1.AddFD(fd);
 449       string const local_sha1 = SHA1.Result();
 450
 451       if(local_sha1 == ServerSha1)
 452       {
 453          // we have the same sha1 as the server so we are done here
 454          if(Debug)
 455             std::clog << "Package file is up-to-date" << std::endl;
 456          // list cleanup needs to know that this file as well as the already
 457          // present index is ours, so we create an empty diff to save it for us
 458          new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 459                               ServerSha1, available_patches);
 460          return true;
 461       }
 462       else
 463       {
 464          if(Debug)
 465             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 466
 467          // check the historie and see what patches we need
 468          string const history = Tags.FindS("SHA1-History");
 469          std::stringstream hist(history);
 470          while(hist >> d.sha1 >> size >> d.file)
 471          {
 472             // read until the first match is found
 473             // from that point on, we probably need all diffs
 474             if(d.sha1 == local_sha1)
 475                found=true;
 476             else if (found == false)
 477                continue;
 478
 479             if(Debug)
 480                std::clog << "Need to get diff: " << d.file << std::endl;
 481             available_patches.push_back(d);
 482          }
 483
 484          if (available_patches.empty() == false)
 485          {
 486             // patching with too many files is rather slow compared to a fast download
 487             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 488             if (fileLimit != 0 && fileLimit < available_patches.size())
 489             {
 490                if (Debug)
 491                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 492                         << ") so fallback to complete download" << std::endl;
 493                return false;
 494             }
 495
 496             // see if the patches are too big
 497             found = false; // it was true and it will be true again at the end
 498             d = *available_patches.begin();
 499             string const firstPatch = d.file;
 500             unsigned long patchesSize = 0;
 501             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 502             while(patches >> d.sha1 >> size >> d.file)
 503             {
 504                if (firstPatch == d.file)
 505                   found = true;
 506                else if (found == false)
 507                   continue;
 508
 509                patchesSize += atol(size.c_str());
 510             }
 511             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 512             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 513             {
 514                if (Debug)
 515                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 516                         << ") so fallback to complete download" << std::endl;
 517                return false;
 518             }
 519          }
 520       }
 521
 522       // we have something, queue the next diff
 523       if(found)
 524       {
 525          // queue the diffs
 526          string::size_type const last_space = Description.rfind(" ");
 527          if(last_space != string::npos)
 528             Description.erase(last_space, Description.size()-last_space);
 529
 530          /* decide if we should download patches one by one or in one go:
 531             The first is good if the server merges patches, but many don't so client
 532             based merging can be attempt in which case the second is better.
 533             "bad things" will happen if patches are merged on the server,
 534             but client side merging is attempt as well */
 535          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 536          if (pdiff_merge == true)
 537          {
 538             // reprepro adds this flag if it has merged patches on the server
 539             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 540             pdiff_merge = (precedence != "merged");
 541          }
 542
 543          if (pdiff_merge == false)
 544          {
 545             new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 546                                  ServerSha1, available_patches);
 547          }
 548          else
 549          {
 550             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 551             for(size_t i = 0; i < available_patches.size(); ++i)
 552                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target,
 553                                                        ExpectedHashes,
 554                                                        MetaIndexParser,
 555                                                        available_patches[i],
 556                                                        diffs);
 557          }
 558
 559          Complete = false;
 560          Status = StatDone;
 561          Dequeue();
 562          return true;
 563       }
 564    }
 565
 566    // Nothing found, report and return false
 567    // Failing here is ok, if we return false later, the full
 568    // IndexFile is queued
 569    if(Debug)
 570       std::clog << "Can't find a patch in the index file" << std::endl;
 571    return false;
 572 }
 573                                                                         /*}}}*/
 574 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 575 {
 576    if(Debug)
 577       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 578                 << "Falling back to normal index file acquire" << std::endl;
 579
 580    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 581
 582    Complete = false;
 583    Status = StatDone;
 584    Dequeue();
 585 }
 586                                                                         /*}}}*/
 587 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
 588                            pkgAcquire::MethodConfig *Cnf)
 589 {
 590    if(Debug)
 591       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 592
 593    Item::Done(Message, Size, Hashes, Cnf);
 594
 595    string FinalFile;
 596    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 597
 598    // success in downloading the index
 599    // rename the index
 600    FinalFile += string(".IndexDiff");
 601    if(Debug)
 602       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 603                 << std::endl;
 604    Rename(DestFile,FinalFile);
 605    chmod(FinalFile.c_str(),0644);
 606    DestFile = FinalFile;
 607
 608    if(!ParseDiffIndex(DestFile))
 609       return Failed("", NULL);
 610
 611    Complete = true;
 612    Status = StatDone;
 613    Dequeue();
 614    return;
 615 }
 616                                                                         /*}}}*/
 617 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 618 // ---------------------------------------------------------------------
 619 /* The package diff is added to the queue. one object is constructed
 620  * for each diff and the index
 621  */
 622 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 623                                    struct IndexTarget const * const Target,
 624                                    HashStringList const &ExpectedHashes,
 625                                    indexRecords *MetaIndexParser,
 626                                    string ServerSha1,
 627                                    vector<DiffInfo> diffs)
 628    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 629      available_patches(diffs), ServerSha1(ServerSha1)
 630 {
 631
 632    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 633    DestFile += URItoFileName(Target->URI);
 634
 635    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 636
 637    RealURI = Target->URI;
 638    Desc.Owner = this;
 639    Description = Target->Description;
 640    Desc.ShortDesc = Target->ShortDesc;
 641
 642    if(available_patches.empty() == true)
 643    {
 644       // we are done (yeah!)
 645       Finish(true);
 646    }
 647    else
 648    {
 649       // get the next diff
 650       State = StateFetchDiff;
 651       QueueNextDiff();
 652    }
 653 }
 654                                                                         /*}}}*/
 655 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 656 {
 657    if(Debug)
 658       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 659                 << "Falling back to normal index file acquire" << std::endl;
 660    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 661    Finish();
 662 }
 663                                                                         /*}}}*/
 664 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 665 void pkgAcqIndexDiffs::Finish(bool allDone)
 666 {
 667    // we restore the original name, this is required, otherwise
 668    // the file will be cleaned
 669    if(allDone)
 670    {
 671       DestFile = _config->FindDir("Dir::State::lists");
 672       DestFile += URItoFileName(RealURI);
 673
 674       if(HashSums().usable() && !HashSums().VerifyFile(DestFile))
 675       {
 676          RenameOnError(HashSumMismatch);
 677          Dequeue();
 678          return;
 679       }
 680
 681       // this is for the "real" finish
 682       Complete = true;
 683       Status = StatDone;
 684       Dequeue();
 685       if(Debug)
 686          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 687       return;
 688    }
 689
 690    if(Debug)
 691       std::clog << "Finishing: " << Desc.URI << std::endl;
 692    Complete = false;
 693    Status = StatDone;
 694    Dequeue();
 695    return;
 696 }
 697                                                                         /*}}}*/
 698 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 699 {
 700
 701    // calc sha1 of the just patched file
 702    string FinalFile = _config->FindDir("Dir::State::lists");
 703    FinalFile += URItoFileName(RealURI);
 704
 705    FileFd fd(FinalFile, FileFd::ReadOnly);
 706    SHA1Summation SHA1;
 707    SHA1.AddFD(fd);
 708    string local_sha1 = string(SHA1.Result());
 709    if(Debug)
 710       std::clog << "QueueNextDiff: "
 711                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 712
 713    // final file reached before all patches are applied
 714    if(local_sha1 == ServerSha1)
 715    {
 716       Finish(true);
 717       return true;
 718    }
 719
 720    // remove all patches until the next matching patch is found
 721    // this requires the Index file to be ordered
 722    for(vector<DiffInfo>::iterator I=available_patches.begin();
 723        available_patches.empty() == false &&
 724           I != available_patches.end() &&
 725           I->sha1 != local_sha1;
 726        ++I)
 727    {
 728       available_patches.erase(I);
 729    }
 730
 731    // error checking and falling back if no patch was found
 732    if(available_patches.empty() == true)
 733    {
 734       Failed("", NULL);
 735       return false;
 736    }
 737
 738    // queue the right diff
 739    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 740    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 741    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 742    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 743
 744    if(Debug)
 745       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 746
 747    QueueURI(Desc);
 748
 749    return true;
 750 }
 751                                                                         /*}}}*/
 752 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes,       /*{{{*/
 753                             pkgAcquire::MethodConfig *Cnf)
 754 {
 755    if(Debug)
 756       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 757
 758    Item::Done(Message, Size, Hashes, Cnf);
 759
 760    string FinalFile;
 761    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 762
 763    // success in downloading a diff, enter ApplyDiff state
 764    if(State == StateFetchDiff)
 765    {
 766
 767       // rred excepts the patch as $FinalFile.ed
 768       Rename(DestFile,FinalFile+".ed");
 769
 770       if(Debug)
 771          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 772
 773       State = StateApplyDiff;
 774       Local = true;
 775       Desc.URI = "rred:" + FinalFile;
 776       QueueURI(Desc);
 777       Mode = "rred";
 778       return;
 779    }
 780
 781
 782    // success in download/apply a diff, queue next (if needed)
 783    if(State == StateApplyDiff)
 784    {
 785       // remove the just applied patch
 786       available_patches.erase(available_patches.begin());
 787       unlink((FinalFile + ".ed").c_str());
 788
 789       // move into place
 790       if(Debug)
 791       {
 792          std::clog << "Moving patched file in place: " << std::endl
 793                    << DestFile << " -> " << FinalFile << std::endl;
 794       }
 795       Rename(DestFile,FinalFile);
 796       chmod(FinalFile.c_str(),0644);
 797
 798       // see if there is more to download
 799       if(available_patches.empty() == false) {
 800          new pkgAcqIndexDiffs(Owner, Target,
 801                               ExpectedHashes, MetaIndexParser,
 802                               ServerSha1, available_patches);
 803          return Finish();
 804       } else
 805          return Finish(true);
 806    }
 807 }
 808                                                                         /*}}}*/
 809 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 810 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 811                                              struct IndexTarget const * const Target,
 812                                              HashStringList const &ExpectedHashes,
 813                                              indexRecords *MetaIndexParser,
 814                                              DiffInfo const &patch,
 815                                              std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 816    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 817      patch(patch), allPatches(allPatches), State(StateFetchDiff)
 818 {
 819
 820    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 821    DestFile += URItoFileName(Target->URI);
 822
 823    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 824
 825    RealURI = Target->URI;
 826    Desc.Owner = this;
 827    Description = Target->Description;
 828    Desc.ShortDesc = Target->ShortDesc;
 829
 830    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 831    Desc.Description = Description + " " + patch.file + string(".pdiff");
 832    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 833    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 834
 835    if(Debug)
 836       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 837
 838    QueueURI(Desc);
 839 }
 840                                                                         /*}}}*/
 841 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 842 {
 843    if(Debug)
 844       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 845    Complete = false;
 846    Status = StatDone;
 847    Dequeue();
 848
 849    // check if we are the first to fail, otherwise we are done here
 850    State = StateDoneDiff;
 851    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 852          I != allPatches->end(); ++I)
 853       if ((*I)->State == StateErrorDiff)
 854          return;
 855
 856    // first failure means we should fallback
 857    State = StateErrorDiff;
 858    std::clog << "Falling back to normal index file acquire" << std::endl;
 859    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 860 }
 861                                                                         /*}}}*/
 862 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes,   /*{{{*/
 863                             pkgAcquire::MethodConfig *Cnf)
 864 {
 865    if(Debug)
 866       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 867
 868    Item::Done(Message,Size,Hashes,Cnf);
 869
 870    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 871
 872    if (State == StateFetchDiff)
 873    {
 874       // rred expects the patch as $FinalFile.ed.$patchname.gz
 875       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 876
 877       // check if this is the last completed diff
 878       State = StateDoneDiff;
 879       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 880             I != allPatches->end(); ++I)
 881          if ((*I)->State != StateDoneDiff)
 882          {
 883             if(Debug)
 884                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 885             return;
 886          }
 887
 888       // this is the last completed diff, so we are ready to apply now
 889       State = StateApplyDiff;
 890
 891       if(Debug)
 892          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 893
 894       Local = true;
 895       Desc.URI = "rred:" + FinalFile;
 896       QueueURI(Desc);
 897       Mode = "rred";
 898       return;
 899    }
 900    // success in download/apply all diffs, clean up
 901    else if (State == StateApplyDiff)
 902    {
 903       // see if we really got the expected file
 904       if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile))
 905       {
 906          RenameOnError(HashSumMismatch);
 907          return;
 908       }
 909
 910       // move the result into place
 911       if(Debug)
 912          std::clog << "Moving patched file in place: " << std::endl
 913                    << DestFile << " -> " << FinalFile << std::endl;
 914       Rename(DestFile, FinalFile);
 915       chmod(FinalFile.c_str(), 0644);
 916
 917       // otherwise lists cleanup will eat the file
 918       DestFile = FinalFile;
 919
 920       // ensure the ed's are gone regardless of list-cleanup
 921       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 922             I != allPatches->end(); ++I)
 923       {
 924             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 925             unlink(patch.c_str());
 926       }
 927
 928       // all set and done
 929       Complete = true;
 930       if(Debug)
 931          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 932    }
 933 }
 934                                                                         /*}}}*/
 935 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* The package file is added to the queue and a second class is
 938    instantiated to fetch the revision file */
 939 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 940                          string URI,string URIDesc,string ShortDesc,
 941                          HashStringList const  &ExpectedHash, string comprExt)
 942    : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI)
 943 {
 944    if(comprExt.empty() == true)
 945    {
 946       // autoselect the compression method
 947       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 948       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 949          comprExt.append(*t).append(" ");
 950       if (comprExt.empty() == false)
 951          comprExt.erase(comprExt.end()-1);
 952    }
 953    CompressionExtension = comprExt;
 954
 955    Verify = true;
 956
 957    Init(URI, URIDesc, ShortDesc);
 958 }
 959 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 960                          HashStringList const &ExpectedHash,
 961                          indexRecords *MetaIndexParser)
 962    : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser),
 963      RealURI(Target->URI)
 964 {
 965    // autoselect the compression method
 966    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 967    CompressionExtension = "";
 968    if (ExpectedHashes.usable())
 969    {
 970       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 971          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 972             CompressionExtension.append(*t).append(" ");
 973    }
 974    else
 975    {
 976       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 977          CompressionExtension.append(*t).append(" ");
 978    }
 979    if (CompressionExtension.empty() == false)
 980       CompressionExtension.erase(CompressionExtension.end()-1);
 981
 982    // only verify non-optional targets, see acquire-item.h for a FIXME
 983    // to make this more flexible
 984    if (Target->IsOptional())
 985      Verify = false;
 986    else
 987      Verify = true;
 988
 989    Init(Target->URI, Target->Description, Target->ShortDesc);
 990 }
 991                                                                         /*}}}*/
 992 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 993 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 994    Decompression = false;
 995    Erase = false;
 996
 997    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 998    DestFile += URItoFileName(URI);
 999
1000    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1001    std::string MetaKey;
1002    if (comprExt == "uncompressed")
1003    {
1004       Desc.URI = URI;
1005       if(Target)
1006          MetaKey = string(Target->MetaKey);
1007    }
1008    else
1009    {
1010       Desc.URI = URI + '.' + comprExt;
1011       if(Target)
1012          MetaKey = string(Target->MetaKey) + '.' + comprExt;
1013    }
1014
1015    // load the filesize
1016    if(MetaIndexParser)
1017    {
1018       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1019       if(Record)
1020          FileSize = Record->Size;
1021
1022       InitByHashIfNeeded(MetaKey);
1023    }
1024
1025    Desc.Description = URIDesc;
1026    Desc.Owner = this;
1027    Desc.ShortDesc = ShortDesc;
1028
1029    QueueURI(Desc);
1030 }
1031                                                                         /*}}}*/
1032 // AcqIndex::AdjustForByHash - modify URI for by-hash support           /*{{{*/
1033 // ---------------------------------------------------------------------
1034 /* */
1035 void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey)
1036 {
1037    // TODO:
1038    //  - (maybe?) add support for by-hash into the sources.list as flag
1039    //  - make apt-ftparchive generate the hashes (and expire?)
1040    std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
1041    if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
1042       _config->FindB(HostKnob, false) == true ||
1043       MetaIndexParser->GetSupportsAcquireByHash())
1044    {
1045       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1046       if(Record)
1047       {
1048          // FIXME: should we really use the best hash here? or a fixed one?
1049          const HashString *TargetHash = Record->Hashes.find("");
1050          std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
1051          size_t trailing_slash = Desc.URI.find_last_of("/");
1052          Desc.URI = Desc.URI.replace(
1053             trailing_slash,
1054             Desc.URI.substr(trailing_slash+1).size()+1,
1055             ByHash);
1056       } else {
1057          _error->Warning(
1058             "Fetching ByHash requested but can not find record for %s",
1059             MetaKey.c_str());
1060       }
1061    }
1062 }
1063                                                                         /*}}}*/
1064 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1065 // ---------------------------------------------------------------------
1066 /* The only header we use is the last-modified header. */
1067 string pkgAcqIndex::Custom600Headers() const
1068 {
1069    string Final = _config->FindDir("Dir::State::lists");
1070    Final += URItoFileName(RealURI);
1071    if (_config->FindB("Acquire::GzipIndexes",false))
1072       Final += ".gz";
1073
1074    string msg = "\nIndex-File: true";
1075    // FIXME: this really should use "IndexTarget::IsOptional()" but that
1076    //        seems to be difficult without breaking ABI
1077    if (ShortDesc().find("Translation") != 0)
1078       msg += "\nFail-Ignore: true";
1079    struct stat Buf;
1080    if (stat(Final.c_str(),&Buf) == 0)
1081       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1082
1083    return msg;
1084 }
1085                                                                         /*}}}*/
1086 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1087 {
1088    size_t const nextExt = CompressionExtension.find(' ');
1089    if (nextExt != std::string::npos)
1090    {
1091       CompressionExtension = CompressionExtension.substr(nextExt+1);
1092       Init(RealURI, Desc.Description, Desc.ShortDesc);
1093       return;
1094    }
1095
1096    // on decompression failure, remove bad versions in partial/
1097    if (Decompression && Erase) {
1098       string s = _config->FindDir("Dir::State::lists") + "partial/";
1099       s.append(URItoFileName(RealURI));
1100       unlink(s.c_str());
1101    }
1102
1103    Item::Failed(Message,Cnf);
1104 }
1105                                                                         /*}}}*/
1106 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1107 // ---------------------------------------------------------------------
1108 /* This goes through a number of states.. On the initial fetch the
1109    method could possibly return an alternate filename which points
1110    to the uncompressed version of the file. If this is so the file
1111    is copied into the partial directory. In all other cases the file
1112    is decompressed with a gzip uri. */
1113 void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,
1114                        pkgAcquire::MethodConfig *Cfg)
1115 {
1116    Item::Done(Message,Size,Hashes,Cfg);
1117
1118    if (Decompression == true)
1119    {
1120       if (ExpectedHashes.usable() && ExpectedHashes != Hashes)
1121       {
1122          RenameOnError(HashSumMismatch);
1123          printHashSumComparision(RealURI, ExpectedHashes, Hashes);
1124          return;
1125       }
1126
1127       /* Verify the index file for correctness (all indexes must
1128        * have a Package field) (LP: #346386) (Closes: #627642) */
1129       if (Verify == true)
1130       {
1131          FileFd fd(DestFile, FileFd::ReadOnly);
1132          // Only test for correctness if the file is not empty (empty is ok)
1133          if (fd.FileSize() > 0)
1134          {
1135             pkgTagSection sec;
1136             pkgTagFile tag(&fd);
1137
1138             // all our current indexes have a field 'Package' in each section
1139             if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1140             {
1141                RenameOnError(InvalidFormat);
1142                return;
1143             }
1144          }
1145       }
1146
1147       // Done, move it into position
1148       string FinalFile = _config->FindDir("Dir::State::lists");
1149       FinalFile += URItoFileName(RealURI);
1150       Rename(DestFile,FinalFile);
1151       chmod(FinalFile.c_str(),0644);
1152
1153       /* We restore the original name to DestFile so that the clean operation
1154          will work OK */
1155       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1156       DestFile += URItoFileName(RealURI);
1157
1158       // Remove the compressed version.
1159       if (Erase == true)
1160          unlink(DestFile.c_str());
1161
1162       return;
1163    }
1164
1165    Erase = false;
1166    Complete = true;
1167
1168    // Handle the unzipd case
1169    string FileName = LookupTag(Message,"Alt-Filename");
1170    if (FileName.empty() == false)
1171    {
1172       // The files timestamp matches
1173       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1174          return;
1175       Decompression = true;
1176       Local = true;
1177       DestFile += ".decomp";
1178       Desc.URI = "copy:" + FileName;
1179       QueueURI(Desc);
1180       Mode = "copy";
1181       return;
1182    }
1183
1184    FileName = LookupTag(Message,"Filename");
1185    if (FileName.empty() == true)
1186    {
1187       Status = StatError;
1188       ErrorText = "Method gave a blank filename";
1189    }
1190
1191    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1192
1193    // The files timestamp matches
1194    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
1195        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1196           // Update DestFile for .gz suffix so that the clean operation keeps it
1197           DestFile += ".gz";
1198       return;
1199     }
1200
1201    if (FileName == DestFile)
1202       Erase = true;
1203    else
1204       Local = true;
1205
1206    string decompProg;
1207
1208    // If we enable compressed indexes and already have gzip, keep it
1209    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
1210       string FinalFile = _config->FindDir("Dir::State::lists");
1211       FinalFile += URItoFileName(RealURI) + ".gz";
1212       Rename(DestFile,FinalFile);
1213       chmod(FinalFile.c_str(),0644);
1214
1215       // Update DestFile for .gz suffix so that the clean operation keeps it
1216       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1217       DestFile += URItoFileName(RealURI) + ".gz";
1218       return;
1219     }
1220
1221    // get the binary name for your used compression type
1222    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1223    if(decompProg.empty() == false);
1224    else if(compExt == "uncompressed")
1225       decompProg = "copy";
1226    else {
1227       _error->Error("Unsupported extension: %s", compExt.c_str());
1228       return;
1229    }
1230
1231    Decompression = true;
1232    DestFile += ".decomp";
1233    Desc.URI = decompProg + ":" + FileName;
1234    QueueURI(Desc);
1235
1236    // FIXME: this points to a c++ string that goes out of scope
1237    Mode = decompProg.c_str();
1238 }
1239                                                                         /*}}}*/
1240 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1241 // ---------------------------------------------------------------------
1242 /* The Translation file is added to the queue */
1243 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1244                             string URI,string URIDesc,string ShortDesc)
1245   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "")
1246 {
1247 }
1248 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target,
1249                          HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser)
1250   : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser)
1251 {
1252    // load the filesize
1253    indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
1254    if(Record)
1255       FileSize = Record->Size;
1256 }
1257                                                                         /*}}}*/
1258 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1259 // ---------------------------------------------------------------------
1260 string pkgAcqIndexTrans::Custom600Headers() const
1261 {
1262    string Final = _config->FindDir("Dir::State::lists");
1263    Final += URItoFileName(RealURI);
1264
1265    struct stat Buf;
1266    if (stat(Final.c_str(),&Buf) != 0)
1267       return "\nFail-Ignore: true\nIndex-File: true";
1268    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1269 }
1270                                                                         /*}}}*/
1271 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1272 // ---------------------------------------------------------------------
1273 /* */
1274 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1275 {
1276    size_t const nextExt = CompressionExtension.find(' ');
1277    if (nextExt != std::string::npos)
1278    {
1279       CompressionExtension = CompressionExtension.substr(nextExt+1);
1280       Init(RealURI, Desc.Description, Desc.ShortDesc);
1281       Status = StatIdle;
1282       return;
1283    }
1284
1285    if (Cnf->LocalOnly == true ||
1286        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1287    {
1288       // Ignore this
1289       Status = StatDone;
1290       Complete = false;
1291       Dequeue();
1292       return;
1293    }
1294
1295    Item::Failed(Message,Cnf);
1296 }
1297                                                                         /*}}}*/
1298 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1299                              string URI,string URIDesc,string ShortDesc,
1300                              string MetaIndexURI, string MetaIndexURIDesc,
1301                              string MetaIndexShortDesc,
1302                              const vector<IndexTarget*>* IndexTargets,
1303                              indexRecords* MetaIndexParser) :
1304    Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI),
1305    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1306    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1307 {
1308    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1309    DestFile += URItoFileName(URI);
1310
1311    // remove any partial downloaded sig-file in partial/.
1312    // it may confuse proxies and is too small to warrant a
1313    // partial download anyway
1314    unlink(DestFile.c_str());
1315
1316    // Create the item
1317    Desc.Description = URIDesc;
1318    Desc.Owner = this;
1319    Desc.ShortDesc = ShortDesc;
1320    Desc.URI = URI;
1321
1322    string Final = _config->FindDir("Dir::State::lists");
1323    Final += URItoFileName(RealURI);
1324    if (RealFileExists(Final) == true)
1325    {
1326       // File was already in place.  It needs to be re-downloaded/verified
1327       // because Release might have changed, we do give it a different
1328       // name than DestFile because otherwise the http method will
1329       // send If-Range requests and there are too many broken servers
1330       // out there that do not understand them
1331       LastGoodSig = DestFile+".reverify";
1332       Rename(Final,LastGoodSig);
1333    }
1334
1335    // we expect the indextargets + one additional Release file
1336    ExpectedAdditionalItems = IndexTargets->size() + 1;
1337
1338    QueueURI(Desc);
1339 }
1340                                                                         /*}}}*/
1341 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1342 {
1343    // if the file was never queued undo file-changes done in the constructor
1344    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1345          LastGoodSig.empty() == false)
1346    {
1347       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1348       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1349          Rename(LastGoodSig, Final);
1350    }
1351
1352 }
1353                                                                         /*}}}*/
1354 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1355 // ---------------------------------------------------------------------
1356 /* The only header we use is the last-modified header. */
1357 string pkgAcqMetaSig::Custom600Headers() const
1358 {
1359    struct stat Buf;
1360    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1361       return "\nIndex-File: true";
1362
1363    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1364 }
1365
1366 void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes,
1367                          pkgAcquire::MethodConfig *Cfg)
1368 {
1369    Item::Done(Message, Size, Hashes, Cfg);
1370
1371    string FileName = LookupTag(Message,"Filename");
1372    if (FileName.empty() == true)
1373    {
1374       Status = StatError;
1375       ErrorText = "Method gave a blank filename";
1376       return;
1377    }
1378
1379    if (FileName != DestFile)
1380    {
1381       // We have to copy it into place
1382       Local = true;
1383       Desc.URI = "copy:" + FileName;
1384       QueueURI(Desc);
1385       return;
1386    }
1387
1388    Complete = true;
1389
1390    // at this point pkgAcqMetaIndex takes over
1391    ExpectedAdditionalItems = 0;
1392
1393    // put the last known good file back on i-m-s hit (it will
1394    // be re-verified again)
1395    // Else do nothing, we have the new file in DestFile then
1396    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1397       Rename(LastGoodSig, DestFile);
1398
1399    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1400    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1401                        MetaIndexShortDesc,  DestFile, IndexTargets,
1402                        MetaIndexParser);
1403
1404 }
1405                                                                         /*}}}*/
1406 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1407 {
1408    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1409
1410    // at this point pkgAcqMetaIndex takes over
1411    ExpectedAdditionalItems = 0;
1412
1413    // if we get a network error we fail gracefully
1414    if(Status == StatTransientNetworkError)
1415    {
1416       Item::Failed(Message,Cnf);
1417       // move the sigfile back on transient network failures
1418       if(FileExists(LastGoodSig))
1419          Rename(LastGoodSig,Final);
1420
1421       // set the status back to , Item::Failed likes to reset it
1422       Status = pkgAcquire::Item::StatTransientNetworkError;
1423       return;
1424    }
1425
1426    // Delete any existing sigfile when the acquire failed
1427    unlink(Final.c_str());
1428
1429    // queue a pkgAcqMetaIndex with no sigfile
1430    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1431                        "", IndexTargets, MetaIndexParser);
1432
1433    if (Cnf->LocalOnly == true ||
1434        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1435    {
1436       // Ignore this
1437       Status = StatDone;
1438       Complete = false;
1439       Dequeue();
1440       return;
1441    }
1442
1443    Item::Failed(Message,Cnf);
1444 }
1445                                                                         /*}}}*/
1446 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1447                                  string URI,string URIDesc,string ShortDesc,
1448                                  string SigFile,
1449                                  const vector<IndexTarget*>* IndexTargets,
1450                                  indexRecords* MetaIndexParser) :
1451    Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1452    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1453 {
1454    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1455    DestFile += URItoFileName(URI);
1456
1457    // Create the item
1458    Desc.Description = URIDesc;
1459    Desc.Owner = this;
1460    Desc.ShortDesc = ShortDesc;
1461    Desc.URI = URI;
1462
1463    // we expect more item
1464    ExpectedAdditionalItems = IndexTargets->size();
1465
1466    QueueURI(Desc);
1467 }
1468                                                                         /*}}}*/
1469 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1470 // ---------------------------------------------------------------------
1471 /* The only header we use is the last-modified header. */
1472 string pkgAcqMetaIndex::Custom600Headers() const
1473 {
1474    string Final = _config->FindDir("Dir::State::lists");
1475    Final += URItoFileName(RealURI);
1476
1477    struct stat Buf;
1478    if (stat(Final.c_str(),&Buf) != 0)
1479       return "\nIndex-File: true";
1480
1481    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1482 }
1483                                                                         /*}}}*/
1484 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
1485                            pkgAcquire::MethodConfig *Cfg)
1486 {
1487    Item::Done(Message,Size,Hashes,Cfg);
1488
1489    // MetaIndexes are done in two passes: one to download the
1490    // metaindex with an appropriate method, and a second to verify it
1491    // with the gpgv method
1492
1493    if (AuthPass == true)
1494    {
1495       AuthDone(Message);
1496
1497       // all cool, move Release file into place
1498       Complete = true;
1499    }
1500    else
1501    {
1502       RetrievalDone(Message);
1503       if (!Complete)
1504          // Still more retrieving to do
1505          return;
1506
1507       if (SigFile == "")
1508       {
1509          // There was no signature file, so we are finished.  Download
1510          // the indexes and do only hashsum verification if possible
1511          MetaIndexParser->Load(DestFile);
1512          QueueIndexes(false);
1513       }
1514       else
1515       {
1516          // FIXME: move this into pkgAcqMetaClearSig::Done on the next
1517          //        ABI break
1518
1519          // if we expect a ClearTextSignature (InRelase), ensure that
1520          // this is what we get and if not fail to queue a
1521          // Release/Release.gpg, see #346386
1522          if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
1523          {
1524             Failed(Message, Cfg);
1525             return;
1526          }
1527
1528          // There was a signature file, so pass it to gpgv for
1529          // verification
1530          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1531             std::cerr << "Metaindex acquired, queueing gpg verification ("
1532                       << SigFile << "," << DestFile << ")\n";
1533          AuthPass = true;
1534          Desc.URI = "gpgv:" + SigFile;
1535          QueueURI(Desc);
1536          Mode = "gpgv";
1537          return;
1538       }
1539    }
1540
1541    if (Complete == true)
1542    {
1543       string FinalFile = _config->FindDir("Dir::State::lists");
1544       FinalFile += URItoFileName(RealURI);
1545       if (SigFile == DestFile)
1546          SigFile = FinalFile;
1547       Rename(DestFile,FinalFile);
1548       chmod(FinalFile.c_str(),0644);
1549       DestFile = FinalFile;
1550    }
1551 }
1552                                                                         /*}}}*/
1553 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1554 {
1555    // We have just finished downloading a Release file (it is not
1556    // verified yet)
1557
1558    string FileName = LookupTag(Message,"Filename");
1559    if (FileName.empty() == true)
1560    {
1561       Status = StatError;
1562       ErrorText = "Method gave a blank filename";
1563       return;
1564    }
1565
1566    if (FileName != DestFile)
1567    {
1568       Local = true;
1569       Desc.URI = "copy:" + FileName;
1570       QueueURI(Desc);
1571       return;
1572    }
1573
1574    // make sure to verify against the right file on I-M-S hit
1575    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1576    if(IMSHit)
1577    {
1578       string FinalFile = _config->FindDir("Dir::State::lists");
1579       FinalFile += URItoFileName(RealURI);
1580       if (SigFile == DestFile)
1581       {
1582          SigFile = FinalFile;
1583          // constructor of pkgAcqMetaClearSig moved it out of the way,
1584          // now move it back in on IMS hit for the 'old' file
1585          string const OldClearSig = DestFile + ".reverify";
1586          if (RealFileExists(OldClearSig) == true)
1587             Rename(OldClearSig, FinalFile);
1588       }
1589       DestFile = FinalFile;
1590    }
1591    Complete = true;
1592 }
1593                                                                         /*}}}*/
1594 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1595 {
1596    // At this point, the gpgv method has succeeded, so there is a
1597    // valid signature from a key in the trusted keyring.  We
1598    // perform additional verification of its contents, and use them
1599    // to verify the indexes we are about to download
1600
1601    if (!MetaIndexParser->Load(DestFile))
1602    {
1603       Status = StatAuthError;
1604       ErrorText = MetaIndexParser->ErrorText;
1605       return;
1606    }
1607
1608    if (!VerifyVendor(Message))
1609    {
1610       return;
1611    }
1612
1613    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1614       std::cerr << "Signature verification succeeded: "
1615                 << DestFile << std::endl;
1616
1617    // Download further indexes with verification
1618    QueueIndexes(true);
1619
1620    // is it a clearsigned MetaIndex file?
1621    if (DestFile == SigFile)
1622       return;
1623
1624    // Done, move signature file into position
1625    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1626       URItoFileName(RealURI) + ".gpg";
1627    Rename(SigFile,VerifiedSigFile);
1628    chmod(VerifiedSigFile.c_str(),0644);
1629 }
1630                                                                         /*}}}*/
1631 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1632 {
1633 #if 0
1634    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1635     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1636    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1637    {
1638       Status = StatError;
1639       ErrorText = MetaIndexParser->ErrorText;
1640       return;
1641    }
1642 #endif
1643    bool transInRelease = false;
1644    {
1645       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1646       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1647          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1648          if (k->find("Translation-") != std::string::npos)
1649          {
1650             transInRelease = true;
1651             break;
1652          }
1653    }
1654
1655    // at this point the real Items are loaded in the fetcher
1656    ExpectedAdditionalItems = 0;
1657    for (vector <IndexTarget*>::const_iterator Target = IndexTargets->begin();
1658         Target != IndexTargets->end();
1659         ++Target)
1660    {
1661       HashStringList ExpectedIndexHashes;
1662       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1663       bool compressedAvailable = false;
1664       if (Record == NULL)
1665       {
1666          if ((*Target)->IsOptional() == true)
1667          {
1668             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1669             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1670                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1671                {
1672                   compressedAvailable = true;
1673                   break;
1674                }
1675          }
1676          else if (verify == true)
1677          {
1678             Status = StatAuthError;
1679             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1680             return;
1681          }
1682       }
1683       else
1684       {
1685          ExpectedIndexHashes = Record->Hashes;
1686          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1687          {
1688             std::cerr << "Queueing: " << (*Target)->URI << std::endl
1689                << "Expected Hash:" << std::endl;
1690             for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
1691                std::cerr <<  "\t- " << hs->toStr() << std::endl;
1692             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1693          }
1694          if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false)
1695          {
1696             Status = StatAuthError;
1697             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1698             return;
1699          }
1700       }
1701
1702       if ((*Target)->IsOptional() == true)
1703       {
1704          if ((*Target)->IsSubIndex() == true)
1705             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1706                                 (*Target)->ShortDesc, ExpectedIndexHashes);
1707          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1708          {
1709             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1710                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1711                new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1712             else
1713                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1714          }
1715          continue;
1716       }
1717
1718       /* Queue Packages file (either diff or full packages files, depending
1719          on the users option) - we also check if the PDiff Index file is listed
1720          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1721          instead, but passing the required info to it is to much hassle */
1722       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1723           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1724          new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1725       else
1726          new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1727    }
1728 }
1729                                                                         /*}}}*/
1730 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1731 {
1732    string::size_type pos;
1733
1734    // check for missing sigs (that where not fatal because otherwise we had
1735    // bombed earlier)
1736    string missingkeys;
1737    string msg = _("There is no public key available for the "
1738                   "following key IDs:\n");
1739    pos = Message.find("NO_PUBKEY ");
1740    if (pos != std::string::npos)
1741    {
1742       string::size_type start = pos+strlen("NO_PUBKEY ");
1743       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1744       missingkeys += (Fingerprint);
1745    }
1746    if(!missingkeys.empty())
1747       _error->Warning("%s", (msg + missingkeys).c_str());
1748
1749    string Transformed = MetaIndexParser->GetExpectedDist();
1750
1751    if (Transformed == "../project/experimental")
1752    {
1753       Transformed = "experimental";
1754    }
1755
1756    pos = Transformed.rfind('/');
1757    if (pos != string::npos)
1758    {
1759       Transformed = Transformed.substr(0, pos);
1760    }
1761
1762    if (Transformed == ".")
1763    {
1764       Transformed = "";
1765    }
1766
1767    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1768        MetaIndexParser->GetValidUntil() > 0) {
1769       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1770       if (invalid_since > 0)
1771          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1772          // the time since then the file is invalid - formated in the same way as in
1773          // the download progress display (e.g. 7d 3h 42min 1s)
1774          return _error->Error(
1775             _("Release file for %s is expired (invalid since %s). "
1776               "Updates for this repository will not be applied."),
1777             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1778    }
1779
1780    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1781    {
1782       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1783       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1784       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1785    }
1786
1787    if (MetaIndexParser->CheckDist(Transformed) == false)
1788    {
1789       // This might become fatal one day
1790 //       Status = StatAuthError;
1791 //       ErrorText = "Conflicting distribution; expected "
1792 //          + MetaIndexParser->GetExpectedDist() + " but got "
1793 //          + MetaIndexParser->GetDist();
1794 //       return false;
1795       if (!Transformed.empty())
1796       {
1797          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1798                          Desc.Description.c_str(),
1799                          Transformed.c_str(),
1800                          MetaIndexParser->GetDist().c_str());
1801       }
1802    }
1803
1804    return true;
1805 }
1806                                                                         /*}}}*/
1807 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1808 // ---------------------------------------------------------------------
1809 /* */
1810 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1811 {
1812    if (AuthPass == true)
1813    {
1814       // gpgv method failed, if we have a good signature
1815       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1816       if (DestFile != SigFile)
1817          LastGoodSigFile.append(".gpg");
1818       LastGoodSigFile.append(".reverify");
1819
1820       if(FileExists(LastGoodSigFile))
1821       {
1822          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1823          if (DestFile != SigFile)
1824             VerifiedSigFile.append(".gpg");
1825          Rename(LastGoodSigFile, VerifiedSigFile);
1826          Status = StatTransientNetworkError;
1827          _error->Warning(_("An error occurred during the signature "
1828                            "verification. The repository is not updated "
1829                            "and the previous index files will be used. "
1830                            "GPG error: %s: %s\n"),
1831                          Desc.Description.c_str(),
1832                          LookupTag(Message,"Message").c_str());
1833          RunScripts("APT::Update::Auth-Failure");
1834          return;
1835       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1836          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1837          _error->Error(_("GPG error: %s: %s"),
1838                          Desc.Description.c_str(),
1839                          LookupTag(Message,"Message").c_str());
1840          return;
1841       } else {
1842          _error->Warning(_("GPG error: %s: %s"),
1843                          Desc.Description.c_str(),
1844                          LookupTag(Message,"Message").c_str());
1845       }
1846       // gpgv method failed
1847       ReportMirrorFailure("GPGFailure");
1848    }
1849
1850    /* Always move the meta index, even if gpgv failed. This ensures
1851     * that PackageFile objects are correctly filled in */
1852    if (FileExists(DestFile)) {
1853       string FinalFile = _config->FindDir("Dir::State::lists");
1854       FinalFile += URItoFileName(RealURI);
1855       /* InRelease files become Release files, otherwise
1856        * they would be considered as trusted later on */
1857       if (SigFile == DestFile) {
1858          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1859                                        "Release");
1860          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1861                                        "Release");
1862          SigFile = FinalFile;
1863       }
1864       Rename(DestFile,FinalFile);
1865       chmod(FinalFile.c_str(),0644);
1866
1867       DestFile = FinalFile;
1868    }
1869
1870    // No Release file was present, or verification failed, so fall
1871    // back to queueing Packages files without verification
1872    QueueIndexes(false);
1873 }
1874                                                                         /*}}}*/
1875 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1876                 string const &URI, string const &URIDesc, string const &ShortDesc,
1877                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1878                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1879                 const vector<IndexTarget*>* IndexTargets,
1880                 indexRecords* MetaIndexParser) :
1881         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1882         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1883         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1884 {
1885    SigFile = DestFile;
1886
1887    // index targets + (worst case:) Release/Release.gpg
1888    ExpectedAdditionalItems = IndexTargets->size() + 2;
1889
1890
1891    // keep the old InRelease around in case of transistent network errors
1892    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1893    if (RealFileExists(Final) == true)
1894    {
1895       string const LastGoodSig = DestFile + ".reverify";
1896       Rename(Final,LastGoodSig);
1897    }
1898 }
1899                                                                         /*}}}*/
1900 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1901 {
1902    // if the file was never queued undo file-changes done in the constructor
1903    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1904    {
1905       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1906       string const LastGoodSig = DestFile + ".reverify";
1907       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1908          Rename(LastGoodSig, Final);
1909    }
1910 }
1911                                                                         /*}}}*/
1912 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1913 // ---------------------------------------------------------------------
1914 // FIXME: this can go away once the InRelease file is used widely
1915 string pkgAcqMetaClearSig::Custom600Headers() const
1916 {
1917    string Final = _config->FindDir("Dir::State::lists");
1918    Final += URItoFileName(RealURI);
1919
1920    struct stat Buf;
1921    if (stat(Final.c_str(),&Buf) != 0)
1922    {
1923       Final = DestFile + ".reverify";
1924       if (stat(Final.c_str(),&Buf) != 0)
1925          return "\nIndex-File: true\nFail-Ignore: true\n";
1926    }
1927
1928    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1929 }
1930                                                                         /*}}}*/
1931 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1932 {
1933    // we failed, we will not get additional items from this method
1934    ExpectedAdditionalItems = 0;
1935
1936    if (AuthPass == false)
1937    {
1938       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1939       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1940       string FinalFile = _config->FindDir("Dir::State::lists");
1941       FinalFile.append(URItoFileName(RealURI));
1942       if (FileExists(FinalFile))
1943          unlink(FinalFile.c_str());
1944
1945       new pkgAcqMetaSig(Owner,
1946                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1947                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1948                         IndexTargets, MetaIndexParser);
1949       if (Cnf->LocalOnly == true ||
1950           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1951          Dequeue();
1952    }
1953    else
1954       pkgAcqMetaIndex::Failed(Message, Cnf);
1955 }
1956                                                                         /*}}}*/
1957 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1958 // ---------------------------------------------------------------------
1959 /* This just sets up the initial fetch environment and queues the first
1960    possibilitiy */
1961 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1962                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1963                              string &StoreFilename) :
1964                Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs),
1965                StoreFilename(StoreFilename), Vf(Version.FileList()),
1966                Trusted(false)
1967 {
1968    Retries = _config->FindI("Acquire::Retries",0);
1969
1970    if (Version.Arch() == 0)
1971    {
1972       _error->Error(_("I wasn't able to locate a file for the %s package. "
1973                       "This might mean you need to manually fix this package. "
1974                       "(due to missing arch)"),
1975                     Version.ParentPkg().FullName().c_str());
1976       return;
1977    }
1978
1979    /* We need to find a filename to determine the extension. We make the
1980       assumption here that all the available sources for this version share
1981       the same extension.. */
1982    // Skip not source sources, they do not have file fields.
1983    for (; Vf.end() == false; ++Vf)
1984    {
1985       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1986          continue;
1987       break;
1988    }
1989
1990    // Does not really matter here.. we are going to fail out below
1991    if (Vf.end() != true)
1992    {
1993       // If this fails to get a file name we will bomb out below.
1994       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1995       if (_error->PendingError() == true)
1996          return;
1997
1998       // Generate the final file name as: package_version_arch.foo
1999       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
2000                       QuoteString(Version.VerStr(),"_:") + '_' +
2001                       QuoteString(Version.Arch(),"_:.") +
2002                       "." + flExtension(Parse.FileName());
2003    }
2004
2005    // check if we have one trusted source for the package. if so, switch
2006    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2007    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
2008    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
2009    bool seenUntrusted = false;
2010    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
2011    {
2012       pkgIndexFile *Index;
2013       if (Sources->FindIndex(i.File(),Index) == false)
2014          continue;
2015
2016       if (debugAuth == true)
2017          std::cerr << "Checking index: " << Index->Describe()
2018                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
2019
2020       if (Index->IsTrusted() == true)
2021       {
2022          Trusted = true;
2023          if (allowUnauth == false)
2024             break;
2025       }
2026       else
2027          seenUntrusted = true;
2028    }
2029
2030    // "allow-unauthenticated" restores apts old fetching behaviour
2031    // that means that e.g. unauthenticated file:// uris are higher
2032    // priority than authenticated http:// uris
2033    if (allowUnauth == true && seenUntrusted == true)
2034       Trusted = false;
2035
2036    // Select a source
2037    if (QueueNext() == false && _error->PendingError() == false)
2038       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
2039                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
2040 }
2041                                                                         /*}}}*/
2042 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
2043 // ---------------------------------------------------------------------
2044 /* This queues the next available file version for download. It checks if
2045    the archive is already available in the cache and stashs the MD5 for
2046    checking later. */
2047 bool pkgAcqArchive::QueueNext()
2048 {
2049    for (; Vf.end() == false; ++Vf)
2050    {
2051       // Ignore not source sources
2052       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2053          continue;
2054
2055       // Try to cross match against the source list
2056       pkgIndexFile *Index;
2057       if (Sources->FindIndex(Vf.File(),Index) == false)
2058             continue;
2059
2060       // only try to get a trusted package from another source if that source
2061       // is also trusted
2062       if(Trusted && !Index->IsTrusted())
2063          continue;
2064
2065       // Grab the text package record
2066       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2067       if (_error->PendingError() == true)
2068          return false;
2069
2070       string PkgFile = Parse.FileName();
2071       ExpectedHashes = Parse.Hashes();
2072
2073       if (PkgFile.empty() == true)
2074          return _error->Error(_("The package index files are corrupted. No Filename: "
2075                               "field for package %s."),
2076                               Version.ParentPkg().Name());
2077
2078       Desc.URI = Index->ArchiveURI(PkgFile);
2079       Desc.Description = Index->ArchiveInfo(Version);
2080       Desc.Owner = this;
2081       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2082
2083       // See if we already have the file. (Legacy filenames)
2084       FileSize = Version->Size;
2085       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2086       struct stat Buf;
2087       if (stat(FinalFile.c_str(),&Buf) == 0)
2088       {
2089          // Make sure the size matches
2090          if ((unsigned long long)Buf.st_size == Version->Size)
2091          {
2092             Complete = true;
2093             Local = true;
2094             Status = StatDone;
2095             StoreFilename = DestFile = FinalFile;
2096             return true;
2097          }
2098
2099          /* Hmm, we have a file and its size does not match, this means it is
2100             an old style mismatched arch */
2101          unlink(FinalFile.c_str());
2102       }
2103
2104       // Check it again using the new style output filenames
2105       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2106       if (stat(FinalFile.c_str(),&Buf) == 0)
2107       {
2108          // Make sure the size matches
2109          if ((unsigned long long)Buf.st_size == Version->Size)
2110          {
2111             Complete = true;
2112             Local = true;
2113             Status = StatDone;
2114             StoreFilename = DestFile = FinalFile;
2115             return true;
2116          }
2117
2118          /* Hmm, we have a file and its size does not match, this shouldn't
2119             happen.. */
2120          unlink(FinalFile.c_str());
2121       }
2122
2123       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2124
2125       // Check the destination file
2126       if (stat(DestFile.c_str(),&Buf) == 0)
2127       {
2128          // Hmm, the partial file is too big, erase it
2129          if ((unsigned long long)Buf.st_size > Version->Size)
2130             unlink(DestFile.c_str());
2131          else
2132             PartialSize = Buf.st_size;
2133       }
2134
2135       // Disables download of archives - useful if no real installation follows,
2136       // e.g. if we are just interested in proposed installation order
2137       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2138       {
2139          Complete = true;
2140          Local = true;
2141          Status = StatDone;
2142          StoreFilename = DestFile = FinalFile;
2143          return true;
2144       }
2145
2146       // Create the item
2147       Local = false;
2148       QueueURI(Desc);
2149
2150       ++Vf;
2151       return true;
2152    }
2153    return false;
2154 }
2155                                                                         /*}}}*/
2156 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2157 // ---------------------------------------------------------------------
2158 /* */
2159 void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes,
2160                          pkgAcquire::MethodConfig *Cfg)
2161 {
2162    Item::Done(Message, Size, CalcHashes, Cfg);
2163
2164    // Check the size
2165    if (Size != Version->Size)
2166    {
2167       RenameOnError(SizeMismatch);
2168       return;
2169    }
2170
2171    // FIXME: could this empty() check impose *any* sort of security issue?
2172    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2173    {
2174       RenameOnError(HashSumMismatch);
2175       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2176       return;
2177    }
2178
2179    // Grab the output filename
2180    string FileName = LookupTag(Message,"Filename");
2181    if (FileName.empty() == true)
2182    {
2183       Status = StatError;
2184       ErrorText = "Method gave a blank filename";
2185       return;
2186    }
2187
2188    Complete = true;
2189
2190    // Reference filename
2191    if (FileName != DestFile)
2192    {
2193       StoreFilename = DestFile = FileName;
2194       Local = true;
2195       return;
2196    }
2197
2198    // Done, move it into position
2199    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2200    FinalFile += flNotDir(StoreFilename);
2201    Rename(DestFile,FinalFile);
2202
2203    StoreFilename = DestFile = FinalFile;
2204    Complete = true;
2205 }
2206                                                                         /*}}}*/
2207 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2208 // ---------------------------------------------------------------------
2209 /* Here we try other sources */
2210 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2211 {
2212    ErrorText = LookupTag(Message,"Message");
2213
2214    /* We don't really want to retry on failed media swaps, this prevents
2215       that. An interesting observation is that permanent failures are not
2216       recorded. */
2217    if (Cnf->Removable == true &&
2218        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2219    {
2220       // Vf = Version.FileList();
2221       while (Vf.end() == false) ++Vf;
2222       StoreFilename = string();
2223       Item::Failed(Message,Cnf);
2224       return;
2225    }
2226
2227    if (QueueNext() == false)
2228    {
2229       // This is the retry counter
2230       if (Retries != 0 &&
2231           Cnf->LocalOnly == false &&
2232           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2233       {
2234          Retries--;
2235          Vf = Version.FileList();
2236          if (QueueNext() == true)
2237             return;
2238       }
2239
2240       StoreFilename = string();
2241       Item::Failed(Message,Cnf);
2242    }
2243 }
2244                                                                         /*}}}*/
2245 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2246 // ---------------------------------------------------------------------
2247 APT_PURE bool pkgAcqArchive::IsTrusted() const
2248 {
2249    return Trusted;
2250 }
2251                                                                         /*}}}*/
2252 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2253 // ---------------------------------------------------------------------
2254 /* */
2255 void pkgAcqArchive::Finished()
2256 {
2257    if (Status == pkgAcquire::Item::StatDone &&
2258        Complete == true)
2259       return;
2260    StoreFilename = string();
2261 }
2262                                                                         /*}}}*/
2263 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2264 // ---------------------------------------------------------------------
2265 /* The file is added to the queue */
2266 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes,
2267                        unsigned long long Size,string Dsc,string ShortDesc,
2268                        const string &DestDir, const string &DestFilename,
2269                        bool IsIndexFile) :
2270                        Item(Owner, Hashes), IsIndexFile(IsIndexFile)
2271 {
2272    Retries = _config->FindI("Acquire::Retries",0);
2273
2274    if(!DestFilename.empty())
2275       DestFile = DestFilename;
2276    else if(!DestDir.empty())
2277       DestFile = DestDir + "/" + flNotDir(URI);
2278    else
2279       DestFile = flNotDir(URI);
2280
2281    // Create the item
2282    Desc.URI = URI;
2283    Desc.Description = Dsc;
2284    Desc.Owner = this;
2285
2286    // Set the short description to the archive component
2287    Desc.ShortDesc = ShortDesc;
2288
2289    // Get the transfer sizes
2290    FileSize = Size;
2291    struct stat Buf;
2292    if (stat(DestFile.c_str(),&Buf) == 0)
2293    {
2294       // Hmm, the partial file is too big, erase it
2295       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2296          unlink(DestFile.c_str());
2297       else
2298          PartialSize = Buf.st_size;
2299    }
2300
2301    QueueURI(Desc);
2302 }
2303                                                                         /*}}}*/
2304 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2305 // ---------------------------------------------------------------------
2306 /* */
2307 void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes,
2308                       pkgAcquire::MethodConfig *Cnf)
2309 {
2310    Item::Done(Message,Size,CalcHashes,Cnf);
2311
2312    // Check the hash
2313    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2314    {
2315       RenameOnError(HashSumMismatch);
2316       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2317       return;
2318    }
2319
2320    string FileName = LookupTag(Message,"Filename");
2321    if (FileName.empty() == true)
2322    {
2323       Status = StatError;
2324       ErrorText = "Method gave a blank filename";
2325       return;
2326    }
2327
2328    Complete = true;
2329
2330    // The files timestamp matches
2331    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2332       return;
2333
2334    // We have to copy it into place
2335    if (FileName != DestFile)
2336    {
2337       Local = true;
2338       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2339           Cnf->Removable == true)
2340       {
2341          Desc.URI = "copy:" + FileName;
2342          QueueURI(Desc);
2343          return;
2344       }
2345
2346       // Erase the file if it is a symlink so we can overwrite it
2347       struct stat St;
2348       if (lstat(DestFile.c_str(),&St) == 0)
2349       {
2350          if (S_ISLNK(St.st_mode) != 0)
2351             unlink(DestFile.c_str());
2352       }
2353
2354       // Symlink the file
2355       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2356       {
2357          ErrorText = "Link to " + DestFile + " failure ";
2358          Status = StatError;
2359          Complete = false;
2360       }
2361    }
2362 }
2363                                                                         /*}}}*/
2364 // AcqFile::Failed - Failure handler                                    /*{{{*/
2365 // ---------------------------------------------------------------------
2366 /* Here we try other sources */
2367 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2368 {
2369    ErrorText = LookupTag(Message,"Message");
2370
2371    // This is the retry counter
2372    if (Retries != 0 &&
2373        Cnf->LocalOnly == false &&
2374        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2375    {
2376       Retries--;
2377       QueueURI(Desc);
2378       return;
2379    }
2380
2381    Item::Failed(Message,Cnf);
2382 }
2383                                                                         /*}}}*/
2384 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2385 // ---------------------------------------------------------------------
2386 /* The only header we use is the last-modified header. */
2387 string pkgAcqFile::Custom600Headers() const
2388 {
2389    if (IsIndexFile)
2390       return "\nIndex-File: true";
2391    return "";
2392 }
2393                                                                         /*}}}*/