apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/
  54 {
  55    if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false)
  56       return;
  57    std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl;
  58    for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs)
  59       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  60    std::cerr << " Actual Hash: " << std::endl;
  61    for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs)
  62       std::cerr <<  "\t- " << hs->toStr() << std::endl;
  63 }
  64                                                                         /*}}}*/
  65
  66 // Acquire::Item::Item - Constructor                                    /*{{{*/
  67 pkgAcquire::Item::Item(pkgAcquire *Owner, HashStringList const &ExpectedHashes) :
  68    Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
  69    Local(false), QueueCounter(0), ExpectedAdditionalItems(0),
  70    ExpectedHashes(ExpectedHashes)
  71 {
  72    Owner->Add(this);
  73    Status = StatIdle;
  74 }
  75                                                                         /*}}}*/
  76 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  77 // ---------------------------------------------------------------------
  78 /* */
  79 pkgAcquire::Item::~Item()
  80 {
  81    Owner->Remove(this);
  82 }
  83                                                                         /*}}}*/
  84 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* We return to an idle state if there are still other queues that could
  87    fetch this object */
  88 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  89 {
  90    Status = StatIdle;
  91    ErrorText = LookupTag(Message,"Message");
  92    UsedMirror =  LookupTag(Message,"UsedMirror");
  93    if (QueueCounter <= 1)
  94    {
  95       /* This indicates that the file is not available right now but might
  96          be sometime later. If we do a retry cycle then this should be
  97          retried [CDROMs] */
  98       if (Cnf->LocalOnly == true &&
  99           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
 100       {
 101          Status = StatIdle;
 102          Dequeue();
 103          return;
 104       }
 105
 106       Status = StatError;
 107       Dequeue();
 108    }
 109
 110    // report mirror failure back to LP if we actually use a mirror
 111    string FailReason = LookupTag(Message, "FailReason");
 112    if(FailReason.size() != 0)
 113       ReportMirrorFailure(FailReason);
 114    else
 115       ReportMirrorFailure(ErrorText);
 116 }
 117                                                                         /*}}}*/
 118 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 119 // ---------------------------------------------------------------------
 120 /* Stash status and the file size. Note that setting Complete means
 121    sub-phases of the acquire process such as decompresion are operating */
 122 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 123 {
 124    Status = StatFetching;
 125    if (FileSize == 0 && Complete == false)
 126       FileSize = Size;
 127 }
 128                                                                         /*}}}*/
 129 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 130 // ---------------------------------------------------------------------
 131 /* */
 132 void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/,
 133                             pkgAcquire::MethodConfig * /*Cnf*/)
 134 {
 135    // We just downloaded something..
 136    string FileName = LookupTag(Message,"Filename");
 137    UsedMirror =  LookupTag(Message,"UsedMirror");
 138    if (Complete == false && !Local && FileName == DestFile)
 139    {
 140       if (Owner->Log != 0)
 141          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 142    }
 143
 144    if (FileSize == 0)
 145       FileSize= Size;
 146    Status = StatDone;
 147    ErrorText = string();
 148    Owner->Dequeue(this);
 149 }
 150                                                                         /*}}}*/
 151 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 152 // ---------------------------------------------------------------------
 153 /* This helper function is used by a lot of item methods as their final
 154    step */
 155 void pkgAcquire::Item::Rename(string From,string To)
 156 {
 157    if (rename(From.c_str(),To.c_str()) != 0)
 158    {
 159       char S[300];
 160       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 161               From.c_str(),To.c_str());
 162       Status = StatError;
 163       ErrorText = S;
 164    }
 165 }
 166                                                                         /*}}}*/
 167 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 168 {
 169    if(FileExists(DestFile))
 170       Rename(DestFile, DestFile + ".FAILED");
 171
 172    switch (error)
 173    {
 174       case HashSumMismatch:
 175          ErrorText = _("Hash Sum mismatch");
 176          Status = StatAuthError;
 177          ReportMirrorFailure("HashChecksumFailure");
 178          break;
 179       case SizeMismatch:
 180          ErrorText = _("Size mismatch");
 181          Status = StatAuthError;
 182          ReportMirrorFailure("SizeFailure");
 183          break;
 184       case InvalidFormat:
 185          ErrorText = _("Invalid file format");
 186          Status = StatError;
 187          // do not report as usually its not the mirrors fault, but Portal/Proxy
 188          break;
 189    }
 190    return false;
 191 }
 192                                                                         /*}}}*/
 193 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 194 // ---------------------------------------------------------------------
 195 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 196 {
 197    // we only act if a mirror was used at all
 198    if(UsedMirror.empty())
 199       return;
 200 #if 0
 201    std::cerr << "\nReportMirrorFailure: "
 202              << UsedMirror
 203              << " Uri: " << DescURI()
 204              << " FailCode: "
 205              << FailCode << std::endl;
 206 #endif
 207    const char *Args[40];
 208    unsigned int i = 0;
 209    string report = _config->Find("Methods::Mirror::ProblemReporting",
 210                                  "/usr/lib/apt/apt-report-mirror-failure");
 211    if(!FileExists(report))
 212       return;
 213    Args[i++] = report.c_str();
 214    Args[i++] = UsedMirror.c_str();
 215    Args[i++] = DescURI().c_str();
 216    Args[i++] = FailCode.c_str();
 217    Args[i++] = NULL;
 218    pid_t pid = ExecFork();
 219    if(pid < 0)
 220    {
 221       _error->Error("ReportMirrorFailure Fork failed");
 222       return;
 223    }
 224    else if(pid == 0)
 225    {
 226       execvp(Args[0], (char**)Args);
 227       std::cerr << "Could not exec " << Args[0] << std::endl;
 228       _exit(100);
 229    }
 230    if(!ExecWait(pid, "report-mirror-failure"))
 231    {
 232       _error->Warning("Couldn't report problem to '%s'",
 233                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 234    }
 235 }
 236                                                                         /*}}}*/
 237 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 238 // ---------------------------------------------------------------------
 239 /* Get a sub-index file based on checksums from a 'master' file and
 240    possibly query additional files */
 241 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 242                                  string const &URIDesc, string const &ShortDesc,
 243                                  HashStringList const &ExpectedHashes)
 244    : Item(Owner, ExpectedHashes)
 245 {
 246    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 247    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 248
 249    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 250    DestFile += URItoFileName(URI);
 251
 252    Desc.URI = URI;
 253    Desc.Description = URIDesc;
 254    Desc.Owner = this;
 255    Desc.ShortDesc = ShortDesc;
 256
 257    QueueURI(Desc);
 258
 259    if(Debug)
 260       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 261 }
 262                                                                         /*}}}*/
 263 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 264 // ---------------------------------------------------------------------
 265 /* The only header we use is the last-modified header. */
 266 string pkgAcqSubIndex::Custom600Headers() const
 267 {
 268    string Final = _config->FindDir("Dir::State::lists");
 269    Final += URItoFileName(Desc.URI);
 270
 271    struct stat Buf;
 272    if (stat(Final.c_str(),&Buf) != 0)
 273       return "\nIndex-File: true\nFail-Ignore: true\n";
 274    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 278 {
 279    if(Debug)
 280       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 281
 282    Complete = false;
 283    Status = StatDone;
 284    Dequeue();
 285
 286    // No good Index is provided
 287 }
 288                                                                         /*}}}*/
 289 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,  /*{{{*/
 290                            pkgAcquire::MethodConfig *Cnf)
 291 {
 292    if(Debug)
 293       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 294
 295    string FileName = LookupTag(Message,"Filename");
 296    if (FileName.empty() == true)
 297    {
 298       Status = StatError;
 299       ErrorText = "Method gave a blank filename";
 300       return;
 301    }
 302
 303    if (FileName != DestFile)
 304    {
 305       Local = true;
 306       Desc.URI = "copy:" + FileName;
 307       QueueURI(Desc);
 308       return;
 309    }
 310
 311    Item::Done(Message, Size, Hashes, Cnf);
 312
 313    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 314
 315    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 316    indexRecords SubIndexParser;
 317    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 318       Status = StatError;
 319       ErrorText = SubIndexParser.ErrorText;
 320       return;
 321    }
 322
 323    // success in downloading the index
 324    // rename the index
 325    if(Debug)
 326       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 327    Rename(DestFile,FinalFile);
 328    chmod(FinalFile.c_str(),0644);
 329    DestFile = FinalFile;
 330
 331    if(ParseIndex(DestFile) == false)
 332       return Failed("", NULL);
 333
 334    Complete = true;
 335    Status = StatDone;
 336    Dequeue();
 337    return;
 338 }
 339                                                                         /*}}}*/
 340 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 341 {
 342    indexRecords SubIndexParser;
 343    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 344       return false;
 345    // so something with the downloaded index
 346    return true;
 347 }
 348                                                                         /*}}}*/
 349 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 350 // ---------------------------------------------------------------------
 351 /* Get the DiffIndex file first and see if there are patches available
 352  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 353  * patches. If anything goes wrong in that process, it will fall back to
 354  * the original packages file
 355  */
 356 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 357                                  IndexTarget const * const Target,
 358                                  HashStringList const &ExpectedHashes,
 359                                  indexRecords *MetaIndexParser)
 360    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser)
 361 {
 362
 363    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 364
 365    RealURI = Target->URI;
 366    Desc.Owner = this;
 367    Desc.Description = Target->Description + "/DiffIndex";
 368    Desc.ShortDesc = Target->ShortDesc;
 369    Desc.URI = Target->URI + ".diff/Index";
 370
 371    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 372    DestFile += URItoFileName(Desc.URI);
 373
 374    if(Debug)
 375       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 376
 377    // look for the current package file
 378    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 379    CurrentPackagesFile += URItoFileName(RealURI);
 380
 381    // FIXME: this file:/ check is a hack to prevent fetching
 382    //        from local sources. this is really silly, and
 383    //        should be fixed cleanly as soon as possible
 384    if(!FileExists(CurrentPackagesFile) ||
 385       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 386    {
 387       // we don't have a pkg file or we don't want to queue
 388       if(Debug)
 389          std::clog << "No index file, local or canceld by user" << std::endl;
 390       Failed("", NULL);
 391       return;
 392    }
 393
 394    if(Debug)
 395       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 396          << CurrentPackagesFile << std::endl;
 397
 398    QueueURI(Desc);
 399
 400 }
 401                                                                         /*}}}*/
 402 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 403 // ---------------------------------------------------------------------
 404 /* The only header we use is the last-modified header. */
 405 string pkgAcqDiffIndex::Custom600Headers() const
 406 {
 407    string Final = _config->FindDir("Dir::State::lists");
 408    Final += URItoFileName(Desc.URI);
 409
 410    if(Debug)
 411       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 412
 413    struct stat Buf;
 414    if (stat(Final.c_str(),&Buf) != 0)
 415       return "\nIndex-File: true";
 416
 417    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 418 }
 419                                                                         /*}}}*/
 420 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 421 {
 422    if(Debug)
 423       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 424          << std::endl;
 425
 426    pkgTagSection Tags;
 427    string ServerSha1;
 428    vector<DiffInfo> available_patches;
 429
 430    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 431    pkgTagFile TF(&Fd);
 432    if (_error->PendingError() == true)
 433       return false;
 434
 435    if(TF.Step(Tags) == true)
 436    {
 437       bool found = false;
 438       DiffInfo d;
 439       string size;
 440
 441       string const tmp = Tags.FindS("SHA1-Current");
 442       std::stringstream ss(tmp);
 443       ss >> ServerSha1 >> size;
 444       unsigned long const ServerSize = atol(size.c_str());
 445
 446       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 447       SHA1Summation SHA1;
 448       SHA1.AddFD(fd);
 449       string const local_sha1 = SHA1.Result();
 450
 451       if(local_sha1 == ServerSha1)
 452       {
 453          // we have the same sha1 as the server so we are done here
 454          if(Debug)
 455             std::clog << "Package file is up-to-date" << std::endl;
 456          // list cleanup needs to know that this file as well as the already
 457          // present index is ours, so we create an empty diff to save it for us
 458          new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 459                               ServerSha1, available_patches);
 460          return true;
 461       }
 462       else
 463       {
 464          if(Debug)
 465             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 466
 467          // check the historie and see what patches we need
 468          string const history = Tags.FindS("SHA1-History");
 469          std::stringstream hist(history);
 470          while(hist >> d.sha1 >> size >> d.file)
 471          {
 472             // read until the first match is found
 473             // from that point on, we probably need all diffs
 474             if(d.sha1 == local_sha1)
 475                found=true;
 476             else if (found == false)
 477                continue;
 478
 479             if(Debug)
 480                std::clog << "Need to get diff: " << d.file << std::endl;
 481             available_patches.push_back(d);
 482          }
 483
 484          if (available_patches.empty() == false)
 485          {
 486             // patching with too many files is rather slow compared to a fast download
 487             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 488             if (fileLimit != 0 && fileLimit < available_patches.size())
 489             {
 490                if (Debug)
 491                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 492                         << ") so fallback to complete download" << std::endl;
 493                return false;
 494             }
 495
 496             // see if the patches are too big
 497             found = false; // it was true and it will be true again at the end
 498             d = *available_patches.begin();
 499             string const firstPatch = d.file;
 500             unsigned long patchesSize = 0;
 501             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 502             while(patches >> d.sha1 >> size >> d.file)
 503             {
 504                if (firstPatch == d.file)
 505                   found = true;
 506                else if (found == false)
 507                   continue;
 508
 509                patchesSize += atol(size.c_str());
 510             }
 511             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 512             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 513             {
 514                if (Debug)
 515                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 516                         << ") so fallback to complete download" << std::endl;
 517                return false;
 518             }
 519          }
 520       }
 521
 522       // we have something, queue the next diff
 523       if(found)
 524       {
 525          // queue the diffs
 526          string::size_type const last_space = Description.rfind(" ");
 527          if(last_space != string::npos)
 528             Description.erase(last_space, Description.size()-last_space);
 529
 530          /* decide if we should download patches one by one or in one go:
 531             The first is good if the server merges patches, but many don't so client
 532             based merging can be attempt in which case the second is better.
 533             "bad things" will happen if patches are merged on the server,
 534             but client side merging is attempt as well */
 535          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 536          if (pdiff_merge == true)
 537          {
 538             // reprepro adds this flag if it has merged patches on the server
 539             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 540             pdiff_merge = (precedence != "merged");
 541          }
 542
 543          if (pdiff_merge == false)
 544          {
 545             new pkgAcqIndexDiffs(Owner, Target, ExpectedHashes, MetaIndexParser,
 546                                  ServerSha1, available_patches);
 547          }
 548          else
 549          {
 550             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 551             for(size_t i = 0; i < available_patches.size(); ++i)
 552                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, Target,
 553                                                        ExpectedHashes,
 554                                                        MetaIndexParser,
 555                                                        available_patches[i],
 556                                                        diffs);
 557          }
 558
 559          Complete = false;
 560          Status = StatDone;
 561          Dequeue();
 562          return true;
 563       }
 564    }
 565
 566    // Nothing found, report and return false
 567    // Failing here is ok, if we return false later, the full
 568    // IndexFile is queued
 569    if(Debug)
 570       std::clog << "Can't find a patch in the index file" << std::endl;
 571    return false;
 572 }
 573                                                                         /*}}}*/
 574 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 575 {
 576    if(Debug)
 577       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 578                 << "Falling back to normal index file acquire" << std::endl;
 579
 580    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 581
 582    Complete = false;
 583    Status = StatDone;
 584    Dequeue();
 585 }
 586                                                                         /*}}}*/
 587 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
 588                            pkgAcquire::MethodConfig *Cnf)
 589 {
 590    if(Debug)
 591       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 592
 593    Item::Done(Message, Size, Hashes, Cnf);
 594
 595    string FinalFile;
 596    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 597
 598    // success in downloading the index
 599    // rename the index
 600    FinalFile += string(".IndexDiff");
 601    if(Debug)
 602       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 603                 << std::endl;
 604    Rename(DestFile,FinalFile);
 605    chmod(FinalFile.c_str(),0644);
 606    DestFile = FinalFile;
 607
 608    if(!ParseDiffIndex(DestFile))
 609       return Failed("", NULL);
 610
 611    Complete = true;
 612    Status = StatDone;
 613    Dequeue();
 614    return;
 615 }
 616                                                                         /*}}}*/
 617 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 618 // ---------------------------------------------------------------------
 619 /* The package diff is added to the queue. one object is constructed
 620  * for each diff and the index
 621  */
 622 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 623                                    struct IndexTarget const * const Target,
 624                                    HashStringList const &ExpectedHashes,
 625                                    indexRecords *MetaIndexParser,
 626                                    string ServerSha1,
 627                                    vector<DiffInfo> diffs)
 628    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 629      available_patches(diffs), ServerSha1(ServerSha1)
 630 {
 631
 632    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 633    DestFile += URItoFileName(Target->URI);
 634
 635    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 636
 637    RealURI = Target->URI;
 638    Desc.Owner = this;
 639    Description = Target->Description;
 640    Desc.ShortDesc = Target->ShortDesc;
 641
 642    if(available_patches.empty() == true)
 643    {
 644       // we are done (yeah!)
 645       Finish(true);
 646    }
 647    else
 648    {
 649       // get the next diff
 650       State = StateFetchDiff;
 651       QueueNextDiff();
 652    }
 653 }
 654                                                                         /*}}}*/
 655 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 656 {
 657    if(Debug)
 658       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 659                 << "Falling back to normal index file acquire" << std::endl;
 660    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 661    Finish();
 662 }
 663                                                                         /*}}}*/
 664 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 665 void pkgAcqIndexDiffs::Finish(bool allDone)
 666 {
 667    // we restore the original name, this is required, otherwise
 668    // the file will be cleaned
 669    if(allDone)
 670    {
 671       DestFile = _config->FindDir("Dir::State::lists");
 672       DestFile += URItoFileName(RealURI);
 673
 674       if(HashSums().usable() && !HashSums().VerifyFile(DestFile))
 675       {
 676          RenameOnError(HashSumMismatch);
 677          Dequeue();
 678          return;
 679       }
 680
 681       // this is for the "real" finish
 682       Complete = true;
 683       Status = StatDone;
 684       Dequeue();
 685       if(Debug)
 686          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 687       return;
 688    }
 689
 690    if(Debug)
 691       std::clog << "Finishing: " << Desc.URI << std::endl;
 692    Complete = false;
 693    Status = StatDone;
 694    Dequeue();
 695    return;
 696 }
 697                                                                         /*}}}*/
 698 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 699 {
 700
 701    // calc sha1 of the just patched file
 702    string FinalFile = _config->FindDir("Dir::State::lists");
 703    FinalFile += URItoFileName(RealURI);
 704
 705    FileFd fd(FinalFile, FileFd::ReadOnly);
 706    SHA1Summation SHA1;
 707    SHA1.AddFD(fd);
 708    string local_sha1 = string(SHA1.Result());
 709    if(Debug)
 710       std::clog << "QueueNextDiff: "
 711                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 712
 713    // final file reached before all patches are applied
 714    if(local_sha1 == ServerSha1)
 715    {
 716       Finish(true);
 717       return true;
 718    }
 719
 720    // remove all patches until the next matching patch is found
 721    // this requires the Index file to be ordered
 722    for(vector<DiffInfo>::iterator I=available_patches.begin();
 723        available_patches.empty() == false &&
 724           I != available_patches.end() &&
 725           I->sha1 != local_sha1;
 726        ++I)
 727    {
 728       available_patches.erase(I);
 729    }
 730
 731    // error checking and falling back if no patch was found
 732    if(available_patches.empty() == true)
 733    {
 734       Failed("", NULL);
 735       return false;
 736    }
 737
 738    // queue the right diff
 739    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 740    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 741    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 742    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 743
 744    if(Debug)
 745       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 746
 747    QueueURI(Desc);
 748
 749    return true;
 750 }
 751                                                                         /*}}}*/
 752 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes,       /*{{{*/
 753                             pkgAcquire::MethodConfig *Cnf)
 754 {
 755    if(Debug)
 756       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 757
 758    Item::Done(Message, Size, Hashes, Cnf);
 759
 760    string FinalFile;
 761    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 762
 763    // success in downloading a diff, enter ApplyDiff state
 764    if(State == StateFetchDiff)
 765    {
 766
 767       // rred excepts the patch as $FinalFile.ed
 768       Rename(DestFile,FinalFile+".ed");
 769
 770       if(Debug)
 771          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 772
 773       State = StateApplyDiff;
 774       Local = true;
 775       Desc.URI = "rred:" + FinalFile;
 776       QueueURI(Desc);
 777       Mode = "rred";
 778       return;
 779    }
 780
 781
 782    // success in download/apply a diff, queue next (if needed)
 783    if(State == StateApplyDiff)
 784    {
 785       // remove the just applied patch
 786       available_patches.erase(available_patches.begin());
 787       unlink((FinalFile + ".ed").c_str());
 788
 789       // move into place
 790       if(Debug)
 791       {
 792          std::clog << "Moving patched file in place: " << std::endl
 793                    << DestFile << " -> " << FinalFile << std::endl;
 794       }
 795       Rename(DestFile,FinalFile);
 796       chmod(FinalFile.c_str(),0644);
 797
 798       // see if there is more to download
 799       if(available_patches.empty() == false) {
 800          new pkgAcqIndexDiffs(Owner, Target,
 801                               ExpectedHashes, MetaIndexParser,
 802                               ServerSha1, available_patches);
 803          return Finish();
 804       } else
 805          return Finish(true);
 806    }
 807 }
 808                                                                         /*}}}*/
 809 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 810 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 811                                              struct IndexTarget const * const Target,
 812                                              HashStringList const &ExpectedHashes,
 813                                              indexRecords *MetaIndexParser,
 814                                              DiffInfo const &patch,
 815                                              std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 816    : pkgAcqBaseIndex(Owner, Target, ExpectedHashes, MetaIndexParser),
 817      patch(patch), allPatches(allPatches), State(StateFetchDiff)
 818 {
 819
 820    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 821    DestFile += URItoFileName(Target->URI);
 822
 823    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 824
 825    RealURI = Target->URI;
 826    Desc.Owner = this;
 827    Description = Target->Description;
 828    Desc.ShortDesc = Target->ShortDesc;
 829
 830    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 831    Desc.Description = Description + " " + patch.file + string(".pdiff");
 832    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 833    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 834
 835    if(Debug)
 836       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 837
 838    QueueURI(Desc);
 839 }
 840                                                                         /*}}}*/
 841 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 842 {
 843    if(Debug)
 844       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 845    Complete = false;
 846    Status = StatDone;
 847    Dequeue();
 848
 849    // check if we are the first to fail, otherwise we are done here
 850    State = StateDoneDiff;
 851    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 852          I != allPatches->end(); ++I)
 853       if ((*I)->State == StateErrorDiff)
 854          return;
 855
 856    // first failure means we should fallback
 857    State = StateErrorDiff;
 858    std::clog << "Falling back to normal index file acquire" << std::endl;
 859    new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser);
 860 }
 861                                                                         /*}}}*/
 862 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes,   /*{{{*/
 863                             pkgAcquire::MethodConfig *Cnf)
 864 {
 865    if(Debug)
 866       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 867
 868    Item::Done(Message,Size,Hashes,Cnf);
 869
 870    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 871
 872    if (State == StateFetchDiff)
 873    {
 874       // rred expects the patch as $FinalFile.ed.$patchname.gz
 875       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 876
 877       // check if this is the last completed diff
 878       State = StateDoneDiff;
 879       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 880             I != allPatches->end(); ++I)
 881          if ((*I)->State != StateDoneDiff)
 882          {
 883             if(Debug)
 884                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 885             return;
 886          }
 887
 888       // this is the last completed diff, so we are ready to apply now
 889       State = StateApplyDiff;
 890
 891       if(Debug)
 892          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 893
 894       Local = true;
 895       Desc.URI = "rred:" + FinalFile;
 896       QueueURI(Desc);
 897       Mode = "rred";
 898       return;
 899    }
 900    // success in download/apply all diffs, clean up
 901    else if (State == StateApplyDiff)
 902    {
 903       // see if we really got the expected file
 904       if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile))
 905       {
 906          RenameOnError(HashSumMismatch);
 907          return;
 908       }
 909
 910       // move the result into place
 911       if(Debug)
 912          std::clog << "Moving patched file in place: " << std::endl
 913                    << DestFile << " -> " << FinalFile << std::endl;
 914       Rename(DestFile, FinalFile);
 915       chmod(FinalFile.c_str(), 0644);
 916
 917       // otherwise lists cleanup will eat the file
 918       DestFile = FinalFile;
 919
 920       // ensure the ed's are gone regardless of list-cleanup
 921       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 922             I != allPatches->end(); ++I)
 923       {
 924             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 925             unlink(patch.c_str());
 926       }
 927
 928       // all set and done
 929       Complete = true;
 930       if(Debug)
 931          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 932    }
 933 }
 934                                                                         /*}}}*/
 935 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* The package file is added to the queue and a second class is
 938    instantiated to fetch the revision file */
 939 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 940                          string URI,string URIDesc,string ShortDesc,
 941                          HashStringList const  &ExpectedHash, string comprExt)
 942    : pkgAcqBaseIndex(Owner, NULL, ExpectedHash, NULL), RealURI(URI)
 943 {
 944    if(comprExt.empty() == true)
 945    {
 946       // autoselect the compression method
 947       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 948       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 949          comprExt.append(*t).append(" ");
 950       if (comprExt.empty() == false)
 951          comprExt.erase(comprExt.end()-1);
 952    }
 953    CompressionExtension = comprExt;
 954
 955    Init(URI, URIDesc, ShortDesc);
 956 }
 957 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 958                          HashStringList const &ExpectedHash,
 959                          indexRecords *MetaIndexParser)
 960    : pkgAcqBaseIndex(Owner, Target, ExpectedHash, MetaIndexParser),
 961      RealURI(Target->URI)
 962 {
 963    // autoselect the compression method
 964    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 965    CompressionExtension = "";
 966    if (ExpectedHashes.usable())
 967    {
 968       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 969          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 970             CompressionExtension.append(*t).append(" ");
 971    }
 972    else
 973    {
 974       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 975          CompressionExtension.append(*t).append(" ");
 976    }
 977    if (CompressionExtension.empty() == false)
 978       CompressionExtension.erase(CompressionExtension.end()-1);
 979
 980    Init(Target->URI, Target->Description, Target->ShortDesc);
 981 }
 982                                                                         /*}}}*/
 983 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 984 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 985    Decompression = false;
 986    Erase = false;
 987
 988    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 989    DestFile += URItoFileName(URI);
 990
 991    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 992    std::string MetaKey;
 993    if (comprExt == "uncompressed")
 994    {
 995       Desc.URI = URI;
 996       if(Target)
 997          MetaKey = string(Target->MetaKey);
 998    }
 999    else
1000    {
1001       Desc.URI = URI + '.' + comprExt;
1002       if(Target)
1003          MetaKey = string(Target->MetaKey) + '.' + comprExt;
1004    }
1005
1006    // load the filesize
1007    if(MetaIndexParser)
1008    {
1009       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1010       if(Record)
1011          FileSize = Record->Size;
1012
1013       InitByHashIfNeeded(MetaKey);
1014    }
1015
1016    Desc.Description = URIDesc;
1017    Desc.Owner = this;
1018    Desc.ShortDesc = ShortDesc;
1019
1020    QueueURI(Desc);
1021 }
1022                                                                         /*}}}*/
1023 // AcqIndex::AdjustForByHash - modify URI for by-hash support           /*{{{*/
1024 // ---------------------------------------------------------------------
1025 /* */
1026 void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey)
1027 {
1028    // TODO:
1029    //  - (maybe?) add support for by-hash into the sources.list as flag
1030    //  - make apt-ftparchive generate the hashes (and expire?)
1031    std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
1032    if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
1033       _config->FindB(HostKnob, false) == true ||
1034       MetaIndexParser->GetSupportsAcquireByHash())
1035    {
1036       indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
1037       if(Record)
1038       {
1039          // FIXME: should we really use the best hash here? or a fixed one?
1040          const HashString *TargetHash = Record->Hashes.find("");
1041          std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
1042          size_t trailing_slash = Desc.URI.find_last_of("/");
1043          Desc.URI = Desc.URI.replace(
1044             trailing_slash,
1045             Desc.URI.substr(trailing_slash+1).size()+1,
1046             ByHash);
1047       } else {
1048          _error->Warning(
1049             "Fetching ByHash requested but can not find record for %s",
1050             MetaKey.c_str());
1051       }
1052    }
1053 }
1054                                                                         /*}}}*/
1055 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1056 // ---------------------------------------------------------------------
1057 /* The only header we use is the last-modified header. */
1058 string pkgAcqIndex::Custom600Headers() const
1059 {
1060    string Final = _config->FindDir("Dir::State::lists");
1061    Final += URItoFileName(RealURI);
1062    if (_config->FindB("Acquire::GzipIndexes",false))
1063       Final += ".gz";
1064
1065    string msg = "\nIndex-File: true";
1066    // FIXME: this really should use "IndexTarget::IsOptional()" but that
1067    //        seems to be difficult without breaking ABI
1068    if (ShortDesc().find("Translation") != 0)
1069       msg += "\nFail-Ignore: true";
1070    struct stat Buf;
1071    if (stat(Final.c_str(),&Buf) == 0)
1072       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1073
1074    return msg;
1075 }
1076                                                                         /*}}}*/
1077 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1078 {
1079    size_t const nextExt = CompressionExtension.find(' ');
1080    if (nextExt != std::string::npos)
1081    {
1082       CompressionExtension = CompressionExtension.substr(nextExt+1);
1083       Init(RealURI, Desc.Description, Desc.ShortDesc);
1084       return;
1085    }
1086
1087    // on decompression failure, remove bad versions in partial/
1088    if (Decompression && Erase) {
1089       string s = _config->FindDir("Dir::State::lists") + "partial/";
1090       s.append(URItoFileName(RealURI));
1091       unlink(s.c_str());
1092    }
1093
1094    Item::Failed(Message,Cnf);
1095 }
1096                                                                         /*}}}*/
1097 // pkgAcqIndex::GetFinalFilename - Return the full final file path      /*{{{*/
1098 std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
1099                                           std::string const &compExt)
1100 {
1101    std::string FinalFile = _config->FindDir("Dir::State::lists");
1102    FinalFile += URItoFileName(URI);
1103    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1104       FinalFile += ".gz";
1105    return FinalFile;
1106 }
1107                                                                         /*}}}*/
1108 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit         /*{{{*/
1109 void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
1110 {
1111    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1112    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1113       DestFile += ".gz";
1114
1115    string FinalFile = GetFinalFilename(RealURI, compExt);
1116    Rename(FinalFile, FileName);
1117    Decompression = true;
1118    Desc.URI = "copy:" + FileName;
1119    QueueURI(Desc);
1120 }
1121                                                                         /*}}}*/
1122 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1123 // ---------------------------------------------------------------------
1124 /* This goes through a number of states.. On the initial fetch the
1125    method could possibly return an alternate filename which points
1126    to the uncompressed version of the file. If this is so the file
1127    is copied into the partial directory. In all other cases the file
1128    is decompressed with a gzip uri. */
1129 void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,
1130                        pkgAcquire::MethodConfig *Cfg)
1131 {
1132    Item::Done(Message,Size,Hash,Cfg);
1133    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1134
1135    if (Decompression == true)
1136    {
1137       if (ExpectedHashes.usable() && ExpectedHashes != Hashes)
1138       {
1139          Desc.URI = RealURI;
1140          RenameOnError(HashSumMismatch);
1141          printHashSumComparision(RealURI, ExpectedHashes, Hashes);
1142          return;
1143       }
1144
1145       // FIXME: this can go away once we only ever download stuff that
1146       //        has a valid hash and we never do GET based probing
1147       //
1148       /* Always verify the index file for correctness (all indexes must
1149        * have a Package field) (LP: #346386) (Closes: #627642)
1150        */
1151       FileFd fd(DestFile, FileFd::ReadOnlyGzip);
1152       // Only test for correctness if the file is not empty (empty is ok)
1153       if (fd.Size() > 0)
1154       {
1155          pkgTagSection sec;
1156          pkgTagFile tag(&fd);
1157
1158          // all our current indexes have a field 'Package' in each section
1159          if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1160          {
1161             RenameOnError(InvalidFormat);
1162             return;
1163          }
1164       }
1165
1166       // Done, move it into position
1167       string FinalFile = GetFinalFilename(RealURI, compExt);
1168       Rename(DestFile,FinalFile);
1169       chmod(FinalFile.c_str(),0644);
1170
1171       /* We restore the original name to DestFile so that the clean operation
1172          will work OK */
1173       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1174       DestFile += URItoFileName(RealURI);
1175       if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1176          DestFile += ".gz";
1177
1178       // Remove the compressed version.
1179       if (Erase == true)
1180          unlink(DestFile.c_str());
1181
1182       return;
1183    }
1184
1185    Erase = false;
1186    Complete = true;
1187
1188    // Handle the unzipd case
1189    string FileName = LookupTag(Message,"Alt-Filename");
1190    if (FileName.empty() == false)
1191    {
1192       Decompression = true;
1193       Local = true;
1194       DestFile += ".decomp";
1195       Desc.URI = "copy:" + FileName;
1196       QueueURI(Desc);
1197       Mode = "copy";
1198       return;
1199    }
1200
1201    FileName = LookupTag(Message,"Filename");
1202    if (FileName.empty() == true)
1203    {
1204       Status = StatError;
1205       ErrorText = "Method gave a blank filename";
1206    }
1207
1208    if (FileName == DestFile)
1209       Erase = true;
1210    else
1211       Local = true;
1212
1213    // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
1214    // file when its doing the indexcopy
1215    if (RealURI.substr(0,6) == "cdrom:" &&
1216        StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1217       return;
1218
1219    // The files timestamp matches, for non-local URLs reverify the local
1220    // file, for local file, uncompress again to ensure the hashsum is still
1221    // matching the Release file
1222    if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1223    {
1224       ReverifyAfterIMS(FileName);
1225       return;
1226    }
1227    string decompProg;
1228
1229    // If we enable compressed indexes, queue for hash verification
1230    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local)
1231    {
1232       DestFile = _config->FindDir("Dir::State::lists");
1233       DestFile += URItoFileName(RealURI) + ".gz";
1234
1235       Decompression = true;
1236       Desc.URI = "copy:" + FileName;
1237       QueueURI(Desc);
1238
1239       return;
1240     }
1241
1242    // get the binary name for your used compression type
1243    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1244    if(decompProg.empty() == false);
1245    else if(compExt == "uncompressed")
1246       decompProg = "copy";
1247    else {
1248       _error->Error("Unsupported extension: %s", compExt.c_str());
1249       return;
1250    }
1251
1252    Decompression = true;
1253    DestFile += ".decomp";
1254    Desc.URI = decompProg + ":" + FileName;
1255    QueueURI(Desc);
1256
1257    // FIXME: this points to a c++ string that goes out of scope
1258    Mode = decompProg.c_str();
1259 }
1260                                                                         /*}}}*/
1261 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1262 // ---------------------------------------------------------------------
1263 /* The Translation file is added to the queue */
1264 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1265                             string URI,string URIDesc,string ShortDesc)
1266   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "")
1267 {
1268 }
1269 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const * const Target,
1270                          HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser)
1271   : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser)
1272 {
1273    // load the filesize
1274    indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey));
1275    if(Record)
1276       FileSize = Record->Size;
1277 }
1278                                                                         /*}}}*/
1279 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1280 // ---------------------------------------------------------------------
1281 string pkgAcqIndexTrans::Custom600Headers() const
1282 {
1283    string Final = _config->FindDir("Dir::State::lists");
1284    Final += URItoFileName(RealURI);
1285
1286    if (_config->FindB("Acquire::GzipIndexes",false))
1287       Final += ".gz";
1288
1289    struct stat Buf;
1290    if (stat(Final.c_str(),&Buf) != 0)
1291       return "\nFail-Ignore: true\nIndex-File: true";
1292    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1293 }
1294                                                                         /*}}}*/
1295 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1296 // ---------------------------------------------------------------------
1297 /* */
1298 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1299 {
1300    size_t const nextExt = CompressionExtension.find(' ');
1301    if (nextExt != std::string::npos)
1302    {
1303       CompressionExtension = CompressionExtension.substr(nextExt+1);
1304       Init(RealURI, Desc.Description, Desc.ShortDesc);
1305       Status = StatIdle;
1306       return;
1307    }
1308
1309    if (Cnf->LocalOnly == true ||
1310        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1311    {
1312       // Ignore this
1313       Status = StatDone;
1314       Complete = false;
1315       Dequeue();
1316       return;
1317    }
1318
1319    Item::Failed(Message,Cnf);
1320 }
1321                                                                         /*}}}*/
1322 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1323                              string URI,string URIDesc,string ShortDesc,
1324                              string MetaIndexURI, string MetaIndexURIDesc,
1325                              string MetaIndexShortDesc,
1326                              const vector<IndexTarget*>* IndexTargets,
1327                              indexRecords* MetaIndexParser) :
1328    Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI),
1329    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1330    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1331 {
1332    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1333    DestFile += URItoFileName(URI);
1334
1335    // remove any partial downloaded sig-file in partial/.
1336    // it may confuse proxies and is too small to warrant a
1337    // partial download anyway
1338    unlink(DestFile.c_str());
1339
1340    // Create the item
1341    Desc.Description = URIDesc;
1342    Desc.Owner = this;
1343    Desc.ShortDesc = ShortDesc;
1344    Desc.URI = URI;
1345
1346    string Final = _config->FindDir("Dir::State::lists");
1347    Final += URItoFileName(RealURI);
1348    if (RealFileExists(Final) == true)
1349    {
1350       // File was already in place.  It needs to be re-downloaded/verified
1351       // because Release might have changed, we do give it a different
1352       // name than DestFile because otherwise the http method will
1353       // send If-Range requests and there are too many broken servers
1354       // out there that do not understand them
1355       LastGoodSig = DestFile+".reverify";
1356       Rename(Final,LastGoodSig);
1357    }
1358
1359    // we expect the indextargets + one additional Release file
1360    ExpectedAdditionalItems = IndexTargets->size() + 1;
1361
1362    QueueURI(Desc);
1363 }
1364                                                                         /*}}}*/
1365 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1366 {
1367    // if the file was never queued undo file-changes done in the constructor
1368    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1369          LastGoodSig.empty() == false)
1370    {
1371       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1372       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1373          Rename(LastGoodSig, Final);
1374    }
1375
1376 }
1377                                                                         /*}}}*/
1378 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1379 // ---------------------------------------------------------------------
1380 /* The only header we use is the last-modified header. */
1381 string pkgAcqMetaSig::Custom600Headers() const
1382 {
1383    struct stat Buf;
1384    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1385       return "\nIndex-File: true";
1386
1387    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1388 }
1389
1390 void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes,
1391                          pkgAcquire::MethodConfig *Cfg)
1392 {
1393    Item::Done(Message, Size, Hashes, Cfg);
1394
1395    string FileName = LookupTag(Message,"Filename");
1396    if (FileName.empty() == true)
1397    {
1398       Status = StatError;
1399       ErrorText = "Method gave a blank filename";
1400       return;
1401    }
1402
1403    if (FileName != DestFile)
1404    {
1405       // We have to copy it into place
1406       Local = true;
1407       Desc.URI = "copy:" + FileName;
1408       QueueURI(Desc);
1409       return;
1410    }
1411
1412    Complete = true;
1413
1414    // at this point pkgAcqMetaIndex takes over
1415    ExpectedAdditionalItems = 0;
1416
1417    // put the last known good file back on i-m-s hit (it will
1418    // be re-verified again)
1419    // Else do nothing, we have the new file in DestFile then
1420    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1421       Rename(LastGoodSig, DestFile);
1422
1423    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1424    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1425                        MetaIndexShortDesc,  DestFile, IndexTargets,
1426                        MetaIndexParser);
1427
1428 }
1429                                                                         /*}}}*/
1430 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1431 {
1432    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1433
1434    // at this point pkgAcqMetaIndex takes over
1435    ExpectedAdditionalItems = 0;
1436
1437    // if we get a network error we fail gracefully
1438    if(Status == StatTransientNetworkError)
1439    {
1440       Item::Failed(Message,Cnf);
1441       // move the sigfile back on transient network failures
1442       if(FileExists(LastGoodSig))
1443          Rename(LastGoodSig,Final);
1444
1445       // set the status back to , Item::Failed likes to reset it
1446       Status = pkgAcquire::Item::StatTransientNetworkError;
1447       return;
1448    }
1449
1450    // Delete any existing sigfile when the acquire failed
1451    unlink(Final.c_str());
1452
1453    // queue a pkgAcqMetaIndex with no sigfile
1454    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1455                        "", IndexTargets, MetaIndexParser);
1456
1457    if (Cnf->LocalOnly == true ||
1458        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1459    {
1460       // Ignore this
1461       Status = StatDone;
1462       Complete = false;
1463       Dequeue();
1464       return;
1465    }
1466
1467    Item::Failed(Message,Cnf);
1468 }
1469                                                                         /*}}}*/
1470 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1471                                  string URI,string URIDesc,string ShortDesc,
1472                                  string SigFile,
1473                                  const vector<IndexTarget*>* IndexTargets,
1474                                  indexRecords* MetaIndexParser) :
1475    Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1476    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1477 {
1478    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1479    DestFile += URItoFileName(URI);
1480
1481    // Create the item
1482    Desc.Description = URIDesc;
1483    Desc.Owner = this;
1484    Desc.ShortDesc = ShortDesc;
1485    Desc.URI = URI;
1486
1487    // we expect more item
1488    ExpectedAdditionalItems = IndexTargets->size();
1489
1490    QueueURI(Desc);
1491 }
1492                                                                         /*}}}*/
1493 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1494 // ---------------------------------------------------------------------
1495 /* The only header we use is the last-modified header. */
1496 string pkgAcqMetaIndex::Custom600Headers() const
1497 {
1498    string Final = _config->FindDir("Dir::State::lists");
1499    Final += URItoFileName(RealURI);
1500
1501    struct stat Buf;
1502    if (stat(Final.c_str(),&Buf) != 0)
1503       return "\nIndex-File: true";
1504
1505    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1506 }
1507                                                                         /*}}}*/
1508 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/
1509                            pkgAcquire::MethodConfig *Cfg)
1510 {
1511    Item::Done(Message,Size,Hashes,Cfg);
1512
1513    // MetaIndexes are done in two passes: one to download the
1514    // metaindex with an appropriate method, and a second to verify it
1515    // with the gpgv method
1516
1517    if (AuthPass == true)
1518    {
1519       AuthDone(Message);
1520
1521       // all cool, move Release file into place
1522       Complete = true;
1523    }
1524    else
1525    {
1526       RetrievalDone(Message);
1527       if (!Complete)
1528          // Still more retrieving to do
1529          return;
1530
1531       if (SigFile == "")
1532       {
1533          // There was no signature file, so we are finished.  Download
1534          // the indexes and do only hashsum verification if possible
1535          MetaIndexParser->Load(DestFile);
1536          QueueIndexes(false);
1537       }
1538       else
1539       {
1540          // FIXME: move this into pkgAcqMetaClearSig::Done on the next
1541          //        ABI break
1542
1543          // if we expect a ClearTextSignature (InRelase), ensure that
1544          // this is what we get and if not fail to queue a
1545          // Release/Release.gpg, see #346386
1546          if (SigFile == DestFile && !StartsWithGPGClearTextSignature(DestFile))
1547          {
1548             Failed(Message, Cfg);
1549             return;
1550          }
1551
1552          // There was a signature file, so pass it to gpgv for
1553          // verification
1554          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1555             std::cerr << "Metaindex acquired, queueing gpg verification ("
1556                       << SigFile << "," << DestFile << ")\n";
1557          AuthPass = true;
1558          Desc.URI = "gpgv:" + SigFile;
1559          QueueURI(Desc);
1560          Mode = "gpgv";
1561          return;
1562       }
1563    }
1564
1565    if (Complete == true)
1566    {
1567       string FinalFile = _config->FindDir("Dir::State::lists");
1568       FinalFile += URItoFileName(RealURI);
1569       if (SigFile == DestFile)
1570          SigFile = FinalFile;
1571       Rename(DestFile,FinalFile);
1572       chmod(FinalFile.c_str(),0644);
1573       DestFile = FinalFile;
1574    }
1575 }
1576                                                                         /*}}}*/
1577 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1578 {
1579    // We have just finished downloading a Release file (it is not
1580    // verified yet)
1581
1582    string FileName = LookupTag(Message,"Filename");
1583    if (FileName.empty() == true)
1584    {
1585       Status = StatError;
1586       ErrorText = "Method gave a blank filename";
1587       return;
1588    }
1589
1590    if (FileName != DestFile)
1591    {
1592       Local = true;
1593       Desc.URI = "copy:" + FileName;
1594       QueueURI(Desc);
1595       return;
1596    }
1597
1598    // make sure to verify against the right file on I-M-S hit
1599    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1600    if(IMSHit)
1601    {
1602       string FinalFile = _config->FindDir("Dir::State::lists");
1603       FinalFile += URItoFileName(RealURI);
1604       if (SigFile == DestFile)
1605       {
1606          SigFile = FinalFile;
1607          // constructor of pkgAcqMetaClearSig moved it out of the way,
1608          // now move it back in on IMS hit for the 'old' file
1609          string const OldClearSig = DestFile + ".reverify";
1610          if (RealFileExists(OldClearSig) == true)
1611             Rename(OldClearSig, FinalFile);
1612       }
1613       DestFile = FinalFile;
1614    }
1615    Complete = true;
1616 }
1617                                                                         /*}}}*/
1618 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1619 {
1620    // At this point, the gpgv method has succeeded, so there is a
1621    // valid signature from a key in the trusted keyring.  We
1622    // perform additional verification of its contents, and use them
1623    // to verify the indexes we are about to download
1624
1625    if (!MetaIndexParser->Load(DestFile))
1626    {
1627       Status = StatAuthError;
1628       ErrorText = MetaIndexParser->ErrorText;
1629       return;
1630    }
1631
1632    if (!VerifyVendor(Message))
1633    {
1634       return;
1635    }
1636
1637    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1638       std::cerr << "Signature verification succeeded: "
1639                 << DestFile << std::endl;
1640
1641    // do not trust any previously unverified content that we may have
1642    string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1643    if (DestFile != SigFile)
1644       LastGoodSigFile.append(".gpg");
1645    LastGoodSigFile.append(".reverify");
1646    if(IMSHit == false && RealFileExists(LastGoodSigFile) == false)
1647    {
1648       for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1649            Target != IndexTargets->end();
1650            ++Target)
1651       {
1652          // remove old indexes
1653          std::string index = _config->FindDir("Dir::State::lists") +
1654             URItoFileName((*Target)->URI);
1655          unlink(index.c_str());
1656          // and also old gzipindexes
1657          index += ".gz";
1658          unlink(index.c_str());
1659       }
1660    }
1661
1662
1663    // Download further indexes with verification
1664    QueueIndexes(true);
1665
1666    // is it a clearsigned MetaIndex file?
1667    if (DestFile == SigFile)
1668       return;
1669
1670    // Done, move signature file into position
1671    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1672       URItoFileName(RealURI) + ".gpg";
1673    Rename(SigFile,VerifiedSigFile);
1674    chmod(VerifiedSigFile.c_str(),0644);
1675 }
1676                                                                         /*}}}*/
1677 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1678 {
1679 #if 0
1680    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1681     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1682    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1683    {
1684       Status = StatError;
1685       ErrorText = MetaIndexParser->ErrorText;
1686       return;
1687    }
1688 #endif
1689    bool transInRelease = false;
1690    {
1691       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1692       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1693          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1694          if (k->find("Translation-") != std::string::npos)
1695          {
1696             transInRelease = true;
1697             break;
1698          }
1699    }
1700
1701    // at this point the real Items are loaded in the fetcher
1702    ExpectedAdditionalItems = 0;
1703    for (vector <IndexTarget*>::const_iterator Target = IndexTargets->begin();
1704         Target != IndexTargets->end();
1705         ++Target)
1706    {
1707       HashStringList ExpectedIndexHashes;
1708       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1709       bool compressedAvailable = false;
1710       if (Record == NULL)
1711       {
1712          if ((*Target)->IsOptional() == true)
1713          {
1714             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1715             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1716                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1717                {
1718                   compressedAvailable = true;
1719                   break;
1720                }
1721          }
1722          else if (verify == true)
1723          {
1724             Status = StatAuthError;
1725             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1726             return;
1727          }
1728       }
1729       else
1730       {
1731          ExpectedIndexHashes = Record->Hashes;
1732          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1733          {
1734             std::cerr << "Queueing: " << (*Target)->URI << std::endl
1735                << "Expected Hash:" << std::endl;
1736             for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
1737                std::cerr <<  "\t- " << hs->toStr() << std::endl;
1738             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1739          }
1740          if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false)
1741          {
1742             Status = StatAuthError;
1743             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1744             return;
1745          }
1746       }
1747
1748       if ((*Target)->IsOptional() == true)
1749       {
1750          if ((*Target)->IsSubIndex() == true)
1751             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1752                                 (*Target)->ShortDesc, ExpectedIndexHashes);
1753          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1754          {
1755             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1756                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1757                new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1758             else
1759                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1760          }
1761          continue;
1762       }
1763
1764       /* Queue Packages file (either diff or full packages files, depending
1765          on the users option) - we also check if the PDiff Index file is listed
1766          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1767          instead, but passing the required info to it is to much hassle */
1768       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1769           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1770          new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1771       else
1772          new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser);
1773    }
1774 }
1775                                                                         /*}}}*/
1776 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1777 {
1778    string::size_type pos;
1779
1780    // check for missing sigs (that where not fatal because otherwise we had
1781    // bombed earlier)
1782    string missingkeys;
1783    string msg = _("There is no public key available for the "
1784                   "following key IDs:\n");
1785    pos = Message.find("NO_PUBKEY ");
1786    if (pos != std::string::npos)
1787    {
1788       string::size_type start = pos+strlen("NO_PUBKEY ");
1789       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1790       missingkeys += (Fingerprint);
1791    }
1792    if(!missingkeys.empty())
1793       _error->Warning("%s", (msg + missingkeys).c_str());
1794
1795    string Transformed = MetaIndexParser->GetExpectedDist();
1796
1797    if (Transformed == "../project/experimental")
1798    {
1799       Transformed = "experimental";
1800    }
1801
1802    pos = Transformed.rfind('/');
1803    if (pos != string::npos)
1804    {
1805       Transformed = Transformed.substr(0, pos);
1806    }
1807
1808    if (Transformed == ".")
1809    {
1810       Transformed = "";
1811    }
1812
1813    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1814        MetaIndexParser->GetValidUntil() > 0) {
1815       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1816       if (invalid_since > 0)
1817          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1818          // the time since then the file is invalid - formated in the same way as in
1819          // the download progress display (e.g. 7d 3h 42min 1s)
1820          return _error->Error(
1821             _("Release file for %s is expired (invalid since %s). "
1822               "Updates for this repository will not be applied."),
1823             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1824    }
1825
1826    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1827    {
1828       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1829       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1830       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1831    }
1832
1833    if (MetaIndexParser->CheckDist(Transformed) == false)
1834    {
1835       // This might become fatal one day
1836 //       Status = StatAuthError;
1837 //       ErrorText = "Conflicting distribution; expected "
1838 //          + MetaIndexParser->GetExpectedDist() + " but got "
1839 //          + MetaIndexParser->GetDist();
1840 //       return false;
1841       if (!Transformed.empty())
1842       {
1843          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1844                          Desc.Description.c_str(),
1845                          Transformed.c_str(),
1846                          MetaIndexParser->GetDist().c_str());
1847       }
1848    }
1849
1850    return true;
1851 }
1852                                                                         /*}}}*/
1853 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1854 // ---------------------------------------------------------------------
1855 /* */
1856 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1857 {
1858    if (AuthPass == true)
1859    {
1860       // gpgv method failed, if we have a good signature
1861       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1862       if (DestFile != SigFile)
1863          LastGoodSigFile.append(".gpg");
1864       LastGoodSigFile.append(".reverify");
1865
1866       if(FileExists(LastGoodSigFile))
1867       {
1868          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1869          if (DestFile != SigFile)
1870             VerifiedSigFile.append(".gpg");
1871          Rename(LastGoodSigFile, VerifiedSigFile);
1872          Status = StatTransientNetworkError;
1873          _error->Warning(_("An error occurred during the signature "
1874                            "verification. The repository is not updated "
1875                            "and the previous index files will be used. "
1876                            "GPG error: %s: %s\n"),
1877                          Desc.Description.c_str(),
1878                          LookupTag(Message,"Message").c_str());
1879          RunScripts("APT::Update::Auth-Failure");
1880          return;
1881       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1882          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1883          _error->Error(_("GPG error: %s: %s"),
1884                          Desc.Description.c_str(),
1885                          LookupTag(Message,"Message").c_str());
1886          return;
1887       } else {
1888          _error->Warning(_("GPG error: %s: %s"),
1889                          Desc.Description.c_str(),
1890                          LookupTag(Message,"Message").c_str());
1891       }
1892       // gpgv method failed
1893       ReportMirrorFailure("GPGFailure");
1894    }
1895
1896    /* Always move the meta index, even if gpgv failed. This ensures
1897     * that PackageFile objects are correctly filled in */
1898    if (FileExists(DestFile)) {
1899       string FinalFile = _config->FindDir("Dir::State::lists");
1900       FinalFile += URItoFileName(RealURI);
1901       /* InRelease files become Release files, otherwise
1902        * they would be considered as trusted later on */
1903       if (SigFile == DestFile) {
1904          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1905                                        "Release");
1906          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1907                                        "Release");
1908          SigFile = FinalFile;
1909       }
1910       Rename(DestFile,FinalFile);
1911       chmod(FinalFile.c_str(),0644);
1912
1913       DestFile = FinalFile;
1914    }
1915
1916    // No Release file was present, or verification failed, so fall
1917    // back to queueing Packages files without verification
1918    QueueIndexes(false);
1919 }
1920                                                                         /*}}}*/
1921 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1922                 string const &URI, string const &URIDesc, string const &ShortDesc,
1923                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1924                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1925                 const vector<IndexTarget*>* IndexTargets,
1926                 indexRecords* MetaIndexParser) :
1927         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1928         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1929         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1930 {
1931    SigFile = DestFile;
1932
1933    // index targets + (worst case:) Release/Release.gpg
1934    ExpectedAdditionalItems = IndexTargets->size() + 2;
1935
1936
1937    // keep the old InRelease around in case of transistent network errors
1938    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1939    if (RealFileExists(Final) == true)
1940    {
1941       string const LastGoodSig = DestFile + ".reverify";
1942       Rename(Final,LastGoodSig);
1943    }
1944 }
1945                                                                         /*}}}*/
1946 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1947 {
1948    // if the file was never queued undo file-changes done in the constructor
1949    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1950    {
1951       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1952       string const LastGoodSig = DestFile + ".reverify";
1953       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1954          Rename(LastGoodSig, Final);
1955    }
1956 }
1957                                                                         /*}}}*/
1958 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1959 // ---------------------------------------------------------------------
1960 // FIXME: this can go away once the InRelease file is used widely
1961 string pkgAcqMetaClearSig::Custom600Headers() const
1962 {
1963    string Final = _config->FindDir("Dir::State::lists");
1964    Final += URItoFileName(RealURI);
1965
1966    struct stat Buf;
1967    if (stat(Final.c_str(),&Buf) != 0)
1968    {
1969       Final = DestFile + ".reverify";
1970       if (stat(Final.c_str(),&Buf) != 0)
1971          return "\nIndex-File: true\nFail-Ignore: true\n";
1972    }
1973
1974    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1975 }
1976                                                                         /*}}}*/
1977 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1978 {
1979    // we failed, we will not get additional items from this method
1980    ExpectedAdditionalItems = 0;
1981
1982    if (AuthPass == false)
1983    {
1984       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1985       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1986       string FinalFile = _config->FindDir("Dir::State::lists");
1987       FinalFile.append(URItoFileName(RealURI));
1988       if (FileExists(FinalFile))
1989          unlink(FinalFile.c_str());
1990
1991       new pkgAcqMetaSig(Owner,
1992                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1993                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1994                         IndexTargets, MetaIndexParser);
1995       if (Cnf->LocalOnly == true ||
1996           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1997          Dequeue();
1998    }
1999    else
2000       pkgAcqMetaIndex::Failed(Message, Cnf);
2001 }
2002                                                                         /*}}}*/
2003 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
2004 // ---------------------------------------------------------------------
2005 /* This just sets up the initial fetch environment and queues the first
2006    possibilitiy */
2007 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
2008                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
2009                              string &StoreFilename) :
2010                Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs),
2011                StoreFilename(StoreFilename), Vf(Version.FileList()),
2012                Trusted(false)
2013 {
2014    Retries = _config->FindI("Acquire::Retries",0);
2015
2016    if (Version.Arch() == 0)
2017    {
2018       _error->Error(_("I wasn't able to locate a file for the %s package. "
2019                       "This might mean you need to manually fix this package. "
2020                       "(due to missing arch)"),
2021                     Version.ParentPkg().FullName().c_str());
2022       return;
2023    }
2024
2025    /* We need to find a filename to determine the extension. We make the
2026       assumption here that all the available sources for this version share
2027       the same extension.. */
2028    // Skip not source sources, they do not have file fields.
2029    for (; Vf.end() == false; ++Vf)
2030    {
2031       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2032          continue;
2033       break;
2034    }
2035
2036    // Does not really matter here.. we are going to fail out below
2037    if (Vf.end() != true)
2038    {
2039       // If this fails to get a file name we will bomb out below.
2040       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2041       if (_error->PendingError() == true)
2042          return;
2043
2044       // Generate the final file name as: package_version_arch.foo
2045       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
2046                       QuoteString(Version.VerStr(),"_:") + '_' +
2047                       QuoteString(Version.Arch(),"_:.") +
2048                       "." + flExtension(Parse.FileName());
2049    }
2050
2051    // check if we have one trusted source for the package. if so, switch
2052    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
2053    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
2054    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
2055    bool seenUntrusted = false;
2056    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
2057    {
2058       pkgIndexFile *Index;
2059       if (Sources->FindIndex(i.File(),Index) == false)
2060          continue;
2061
2062       if (debugAuth == true)
2063          std::cerr << "Checking index: " << Index->Describe()
2064                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
2065
2066       if (Index->IsTrusted() == true)
2067       {
2068          Trusted = true;
2069          if (allowUnauth == false)
2070             break;
2071       }
2072       else
2073          seenUntrusted = true;
2074    }
2075
2076    // "allow-unauthenticated" restores apts old fetching behaviour
2077    // that means that e.g. unauthenticated file:// uris are higher
2078    // priority than authenticated http:// uris
2079    if (allowUnauth == true && seenUntrusted == true)
2080       Trusted = false;
2081
2082    // Select a source
2083    if (QueueNext() == false && _error->PendingError() == false)
2084       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
2085                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
2086 }
2087                                                                         /*}}}*/
2088 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
2089 // ---------------------------------------------------------------------
2090 /* This queues the next available file version for download. It checks if
2091    the archive is already available in the cache and stashs the MD5 for
2092    checking later. */
2093 bool pkgAcqArchive::QueueNext()
2094 {
2095    for (; Vf.end() == false; ++Vf)
2096    {
2097       // Ignore not source sources
2098       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2099          continue;
2100
2101       // Try to cross match against the source list
2102       pkgIndexFile *Index;
2103       if (Sources->FindIndex(Vf.File(),Index) == false)
2104             continue;
2105
2106       // only try to get a trusted package from another source if that source
2107       // is also trusted
2108       if(Trusted && !Index->IsTrusted())
2109          continue;
2110
2111       // Grab the text package record
2112       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2113       if (_error->PendingError() == true)
2114          return false;
2115
2116       string PkgFile = Parse.FileName();
2117       ExpectedHashes = Parse.Hashes();
2118
2119       if (PkgFile.empty() == true)
2120          return _error->Error(_("The package index files are corrupted. No Filename: "
2121                               "field for package %s."),
2122                               Version.ParentPkg().Name());
2123
2124       Desc.URI = Index->ArchiveURI(PkgFile);
2125       Desc.Description = Index->ArchiveInfo(Version);
2126       Desc.Owner = this;
2127       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2128
2129       // See if we already have the file. (Legacy filenames)
2130       FileSize = Version->Size;
2131       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2132       struct stat Buf;
2133       if (stat(FinalFile.c_str(),&Buf) == 0)
2134       {
2135          // Make sure the size matches
2136          if ((unsigned long long)Buf.st_size == Version->Size)
2137          {
2138             Complete = true;
2139             Local = true;
2140             Status = StatDone;
2141             StoreFilename = DestFile = FinalFile;
2142             return true;
2143          }
2144
2145          /* Hmm, we have a file and its size does not match, this means it is
2146             an old style mismatched arch */
2147          unlink(FinalFile.c_str());
2148       }
2149
2150       // Check it again using the new style output filenames
2151       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2152       if (stat(FinalFile.c_str(),&Buf) == 0)
2153       {
2154          // Make sure the size matches
2155          if ((unsigned long long)Buf.st_size == Version->Size)
2156          {
2157             Complete = true;
2158             Local = true;
2159             Status = StatDone;
2160             StoreFilename = DestFile = FinalFile;
2161             return true;
2162          }
2163
2164          /* Hmm, we have a file and its size does not match, this shouldn't
2165             happen.. */
2166          unlink(FinalFile.c_str());
2167       }
2168
2169       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2170
2171       // Check the destination file
2172       if (stat(DestFile.c_str(),&Buf) == 0)
2173       {
2174          // Hmm, the partial file is too big, erase it
2175          if ((unsigned long long)Buf.st_size > Version->Size)
2176             unlink(DestFile.c_str());
2177          else
2178             PartialSize = Buf.st_size;
2179       }
2180
2181       // Disables download of archives - useful if no real installation follows,
2182       // e.g. if we are just interested in proposed installation order
2183       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2184       {
2185          Complete = true;
2186          Local = true;
2187          Status = StatDone;
2188          StoreFilename = DestFile = FinalFile;
2189          return true;
2190       }
2191
2192       // Create the item
2193       Local = false;
2194       QueueURI(Desc);
2195
2196       ++Vf;
2197       return true;
2198    }
2199    return false;
2200 }
2201                                                                         /*}}}*/
2202 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2203 // ---------------------------------------------------------------------
2204 /* */
2205 void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes,
2206                          pkgAcquire::MethodConfig *Cfg)
2207 {
2208    Item::Done(Message, Size, CalcHashes, Cfg);
2209
2210    // Check the size
2211    if (Size != Version->Size)
2212    {
2213       RenameOnError(SizeMismatch);
2214       return;
2215    }
2216
2217    // FIXME: could this empty() check impose *any* sort of security issue?
2218    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2219    {
2220       RenameOnError(HashSumMismatch);
2221       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2222       return;
2223    }
2224
2225    // Grab the output filename
2226    string FileName = LookupTag(Message,"Filename");
2227    if (FileName.empty() == true)
2228    {
2229       Status = StatError;
2230       ErrorText = "Method gave a blank filename";
2231       return;
2232    }
2233
2234    Complete = true;
2235
2236    // Reference filename
2237    if (FileName != DestFile)
2238    {
2239       StoreFilename = DestFile = FileName;
2240       Local = true;
2241       return;
2242    }
2243
2244    // Done, move it into position
2245    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2246    FinalFile += flNotDir(StoreFilename);
2247    Rename(DestFile,FinalFile);
2248
2249    StoreFilename = DestFile = FinalFile;
2250    Complete = true;
2251 }
2252                                                                         /*}}}*/
2253 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2254 // ---------------------------------------------------------------------
2255 /* Here we try other sources */
2256 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2257 {
2258    ErrorText = LookupTag(Message,"Message");
2259
2260    /* We don't really want to retry on failed media swaps, this prevents
2261       that. An interesting observation is that permanent failures are not
2262       recorded. */
2263    if (Cnf->Removable == true &&
2264        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2265    {
2266       // Vf = Version.FileList();
2267       while (Vf.end() == false) ++Vf;
2268       StoreFilename = string();
2269       Item::Failed(Message,Cnf);
2270       return;
2271    }
2272
2273    if (QueueNext() == false)
2274    {
2275       // This is the retry counter
2276       if (Retries != 0 &&
2277           Cnf->LocalOnly == false &&
2278           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2279       {
2280          Retries--;
2281          Vf = Version.FileList();
2282          if (QueueNext() == true)
2283             return;
2284       }
2285
2286       StoreFilename = string();
2287       Item::Failed(Message,Cnf);
2288    }
2289 }
2290                                                                         /*}}}*/
2291 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2292 // ---------------------------------------------------------------------
2293 APT_PURE bool pkgAcqArchive::IsTrusted() const
2294 {
2295    return Trusted;
2296 }
2297                                                                         /*}}}*/
2298 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2299 // ---------------------------------------------------------------------
2300 /* */
2301 void pkgAcqArchive::Finished()
2302 {
2303    if (Status == pkgAcquire::Item::StatDone &&
2304        Complete == true)
2305       return;
2306    StoreFilename = string();
2307 }
2308                                                                         /*}}}*/
2309 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2310 // ---------------------------------------------------------------------
2311 /* The file is added to the queue */
2312 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes,
2313                        unsigned long long Size,string Dsc,string ShortDesc,
2314                        const string &DestDir, const string &DestFilename,
2315                        bool IsIndexFile) :
2316                        Item(Owner, Hashes), IsIndexFile(IsIndexFile)
2317 {
2318    Retries = _config->FindI("Acquire::Retries",0);
2319
2320    if(!DestFilename.empty())
2321       DestFile = DestFilename;
2322    else if(!DestDir.empty())
2323       DestFile = DestDir + "/" + flNotDir(URI);
2324    else
2325       DestFile = flNotDir(URI);
2326
2327    // Create the item
2328    Desc.URI = URI;
2329    Desc.Description = Dsc;
2330    Desc.Owner = this;
2331
2332    // Set the short description to the archive component
2333    Desc.ShortDesc = ShortDesc;
2334
2335    // Get the transfer sizes
2336    FileSize = Size;
2337    struct stat Buf;
2338    if (stat(DestFile.c_str(),&Buf) == 0)
2339    {
2340       // Hmm, the partial file is too big, erase it
2341       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2342          unlink(DestFile.c_str());
2343       else
2344          PartialSize = Buf.st_size;
2345    }
2346
2347    QueueURI(Desc);
2348 }
2349                                                                         /*}}}*/
2350 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2351 // ---------------------------------------------------------------------
2352 /* */
2353 void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes,
2354                       pkgAcquire::MethodConfig *Cnf)
2355 {
2356    Item::Done(Message,Size,CalcHashes,Cnf);
2357
2358    // Check the hash
2359    if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
2360    {
2361       RenameOnError(HashSumMismatch);
2362       printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
2363       return;
2364    }
2365
2366    string FileName = LookupTag(Message,"Filename");
2367    if (FileName.empty() == true)
2368    {
2369       Status = StatError;
2370       ErrorText = "Method gave a blank filename";
2371       return;
2372    }
2373
2374    Complete = true;
2375
2376    // The files timestamp matches
2377    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2378       return;
2379
2380    // We have to copy it into place
2381    if (FileName != DestFile)
2382    {
2383       Local = true;
2384       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2385           Cnf->Removable == true)
2386       {
2387          Desc.URI = "copy:" + FileName;
2388          QueueURI(Desc);
2389          return;
2390       }
2391
2392       // Erase the file if it is a symlink so we can overwrite it
2393       struct stat St;
2394       if (lstat(DestFile.c_str(),&St) == 0)
2395       {
2396          if (S_ISLNK(St.st_mode) != 0)
2397             unlink(DestFile.c_str());
2398       }
2399
2400       // Symlink the file
2401       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2402       {
2403          ErrorText = "Link to " + DestFile + " failure ";
2404          Status = StatError;
2405          Complete = false;
2406       }
2407    }
2408 }
2409                                                                         /*}}}*/
2410 // AcqFile::Failed - Failure handler                                    /*{{{*/
2411 // ---------------------------------------------------------------------
2412 /* Here we try other sources */
2413 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2414 {
2415    ErrorText = LookupTag(Message,"Message");
2416
2417    // This is the retry counter
2418    if (Retries != 0 &&
2419        Cnf->LocalOnly == false &&
2420        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2421    {
2422       Retries--;
2423       QueueURI(Desc);
2424       return;
2425    }
2426
2427    Item::Failed(Message,Cnf);
2428 }
2429                                                                         /*}}}*/
2430 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2431 // ---------------------------------------------------------------------
2432 /* The only header we use is the last-modified header. */
2433 string pkgAcqFile::Custom600Headers() const
2434 {
2435    if (IsIndexFile)
2436       return "\nIndex-File: true";
2437    return "";
2438 }
2439                                                                         /*}}}*/