apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(Desc.URI);
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(Desc.URI);
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Init(URI, URIDesc, ShortDesc);
 936 }
 937 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 938                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 939    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 940 {
 941    // autoselect the compression method
 942    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 943    CompressionExtension = "";
 944    if (ExpectedHash.empty() == false)
 945    {
 946       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 947          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 948             CompressionExtension.append(*t).append(" ");
 949    }
 950    else
 951    {
 952       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 953          CompressionExtension.append(*t).append(" ");
 954    }
 955    if (CompressionExtension.empty() == false)
 956       CompressionExtension.erase(CompressionExtension.end()-1);
 957
 958    Init(Target->URI, Target->Description, Target->ShortDesc);
 959 }
 960                                                                         /*}}}*/
 961 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 962 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 963    Decompression = false;
 964    Erase = false;
 965
 966    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 967    DestFile += URItoFileName(URI);
 968
 969    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 970    if (comprExt == "uncompressed")
 971       Desc.URI = URI;
 972    else {
 973       Desc.URI = URI + '.' + comprExt;
 974       DestFile = DestFile + '.' + comprExt;
 975    }
 976
 977    Desc.Description = URIDesc;
 978    Desc.Owner = this;
 979    Desc.ShortDesc = ShortDesc;
 980
 981    QueueURI(Desc);
 982 }
 983                                                                         /*}}}*/
 984 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 985 // ---------------------------------------------------------------------
 986 /* The only header we use is the last-modified header. */
 987 string pkgAcqIndex::Custom600Headers()
 988 {
 989    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 990    string Final = _config->FindDir("Dir::State::lists");
 991    Final += URItoFileName(RealURI);
 992    if (_config->FindB("Acquire::GzipIndexes",false))
 993       Final += compExt;
 994
 995    string msg = "\nIndex-File: true";
 996    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 997    //        seems to be difficult without breaking ABI
 998    if (ShortDesc().find("Translation") != 0)
 999       msg += "\nFail-Ignore: true";
1000    struct stat Buf;
1001    if (stat(Final.c_str(),&Buf) == 0)
1002       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1003
1004    return msg;
1005 }
1006                                                                         /*}}}*/
1007 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1008 {
1009    size_t const nextExt = CompressionExtension.find(' ');
1010    if (nextExt != std::string::npos)
1011    {
1012       CompressionExtension = CompressionExtension.substr(nextExt+1);
1013       Init(RealURI, Desc.Description, Desc.ShortDesc);
1014       return;
1015    }
1016
1017    // on decompression failure, remove bad versions in partial/
1018    if (Decompression && Erase) {
1019       string s = _config->FindDir("Dir::State::lists") + "partial/";
1020       s.append(URItoFileName(RealURI));
1021       unlink(s.c_str());
1022    }
1023
1024    Item::Failed(Message,Cnf);
1025 }
1026                                                                         /*}}}*/
1027 // pkgAcqIndex::GetFinalFilename - Return the full final file path      /*{{{*/
1028 std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
1029                                           std::string const &compExt)
1030 {
1031    std::string FinalFile = _config->FindDir("Dir::State::lists");
1032    FinalFile += URItoFileName(URI);
1033    if (_config->FindB("Acquire::GzipIndexes",false) == true)
1034       FinalFile += '.' + compExt;
1035    return FinalFile;
1036 }
1037                                                                         /*}}}*/
1038 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit         /*{{{*/
1039 void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
1040 {
1041    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1042    if (_config->FindB("Acquire::GzipIndexes",false) == true)
1043       DestFile += compExt;
1044
1045    string FinalFile = GetFinalFilename(RealURI, compExt);
1046    Rename(FinalFile, FileName);
1047    Decompression = true;
1048    Desc.URI = "copy:" + FileName;
1049    QueueURI(Desc);
1050 }
1051                                                                         /*}}}*/
1052 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1053 // ---------------------------------------------------------------------
1054 /* This goes through a number of states.. On the initial fetch the
1055    method could possibly return an alternate filename which points
1056    to the uncompressed version of the file. If this is so the file
1057    is copied into the partial directory. In all other cases the file
1058    is decompressed with a gzip uri. */
1059 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1060                        pkgAcquire::MethodConfig *Cfg)
1061 {
1062    Item::Done(Message,Size,Hash,Cfg);
1063    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1064
1065    if (Decompression == true)
1066    {
1067       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1068       {
1069          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1070          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1071       }
1072
1073       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1074       {
1075          Desc.URI = RealURI;
1076          RenameOnError(HashSumMismatch);
1077          return;
1078       }
1079
1080       // FIXME: this can go away once we only ever download stuff that
1081       //        has a valid hash and we never do GET based probing
1082       //
1083       /* Always verify the index file for correctness (all indexes must
1084        * have a Package field) (LP: #346386) (Closes: #627642)
1085        */
1086       FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension);
1087       // Only test for correctness if the file is not empty (empty is ok)
1088       if (fd.Size() > 0)
1089       {
1090          pkgTagSection sec;
1091          pkgTagFile tag(&fd);
1092
1093          // all our current indexes have a field 'Package' in each section
1094          if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1095          {
1096             RenameOnError(InvalidFormat);
1097             return;
1098          }
1099       }
1100
1101       // Done, move it into position
1102       string FinalFile = GetFinalFilename(RealURI, compExt);
1103       Rename(DestFile,FinalFile);
1104       chmod(FinalFile.c_str(),0644);
1105
1106       /* We restore the original name to DestFile so that the clean operation
1107          will work OK */
1108       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1109       DestFile += URItoFileName(RealURI);
1110       if (_config->FindB("Acquire::GzipIndexes",false))
1111          DestFile += '.' + compExt;
1112
1113       // Remove the compressed version.
1114       if (Erase == true)
1115          unlink(DestFile.c_str());
1116       return;
1117    }
1118
1119    Erase = false;
1120    Complete = true;
1121
1122    // Handle the unzipd case
1123    string FileName = LookupTag(Message,"Alt-Filename");
1124    if (FileName.empty() == false)
1125    {
1126       Decompression = true;
1127       Local = true;
1128       DestFile += ".decomp";
1129       Desc.URI = "copy:" + FileName;
1130       QueueURI(Desc);
1131       Mode = "copy";
1132       return;
1133    }
1134
1135    FileName = LookupTag(Message,"Filename");
1136    if (FileName.empty() == true)
1137    {
1138       Status = StatError;
1139       ErrorText = "Method gave a blank filename";
1140    }
1141
1142    if (FileName == DestFile)
1143       Erase = true;
1144    else
1145       Local = true;
1146
1147    // The files timestamp matches, for non-local URLs reverify the local
1148    // file, for local file, uncompress again to ensure the hashsum is still
1149    // matching the Release file
1150    bool const IsCDROM = RealURI.substr(0,6) == "cdrom:";
1151    if ((Local == false || IsCDROM == true) &&
1152          StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1153    {
1154       // set destfile to the final destfile
1155       if(_config->FindB("Acquire::GzipIndexes",false) == false)
1156       {
1157          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1158          DestFile += URItoFileName(RealURI);
1159       }
1160
1161       // do not reverify cdrom sources as apt-cdrom may rewrite the Packages
1162       // file when its doing the indexcopy
1163       if (IsCDROM == false)
1164          ReverifyAfterIMS(FileName);
1165       return;
1166    }
1167    string decompProg;
1168
1169    // If we enable compressed indexes, queue for hash verification
1170    if (_config->FindB("Acquire::GzipIndexes",false))
1171    {
1172       DestFile = _config->FindDir("Dir::State::lists");
1173       DestFile += URItoFileName(RealURI) + '.' + compExt;
1174
1175       Decompression = true;
1176       Desc.URI = "copy:" + FileName;
1177       QueueURI(Desc);
1178
1179       return;
1180     }
1181
1182    // get the binary name for your used compression type
1183    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1184    if(decompProg.empty() == false);
1185    else if(compExt == "uncompressed")
1186       decompProg = "copy";
1187    else {
1188       _error->Error("Unsupported extension: %s", compExt.c_str());
1189       return;
1190    }
1191
1192    Decompression = true;
1193    DestFile += ".decomp";
1194    Desc.URI = decompProg + ":" + FileName;
1195    QueueURI(Desc);
1196
1197    // FIXME: this points to a c++ string that goes out of scope
1198    Mode = decompProg.c_str();
1199 }
1200                                                                         /*}}}*/
1201 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1202 // ---------------------------------------------------------------------
1203 /* The Translation file is added to the queue */
1204 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1205                             string URI,string URIDesc,string ShortDesc)
1206   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1207 {
1208 }
1209 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1210                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1211   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1212 {
1213 }
1214                                                                         /*}}}*/
1215 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1216 // ---------------------------------------------------------------------
1217 string pkgAcqIndexTrans::Custom600Headers()
1218 {
1219    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1220    string Final = _config->FindDir("Dir::State::lists");
1221    Final += URItoFileName(RealURI);
1222    if (_config->FindB("Acquire::GzipIndexes",false))
1223       Final += compExt;
1224
1225    struct stat Buf;
1226    if (stat(Final.c_str(),&Buf) != 0)
1227       return "\nFail-Ignore: true\nIndex-File: true";
1228    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1229 }
1230                                                                         /*}}}*/
1231 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1232 // ---------------------------------------------------------------------
1233 /* */
1234 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1235 {
1236    size_t const nextExt = CompressionExtension.find(' ');
1237    if (nextExt != std::string::npos)
1238    {
1239       CompressionExtension = CompressionExtension.substr(nextExt+1);
1240       Init(RealURI, Desc.Description, Desc.ShortDesc);
1241       Status = StatIdle;
1242       return;
1243    }
1244
1245    if (Cnf->LocalOnly == true ||
1246        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1247    {
1248       // Ignore this
1249       Status = StatDone;
1250       Complete = false;
1251       Dequeue();
1252       return;
1253    }
1254
1255    Item::Failed(Message,Cnf);
1256 }
1257                                                                         /*}}}*/
1258 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1259                              string URI,string URIDesc,string ShortDesc,
1260                              string MetaIndexURI, string MetaIndexURIDesc,
1261                              string MetaIndexShortDesc,
1262                              const vector<IndexTarget*>* IndexTargets,
1263                              indexRecords* MetaIndexParser) :
1264    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1265    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1266    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1267 {
1268    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1269    DestFile += URItoFileName(URI);
1270
1271    // remove any partial downloaded sig-file in partial/.
1272    // it may confuse proxies and is too small to warrant a
1273    // partial download anyway
1274    unlink(DestFile.c_str());
1275
1276    // Create the item
1277    Desc.Description = URIDesc;
1278    Desc.Owner = this;
1279    Desc.ShortDesc = ShortDesc;
1280    Desc.URI = URI;
1281
1282    string Final = _config->FindDir("Dir::State::lists");
1283    Final += URItoFileName(RealURI);
1284    if (RealFileExists(Final) == true)
1285    {
1286       // File was already in place.  It needs to be re-downloaded/verified
1287       // because Release might have changed, we do give it a different
1288       // name than DestFile because otherwise the http method will
1289       // send If-Range requests and there are too many broken servers
1290       // out there that do not understand them
1291       LastGoodSig = DestFile+".reverify";
1292       Rename(Final,LastGoodSig);
1293    }
1294
1295    QueueURI(Desc);
1296 }
1297                                                                         /*}}}*/
1298 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1299 {
1300    // if the file was never queued undo file-changes done in the constructor
1301    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1302          LastGoodSig.empty() == false)
1303    {
1304       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1305       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1306          Rename(LastGoodSig, Final);
1307    }
1308
1309 }
1310                                                                         /*}}}*/
1311 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1312 // ---------------------------------------------------------------------
1313 /* The only header we use is the last-modified header. */
1314 string pkgAcqMetaSig::Custom600Headers()
1315 {
1316    struct stat Buf;
1317    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1318       return "\nIndex-File: true";
1319
1320    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1321 }
1322
1323 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1324                          pkgAcquire::MethodConfig *Cfg)
1325 {
1326    Item::Done(Message,Size,MD5,Cfg);
1327
1328    string FileName = LookupTag(Message,"Filename");
1329    if (FileName.empty() == true)
1330    {
1331       Status = StatError;
1332       ErrorText = "Method gave a blank filename";
1333       return;
1334    }
1335
1336    if (FileName != DestFile)
1337    {
1338       // We have to copy it into place
1339       Local = true;
1340       Desc.URI = "copy:" + FileName;
1341       QueueURI(Desc);
1342       return;
1343    }
1344
1345    Complete = true;
1346
1347    // put the last known good file back on i-m-s hit (it will
1348    // be re-verified again)
1349    // Else do nothing, we have the new file in DestFile then
1350    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1351       Rename(LastGoodSig, DestFile);
1352
1353    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1354    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1355                        MetaIndexShortDesc,  DestFile, IndexTargets,
1356                        MetaIndexParser);
1357
1358 }
1359                                                                         /*}}}*/
1360 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1361 {
1362    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1363
1364    // if we get a network error we fail gracefully
1365    if(Status == StatTransientNetworkError)
1366    {
1367       Item::Failed(Message,Cnf);
1368       // move the sigfile back on transient network failures
1369       if(FileExists(LastGoodSig))
1370          Rename(LastGoodSig,Final);
1371
1372       // set the status back to , Item::Failed likes to reset it
1373       Status = pkgAcquire::Item::StatTransientNetworkError;
1374       return;
1375    }
1376
1377    // Delete any existing sigfile when the acquire failed
1378    unlink(Final.c_str());
1379
1380    // queue a pkgAcqMetaIndex with no sigfile
1381    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1382                        "", IndexTargets, MetaIndexParser);
1383
1384    if (Cnf->LocalOnly == true ||
1385        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1386    {
1387       // Ignore this
1388       Status = StatDone;
1389       Complete = false;
1390       Dequeue();
1391       return;
1392    }
1393
1394    Item::Failed(Message,Cnf);
1395 }
1396                                                                         /*}}}*/
1397 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1398                                  string URI,string URIDesc,string ShortDesc,
1399                                  string SigFile,
1400                                  const vector<struct IndexTarget*>* IndexTargets,
1401                                  indexRecords* MetaIndexParser) :
1402    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1403    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1404 {
1405    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1406    DestFile += URItoFileName(URI);
1407
1408    // Create the item
1409    Desc.Description = URIDesc;
1410    Desc.Owner = this;
1411    Desc.ShortDesc = ShortDesc;
1412    Desc.URI = URI;
1413
1414    QueueURI(Desc);
1415 }
1416                                                                         /*}}}*/
1417 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1418 // ---------------------------------------------------------------------
1419 /* The only header we use is the last-modified header. */
1420 string pkgAcqMetaIndex::Custom600Headers()
1421 {
1422    string Final = _config->FindDir("Dir::State::lists");
1423    Final += URItoFileName(RealURI);
1424
1425    struct stat Buf;
1426    if (stat(Final.c_str(),&Buf) != 0)
1427       return "\nIndex-File: true";
1428
1429    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1430 }
1431                                                                         /*}}}*/
1432 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1433                            pkgAcquire::MethodConfig *Cfg)
1434 {
1435    Item::Done(Message,Size,Hash,Cfg);
1436
1437    // MetaIndexes are done in two passes: one to download the
1438    // metaindex with an appropriate method, and a second to verify it
1439    // with the gpgv method
1440
1441    if (AuthPass == true)
1442    {
1443       AuthDone(Message);
1444
1445       // all cool, move Release file into place
1446       Complete = true;
1447    }
1448    else
1449    {
1450       RetrievalDone(Message);
1451       if (!Complete)
1452          // Still more retrieving to do
1453          return;
1454
1455       if (SigFile == "")
1456       {
1457          // There was no signature file, so we are finished.  Download
1458          // the indexes and do only hashsum verification if possible
1459          MetaIndexParser->Load(DestFile);
1460          QueueIndexes(false);
1461       }
1462       else
1463       {
1464          // There was a signature file, so pass it to gpgv for
1465          // verification
1466
1467          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1468             std::cerr << "Metaindex acquired, queueing gpg verification ("
1469                       << SigFile << "," << DestFile << ")\n";
1470          AuthPass = true;
1471          Desc.URI = "gpgv:" + SigFile;
1472          QueueURI(Desc);
1473          Mode = "gpgv";
1474          return;
1475       }
1476    }
1477
1478    if (Complete == true)
1479    {
1480       string FinalFile = _config->FindDir("Dir::State::lists");
1481       FinalFile += URItoFileName(RealURI);
1482       if (SigFile == DestFile)
1483          SigFile = FinalFile;
1484       Rename(DestFile,FinalFile);
1485       chmod(FinalFile.c_str(),0644);
1486       DestFile = FinalFile;
1487    }
1488 }
1489                                                                         /*}}}*/
1490 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1491 {
1492    // We have just finished downloading a Release file (it is not
1493    // verified yet)
1494
1495    string FileName = LookupTag(Message,"Filename");
1496    if (FileName.empty() == true)
1497    {
1498       Status = StatError;
1499       ErrorText = "Method gave a blank filename";
1500       return;
1501    }
1502
1503    if (FileName != DestFile)
1504    {
1505       Local = true;
1506       Desc.URI = "copy:" + FileName;
1507       QueueURI(Desc);
1508       return;
1509    }
1510
1511    // make sure to verify against the right file on I-M-S hit
1512    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1513    if(IMSHit)
1514    {
1515       string FinalFile = _config->FindDir("Dir::State::lists");
1516       FinalFile += URItoFileName(RealURI);
1517       if (SigFile == DestFile)
1518       {
1519          SigFile = FinalFile;
1520          // constructor of pkgAcqMetaClearSig moved it out of the way,
1521          // now move it back in on IMS hit for the 'old' file
1522          string const OldClearSig = DestFile + ".reverify";
1523          if (RealFileExists(OldClearSig) == true)
1524             Rename(OldClearSig, FinalFile);
1525       }
1526       DestFile = FinalFile;
1527    }
1528    Complete = true;
1529 }
1530                                                                         /*}}}*/
1531 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1532 {
1533    // At this point, the gpgv method has succeeded, so there is a
1534    // valid signature from a key in the trusted keyring.  We
1535    // perform additional verification of its contents, and use them
1536    // to verify the indexes we are about to download
1537
1538    if (!MetaIndexParser->Load(DestFile))
1539    {
1540       Status = StatAuthError;
1541       ErrorText = MetaIndexParser->ErrorText;
1542       return;
1543    }
1544
1545    if (!VerifyVendor(Message))
1546    {
1547       return;
1548    }
1549
1550    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1551       std::cerr << "Signature verification succeeded: "
1552                 << DestFile << std::endl;
1553
1554    // do not trust any previously unverified content that we may have
1555    string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1556    if (DestFile != SigFile)
1557       LastGoodSigFile.append(".gpg");
1558    LastGoodSigFile.append(".reverify");
1559    if(IMSHit == false && RealFileExists(LastGoodSigFile) == false)
1560    {
1561       for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1562            Target != IndexTargets->end();
1563            ++Target)
1564       {
1565          // remove old indexes
1566          std::string index = _config->FindDir("Dir::State::lists") +
1567             URItoFileName((*Target)->URI);
1568          unlink(index.c_str());
1569          // and also old gzipindexes
1570          std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1571          for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1572          {
1573             index += '.' + (*t);
1574             unlink(index.c_str());
1575          }
1576       }
1577    }
1578
1579
1580    // Download further indexes with verification
1581    QueueIndexes(true);
1582
1583    // is it a clearsigned MetaIndex file?
1584    if (DestFile == SigFile)
1585       return;
1586
1587    // Done, move signature file into position
1588    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1589       URItoFileName(RealURI) + ".gpg";
1590    Rename(SigFile,VerifiedSigFile);
1591    chmod(VerifiedSigFile.c_str(),0644);
1592 }
1593                                                                         /*}}}*/
1594 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1595 {
1596 #if 0
1597    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1598     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1599    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1600    {
1601       Status = StatError;
1602       ErrorText = MetaIndexParser->ErrorText;
1603       return;
1604    }
1605 #endif
1606    bool transInRelease = false;
1607    {
1608       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1609       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1610          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1611          if (k->find("Translation-") != std::string::npos)
1612          {
1613             transInRelease = true;
1614             break;
1615          }
1616    }
1617
1618    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1619         Target != IndexTargets->end();
1620         ++Target)
1621    {
1622       HashString ExpectedIndexHash;
1623       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1624       bool compressedAvailable = false;
1625       if (Record == NULL)
1626       {
1627          if ((*Target)->IsOptional() == true)
1628          {
1629             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1630             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1631                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1632                {
1633                   compressedAvailable = true;
1634                   break;
1635                }
1636          }
1637          else if (verify == true)
1638          {
1639             Status = StatAuthError;
1640             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1641             return;
1642          }
1643       }
1644       else
1645       {
1646          ExpectedIndexHash = Record->Hash;
1647          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1648          {
1649             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1650             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1651             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1652          }
1653          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1654          {
1655             Status = StatAuthError;
1656             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1657             return;
1658          }
1659       }
1660
1661       if ((*Target)->IsOptional() == true)
1662       {
1663          if ((*Target)->IsSubIndex() == true)
1664             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1665                                 (*Target)->ShortDesc, ExpectedIndexHash);
1666          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1667          {
1668             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1669                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1670                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1671                                    (*Target)->ShortDesc, ExpectedIndexHash);
1672             else
1673                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1674          }
1675          continue;
1676       }
1677
1678       /* Queue Packages file (either diff or full packages files, depending
1679          on the users option) - we also check if the PDiff Index file is listed
1680          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1681          instead, but passing the required info to it is to much hassle */
1682       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1683           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1684          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1685                              (*Target)->ShortDesc, ExpectedIndexHash);
1686       else
1687          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1688    }
1689 }
1690                                                                         /*}}}*/
1691 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1692 {
1693    string::size_type pos;
1694
1695    // check for missing sigs (that where not fatal because otherwise we had
1696    // bombed earlier)
1697    string missingkeys;
1698    string msg = _("There is no public key available for the "
1699                   "following key IDs:\n");
1700    pos = Message.find("NO_PUBKEY ");
1701    if (pos != std::string::npos)
1702    {
1703       string::size_type start = pos+strlen("NO_PUBKEY ");
1704       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1705       missingkeys += (Fingerprint);
1706    }
1707    if(!missingkeys.empty())
1708       _error->Warning("%s", (msg + missingkeys).c_str());
1709
1710    string Transformed = MetaIndexParser->GetExpectedDist();
1711
1712    if (Transformed == "../project/experimental")
1713    {
1714       Transformed = "experimental";
1715    }
1716
1717    pos = Transformed.rfind('/');
1718    if (pos != string::npos)
1719    {
1720       Transformed = Transformed.substr(0, pos);
1721    }
1722
1723    if (Transformed == ".")
1724    {
1725       Transformed = "";
1726    }
1727
1728    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1729        MetaIndexParser->GetValidUntil() > 0) {
1730       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1731       if (invalid_since > 0)
1732          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1733          // the time since then the file is invalid - formated in the same way as in
1734          // the download progress display (e.g. 7d 3h 42min 1s)
1735          return _error->Error(
1736             _("Release file for %s is expired (invalid since %s). "
1737               "Updates for this repository will not be applied."),
1738             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1739    }
1740
1741    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1742    {
1743       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1744       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1745       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1746    }
1747
1748    if (MetaIndexParser->CheckDist(Transformed) == false)
1749    {
1750       // This might become fatal one day
1751 //       Status = StatAuthError;
1752 //       ErrorText = "Conflicting distribution; expected "
1753 //          + MetaIndexParser->GetExpectedDist() + " but got "
1754 //          + MetaIndexParser->GetDist();
1755 //       return false;
1756       if (!Transformed.empty())
1757       {
1758          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1759                          Desc.Description.c_str(),
1760                          Transformed.c_str(),
1761                          MetaIndexParser->GetDist().c_str());
1762       }
1763    }
1764
1765    return true;
1766 }
1767                                                                         /*}}}*/
1768 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1769 // ---------------------------------------------------------------------
1770 /* */
1771 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1772 {
1773    if (AuthPass == true)
1774    {
1775       // gpgv method failed, if we have a good signature
1776       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1777       if (DestFile != SigFile)
1778          LastGoodSigFile.append(".gpg");
1779       LastGoodSigFile.append(".reverify");
1780
1781       if(FileExists(LastGoodSigFile))
1782       {
1783          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1784          if (DestFile != SigFile)
1785             VerifiedSigFile.append(".gpg");
1786          Rename(LastGoodSigFile, VerifiedSigFile);
1787          Status = StatTransientNetworkError;
1788          _error->Warning(_("An error occurred during the signature "
1789                            "verification. The repository is not updated "
1790                            "and the previous index files will be used. "
1791                            "GPG error: %s: %s\n"),
1792                          Desc.Description.c_str(),
1793                          LookupTag(Message,"Message").c_str());
1794          RunScripts("APT::Update::Auth-Failure");
1795          return;
1796       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1797          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1798          _error->Error(_("GPG error: %s: %s"),
1799                          Desc.Description.c_str(),
1800                          LookupTag(Message,"Message").c_str());
1801          return;
1802       } else {
1803          _error->Warning(_("GPG error: %s: %s"),
1804                          Desc.Description.c_str(),
1805                          LookupTag(Message,"Message").c_str());
1806       }
1807       // gpgv method failed
1808       ReportMirrorFailure("GPGFailure");
1809    }
1810
1811    /* Always move the meta index, even if gpgv failed. This ensures
1812     * that PackageFile objects are correctly filled in */
1813    if (FileExists(DestFile)) {
1814       string FinalFile = _config->FindDir("Dir::State::lists");
1815       FinalFile += URItoFileName(RealURI);
1816       /* InRelease files become Release files, otherwise
1817        * they would be considered as trusted later on */
1818       if (SigFile == DestFile) {
1819          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1820                                        "Release");
1821          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1822                                        "Release");
1823          SigFile = FinalFile;
1824       }
1825       Rename(DestFile,FinalFile);
1826       chmod(FinalFile.c_str(),0644);
1827
1828       DestFile = FinalFile;
1829    }
1830
1831    // No Release file was present, or verification failed, so fall
1832    // back to queueing Packages files without verification
1833    QueueIndexes(false);
1834 }
1835                                                                         /*}}}*/
1836 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1837                 string const &URI, string const &URIDesc, string const &ShortDesc,
1838                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1839                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1840                 const vector<struct IndexTarget*>* IndexTargets,
1841                 indexRecords* MetaIndexParser) :
1842         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1843         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1844         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1845 {
1846    SigFile = DestFile;
1847
1848    // keep the old InRelease around in case of transistent network errors
1849    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1850    if (RealFileExists(Final) == true)
1851    {
1852       string const LastGoodSig = DestFile + ".reverify";
1853       Rename(Final,LastGoodSig);
1854    }
1855 }
1856                                                                         /*}}}*/
1857 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1858 {
1859    // if the file was never queued undo file-changes done in the constructor
1860    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1861    {
1862       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1863       string const LastGoodSig = DestFile + ".reverify";
1864       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1865          Rename(LastGoodSig, Final);
1866    }
1867 }
1868                                                                         /*}}}*/
1869 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1870 // ---------------------------------------------------------------------
1871 // FIXME: this can go away once the InRelease file is used widely
1872 string pkgAcqMetaClearSig::Custom600Headers()
1873 {
1874    string Final = _config->FindDir("Dir::State::lists");
1875    Final += URItoFileName(RealURI);
1876
1877    struct stat Buf;
1878    if (stat(Final.c_str(),&Buf) != 0)
1879    {
1880       Final = DestFile + ".reverify";
1881       if (stat(Final.c_str(),&Buf) != 0)
1882          return "\nIndex-File: true\nFail-Ignore: true\n";
1883    }
1884
1885    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1886 }
1887                                                                         /*}}}*/
1888 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1889 {
1890    if (AuthPass == false)
1891    {
1892       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1893       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1894       string FinalFile = _config->FindDir("Dir::State::lists");
1895       FinalFile.append(URItoFileName(RealURI));
1896       if (FileExists(FinalFile))
1897          unlink(FinalFile.c_str());
1898
1899       new pkgAcqMetaSig(Owner,
1900                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1901                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1902                         IndexTargets, MetaIndexParser);
1903       if (Cnf->LocalOnly == true ||
1904           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1905          Dequeue();
1906    }
1907    else
1908       pkgAcqMetaIndex::Failed(Message, Cnf);
1909 }
1910                                                                         /*}}}*/
1911 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1912 // ---------------------------------------------------------------------
1913 /* This just sets up the initial fetch environment and queues the first
1914    possibilitiy */
1915 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1916                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1917                              string &StoreFilename) :
1918                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1919                StoreFilename(StoreFilename), Vf(Version.FileList()),
1920                Trusted(false)
1921 {
1922    Retries = _config->FindI("Acquire::Retries",0);
1923
1924    if (Version.Arch() == 0)
1925    {
1926       _error->Error(_("I wasn't able to locate a file for the %s package. "
1927                       "This might mean you need to manually fix this package. "
1928                       "(due to missing arch)"),
1929                     Version.ParentPkg().FullName().c_str());
1930       return;
1931    }
1932
1933    /* We need to find a filename to determine the extension. We make the
1934       assumption here that all the available sources for this version share
1935       the same extension.. */
1936    // Skip not source sources, they do not have file fields.
1937    for (; Vf.end() == false; ++Vf)
1938    {
1939       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1940          continue;
1941       break;
1942    }
1943
1944    // Does not really matter here.. we are going to fail out below
1945    if (Vf.end() != true)
1946    {
1947       // If this fails to get a file name we will bomb out below.
1948       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1949       if (_error->PendingError() == true)
1950          return;
1951
1952       // Generate the final file name as: package_version_arch.foo
1953       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1954                       QuoteString(Version.VerStr(),"_:") + '_' +
1955                       QuoteString(Version.Arch(),"_:.") +
1956                       "." + flExtension(Parse.FileName());
1957    }
1958
1959    // check if we have one trusted source for the package. if so, switch
1960    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1961    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1962    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1963    bool seenUntrusted = false;
1964    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1965    {
1966       pkgIndexFile *Index;
1967       if (Sources->FindIndex(i.File(),Index) == false)
1968          continue;
1969
1970       if (debugAuth == true)
1971          std::cerr << "Checking index: " << Index->Describe()
1972                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1973
1974       if (Index->IsTrusted() == true)
1975       {
1976          Trusted = true;
1977          if (allowUnauth == false)
1978             break;
1979       }
1980       else
1981          seenUntrusted = true;
1982    }
1983
1984    // "allow-unauthenticated" restores apts old fetching behaviour
1985    // that means that e.g. unauthenticated file:// uris are higher
1986    // priority than authenticated http:// uris
1987    if (allowUnauth == true && seenUntrusted == true)
1988       Trusted = false;
1989
1990    // Select a source
1991    if (QueueNext() == false && _error->PendingError() == false)
1992       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1993                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1994 }
1995                                                                         /*}}}*/
1996 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1997 // ---------------------------------------------------------------------
1998 /* This queues the next available file version for download. It checks if
1999    the archive is already available in the cache and stashs the MD5 for
2000    checking later. */
2001 bool pkgAcqArchive::QueueNext()
2002 {
2003    string const ForceHash = _config->Find("Acquire::ForceHash");
2004    for (; Vf.end() == false; ++Vf)
2005    {
2006       // Ignore not source sources
2007       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
2008          continue;
2009
2010       // Try to cross match against the source list
2011       pkgIndexFile *Index;
2012       if (Sources->FindIndex(Vf.File(),Index) == false)
2013             continue;
2014
2015       // only try to get a trusted package from another source if that source
2016       // is also trusted
2017       if(Trusted && !Index->IsTrusted())
2018          continue;
2019
2020       // Grab the text package record
2021       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2022       if (_error->PendingError() == true)
2023          return false;
2024
2025       string PkgFile = Parse.FileName();
2026       if (ForceHash.empty() == false)
2027       {
2028          if(stringcasecmp(ForceHash, "sha512") == 0)
2029             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
2030          else if(stringcasecmp(ForceHash, "sha256") == 0)
2031             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
2032          else if (stringcasecmp(ForceHash, "sha1") == 0)
2033             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
2034          else
2035             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2036       }
2037       else
2038       {
2039          string Hash;
2040          if ((Hash = Parse.SHA512Hash()).empty() == false)
2041             ExpectedHash = HashString("SHA512", Hash);
2042          else if ((Hash = Parse.SHA256Hash()).empty() == false)
2043             ExpectedHash = HashString("SHA256", Hash);
2044          else if ((Hash = Parse.SHA1Hash()).empty() == false)
2045             ExpectedHash = HashString("SHA1", Hash);
2046          else
2047             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2048       }
2049       if (PkgFile.empty() == true)
2050          return _error->Error(_("The package index files are corrupted. No Filename: "
2051                               "field for package %s."),
2052                               Version.ParentPkg().Name());
2053
2054       Desc.URI = Index->ArchiveURI(PkgFile);
2055       Desc.Description = Index->ArchiveInfo(Version);
2056       Desc.Owner = this;
2057       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2058
2059       // See if we already have the file. (Legacy filenames)
2060       FileSize = Version->Size;
2061       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2062       struct stat Buf;
2063       if (stat(FinalFile.c_str(),&Buf) == 0)
2064       {
2065          // Make sure the size matches
2066          if ((unsigned long long)Buf.st_size == Version->Size)
2067          {
2068             Complete = true;
2069             Local = true;
2070             Status = StatDone;
2071             StoreFilename = DestFile = FinalFile;
2072             return true;
2073          }
2074
2075          /* Hmm, we have a file and its size does not match, this means it is
2076             an old style mismatched arch */
2077          unlink(FinalFile.c_str());
2078       }
2079
2080       // Check it again using the new style output filenames
2081       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2082       if (stat(FinalFile.c_str(),&Buf) == 0)
2083       {
2084          // Make sure the size matches
2085          if ((unsigned long long)Buf.st_size == Version->Size)
2086          {
2087             Complete = true;
2088             Local = true;
2089             Status = StatDone;
2090             StoreFilename = DestFile = FinalFile;
2091             return true;
2092          }
2093
2094          /* Hmm, we have a file and its size does not match, this shouldn't
2095             happen.. */
2096          unlink(FinalFile.c_str());
2097       }
2098
2099       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2100
2101       // Check the destination file
2102       if (stat(DestFile.c_str(),&Buf) == 0)
2103       {
2104          // Hmm, the partial file is too big, erase it
2105          if ((unsigned long long)Buf.st_size > Version->Size)
2106             unlink(DestFile.c_str());
2107          else
2108             PartialSize = Buf.st_size;
2109       }
2110
2111       // Disables download of archives - useful if no real installation follows,
2112       // e.g. if we are just interested in proposed installation order
2113       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2114       {
2115          Complete = true;
2116          Local = true;
2117          Status = StatDone;
2118          StoreFilename = DestFile = FinalFile;
2119          return true;
2120       }
2121
2122       // Create the item
2123       Local = false;
2124       QueueURI(Desc);
2125
2126       ++Vf;
2127       return true;
2128    }
2129    return false;
2130 }
2131                                                                         /*}}}*/
2132 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2133 // ---------------------------------------------------------------------
2134 /* */
2135 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2136                          pkgAcquire::MethodConfig *Cfg)
2137 {
2138    Item::Done(Message,Size,CalcHash,Cfg);
2139
2140    // Check the size
2141    if (Size != Version->Size)
2142    {
2143       RenameOnError(SizeMismatch);
2144       return;
2145    }
2146
2147    // Check the hash
2148    if(ExpectedHash.toStr() != CalcHash)
2149    {
2150       RenameOnError(HashSumMismatch);
2151       return;
2152    }
2153
2154    // Grab the output filename
2155    string FileName = LookupTag(Message,"Filename");
2156    if (FileName.empty() == true)
2157    {
2158       Status = StatError;
2159       ErrorText = "Method gave a blank filename";
2160       return;
2161    }
2162
2163    Complete = true;
2164
2165    // Reference filename
2166    if (FileName != DestFile)
2167    {
2168       StoreFilename = DestFile = FileName;
2169       Local = true;
2170       return;
2171    }
2172
2173    // Done, move it into position
2174    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2175    FinalFile += flNotDir(StoreFilename);
2176    Rename(DestFile,FinalFile);
2177
2178    StoreFilename = DestFile = FinalFile;
2179    Complete = true;
2180 }
2181                                                                         /*}}}*/
2182 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2183 // ---------------------------------------------------------------------
2184 /* Here we try other sources */
2185 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2186 {
2187    ErrorText = LookupTag(Message,"Message");
2188
2189    /* We don't really want to retry on failed media swaps, this prevents
2190       that. An interesting observation is that permanent failures are not
2191       recorded. */
2192    if (Cnf->Removable == true &&
2193        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2194    {
2195       // Vf = Version.FileList();
2196       while (Vf.end() == false) ++Vf;
2197       StoreFilename = string();
2198       Item::Failed(Message,Cnf);
2199       return;
2200    }
2201
2202    if (QueueNext() == false)
2203    {
2204       // This is the retry counter
2205       if (Retries != 0 &&
2206           Cnf->LocalOnly == false &&
2207           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2208       {
2209          Retries--;
2210          Vf = Version.FileList();
2211          if (QueueNext() == true)
2212             return;
2213       }
2214
2215       StoreFilename = string();
2216       Item::Failed(Message,Cnf);
2217    }
2218 }
2219                                                                         /*}}}*/
2220 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2221 // ---------------------------------------------------------------------
2222 APT_PURE bool pkgAcqArchive::IsTrusted()
2223 {
2224    return Trusted;
2225 }
2226                                                                         /*}}}*/
2227 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2228 // ---------------------------------------------------------------------
2229 /* */
2230 void pkgAcqArchive::Finished()
2231 {
2232    if (Status == pkgAcquire::Item::StatDone &&
2233        Complete == true)
2234       return;
2235    StoreFilename = string();
2236 }
2237                                                                         /*}}}*/
2238 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2239 // ---------------------------------------------------------------------
2240 /* The file is added to the queue */
2241 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2242                        unsigned long long Size,string Dsc,string ShortDesc,
2243                        const string &DestDir, const string &DestFilename,
2244                        bool IsIndexFile) :
2245                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2246 {
2247    Retries = _config->FindI("Acquire::Retries",0);
2248
2249    if(!DestFilename.empty())
2250       DestFile = DestFilename;
2251    else if(!DestDir.empty())
2252       DestFile = DestDir + "/" + flNotDir(URI);
2253    else
2254       DestFile = flNotDir(URI);
2255
2256    // Create the item
2257    Desc.URI = URI;
2258    Desc.Description = Dsc;
2259    Desc.Owner = this;
2260
2261    // Set the short description to the archive component
2262    Desc.ShortDesc = ShortDesc;
2263
2264    // Get the transfer sizes
2265    FileSize = Size;
2266    struct stat Buf;
2267    if (stat(DestFile.c_str(),&Buf) == 0)
2268    {
2269       // Hmm, the partial file is too big, erase it
2270       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2271          unlink(DestFile.c_str());
2272       else
2273          PartialSize = Buf.st_size;
2274    }
2275
2276    QueueURI(Desc);
2277 }
2278                                                                         /*}}}*/
2279 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2280 // ---------------------------------------------------------------------
2281 /* */
2282 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2283                       pkgAcquire::MethodConfig *Cnf)
2284 {
2285    Item::Done(Message,Size,CalcHash,Cnf);
2286
2287    // Check the hash
2288    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2289    {
2290       RenameOnError(HashSumMismatch);
2291       return;
2292    }
2293
2294    string FileName = LookupTag(Message,"Filename");
2295    if (FileName.empty() == true)
2296    {
2297       Status = StatError;
2298       ErrorText = "Method gave a blank filename";
2299       return;
2300    }
2301
2302    Complete = true;
2303
2304    // The files timestamp matches
2305    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2306       return;
2307
2308    // We have to copy it into place
2309    if (FileName != DestFile)
2310    {
2311       Local = true;
2312       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2313           Cnf->Removable == true)
2314       {
2315          Desc.URI = "copy:" + FileName;
2316          QueueURI(Desc);
2317          return;
2318       }
2319
2320       // Erase the file if it is a symlink so we can overwrite it
2321       struct stat St;
2322       if (lstat(DestFile.c_str(),&St) == 0)
2323       {
2324          if (S_ISLNK(St.st_mode) != 0)
2325             unlink(DestFile.c_str());
2326       }
2327
2328       // Symlink the file
2329       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2330       {
2331          ErrorText = "Link to " + DestFile + " failure ";
2332          Status = StatError;
2333          Complete = false;
2334       }
2335    }
2336 }
2337                                                                         /*}}}*/
2338 // AcqFile::Failed - Failure handler                                    /*{{{*/
2339 // ---------------------------------------------------------------------
2340 /* Here we try other sources */
2341 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2342 {
2343    ErrorText = LookupTag(Message,"Message");
2344
2345    // This is the retry counter
2346    if (Retries != 0 &&
2347        Cnf->LocalOnly == false &&
2348        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2349    {
2350       Retries--;
2351       QueueURI(Desc);
2352       return;
2353    }
2354
2355    Item::Failed(Message,Cnf);
2356 }
2357                                                                         /*}}}*/
2358 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2359 // ---------------------------------------------------------------------
2360 /* The only header we use is the last-modified header. */
2361 string pkgAcqFile::Custom600Headers()
2362 {
2363    if (IsIndexFile)
2364       return "\nIndex-File: true";
2365    return "";
2366 }
2367                                                                         /*}}}*/