apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <config.h>
  17
  18 #include <apt-pkg/acquire-item.h>
  19 #include <apt-pkg/configuration.h>
  20 #include <apt-pkg/aptconfiguration.h>
  21 #include <apt-pkg/sourcelist.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/strutl.h>
  24 #include <apt-pkg/fileutl.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27 #include <apt-pkg/indexrecords.h>
  28 #include <apt-pkg/acquire.h>
  29 #include <apt-pkg/hashes.h>
  30 #include <apt-pkg/indexfile.h>
  31 #include <apt-pkg/pkgcache.h>
  32 #include <apt-pkg/cacheiterators.h>
  33 #include <apt-pkg/pkgrecords.h>
  34
  35 #include <stddef.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <iostream>
  39 #include <vector>
  40 #include <sys/stat.h>
  41 #include <unistd.h>
  42 #include <errno.h>
  43 #include <string>
  44 #include <sstream>
  45 #include <stdio.h>
  46 #include <ctime>
  47
  48 #include <apti18n.h>
  49                                                                         /*}}}*/
  50
  51 using namespace std;
  52
  53 // Acquire::Item::Item - Constructor                                    /*{{{*/
  54 // ---------------------------------------------------------------------
  55 /* */
  56 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  57                        PartialSize(0), Mode(0), ID(0), Complete(false),
  58                        Local(false), QueueCounter(0)
  59 {
  60    Owner->Add(this);
  61    Status = StatIdle;
  62 }
  63                                                                         /*}}}*/
  64 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  65 // ---------------------------------------------------------------------
  66 /* */
  67 pkgAcquire::Item::~Item()
  68 {
  69    Owner->Remove(this);
  70 }
  71                                                                         /*}}}*/
  72 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  73 // ---------------------------------------------------------------------
  74 /* We return to an idle state if there are still other queues that could
  75    fetch this object */
  76 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  77 {
  78    Status = StatIdle;
  79    ErrorText = LookupTag(Message,"Message");
  80    UsedMirror =  LookupTag(Message,"UsedMirror");
  81    if (QueueCounter <= 1)
  82    {
  83       /* This indicates that the file is not available right now but might
  84          be sometime later. If we do a retry cycle then this should be
  85          retried [CDROMs] */
  86       if (Cnf->LocalOnly == true &&
  87           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  88       {
  89          Status = StatIdle;
  90          Dequeue();
  91          return;
  92       }
  93
  94       Status = StatError;
  95       Dequeue();
  96    }
  97
  98    // report mirror failure back to LP if we actually use a mirror
  99    string FailReason = LookupTag(Message, "FailReason");
 100    if(FailReason.size() != 0)
 101       ReportMirrorFailure(FailReason);
 102    else
 103       ReportMirrorFailure(ErrorText);
 104 }
 105                                                                         /*}}}*/
 106 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
 107 // ---------------------------------------------------------------------
 108 /* Stash status and the file size. Note that setting Complete means
 109    sub-phases of the acquire process such as decompresion are operating */
 110 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 111 {
 112    Status = StatFetching;
 113    if (FileSize == 0 && Complete == false)
 114       FileSize = Size;
 115 }
 116                                                                         /*}}}*/
 117 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* */
 120 void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/,
 121                             pkgAcquire::MethodConfig * /*Cnf*/)
 122 {
 123    // We just downloaded something..
 124    string FileName = LookupTag(Message,"Filename");
 125    UsedMirror =  LookupTag(Message,"UsedMirror");
 126    if (Complete == false && !Local && FileName == DestFile)
 127    {
 128       if (Owner->Log != 0)
 129          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 130    }
 131
 132    if (FileSize == 0)
 133       FileSize= Size;
 134    Status = StatDone;
 135    ErrorText = string();
 136    Owner->Dequeue(this);
 137 }
 138                                                                         /*}}}*/
 139 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This helper function is used by a lot of item methods as their final
 142    step */
 143 void pkgAcquire::Item::Rename(string From,string To)
 144 {
 145    if (rename(From.c_str(),To.c_str()) != 0)
 146    {
 147       char S[300];
 148       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 149               From.c_str(),To.c_str());
 150       Status = StatError;
 151       ErrorText = S;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 156 {
 157    if(FileExists(DestFile))
 158       Rename(DestFile, DestFile + ".FAILED");
 159
 160    switch (error)
 161    {
 162       case HashSumMismatch:
 163          ErrorText = _("Hash Sum mismatch");
 164          Status = StatAuthError;
 165          ReportMirrorFailure("HashChecksumFailure");
 166          break;
 167       case SizeMismatch:
 168          ErrorText = _("Size mismatch");
 169          Status = StatAuthError;
 170          ReportMirrorFailure("SizeFailure");
 171          break;
 172       case InvalidFormat:
 173          ErrorText = _("Invalid file format");
 174          Status = StatError;
 175          // do not report as usually its not the mirrors fault, but Portal/Proxy
 176          break;
 177    }
 178    return false;
 179 }
 180                                                                         /*}}}*/
 181 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 182 // ---------------------------------------------------------------------
 183 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 184 {
 185    // we only act if a mirror was used at all
 186    if(UsedMirror.empty())
 187       return;
 188 #if 0
 189    std::cerr << "\nReportMirrorFailure: "
 190              << UsedMirror
 191              << " Uri: " << DescURI()
 192              << " FailCode: "
 193              << FailCode << std::endl;
 194 #endif
 195    const char *Args[40];
 196    unsigned int i = 0;
 197    string report = _config->Find("Methods::Mirror::ProblemReporting",
 198                                  "/usr/lib/apt/apt-report-mirror-failure");
 199    if(!FileExists(report))
 200       return;
 201    Args[i++] = report.c_str();
 202    Args[i++] = UsedMirror.c_str();
 203    Args[i++] = DescURI().c_str();
 204    Args[i++] = FailCode.c_str();
 205    Args[i++] = NULL;
 206    pid_t pid = ExecFork();
 207    if(pid < 0)
 208    {
 209       _error->Error("ReportMirrorFailure Fork failed");
 210       return;
 211    }
 212    else if(pid == 0)
 213    {
 214       execvp(Args[0], (char**)Args);
 215       std::cerr << "Could not exec " << Args[0] << std::endl;
 216       _exit(100);
 217    }
 218    if(!ExecWait(pid, "report-mirror-failure"))
 219    {
 220       _error->Warning("Couldn't report problem to '%s'",
 221                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 222    }
 223 }
 224                                                                         /*}}}*/
 225 // AcqSubIndex::AcqSubIndex - Constructor                               /*{{{*/
 226 // ---------------------------------------------------------------------
 227 /* Get a sub-index file based on checksums from a 'master' file and
 228    possibly query additional files */
 229 pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
 230                                  string const &URIDesc, string const &ShortDesc,
 231                                  HashString const &ExpectedHash)
 232    : Item(Owner), ExpectedHash(ExpectedHash)
 233 {
 234    /* XXX: Beware: Currently this class does nothing (of value) anymore ! */
 235    Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
 236
 237    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 238    DestFile += URItoFileName(URI);
 239
 240    Desc.URI = URI;
 241    Desc.Description = URIDesc;
 242    Desc.Owner = this;
 243    Desc.ShortDesc = ShortDesc;
 244
 245    QueueURI(Desc);
 246
 247    if(Debug)
 248       std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
 249 }
 250                                                                         /*}}}*/
 251 // AcqSubIndex::Custom600Headers - Insert custom request headers        /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* The only header we use is the last-modified header. */
 254 string pkgAcqSubIndex::Custom600Headers()
 255 {
 256    string Final = _config->FindDir("Dir::State::lists");
 257    Final += URItoFileName(Desc.URI);
 258
 259    struct stat Buf;
 260    if (stat(Final.c_str(),&Buf) != 0)
 261       return "\nIndex-File: true\nFail-Ignore: true\n";
 262    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 263 }
 264                                                                         /*}}}*/
 265 void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 266 {
 267    if(Debug)
 268       std::clog << "pkgAcqSubIndex failed: " << Desc.URI << " with " << Message << std::endl;
 269
 270    Complete = false;
 271    Status = StatDone;
 272    Dequeue();
 273
 274    // No good Index is provided
 275 }
 276                                                                         /*}}}*/
 277 void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash,        /*{{{*/
 278                            pkgAcquire::MethodConfig *Cnf)
 279 {
 280    if(Debug)
 281       std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
 282
 283    string FileName = LookupTag(Message,"Filename");
 284    if (FileName.empty() == true)
 285    {
 286       Status = StatError;
 287       ErrorText = "Method gave a blank filename";
 288       return;
 289    }
 290
 291    if (FileName != DestFile)
 292    {
 293       Local = true;
 294       Desc.URI = "copy:" + FileName;
 295       QueueURI(Desc);
 296       return;
 297    }
 298
 299    Item::Done(Message,Size,Md5Hash,Cnf);
 300
 301    string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
 302
 303    /* Downloaded invalid transindex => Error (LP: #346386) (Closes: #627642) */
 304    indexRecords SubIndexParser;
 305    if (FileExists(DestFile) == true && !SubIndexParser.Load(DestFile)) {
 306       Status = StatError;
 307       ErrorText = SubIndexParser.ErrorText;
 308       return;
 309    }
 310
 311    // success in downloading the index
 312    // rename the index
 313    if(Debug)
 314       std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
 315    Rename(DestFile,FinalFile);
 316    chmod(FinalFile.c_str(),0644);
 317    DestFile = FinalFile;
 318
 319    if(ParseIndex(DestFile) == false)
 320       return Failed("", NULL);
 321
 322    Complete = true;
 323    Status = StatDone;
 324    Dequeue();
 325    return;
 326 }
 327                                                                         /*}}}*/
 328 bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)                /*{{{*/
 329 {
 330    indexRecords SubIndexParser;
 331    if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
 332       return false;
 333    // so something with the downloaded index
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* Get the DiffIndex file first and see if there are patches available
 340  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 341  * patches. If anything goes wrong in that process, it will fall back to
 342  * the original packages file
 343  */
 344 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 345                                  string URI,string URIDesc,string ShortDesc,
 346                                  HashString ExpectedHash)
 347    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 348      Description(URIDesc)
 349 {
 350
 351    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 352
 353    Desc.Description = URIDesc + "/DiffIndex";
 354    Desc.Owner = this;
 355    Desc.ShortDesc = ShortDesc;
 356    Desc.URI = URI + ".diff/Index";
 357
 358    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 359    DestFile += URItoFileName(Desc.URI);
 360
 361    if(Debug)
 362       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 363
 364    // look for the current package file
 365    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 366    CurrentPackagesFile += URItoFileName(RealURI);
 367
 368    // FIXME: this file:/ check is a hack to prevent fetching
 369    //        from local sources. this is really silly, and
 370    //        should be fixed cleanly as soon as possible
 371    if(!FileExists(CurrentPackagesFile) ||
 372       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 373    {
 374       // we don't have a pkg file or we don't want to queue
 375       if(Debug)
 376          std::clog << "No index file, local or canceld by user" << std::endl;
 377       Failed("", NULL);
 378       return;
 379    }
 380
 381    if(Debug)
 382       std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
 383          << CurrentPackagesFile << std::endl;
 384
 385    QueueURI(Desc);
 386
 387 }
 388                                                                         /*}}}*/
 389 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* The only header we use is the last-modified header. */
 392 string pkgAcqDiffIndex::Custom600Headers()
 393 {
 394    string Final = _config->FindDir("Dir::State::lists");
 395    Final += URItoFileName(Desc.URI);
 396
 397    if(Debug)
 398       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 399
 400    struct stat Buf;
 401    if (stat(Final.c_str(),&Buf) != 0)
 402       return "\nIndex-File: true";
 403
 404    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 405 }
 406                                                                         /*}}}*/
 407 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 408 {
 409    if(Debug)
 410       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 411          << std::endl;
 412
 413    pkgTagSection Tags;
 414    string ServerSha1;
 415    vector<DiffInfo> available_patches;
 416
 417    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 418    pkgTagFile TF(&Fd);
 419    if (_error->PendingError() == true)
 420       return false;
 421
 422    if(TF.Step(Tags) == true)
 423    {
 424       bool found = false;
 425       DiffInfo d;
 426       string size;
 427
 428       string const tmp = Tags.FindS("SHA1-Current");
 429       std::stringstream ss(tmp);
 430       ss >> ServerSha1 >> size;
 431       unsigned long const ServerSize = atol(size.c_str());
 432
 433       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 434       SHA1Summation SHA1;
 435       SHA1.AddFD(fd);
 436       string const local_sha1 = SHA1.Result();
 437
 438       if(local_sha1 == ServerSha1)
 439       {
 440          // we have the same sha1 as the server so we are done here
 441          if(Debug)
 442             std::clog << "Package file is up-to-date" << std::endl;
 443          // list cleanup needs to know that this file as well as the already
 444          // present index is ours, so we create an empty diff to save it for us
 445          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 446                ExpectedHash, ServerSha1, available_patches);
 447          return true;
 448       }
 449       else
 450       {
 451          if(Debug)
 452             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 453
 454          // check the historie and see what patches we need
 455          string const history = Tags.FindS("SHA1-History");
 456          std::stringstream hist(history);
 457          while(hist >> d.sha1 >> size >> d.file)
 458          {
 459             // read until the first match is found
 460             // from that point on, we probably need all diffs
 461             if(d.sha1 == local_sha1)
 462                found=true;
 463             else if (found == false)
 464                continue;
 465
 466             if(Debug)
 467                std::clog << "Need to get diff: " << d.file << std::endl;
 468             available_patches.push_back(d);
 469          }
 470
 471          if (available_patches.empty() == false)
 472          {
 473             // patching with too many files is rather slow compared to a fast download
 474             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 475             if (fileLimit != 0 && fileLimit < available_patches.size())
 476             {
 477                if (Debug)
 478                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 479                         << ") so fallback to complete download" << std::endl;
 480                return false;
 481             }
 482
 483             // see if the patches are too big
 484             found = false; // it was true and it will be true again at the end
 485             d = *available_patches.begin();
 486             string const firstPatch = d.file;
 487             unsigned long patchesSize = 0;
 488             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 489             while(patches >> d.sha1 >> size >> d.file)
 490             {
 491                if (firstPatch == d.file)
 492                   found = true;
 493                else if (found == false)
 494                   continue;
 495
 496                patchesSize += atol(size.c_str());
 497             }
 498             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 499             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 500             {
 501                if (Debug)
 502                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 503                         << ") so fallback to complete download" << std::endl;
 504                return false;
 505             }
 506          }
 507       }
 508
 509       // we have something, queue the next diff
 510       if(found)
 511       {
 512          // queue the diffs
 513          string::size_type const last_space = Description.rfind(" ");
 514          if(last_space != string::npos)
 515             Description.erase(last_space, Description.size()-last_space);
 516
 517          /* decide if we should download patches one by one or in one go:
 518             The first is good if the server merges patches, but many don't so client
 519             based merging can be attempt in which case the second is better.
 520             "bad things" will happen if patches are merged on the server,
 521             but client side merging is attempt as well */
 522          bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
 523          if (pdiff_merge == true)
 524          {
 525             // reprepro adds this flag if it has merged patches on the server
 526             std::string const precedence = Tags.FindS("X-Patch-Precedence");
 527             pdiff_merge = (precedence != "merged");
 528          }
 529
 530          if (pdiff_merge == false)
 531             new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 532                   ExpectedHash, ServerSha1, available_patches);
 533          else
 534          {
 535             std::vector<pkgAcqIndexMergeDiffs*> *diffs = new std::vector<pkgAcqIndexMergeDiffs*>(available_patches.size());
 536             for(size_t i = 0; i < available_patches.size(); ++i)
 537                (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash,
 538                      available_patches[i], diffs);
 539          }
 540
 541          Complete = false;
 542          Status = StatDone;
 543          Dequeue();
 544          return true;
 545       }
 546    }
 547
 548    // Nothing found, report and return false
 549    // Failing here is ok, if we return false later, the full
 550    // IndexFile is queued
 551    if(Debug)
 552       std::clog << "Can't find a patch in the index file" << std::endl;
 553    return false;
 554 }
 555                                                                         /*}}}*/
 556 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 557 {
 558    if(Debug)
 559       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 560                 << "Falling back to normal index file acquire" << std::endl;
 561
 562    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 563                    ExpectedHash);
 564
 565    Complete = false;
 566    Status = StatDone;
 567    Dequeue();
 568 }
 569                                                                         /*}}}*/
 570 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash,       /*{{{*/
 571                            pkgAcquire::MethodConfig *Cnf)
 572 {
 573    if(Debug)
 574       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 575
 576    Item::Done(Message,Size,Md5Hash,Cnf);
 577
 578    string FinalFile;
 579    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 580
 581    // success in downloading the index
 582    // rename the index
 583    FinalFile += string(".IndexDiff");
 584    if(Debug)
 585       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 586                 << std::endl;
 587    Rename(DestFile,FinalFile);
 588    chmod(FinalFile.c_str(),0644);
 589    DestFile = FinalFile;
 590
 591    if(!ParseDiffIndex(DestFile))
 592       return Failed("", NULL);
 593
 594    Complete = true;
 595    Status = StatDone;
 596    Dequeue();
 597    return;
 598 }
 599                                                                         /*}}}*/
 600 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 601 // ---------------------------------------------------------------------
 602 /* The package diff is added to the queue. one object is constructed
 603  * for each diff and the index
 604  */
 605 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 606                                    string URI,string URIDesc,string ShortDesc,
 607                                    HashString ExpectedHash,
 608                                    string ServerSha1,
 609                                    vector<DiffInfo> diffs)
 610    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 611      available_patches(diffs), ServerSha1(ServerSha1)
 612 {
 613
 614    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 615    DestFile += URItoFileName(URI);
 616
 617    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 618
 619    Description = URIDesc;
 620    Desc.Owner = this;
 621    Desc.ShortDesc = ShortDesc;
 622
 623    if(available_patches.empty() == true)
 624    {
 625       // we are done (yeah!)
 626       Finish(true);
 627    }
 628    else
 629    {
 630       // get the next diff
 631       State = StateFetchDiff;
 632       QueueNextDiff();
 633    }
 634 }
 635                                                                         /*}}}*/
 636 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 637 {
 638    if(Debug)
 639       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 640                 << "Falling back to normal index file acquire" << std::endl;
 641    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 642                    ExpectedHash);
 643    Finish();
 644 }
 645                                                                         /*}}}*/
 646 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 647 void pkgAcqIndexDiffs::Finish(bool allDone)
 648 {
 649    // we restore the original name, this is required, otherwise
 650    // the file will be cleaned
 651    if(allDone)
 652    {
 653       DestFile = _config->FindDir("Dir::State::lists");
 654       DestFile += URItoFileName(RealURI);
 655
 656       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 657       {
 658          RenameOnError(HashSumMismatch);
 659          Dequeue();
 660          return;
 661       }
 662
 663       // this is for the "real" finish
 664       Complete = true;
 665       Status = StatDone;
 666       Dequeue();
 667       if(Debug)
 668          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 669       return;
 670    }
 671
 672    if(Debug)
 673       std::clog << "Finishing: " << Desc.URI << std::endl;
 674    Complete = false;
 675    Status = StatDone;
 676    Dequeue();
 677    return;
 678 }
 679                                                                         /*}}}*/
 680 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 681 {
 682
 683    // calc sha1 of the just patched file
 684    string FinalFile = _config->FindDir("Dir::State::lists");
 685    FinalFile += URItoFileName(RealURI);
 686
 687    FileFd fd(FinalFile, FileFd::ReadOnly);
 688    SHA1Summation SHA1;
 689    SHA1.AddFD(fd);
 690    string local_sha1 = string(SHA1.Result());
 691    if(Debug)
 692       std::clog << "QueueNextDiff: "
 693                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 694
 695    // final file reached before all patches are applied
 696    if(local_sha1 == ServerSha1)
 697    {
 698       Finish(true);
 699       return true;
 700    }
 701
 702    // remove all patches until the next matching patch is found
 703    // this requires the Index file to be ordered
 704    for(vector<DiffInfo>::iterator I=available_patches.begin();
 705        available_patches.empty() == false &&
 706           I != available_patches.end() &&
 707           I->sha1 != local_sha1;
 708        ++I)
 709    {
 710       available_patches.erase(I);
 711    }
 712
 713    // error checking and falling back if no patch was found
 714    if(available_patches.empty() == true)
 715    {
 716       Failed("", NULL);
 717       return false;
 718    }
 719
 720    // queue the right diff
 721    Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
 722    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 723    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 724    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 725
 726    if(Debug)
 727       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 728
 729    QueueURI(Desc);
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash,      /*{{{*/
 735                             pkgAcquire::MethodConfig *Cnf)
 736 {
 737    if(Debug)
 738       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 739
 740    Item::Done(Message,Size,Md5Hash,Cnf);
 741
 742    string FinalFile;
 743    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 744
 745    // success in downloading a diff, enter ApplyDiff state
 746    if(State == StateFetchDiff)
 747    {
 748
 749       // rred excepts the patch as $FinalFile.ed
 750       Rename(DestFile,FinalFile+".ed");
 751
 752       if(Debug)
 753          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 754
 755       State = StateApplyDiff;
 756       Local = true;
 757       Desc.URI = "rred:" + FinalFile;
 758       QueueURI(Desc);
 759       Mode = "rred";
 760       return;
 761    }
 762
 763
 764    // success in download/apply a diff, queue next (if needed)
 765    if(State == StateApplyDiff)
 766    {
 767       // remove the just applied patch
 768       available_patches.erase(available_patches.begin());
 769       unlink((FinalFile + ".ed").c_str());
 770
 771       // move into place
 772       if(Debug)
 773       {
 774          std::clog << "Moving patched file in place: " << std::endl
 775                    << DestFile << " -> " << FinalFile << std::endl;
 776       }
 777       Rename(DestFile,FinalFile);
 778       chmod(FinalFile.c_str(),0644);
 779
 780       // see if there is more to download
 781       if(available_patches.empty() == false) {
 782          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 783                               ExpectedHash, ServerSha1, available_patches);
 784          return Finish();
 785       } else
 786          return Finish(true);
 787    }
 788 }
 789                                                                         /*}}}*/
 790 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor                 /*{{{*/
 791 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
 792                                    string const &URI, string const &URIDesc,
 793                                    string const &ShortDesc, HashString const &ExpectedHash,
 794                                    DiffInfo const &patch,
 795                                    std::vector<pkgAcqIndexMergeDiffs*> const * const allPatches)
 796    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 797      patch(patch),allPatches(allPatches), State(StateFetchDiff)
 798 {
 799
 800    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 801    DestFile += URItoFileName(URI);
 802
 803    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 804
 805    Description = URIDesc;
 806    Desc.Owner = this;
 807    Desc.ShortDesc = ShortDesc;
 808
 809    Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
 810    Desc.Description = Description + " " + patch.file + string(".pdiff");
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(RealURI + ".diff/" + patch.file);
 813
 814    if(Debug)
 815       std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
 816
 817    QueueURI(Desc);
 818 }
 819                                                                         /*}}}*/
 820 void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
 821 {
 822    if(Debug)
 823       std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 824    Complete = false;
 825    Status = StatDone;
 826    Dequeue();
 827
 828    // check if we are the first to fail, otherwise we are done here
 829    State = StateDoneDiff;
 830    for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 831          I != allPatches->end(); ++I)
 832       if ((*I)->State == StateErrorDiff)
 833          return;
 834
 835    // first failure means we should fallback
 836    State = StateErrorDiff;
 837    std::clog << "Falling back to normal index file acquire" << std::endl;
 838    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 839                    ExpectedHash);
 840 }
 841                                                                         /*}}}*/
 842 void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/
 843                             pkgAcquire::MethodConfig *Cnf)
 844 {
 845    if(Debug)
 846       std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 847
 848    Item::Done(Message,Size,Md5Hash,Cnf);
 849
 850    string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 851
 852    if (State == StateFetchDiff)
 853    {
 854       // rred expects the patch as $FinalFile.ed.$patchname.gz
 855       Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
 856
 857       // check if this is the last completed diff
 858       State = StateDoneDiff;
 859       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 860             I != allPatches->end(); ++I)
 861          if ((*I)->State != StateDoneDiff)
 862          {
 863             if(Debug)
 864                std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
 865             return;
 866          }
 867
 868       // this is the last completed diff, so we are ready to apply now
 869       State = StateApplyDiff;
 870
 871       if(Debug)
 872          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 873
 874       Local = true;
 875       Desc.URI = "rred:" + FinalFile;
 876       QueueURI(Desc);
 877       Mode = "rred";
 878       return;
 879    }
 880    // success in download/apply all diffs, clean up
 881    else if (State == StateApplyDiff)
 882    {
 883       // see if we really got the expected file
 884       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 885       {
 886          RenameOnError(HashSumMismatch);
 887          return;
 888       }
 889
 890       // move the result into place
 891       if(Debug)
 892          std::clog << "Moving patched file in place: " << std::endl
 893                    << DestFile << " -> " << FinalFile << std::endl;
 894       Rename(DestFile, FinalFile);
 895       chmod(FinalFile.c_str(), 0644);
 896
 897       // otherwise lists cleanup will eat the file
 898       DestFile = FinalFile;
 899
 900       // ensure the ed's are gone regardless of list-cleanup
 901       for (std::vector<pkgAcqIndexMergeDiffs *>::const_iterator I = allPatches->begin();
 902             I != allPatches->end(); ++I)
 903       {
 904             std::string patch = FinalFile + ".ed." + (*I)->patch.file + ".gz";
 905             unlink(patch.c_str());
 906       }
 907
 908       // all set and done
 909       Complete = true;
 910       if(Debug)
 911          std::clog << "allDone: " << DestFile << "\n" << std::endl;
 912    }
 913 }
 914                                                                         /*}}}*/
 915 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 916 // ---------------------------------------------------------------------
 917 /* The package file is added to the queue and a second class is
 918    instantiated to fetch the revision file */
 919 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 920                          string URI,string URIDesc,string ShortDesc,
 921                          HashString ExpectedHash, string comprExt)
 922    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 923 {
 924    if(comprExt.empty() == true)
 925    {
 926       // autoselect the compression method
 927       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 928       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 929          comprExt.append(*t).append(" ");
 930       if (comprExt.empty() == false)
 931          comprExt.erase(comprExt.end()-1);
 932    }
 933    CompressionExtension = comprExt;
 934
 935    Init(URI, URIDesc, ShortDesc);
 936 }
 937 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 938                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 939    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 940 {
 941    // autoselect the compression method
 942    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 943    CompressionExtension = "";
 944    if (ExpectedHash.empty() == false)
 945    {
 946       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 947          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 948             CompressionExtension.append(*t).append(" ");
 949    }
 950    else
 951    {
 952       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 953          CompressionExtension.append(*t).append(" ");
 954    }
 955    if (CompressionExtension.empty() == false)
 956       CompressionExtension.erase(CompressionExtension.end()-1);
 957
 958    Init(Target->URI, Target->Description, Target->ShortDesc);
 959 }
 960                                                                         /*}}}*/
 961 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 962 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 963    Decompression = false;
 964    Erase = false;
 965
 966    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 967    DestFile += URItoFileName(URI);
 968
 969    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 970    if (comprExt == "uncompressed")
 971       Desc.URI = URI;
 972    else
 973       Desc.URI = URI + '.' + comprExt;
 974
 975    Desc.Description = URIDesc;
 976    Desc.Owner = this;
 977    Desc.ShortDesc = ShortDesc;
 978
 979    QueueURI(Desc);
 980 }
 981                                                                         /*}}}*/
 982 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* The only header we use is the last-modified header. */
 985 string pkgAcqIndex::Custom600Headers()
 986 {
 987    string Final = _config->FindDir("Dir::State::lists");
 988    Final += URItoFileName(RealURI);
 989    if (_config->FindB("Acquire::GzipIndexes",false))
 990       Final += ".gz";
 991
 992    string msg = "\nIndex-File: true";
 993    // FIXME: this really should use "IndexTarget::IsOptional()" but that
 994    //        seems to be difficult without breaking ABI
 995    if (ShortDesc().find("Translation") != 0)
 996       msg += "\nFail-Ignore: true";
 997    struct stat Buf;
 998    if (stat(Final.c_str(),&Buf) == 0)
 999       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1000
1001    return msg;
1002 }
1003                                                                         /*}}}*/
1004 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
1005 {
1006    size_t const nextExt = CompressionExtension.find(' ');
1007    if (nextExt != std::string::npos)
1008    {
1009       CompressionExtension = CompressionExtension.substr(nextExt+1);
1010       Init(RealURI, Desc.Description, Desc.ShortDesc);
1011       return;
1012    }
1013
1014    // on decompression failure, remove bad versions in partial/
1015    if (Decompression && Erase) {
1016       string s = _config->FindDir("Dir::State::lists") + "partial/";
1017       s.append(URItoFileName(RealURI));
1018       unlink(s.c_str());
1019    }
1020
1021    Item::Failed(Message,Cnf);
1022 }
1023                                                                         /*}}}*/
1024 // pkgAcqIndex::GetFinalFilename - Return the full final file path      /*{{{*/
1025 std::string pkgAcqIndex::GetFinalFilename(std::string const &URI,
1026                                           std::string const &compExt)
1027 {
1028    std::string FinalFile = _config->FindDir("Dir::State::lists");
1029    FinalFile += URItoFileName(URI);
1030    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1031       FinalFile += ".gz";
1032    return FinalFile;
1033 }
1034                                                                         /*}}}*/
1035 // AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit         /*{{{*/
1036 void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName)
1037 {
1038    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1039    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1040       DestFile += ".gz";
1041
1042    string FinalFile = GetFinalFilename(RealURI, compExt);
1043    Rename(FinalFile, FileName);
1044    Decompression = true;
1045    Desc.URI = "copy:" + FileName;
1046    QueueURI(Desc);
1047 }
1048                                                                         /*}}}*/
1049 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
1050 // ---------------------------------------------------------------------
1051 /* This goes through a number of states.. On the initial fetch the
1052    method could possibly return an alternate filename which points
1053    to the uncompressed version of the file. If this is so the file
1054    is copied into the partial directory. In all other cases the file
1055    is decompressed with a gzip uri. */
1056 void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash,
1057                        pkgAcquire::MethodConfig *Cfg)
1058 {
1059    Item::Done(Message,Size,Hash,Cfg);
1060    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
1061
1062    if (Decompression == true)
1063    {
1064       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1065       {
1066          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
1067          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
1068       }
1069
1070       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
1071       {
1072          Desc.URI = RealURI;
1073          RenameOnError(HashSumMismatch);
1074          return;
1075       }
1076
1077       // FIXME: this can go away once we only ever download stuff that
1078       //        has a valid hash and we never do GET based probing
1079       //
1080       /* Always verify the index file for correctness (all indexes must
1081        * have a Package field) (LP: #346386) (Closes: #627642)
1082        */
1083       FileFd fd(DestFile, FileFd::ReadOnlyGzip);
1084       // Only test for correctness if the file is not empty (empty is ok)
1085       if (fd.Size() > 0)
1086       {
1087          pkgTagSection sec;
1088          pkgTagFile tag(&fd);
1089
1090          // all our current indexes have a field 'Package' in each section
1091          if (_error->PendingError() == true || tag.Step(sec) == false || sec.Exists("Package") == false)
1092          {
1093             RenameOnError(InvalidFormat);
1094             return;
1095          }
1096       }
1097
1098       // Done, move it into position
1099       string FinalFile = GetFinalFilename(RealURI, compExt);
1100       Rename(DestFile,FinalFile);
1101       chmod(FinalFile.c_str(),0644);
1102
1103       /* We restore the original name to DestFile so that the clean operation
1104          will work OK */
1105       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1106       DestFile += URItoFileName(RealURI);
1107       if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
1108          DestFile += ".gz";
1109
1110       // Remove the compressed version.
1111       if (Erase == true)
1112          unlink(DestFile.c_str());
1113       return;
1114    }
1115
1116    Erase = false;
1117    Complete = true;
1118
1119    // Handle the unzipd case
1120    string FileName = LookupTag(Message,"Alt-Filename");
1121    if (FileName.empty() == false)
1122    {
1123       // The files timestamp matches
1124       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
1125       {
1126          ReverifyAfterIMS(FileName);
1127          return;
1128       }
1129       Decompression = true;
1130       Local = true;
1131       DestFile += ".decomp";
1132       Desc.URI = "copy:" + FileName;
1133       QueueURI(Desc);
1134       Mode = "copy";
1135       return;
1136    }
1137
1138    FileName = LookupTag(Message,"Filename");
1139    if (FileName.empty() == true)
1140    {
1141       Status = StatError;
1142       ErrorText = "Method gave a blank filename";
1143    }
1144
1145    // The files timestamp matches
1146    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1147    {
1148       ReverifyAfterIMS(FileName);
1149       return;
1150    }
1151
1152    if (FileName == DestFile)
1153       Erase = true;
1154    else
1155       Local = true;
1156
1157    string decompProg;
1158
1159    // If we enable compressed indexes, queue for hash verification
1160    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local)
1161    {
1162       DestFile = _config->FindDir("Dir::State::lists");
1163       DestFile += URItoFileName(RealURI) + ".gz";
1164
1165       Decompression = true;
1166       Desc.URI = "copy:" + FileName;
1167       QueueURI(Desc);
1168
1169       return;
1170     }
1171
1172    // get the binary name for your used compression type
1173    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
1174    if(decompProg.empty() == false);
1175    else if(compExt == "uncompressed")
1176       decompProg = "copy";
1177    else {
1178       _error->Error("Unsupported extension: %s", compExt.c_str());
1179       return;
1180    }
1181
1182    Decompression = true;
1183    DestFile += ".decomp";
1184    Desc.URI = decompProg + ":" + FileName;
1185    QueueURI(Desc);
1186
1187    // FIXME: this points to a c++ string that goes out of scope
1188    Mode = decompProg.c_str();
1189 }
1190                                                                         /*}}}*/
1191 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
1192 // ---------------------------------------------------------------------
1193 /* The Translation file is added to the queue */
1194 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
1195                             string URI,string URIDesc,string ShortDesc)
1196   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
1197 {
1198 }
1199 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
1200                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
1201   : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
1202 {
1203 }
1204                                                                         /*}}}*/
1205 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
1206 // ---------------------------------------------------------------------
1207 string pkgAcqIndexTrans::Custom600Headers()
1208 {
1209    string Final = _config->FindDir("Dir::State::lists");
1210    Final += URItoFileName(RealURI);
1211
1212    if (_config->FindB("Acquire::GzipIndexes",false))
1213       Final += ".gz";
1214
1215    struct stat Buf;
1216    if (stat(Final.c_str(),&Buf) != 0)
1217       return "\nFail-Ignore: true\nIndex-File: true";
1218    return "\nFail-Ignore: true\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1219 }
1220                                                                         /*}}}*/
1221 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
1222 // ---------------------------------------------------------------------
1223 /* */
1224 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1225 {
1226    size_t const nextExt = CompressionExtension.find(' ');
1227    if (nextExt != std::string::npos)
1228    {
1229       CompressionExtension = CompressionExtension.substr(nextExt+1);
1230       Init(RealURI, Desc.Description, Desc.ShortDesc);
1231       Status = StatIdle;
1232       return;
1233    }
1234
1235    if (Cnf->LocalOnly == true ||
1236        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1237    {
1238       // Ignore this
1239       Status = StatDone;
1240       Complete = false;
1241       Dequeue();
1242       return;
1243    }
1244
1245    Item::Failed(Message,Cnf);
1246 }
1247                                                                         /*}}}*/
1248 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
1249                              string URI,string URIDesc,string ShortDesc,
1250                              string MetaIndexURI, string MetaIndexURIDesc,
1251                              string MetaIndexShortDesc,
1252                              const vector<IndexTarget*>* IndexTargets,
1253                              indexRecords* MetaIndexParser) :
1254    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
1255    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1256    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
1257 {
1258    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1259    DestFile += URItoFileName(URI);
1260
1261    // remove any partial downloaded sig-file in partial/.
1262    // it may confuse proxies and is too small to warrant a
1263    // partial download anyway
1264    unlink(DestFile.c_str());
1265
1266    // Create the item
1267    Desc.Description = URIDesc;
1268    Desc.Owner = this;
1269    Desc.ShortDesc = ShortDesc;
1270    Desc.URI = URI;
1271
1272    string Final = _config->FindDir("Dir::State::lists");
1273    Final += URItoFileName(RealURI);
1274    if (RealFileExists(Final) == true)
1275    {
1276       // File was already in place.  It needs to be re-downloaded/verified
1277       // because Release might have changed, we do give it a different
1278       // name than DestFile because otherwise the http method will
1279       // send If-Range requests and there are too many broken servers
1280       // out there that do not understand them
1281       LastGoodSig = DestFile+".reverify";
1282       Rename(Final,LastGoodSig);
1283    }
1284
1285    QueueURI(Desc);
1286 }
1287                                                                         /*}}}*/
1288 pkgAcqMetaSig::~pkgAcqMetaSig()                                         /*{{{*/
1289 {
1290    // if the file was never queued undo file-changes done in the constructor
1291    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false &&
1292          LastGoodSig.empty() == false)
1293    {
1294       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1295       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1296          Rename(LastGoodSig, Final);
1297    }
1298
1299 }
1300                                                                         /*}}}*/
1301 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
1302 // ---------------------------------------------------------------------
1303 /* The only header we use is the last-modified header. */
1304 string pkgAcqMetaSig::Custom600Headers()
1305 {
1306    struct stat Buf;
1307    if (stat(LastGoodSig.c_str(),&Buf) != 0)
1308       return "\nIndex-File: true";
1309
1310    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1311 }
1312
1313 void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5,
1314                          pkgAcquire::MethodConfig *Cfg)
1315 {
1316    Item::Done(Message,Size,MD5,Cfg);
1317
1318    string FileName = LookupTag(Message,"Filename");
1319    if (FileName.empty() == true)
1320    {
1321       Status = StatError;
1322       ErrorText = "Method gave a blank filename";
1323       return;
1324    }
1325
1326    if (FileName != DestFile)
1327    {
1328       // We have to copy it into place
1329       Local = true;
1330       Desc.URI = "copy:" + FileName;
1331       QueueURI(Desc);
1332       return;
1333    }
1334
1335    Complete = true;
1336
1337    // put the last known good file back on i-m-s hit (it will
1338    // be re-verified again)
1339    // Else do nothing, we have the new file in DestFile then
1340    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1341       Rename(LastGoodSig, DestFile);
1342
1343    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
1344    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
1345                        MetaIndexShortDesc,  DestFile, IndexTargets,
1346                        MetaIndexParser);
1347
1348 }
1349                                                                         /*}}}*/
1350 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
1351 {
1352    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1353
1354    // if we get a network error we fail gracefully
1355    if(Status == StatTransientNetworkError)
1356    {
1357       Item::Failed(Message,Cnf);
1358       // move the sigfile back on transient network failures
1359       if(FileExists(LastGoodSig))
1360          Rename(LastGoodSig,Final);
1361
1362       // set the status back to , Item::Failed likes to reset it
1363       Status = pkgAcquire::Item::StatTransientNetworkError;
1364       return;
1365    }
1366
1367    // Delete any existing sigfile when the acquire failed
1368    unlink(Final.c_str());
1369
1370    // queue a pkgAcqMetaIndex with no sigfile
1371    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1372                        "", IndexTargets, MetaIndexParser);
1373
1374    if (Cnf->LocalOnly == true ||
1375        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1376    {
1377       // Ignore this
1378       Status = StatDone;
1379       Complete = false;
1380       Dequeue();
1381       return;
1382    }
1383
1384    Item::Failed(Message,Cnf);
1385 }
1386                                                                         /*}}}*/
1387 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1388                                  string URI,string URIDesc,string ShortDesc,
1389                                  string SigFile,
1390                                  const vector<struct IndexTarget*>* IndexTargets,
1391                                  indexRecords* MetaIndexParser) :
1392    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1393    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1394 {
1395    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1396    DestFile += URItoFileName(URI);
1397
1398    // Create the item
1399    Desc.Description = URIDesc;
1400    Desc.Owner = this;
1401    Desc.ShortDesc = ShortDesc;
1402    Desc.URI = URI;
1403
1404    QueueURI(Desc);
1405 }
1406                                                                         /*}}}*/
1407 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1408 // ---------------------------------------------------------------------
1409 /* The only header we use is the last-modified header. */
1410 string pkgAcqMetaIndex::Custom600Headers()
1411 {
1412    string Final = _config->FindDir("Dir::State::lists");
1413    Final += URItoFileName(RealURI);
1414
1415    struct stat Buf;
1416    if (stat(Final.c_str(),&Buf) != 0)
1417       return "\nIndex-File: true";
1418
1419    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1420 }
1421                                                                         /*}}}*/
1422 void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash,  /*{{{*/
1423                            pkgAcquire::MethodConfig *Cfg)
1424 {
1425    Item::Done(Message,Size,Hash,Cfg);
1426
1427    // MetaIndexes are done in two passes: one to download the
1428    // metaindex with an appropriate method, and a second to verify it
1429    // with the gpgv method
1430
1431    if (AuthPass == true)
1432    {
1433       AuthDone(Message);
1434
1435       // all cool, move Release file into place
1436       Complete = true;
1437    }
1438    else
1439    {
1440       RetrievalDone(Message);
1441       if (!Complete)
1442          // Still more retrieving to do
1443          return;
1444
1445       if (SigFile == "")
1446       {
1447          // There was no signature file, so we are finished.  Download
1448          // the indexes and do only hashsum verification if possible
1449          MetaIndexParser->Load(DestFile);
1450          QueueIndexes(false);
1451       }
1452       else
1453       {
1454          // There was a signature file, so pass it to gpgv for
1455          // verification
1456
1457          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1458             std::cerr << "Metaindex acquired, queueing gpg verification ("
1459                       << SigFile << "," << DestFile << ")\n";
1460          AuthPass = true;
1461          Desc.URI = "gpgv:" + SigFile;
1462          QueueURI(Desc);
1463          Mode = "gpgv";
1464          return;
1465       }
1466    }
1467
1468    if (Complete == true)
1469    {
1470       string FinalFile = _config->FindDir("Dir::State::lists");
1471       FinalFile += URItoFileName(RealURI);
1472       if (SigFile == DestFile)
1473          SigFile = FinalFile;
1474       Rename(DestFile,FinalFile);
1475       chmod(FinalFile.c_str(),0644);
1476       DestFile = FinalFile;
1477    }
1478 }
1479                                                                         /*}}}*/
1480 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1481 {
1482    // We have just finished downloading a Release file (it is not
1483    // verified yet)
1484
1485    string FileName = LookupTag(Message,"Filename");
1486    if (FileName.empty() == true)
1487    {
1488       Status = StatError;
1489       ErrorText = "Method gave a blank filename";
1490       return;
1491    }
1492
1493    if (FileName != DestFile)
1494    {
1495       Local = true;
1496       Desc.URI = "copy:" + FileName;
1497       QueueURI(Desc);
1498       return;
1499    }
1500
1501    // make sure to verify against the right file on I-M-S hit
1502    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1503    if(IMSHit)
1504    {
1505       string FinalFile = _config->FindDir("Dir::State::lists");
1506       FinalFile += URItoFileName(RealURI);
1507       if (SigFile == DestFile)
1508       {
1509          SigFile = FinalFile;
1510          // constructor of pkgAcqMetaClearSig moved it out of the way,
1511          // now move it back in on IMS hit for the 'old' file
1512          string const OldClearSig = DestFile + ".reverify";
1513          if (RealFileExists(OldClearSig) == true)
1514             Rename(OldClearSig, FinalFile);
1515       }
1516       DestFile = FinalFile;
1517    }
1518    Complete = true;
1519 }
1520                                                                         /*}}}*/
1521 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1522 {
1523    // At this point, the gpgv method has succeeded, so there is a
1524    // valid signature from a key in the trusted keyring.  We
1525    // perform additional verification of its contents, and use them
1526    // to verify the indexes we are about to download
1527
1528    if (!MetaIndexParser->Load(DestFile))
1529    {
1530       Status = StatAuthError;
1531       ErrorText = MetaIndexParser->ErrorText;
1532       return;
1533    }
1534
1535    if (!VerifyVendor(Message))
1536    {
1537       return;
1538    }
1539
1540    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1541       std::cerr << "Signature verification succeeded: "
1542                 << DestFile << std::endl;
1543
1544    // do not trust any previously unverified content that we may have
1545    string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1546    if (DestFile != SigFile)
1547       LastGoodSigFile.append(".gpg");
1548    LastGoodSigFile.append(".reverify");
1549    if(IMSHit == false && RealFileExists(LastGoodSigFile) == false)
1550    {
1551       for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1552            Target != IndexTargets->end();
1553            ++Target)
1554       {
1555          // remove old indexes
1556          std::string index = _config->FindDir("Dir::State::lists") +
1557             URItoFileName((*Target)->URI);
1558          unlink(index.c_str());
1559          // and also old gzipindexes
1560          index += ".gz";
1561          unlink(index.c_str());
1562       }
1563    }
1564
1565
1566    // Download further indexes with verification
1567    QueueIndexes(true);
1568
1569    // is it a clearsigned MetaIndex file?
1570    if (DestFile == SigFile)
1571       return;
1572
1573    // Done, move signature file into position
1574    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1575       URItoFileName(RealURI) + ".gpg";
1576    Rename(SigFile,VerifiedSigFile);
1577    chmod(VerifiedSigFile.c_str(),0644);
1578 }
1579                                                                         /*}}}*/
1580 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1581 {
1582 #if 0
1583    /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642)
1584     * FIXME: Disabled; it breaks unsigned repositories without hashes */
1585    if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile))
1586    {
1587       Status = StatError;
1588       ErrorText = MetaIndexParser->ErrorText;
1589       return;
1590    }
1591 #endif
1592    bool transInRelease = false;
1593    {
1594       std::vector<std::string> const keys = MetaIndexParser->MetaKeys();
1595       for (std::vector<std::string>::const_iterator k = keys.begin(); k != keys.end(); ++k)
1596          // FIXME: Feels wrong to check for hardcoded string here, but what should we do else…
1597          if (k->find("Translation-") != std::string::npos)
1598          {
1599             transInRelease = true;
1600             break;
1601          }
1602    }
1603
1604    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1605         Target != IndexTargets->end();
1606         ++Target)
1607    {
1608       HashString ExpectedIndexHash;
1609       const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1610       bool compressedAvailable = false;
1611       if (Record == NULL)
1612       {
1613          if ((*Target)->IsOptional() == true)
1614          {
1615             std::vector<std::string> types = APT::Configuration::getCompressionTypes();
1616             for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
1617                if (MetaIndexParser->Exists((*Target)->MetaKey + "." + *t) == true)
1618                {
1619                   compressedAvailable = true;
1620                   break;
1621                }
1622          }
1623          else if (verify == true)
1624          {
1625             Status = StatAuthError;
1626             strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
1627             return;
1628          }
1629       }
1630       else
1631       {
1632          ExpectedIndexHash = Record->Hash;
1633          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1634          {
1635             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1636             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1637             std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
1638          }
1639          if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
1640          {
1641             Status = StatAuthError;
1642             strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
1643             return;
1644          }
1645       }
1646
1647       if ((*Target)->IsOptional() == true)
1648       {
1649          if ((*Target)->IsSubIndex() == true)
1650             new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
1651                                 (*Target)->ShortDesc, ExpectedIndexHash);
1652          else if (transInRelease == false || Record != NULL || compressedAvailable == true)
1653          {
1654             if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true &&
1655                 MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)
1656                new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1657                                    (*Target)->ShortDesc, ExpectedIndexHash);
1658             else
1659                new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1660          }
1661          continue;
1662       }
1663
1664       /* Queue Packages file (either diff or full packages files, depending
1665          on the users option) - we also check if the PDiff Index file is listed
1666          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1667          instead, but passing the required info to it is to much hassle */
1668       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1669           MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
1670          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1671                              (*Target)->ShortDesc, ExpectedIndexHash);
1672       else
1673          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1674    }
1675 }
1676                                                                         /*}}}*/
1677 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1678 {
1679    string::size_type pos;
1680
1681    // check for missing sigs (that where not fatal because otherwise we had
1682    // bombed earlier)
1683    string missingkeys;
1684    string msg = _("There is no public key available for the "
1685                   "following key IDs:\n");
1686    pos = Message.find("NO_PUBKEY ");
1687    if (pos != std::string::npos)
1688    {
1689       string::size_type start = pos+strlen("NO_PUBKEY ");
1690       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1691       missingkeys += (Fingerprint);
1692    }
1693    if(!missingkeys.empty())
1694       _error->Warning("%s", (msg + missingkeys).c_str());
1695
1696    string Transformed = MetaIndexParser->GetExpectedDist();
1697
1698    if (Transformed == "../project/experimental")
1699    {
1700       Transformed = "experimental";
1701    }
1702
1703    pos = Transformed.rfind('/');
1704    if (pos != string::npos)
1705    {
1706       Transformed = Transformed.substr(0, pos);
1707    }
1708
1709    if (Transformed == ".")
1710    {
1711       Transformed = "";
1712    }
1713
1714    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1715        MetaIndexParser->GetValidUntil() > 0) {
1716       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1717       if (invalid_since > 0)
1718          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1719          // the time since then the file is invalid - formated in the same way as in
1720          // the download progress display (e.g. 7d 3h 42min 1s)
1721          return _error->Error(
1722             _("Release file for %s is expired (invalid since %s). "
1723               "Updates for this repository will not be applied."),
1724             RealURI.c_str(), TimeToStr(invalid_since).c_str());
1725    }
1726
1727    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1728    {
1729       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1730       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1731       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1732    }
1733
1734    if (MetaIndexParser->CheckDist(Transformed) == false)
1735    {
1736       // This might become fatal one day
1737 //       Status = StatAuthError;
1738 //       ErrorText = "Conflicting distribution; expected "
1739 //          + MetaIndexParser->GetExpectedDist() + " but got "
1740 //          + MetaIndexParser->GetDist();
1741 //       return false;
1742       if (!Transformed.empty())
1743       {
1744          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1745                          Desc.Description.c_str(),
1746                          Transformed.c_str(),
1747                          MetaIndexParser->GetDist().c_str());
1748       }
1749    }
1750
1751    return true;
1752 }
1753                                                                         /*}}}*/
1754 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1755 // ---------------------------------------------------------------------
1756 /* */
1757 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)
1758 {
1759    if (AuthPass == true)
1760    {
1761       // gpgv method failed, if we have a good signature
1762       string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI));
1763       if (DestFile != SigFile)
1764          LastGoodSigFile.append(".gpg");
1765       LastGoodSigFile.append(".reverify");
1766
1767       if(FileExists(LastGoodSigFile))
1768       {
1769          string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1770          if (DestFile != SigFile)
1771             VerifiedSigFile.append(".gpg");
1772          Rename(LastGoodSigFile, VerifiedSigFile);
1773          Status = StatTransientNetworkError;
1774          _error->Warning(_("An error occurred during the signature "
1775                            "verification. The repository is not updated "
1776                            "and the previous index files will be used. "
1777                            "GPG error: %s: %s\n"),
1778                          Desc.Description.c_str(),
1779                          LookupTag(Message,"Message").c_str());
1780          RunScripts("APT::Update::Auth-Failure");
1781          return;
1782       } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
1783          /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
1784          _error->Error(_("GPG error: %s: %s"),
1785                          Desc.Description.c_str(),
1786                          LookupTag(Message,"Message").c_str());
1787          return;
1788       } else {
1789          _error->Warning(_("GPG error: %s: %s"),
1790                          Desc.Description.c_str(),
1791                          LookupTag(Message,"Message").c_str());
1792       }
1793       // gpgv method failed
1794       ReportMirrorFailure("GPGFailure");
1795    }
1796
1797    /* Always move the meta index, even if gpgv failed. This ensures
1798     * that PackageFile objects are correctly filled in */
1799    if (FileExists(DestFile)) {
1800       string FinalFile = _config->FindDir("Dir::State::lists");
1801       FinalFile += URItoFileName(RealURI);
1802       /* InRelease files become Release files, otherwise
1803        * they would be considered as trusted later on */
1804       if (SigFile == DestFile) {
1805          RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
1806                                        "Release");
1807          FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
1808                                        "Release");
1809          SigFile = FinalFile;
1810       }
1811       Rename(DestFile,FinalFile);
1812       chmod(FinalFile.c_str(),0644);
1813
1814       DestFile = FinalFile;
1815    }
1816
1817    // No Release file was present, or verification failed, so fall
1818    // back to queueing Packages files without verification
1819    QueueIndexes(false);
1820 }
1821                                                                         /*}}}*/
1822 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1823                 string const &URI, string const &URIDesc, string const &ShortDesc,
1824                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1825                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1826                 const vector<struct IndexTarget*>* IndexTargets,
1827                 indexRecords* MetaIndexParser) :
1828         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1829         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1830         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1831 {
1832    SigFile = DestFile;
1833
1834    // keep the old InRelease around in case of transistent network errors
1835    string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1836    if (RealFileExists(Final) == true)
1837    {
1838       string const LastGoodSig = DestFile + ".reverify";
1839       Rename(Final,LastGoodSig);
1840    }
1841 }
1842                                                                         /*}}}*/
1843 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()                               /*{{{*/
1844 {
1845    // if the file was never queued undo file-changes done in the constructor
1846    if (QueueCounter == 1 && Status == StatIdle && FileSize == 0 && Complete == false)
1847    {
1848       string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
1849       string const LastGoodSig = DestFile + ".reverify";
1850       if (RealFileExists(Final) == false && RealFileExists(LastGoodSig) == true)
1851          Rename(LastGoodSig, Final);
1852    }
1853 }
1854                                                                         /*}}}*/
1855 // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/
1856 // ---------------------------------------------------------------------
1857 // FIXME: this can go away once the InRelease file is used widely
1858 string pkgAcqMetaClearSig::Custom600Headers()
1859 {
1860    string Final = _config->FindDir("Dir::State::lists");
1861    Final += URItoFileName(RealURI);
1862
1863    struct stat Buf;
1864    if (stat(Final.c_str(),&Buf) != 0)
1865    {
1866       Final = DestFile + ".reverify";
1867       if (stat(Final.c_str(),&Buf) != 0)
1868          return "\nIndex-File: true\nFail-Ignore: true\n";
1869    }
1870
1871    return "\nIndex-File: true\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1872 }
1873                                                                         /*}}}*/
1874 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1875 {
1876    if (AuthPass == false)
1877    {
1878       // Remove the 'old' InRelease file if we try Release.gpg now as otherwise
1879       // the file will stay around and gives a false-auth impression (CVE-2012-0214)
1880       string FinalFile = _config->FindDir("Dir::State::lists");
1881       FinalFile.append(URItoFileName(RealURI));
1882       if (FileExists(FinalFile))
1883          unlink(FinalFile.c_str());
1884
1885       new pkgAcqMetaSig(Owner,
1886                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1887                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1888                         IndexTargets, MetaIndexParser);
1889       if (Cnf->LocalOnly == true ||
1890           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1891          Dequeue();
1892    }
1893    else
1894       pkgAcqMetaIndex::Failed(Message, Cnf);
1895 }
1896                                                                         /*}}}*/
1897 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1898 // ---------------------------------------------------------------------
1899 /* This just sets up the initial fetch environment and queues the first
1900    possibilitiy */
1901 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1902                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1903                              string &StoreFilename) :
1904                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1905                StoreFilename(StoreFilename), Vf(Version.FileList()),
1906                Trusted(false)
1907 {
1908    Retries = _config->FindI("Acquire::Retries",0);
1909
1910    if (Version.Arch() == 0)
1911    {
1912       _error->Error(_("I wasn't able to locate a file for the %s package. "
1913                       "This might mean you need to manually fix this package. "
1914                       "(due to missing arch)"),
1915                     Version.ParentPkg().FullName().c_str());
1916       return;
1917    }
1918
1919    /* We need to find a filename to determine the extension. We make the
1920       assumption here that all the available sources for this version share
1921       the same extension.. */
1922    // Skip not source sources, they do not have file fields.
1923    for (; Vf.end() == false; ++Vf)
1924    {
1925       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1926          continue;
1927       break;
1928    }
1929
1930    // Does not really matter here.. we are going to fail out below
1931    if (Vf.end() != true)
1932    {
1933       // If this fails to get a file name we will bomb out below.
1934       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1935       if (_error->PendingError() == true)
1936          return;
1937
1938       // Generate the final file name as: package_version_arch.foo
1939       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1940                       QuoteString(Version.VerStr(),"_:") + '_' +
1941                       QuoteString(Version.Arch(),"_:.") +
1942                       "." + flExtension(Parse.FileName());
1943    }
1944
1945    // check if we have one trusted source for the package. if so, switch
1946    // to "TrustedOnly" mode - but only if not in AllowUnauthenticated mode
1947    bool const allowUnauth = _config->FindB("APT::Get::AllowUnauthenticated", false);
1948    bool const debugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
1949    bool seenUntrusted = false;
1950    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; ++i)
1951    {
1952       pkgIndexFile *Index;
1953       if (Sources->FindIndex(i.File(),Index) == false)
1954          continue;
1955
1956       if (debugAuth == true)
1957          std::cerr << "Checking index: " << Index->Describe()
1958                    << "(Trusted=" << Index->IsTrusted() << ")" << std::endl;
1959
1960       if (Index->IsTrusted() == true)
1961       {
1962          Trusted = true;
1963          if (allowUnauth == false)
1964             break;
1965       }
1966       else
1967          seenUntrusted = true;
1968    }
1969
1970    // "allow-unauthenticated" restores apts old fetching behaviour
1971    // that means that e.g. unauthenticated file:// uris are higher
1972    // priority than authenticated http:// uris
1973    if (allowUnauth == true && seenUntrusted == true)
1974       Trusted = false;
1975
1976    // Select a source
1977    if (QueueNext() == false && _error->PendingError() == false)
1978       _error->Error(_("Can't find a source to download version '%s' of '%s'"),
1979                     Version.VerStr(), Version.ParentPkg().FullName(false).c_str());
1980 }
1981                                                                         /*}}}*/
1982 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1983 // ---------------------------------------------------------------------
1984 /* This queues the next available file version for download. It checks if
1985    the archive is already available in the cache and stashs the MD5 for
1986    checking later. */
1987 bool pkgAcqArchive::QueueNext()
1988 {
1989    string const ForceHash = _config->Find("Acquire::ForceHash");
1990    for (; Vf.end() == false; ++Vf)
1991    {
1992       // Ignore not source sources
1993       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1994          continue;
1995
1996       // Try to cross match against the source list
1997       pkgIndexFile *Index;
1998       if (Sources->FindIndex(Vf.File(),Index) == false)
1999             continue;
2000
2001       // only try to get a trusted package from another source if that source
2002       // is also trusted
2003       if(Trusted && !Index->IsTrusted())
2004          continue;
2005
2006       // Grab the text package record
2007       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
2008       if (_error->PendingError() == true)
2009          return false;
2010
2011       string PkgFile = Parse.FileName();
2012       if (ForceHash.empty() == false)
2013       {
2014          if(stringcasecmp(ForceHash, "sha512") == 0)
2015             ExpectedHash = HashString("SHA512", Parse.SHA512Hash());
2016          else if(stringcasecmp(ForceHash, "sha256") == 0)
2017             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
2018          else if (stringcasecmp(ForceHash, "sha1") == 0)
2019             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
2020          else
2021             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2022       }
2023       else
2024       {
2025          string Hash;
2026          if ((Hash = Parse.SHA512Hash()).empty() == false)
2027             ExpectedHash = HashString("SHA512", Hash);
2028          else if ((Hash = Parse.SHA256Hash()).empty() == false)
2029             ExpectedHash = HashString("SHA256", Hash);
2030          else if ((Hash = Parse.SHA1Hash()).empty() == false)
2031             ExpectedHash = HashString("SHA1", Hash);
2032          else
2033             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
2034       }
2035       if (PkgFile.empty() == true)
2036          return _error->Error(_("The package index files are corrupted. No Filename: "
2037                               "field for package %s."),
2038                               Version.ParentPkg().Name());
2039
2040       Desc.URI = Index->ArchiveURI(PkgFile);
2041       Desc.Description = Index->ArchiveInfo(Version);
2042       Desc.Owner = this;
2043       Desc.ShortDesc = Version.ParentPkg().FullName(true);
2044
2045       // See if we already have the file. (Legacy filenames)
2046       FileSize = Version->Size;
2047       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
2048       struct stat Buf;
2049       if (stat(FinalFile.c_str(),&Buf) == 0)
2050       {
2051          // Make sure the size matches
2052          if ((unsigned long long)Buf.st_size == Version->Size)
2053          {
2054             Complete = true;
2055             Local = true;
2056             Status = StatDone;
2057             StoreFilename = DestFile = FinalFile;
2058             return true;
2059          }
2060
2061          /* Hmm, we have a file and its size does not match, this means it is
2062             an old style mismatched arch */
2063          unlink(FinalFile.c_str());
2064       }
2065
2066       // Check it again using the new style output filenames
2067       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
2068       if (stat(FinalFile.c_str(),&Buf) == 0)
2069       {
2070          // Make sure the size matches
2071          if ((unsigned long long)Buf.st_size == Version->Size)
2072          {
2073             Complete = true;
2074             Local = true;
2075             Status = StatDone;
2076             StoreFilename = DestFile = FinalFile;
2077             return true;
2078          }
2079
2080          /* Hmm, we have a file and its size does not match, this shouldn't
2081             happen.. */
2082          unlink(FinalFile.c_str());
2083       }
2084
2085       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
2086
2087       // Check the destination file
2088       if (stat(DestFile.c_str(),&Buf) == 0)
2089       {
2090          // Hmm, the partial file is too big, erase it
2091          if ((unsigned long long)Buf.st_size > Version->Size)
2092             unlink(DestFile.c_str());
2093          else
2094             PartialSize = Buf.st_size;
2095       }
2096
2097       // Disables download of archives - useful if no real installation follows,
2098       // e.g. if we are just interested in proposed installation order
2099       if (_config->FindB("Debug::pkgAcqArchive::NoQueue", false) == true)
2100       {
2101          Complete = true;
2102          Local = true;
2103          Status = StatDone;
2104          StoreFilename = DestFile = FinalFile;
2105          return true;
2106       }
2107
2108       // Create the item
2109       Local = false;
2110       QueueURI(Desc);
2111
2112       ++Vf;
2113       return true;
2114    }
2115    return false;
2116 }
2117                                                                         /*}}}*/
2118 // AcqArchive::Done - Finished fetching                                 /*{{{*/
2119 // ---------------------------------------------------------------------
2120 /* */
2121 void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash,
2122                          pkgAcquire::MethodConfig *Cfg)
2123 {
2124    Item::Done(Message,Size,CalcHash,Cfg);
2125
2126    // Check the size
2127    if (Size != Version->Size)
2128    {
2129       RenameOnError(SizeMismatch);
2130       return;
2131    }
2132
2133    // Check the hash
2134    if(ExpectedHash.toStr() != CalcHash)
2135    {
2136       RenameOnError(HashSumMismatch);
2137       return;
2138    }
2139
2140    // Grab the output filename
2141    string FileName = LookupTag(Message,"Filename");
2142    if (FileName.empty() == true)
2143    {
2144       Status = StatError;
2145       ErrorText = "Method gave a blank filename";
2146       return;
2147    }
2148
2149    Complete = true;
2150
2151    // Reference filename
2152    if (FileName != DestFile)
2153    {
2154       StoreFilename = DestFile = FileName;
2155       Local = true;
2156       return;
2157    }
2158
2159    // Done, move it into position
2160    string FinalFile = _config->FindDir("Dir::Cache::Archives");
2161    FinalFile += flNotDir(StoreFilename);
2162    Rename(DestFile,FinalFile);
2163
2164    StoreFilename = DestFile = FinalFile;
2165    Complete = true;
2166 }
2167                                                                         /*}}}*/
2168 // AcqArchive::Failed - Failure handler                                 /*{{{*/
2169 // ---------------------------------------------------------------------
2170 /* Here we try other sources */
2171 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2172 {
2173    ErrorText = LookupTag(Message,"Message");
2174
2175    /* We don't really want to retry on failed media swaps, this prevents
2176       that. An interesting observation is that permanent failures are not
2177       recorded. */
2178    if (Cnf->Removable == true &&
2179        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2180    {
2181       // Vf = Version.FileList();
2182       while (Vf.end() == false) ++Vf;
2183       StoreFilename = string();
2184       Item::Failed(Message,Cnf);
2185       return;
2186    }
2187
2188    if (QueueNext() == false)
2189    {
2190       // This is the retry counter
2191       if (Retries != 0 &&
2192           Cnf->LocalOnly == false &&
2193           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2194       {
2195          Retries--;
2196          Vf = Version.FileList();
2197          if (QueueNext() == true)
2198             return;
2199       }
2200
2201       StoreFilename = string();
2202       Item::Failed(Message,Cnf);
2203    }
2204 }
2205                                                                         /*}}}*/
2206 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
2207 // ---------------------------------------------------------------------
2208 APT_PURE bool pkgAcqArchive::IsTrusted()
2209 {
2210    return Trusted;
2211 }
2212                                                                         /*}}}*/
2213 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
2214 // ---------------------------------------------------------------------
2215 /* */
2216 void pkgAcqArchive::Finished()
2217 {
2218    if (Status == pkgAcquire::Item::StatDone &&
2219        Complete == true)
2220       return;
2221    StoreFilename = string();
2222 }
2223                                                                         /*}}}*/
2224 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
2225 // ---------------------------------------------------------------------
2226 /* The file is added to the queue */
2227 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
2228                        unsigned long long Size,string Dsc,string ShortDesc,
2229                        const string &DestDir, const string &DestFilename,
2230                        bool IsIndexFile) :
2231                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
2232 {
2233    Retries = _config->FindI("Acquire::Retries",0);
2234
2235    if(!DestFilename.empty())
2236       DestFile = DestFilename;
2237    else if(!DestDir.empty())
2238       DestFile = DestDir + "/" + flNotDir(URI);
2239    else
2240       DestFile = flNotDir(URI);
2241
2242    // Create the item
2243    Desc.URI = URI;
2244    Desc.Description = Dsc;
2245    Desc.Owner = this;
2246
2247    // Set the short description to the archive component
2248    Desc.ShortDesc = ShortDesc;
2249
2250    // Get the transfer sizes
2251    FileSize = Size;
2252    struct stat Buf;
2253    if (stat(DestFile.c_str(),&Buf) == 0)
2254    {
2255       // Hmm, the partial file is too big, erase it
2256       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
2257          unlink(DestFile.c_str());
2258       else
2259          PartialSize = Buf.st_size;
2260    }
2261
2262    QueueURI(Desc);
2263 }
2264                                                                         /*}}}*/
2265 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
2266 // ---------------------------------------------------------------------
2267 /* */
2268 void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash,
2269                       pkgAcquire::MethodConfig *Cnf)
2270 {
2271    Item::Done(Message,Size,CalcHash,Cnf);
2272
2273    // Check the hash
2274    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
2275    {
2276       RenameOnError(HashSumMismatch);
2277       return;
2278    }
2279
2280    string FileName = LookupTag(Message,"Filename");
2281    if (FileName.empty() == true)
2282    {
2283       Status = StatError;
2284       ErrorText = "Method gave a blank filename";
2285       return;
2286    }
2287
2288    Complete = true;
2289
2290    // The files timestamp matches
2291    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
2292       return;
2293
2294    // We have to copy it into place
2295    if (FileName != DestFile)
2296    {
2297       Local = true;
2298       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
2299           Cnf->Removable == true)
2300       {
2301          Desc.URI = "copy:" + FileName;
2302          QueueURI(Desc);
2303          return;
2304       }
2305
2306       // Erase the file if it is a symlink so we can overwrite it
2307       struct stat St;
2308       if (lstat(DestFile.c_str(),&St) == 0)
2309       {
2310          if (S_ISLNK(St.st_mode) != 0)
2311             unlink(DestFile.c_str());
2312       }
2313
2314       // Symlink the file
2315       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
2316       {
2317          ErrorText = "Link to " + DestFile + " failure ";
2318          Status = StatError;
2319          Complete = false;
2320       }
2321    }
2322 }
2323                                                                         /*}}}*/
2324 // AcqFile::Failed - Failure handler                                    /*{{{*/
2325 // ---------------------------------------------------------------------
2326 /* Here we try other sources */
2327 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
2328 {
2329    ErrorText = LookupTag(Message,"Message");
2330
2331    // This is the retry counter
2332    if (Retries != 0 &&
2333        Cnf->LocalOnly == false &&
2334        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
2335    {
2336       Retries--;
2337       QueueURI(Desc);
2338       return;
2339    }
2340
2341    Item::Failed(Message,Cnf);
2342 }
2343                                                                         /*}}}*/
2344 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
2345 // ---------------------------------------------------------------------
2346 /* The only header we use is the last-modified header. */
2347 string pkgAcqFile::Custom600Headers()
2348 {
2349    if (IsIndexFile)
2350       return "\nIndex-File: true";
2351    return "";
2352 }
2353                                                                         /*}}}*/