apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Acquire::Item::Item - Constructor                                    /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  44                        PartialSize(0), Mode(0), ID(0), Complete(false),
  45                        Local(false), QueueCounter(0)
  46 {
  47    Owner->Add(this);
  48    Status = StatIdle;
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* */
  54 pkgAcquire::Item::~Item()
  55 {
  56    Owner->Remove(this);
  57 }
  58                                                                         /*}}}*/
  59 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* We return to an idle state if there are still other queues that could
  62    fetch this object */
  63 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  64 {
  65    Status = StatIdle;
  66    ErrorText = LookupTag(Message,"Message");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83 }
  84                                                                         /*}}}*/
  85 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Stash status and the file size. Note that setting Complete means
  88    sub-phases of the acquire process such as decompresion are operating */
  89 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  90 {
  91    Status = StatFetching;
  92    if (FileSize == 0 && Complete == false)
  93       FileSize = Size;
  94 }
  95                                                                         /*}}}*/
  96 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 100                             pkgAcquire::MethodConfig *Cnf)
 101 {
 102    // We just downloaded something..
 103    string FileName = LookupTag(Message,"Filename");
 104    // we only inform the Log class if it was actually not a local thing
 105    if (Complete == false && !Local && FileName == DestFile)
 106    {
 107       if (Owner->Log != 0)
 108          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 109    }
 110
 111    if (FileSize == 0)
 112       FileSize= Size;
 113
 114    Status = StatDone;
 115    ErrorText = string();
 116    Owner->Dequeue(this);
 117 }
 118                                                                         /*}}}*/
 119 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This helper function is used by alot of item methods as thier final
 122    step */
 123 void pkgAcquire::Item::Rename(string From,string To)
 124 {
 125    if (rename(From.c_str(),To.c_str()) != 0)
 126    {
 127       char S[300];
 128       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 129               From.c_str(),To.c_str());
 130       Status = StatError;
 131       ErrorText = S;
 132    }
 133 }
 134                                                                         /*}}}*/
 135 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* Get the DiffIndex file first and see if there are patches availabe
 138  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 139  * patches. If anything goes wrong in that process, it will fall back to
 140  * the original packages file
 141  */
 142 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 143                                  string URI,string URIDesc,string ShortDesc,
 144                                  HashString ExpectedHash)
 145    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 146      Description(URIDesc)
 147 {
 148
 149    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 150
 151    Desc.Description = URIDesc + "/DiffIndex";
 152    Desc.Owner = this;
 153    Desc.ShortDesc = ShortDesc;
 154    Desc.URI = URI + ".diff/Index";
 155
 156    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 157    DestFile += URItoFileName(URI) + string(".DiffIndex");
 158
 159    if(Debug)
 160       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 161
 162    // look for the current package file
 163    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 164    CurrentPackagesFile += URItoFileName(RealURI);
 165
 166    // FIXME: this file:/ check is a hack to prevent fetching
 167    //        from local sources. this is really silly, and
 168    //        should be fixed cleanly as soon as possible
 169    if(!FileExists(CurrentPackagesFile) ||
 170       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 171    {
 172       // we don't have a pkg file or we don't want to queue
 173       if(Debug)
 174          std::clog << "No index file, local or canceld by user" << std::endl;
 175       Failed("", NULL);
 176       return;
 177    }
 178
 179    if(Debug)
 180       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 181                 << CurrentPackagesFile << std::endl;
 182
 183    QueueURI(Desc);
 184
 185 }
 186                                                                         /*}}}*/
 187 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* The only header we use is the last-modified header. */
 190 string pkgAcqDiffIndex::Custom600Headers()
 191 {
 192    string Final = _config->FindDir("Dir::State::lists");
 193    Final += URItoFileName(RealURI) + string(".IndexDiff");
 194
 195    if(Debug)
 196       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 197
 198    struct stat Buf;
 199    if (stat(Final.c_str(),&Buf) != 0)
 200       return "\nIndex-File: true";
 201
 202    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 203 }
 204                                                                         /*}}}*/
 205 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 206 {
 207    if(Debug)
 208       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 209                 << std::endl;
 210
 211    pkgTagSection Tags;
 212    string ServerSha1;
 213    vector<DiffInfo> available_patches;
 214
 215    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 216    pkgTagFile TF(&Fd);
 217    if (_error->PendingError() == true)
 218       return false;
 219
 220    if(TF.Step(Tags) == true)
 221    {
 222       bool found = false;
 223       DiffInfo d;
 224       string size;
 225
 226       string const tmp = Tags.FindS("SHA1-Current");
 227       std::stringstream ss(tmp);
 228       ss >> ServerSha1 >> size;
 229       unsigned long const ServerSize = atol(size.c_str());
 230
 231       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 232       SHA1Summation SHA1;
 233       SHA1.AddFD(fd.Fd(), fd.Size());
 234       string const local_sha1 = SHA1.Result();
 235
 236       if(local_sha1 == ServerSha1)
 237       {
 238          // we have the same sha1 as the server
 239          if(Debug)
 240             std::clog << "Package file is up-to-date" << std::endl;
 241          // set found to true, this will queue a pkgAcqIndexDiffs with
 242          // a empty availabe_patches
 243          found = true;
 244       }
 245       else
 246       {
 247          if(Debug)
 248             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 249
 250          // check the historie and see what patches we need
 251          string const history = Tags.FindS("SHA1-History");
 252          std::stringstream hist(history);
 253          while(hist >> d.sha1 >> size >> d.file)
 254          {
 255             // read until the first match is found
 256             // from that point on, we probably need all diffs
 257             if(d.sha1 == local_sha1)
 258                found=true;
 259             else if (found == false)
 260                continue;
 261
 262             if(Debug)
 263                std::clog << "Need to get diff: " << d.file << std::endl;
 264             available_patches.push_back(d);
 265          }
 266
 267          if (available_patches.empty() == false)
 268          {
 269             // patching with too many files is rather slow compared to a fast download
 270             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 271             if (fileLimit != 0 && fileLimit < available_patches.size())
 272             {
 273                if (Debug)
 274                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 275                         << ") so fallback to complete download" << std::endl;
 276                return false;
 277             }
 278
 279             // see if the patches are too big
 280             found = false; // it was true and it will be true again at the end
 281             d = *available_patches.begin();
 282             string const firstPatch = d.file;
 283             unsigned long patchesSize = 0;
 284             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 285             while(patches >> d.sha1 >> size >> d.file)
 286             {
 287                if (firstPatch == d.file)
 288                   found = true;
 289                else if (found == false)
 290                   continue;
 291
 292                patchesSize += atol(size.c_str());
 293             }
 294             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 295             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 296             {
 297                if (Debug)
 298                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 299                         << ") so fallback to complete download" << std::endl;
 300                return false;
 301             }
 302          }
 303       }
 304
 305       // we have something, queue the next diff
 306       if(found)
 307       {
 308          // queue the diffs
 309          string::size_type const last_space = Description.rfind(" ");
 310          if(last_space != string::npos)
 311             Description.erase(last_space, Description.size()-last_space);
 312          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 313                               ExpectedHash, available_patches);
 314          Complete = false;
 315          Status = StatDone;
 316          Dequeue();
 317          return true;
 318       }
 319    }
 320
 321    // Nothing found, report and return false
 322    // Failing here is ok, if we return false later, the full
 323    // IndexFile is queued
 324    if(Debug)
 325       std::clog << "Can't find a patch in the index file" << std::endl;
 326    return false;
 327 }
 328                                                                         /*}}}*/
 329 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 330 {
 331    if(Debug)
 332       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 333                 << "Falling back to normal index file aquire" << std::endl;
 334
 335    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 336                    ExpectedHash);
 337
 338    Complete = false;
 339    Status = StatDone;
 340    Dequeue();
 341 }
 342                                                                         /*}}}*/
 343 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 344                            pkgAcquire::MethodConfig *Cnf)
 345 {
 346    if(Debug)
 347       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 348
 349    Item::Done(Message,Size,Md5Hash,Cnf);
 350
 351    string FinalFile;
 352    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 353
 354    // sucess in downloading the index
 355    // rename the index
 356    FinalFile += string(".IndexDiff");
 357    if(Debug)
 358       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 359                 << std::endl;
 360    Rename(DestFile,FinalFile);
 361    chmod(FinalFile.c_str(),0644);
 362    DestFile = FinalFile;
 363
 364    if(!ParseDiffIndex(DestFile))
 365       return Failed("", NULL);
 366
 367    Complete = true;
 368    Status = StatDone;
 369    Dequeue();
 370    return;
 371 }
 372                                                                         /*}}}*/
 373 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* The package diff is added to the queue. one object is constructed
 376  * for each diff and the index
 377  */
 378 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 379                                    string URI,string URIDesc,string ShortDesc,
 380                                    HashString ExpectedHash,
 381                                    vector<DiffInfo> diffs)
 382    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 383      available_patches(diffs)
 384 {
 385
 386    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 387    DestFile += URItoFileName(URI);
 388
 389    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 390
 391    Description = URIDesc;
 392    Desc.Owner = this;
 393    Desc.ShortDesc = ShortDesc;
 394
 395    if(available_patches.size() == 0)
 396    {
 397       // we are done (yeah!)
 398       Finish(true);
 399    }
 400    else
 401    {
 402       // get the next diff
 403       State = StateFetchDiff;
 404       QueueNextDiff();
 405    }
 406 }
 407                                                                         /*}}}*/
 408 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 409 {
 410    if(Debug)
 411       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 412                 << "Falling back to normal index file aquire" << std::endl;
 413    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 414                    ExpectedHash);
 415    Finish();
 416 }
 417                                                                         /*}}}*/
 418 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 419 void pkgAcqIndexDiffs::Finish(bool allDone)
 420 {
 421    // we restore the original name, this is required, otherwise
 422    // the file will be cleaned
 423    if(allDone)
 424    {
 425       DestFile = _config->FindDir("Dir::State::lists");
 426       DestFile += URItoFileName(RealURI);
 427
 428       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 429       {
 430          Status = StatAuthError;
 431          ErrorText = _("MD5Sum mismatch");
 432          Rename(DestFile,DestFile + ".FAILED");
 433          Dequeue();
 434          return;
 435       }
 436
 437       // this is for the "real" finish
 438       Complete = true;
 439       Status = StatDone;
 440       Dequeue();
 441       if(Debug)
 442          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 443       return;
 444    }
 445
 446    if(Debug)
 447       std::clog << "Finishing: " << Desc.URI << std::endl;
 448    Complete = false;
 449    Status = StatDone;
 450    Dequeue();
 451    return;
 452 }
 453                                                                         /*}}}*/
 454 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 455 {
 456
 457    // calc sha1 of the just patched file
 458    string FinalFile = _config->FindDir("Dir::State::lists");
 459    FinalFile += URItoFileName(RealURI);
 460
 461    FileFd fd(FinalFile, FileFd::ReadOnly);
 462    SHA1Summation SHA1;
 463    SHA1.AddFD(fd.Fd(), fd.Size());
 464    string local_sha1 = string(SHA1.Result());
 465    if(Debug)
 466       std::clog << "QueueNextDiff: "
 467                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 468
 469    // remove all patches until the next matching patch is found
 470    // this requires the Index file to be ordered
 471    for(vector<DiffInfo>::iterator I=available_patches.begin();
 472        available_patches.size() > 0 &&
 473           I != available_patches.end() &&
 474           (*I).sha1 != local_sha1;
 475        I++)
 476    {
 477       available_patches.erase(I);
 478    }
 479
 480    // error checking and falling back if no patch was found
 481    if(available_patches.size() == 0)
 482    {
 483       Failed("", NULL);
 484       return false;
 485    }
 486
 487    // queue the right diff
 488    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 489    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 490    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 491    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 492
 493    if(Debug)
 494       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 495
 496    QueueURI(Desc);
 497
 498    return true;
 499 }
 500                                                                         /*}}}*/
 501 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 502                             pkgAcquire::MethodConfig *Cnf)
 503 {
 504    if(Debug)
 505       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 506
 507    Item::Done(Message,Size,Md5Hash,Cnf);
 508
 509    string FinalFile;
 510    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 511
 512    // sucess in downloading a diff, enter ApplyDiff state
 513    if(State == StateFetchDiff)
 514    {
 515
 516       if(Debug)
 517          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 518
 519       string FileName = LookupTag(Message,"Filename");
 520       State = StateUnzipDiff;
 521       Local = true;
 522       Desc.URI = "gzip:" + FileName;
 523       DestFile += ".decomp";
 524       QueueURI(Desc);
 525       Mode = "gzip";
 526       return;
 527    }
 528
 529    // sucess in downloading a diff, enter ApplyDiff state
 530    if(State == StateUnzipDiff)
 531    {
 532
 533       // rred excepts the patch as $FinalFile.ed
 534       Rename(DestFile,FinalFile+".ed");
 535
 536       if(Debug)
 537          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 538
 539       State = StateApplyDiff;
 540       Local = true;
 541       Desc.URI = "rred:" + FinalFile;
 542       QueueURI(Desc);
 543       Mode = "rred";
 544       return;
 545    }
 546
 547
 548    // success in download/apply a diff, queue next (if needed)
 549    if(State == StateApplyDiff)
 550    {
 551       // remove the just applied patch
 552       available_patches.erase(available_patches.begin());
 553
 554       // move into place
 555       if(Debug)
 556       {
 557          std::clog << "Moving patched file in place: " << std::endl
 558                    << DestFile << " -> " << FinalFile << std::endl;
 559       }
 560       Rename(DestFile,FinalFile);
 561       chmod(FinalFile.c_str(),0644);
 562
 563       // see if there is more to download
 564       if(available_patches.size() > 0) {
 565          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 566                               ExpectedHash, available_patches);
 567          return Finish();
 568       } else
 569          return Finish(true);
 570    }
 571 }
 572                                                                         /*}}}*/
 573 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 574 // ---------------------------------------------------------------------
 575 /* The package file is added to the queue and a second class is
 576    instantiated to fetch the revision file */
 577 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 578                          string URI,string URIDesc,string ShortDesc,
 579                          HashString ExpectedHash, string comprExt)
 580    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 581 {
 582    Decompression = false;
 583    Erase = false;
 584
 585    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 586    DestFile += URItoFileName(URI);
 587
 588    if(comprExt.empty())
 589    {
 590       // autoselect the compression method
 591       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 592       if (types.empty() == true)
 593          comprExt = "plain";
 594       else
 595          comprExt = "." + types[0];
 596    }
 597    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 598
 599    Desc.URI = URI + CompressionExtension;
 600
 601    Desc.Description = URIDesc;
 602    Desc.Owner = this;
 603    Desc.ShortDesc = ShortDesc;
 604
 605    QueueURI(Desc);
 606 }
 607                                                                         /*}}}*/
 608 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 609 // ---------------------------------------------------------------------
 610 /* The only header we use is the last-modified header. */
 611 string pkgAcqIndex::Custom600Headers()
 612 {
 613    string Final = _config->FindDir("Dir::State::lists");
 614    Final += URItoFileName(RealURI);
 615
 616    struct stat Buf;
 617    if (stat(Final.c_str(),&Buf) != 0)
 618       return "\nIndex-File: true";
 619
 620    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 621 }
 622                                                                         /*}}}*/
 623 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 624 {
 625    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 626
 627    for (std::vector<std::string>::const_iterator t = types.begin();
 628         t != types.end(); t++)
 629    {
 630       // jump over all already tried compression types
 631       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 632       if(Desc.URI.substr(nameLen) != *t)
 633          continue;
 634
 635       // we want to try it with the next extension (and make sure to
 636       // not skip over the end)
 637       t++;
 638       if (t == types.end())
 639          break;
 640
 641       // queue new download
 642       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 643       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 644       ExpectedHash, string(".").append(*t));
 645
 646       Status = StatDone;
 647       Complete = false;
 648       Dequeue();
 649       return;
 650    }
 651
 652    // on decompression failure, remove bad versions in partial/
 653    if(Decompression && Erase) {
 654       string s = _config->FindDir("Dir::State::lists") + "partial/";
 655       s += URItoFileName(RealURI);
 656       unlink(s.c_str());
 657    }
 658
 659    Item::Failed(Message,Cnf);
 660 }
 661                                                                         /*}}}*/
 662 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 663 // ---------------------------------------------------------------------
 664 /* This goes through a number of states.. On the initial fetch the
 665    method could possibly return an alternate filename which points
 666    to the uncompressed version of the file. If this is so the file
 667    is copied into the partial directory. In all other cases the file
 668    is decompressed with a gzip uri. */
 669 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 670                        pkgAcquire::MethodConfig *Cfg)
 671 {
 672    Item::Done(Message,Size,Hash,Cfg);
 673
 674    if (Decompression == true)
 675    {
 676       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 677       {
 678          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 679          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 680       }
 681
 682       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 683       {
 684          Status = StatAuthError;
 685          ErrorText = _("Hash Sum mismatch");
 686          Rename(DestFile,DestFile + ".FAILED");
 687          return;
 688       }
 689       // Done, move it into position
 690       string FinalFile = _config->FindDir("Dir::State::lists");
 691       FinalFile += URItoFileName(RealURI);
 692       Rename(DestFile,FinalFile);
 693       chmod(FinalFile.c_str(),0644);
 694
 695       /* We restore the original name to DestFile so that the clean operation
 696          will work OK */
 697       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 698       DestFile += URItoFileName(RealURI);
 699
 700       // Remove the compressed version.
 701       if (Erase == true)
 702          unlink(DestFile.c_str());
 703       return;
 704    }
 705
 706    Erase = false;
 707    Complete = true;
 708
 709    // Handle the unzipd case
 710    string FileName = LookupTag(Message,"Alt-Filename");
 711    if (FileName.empty() == false)
 712    {
 713       // The files timestamp matches
 714       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 715          return;
 716       Decompression = true;
 717       Local = true;
 718       DestFile += ".decomp";
 719       Desc.URI = "copy:" + FileName;
 720       QueueURI(Desc);
 721       Mode = "copy";
 722       return;
 723    }
 724
 725    FileName = LookupTag(Message,"Filename");
 726    if (FileName.empty() == true)
 727    {
 728       Status = StatError;
 729       ErrorText = "Method gave a blank filename";
 730    }
 731
 732    // The files timestamp matches
 733    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 734       return;
 735
 736    if (FileName == DestFile)
 737       Erase = true;
 738    else
 739       Local = true;
 740
 741    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 742    string decompProg;
 743
 744    // get the binary name for your used compression type
 745    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 746    if(decompProg.empty() == false);
 747    // flExtensions returns the full name if no extension is found
 748    // this is why we have this complicated compare operation here
 749    // FIMXE: add a new flJustExtension() that return "" if no
 750    //        extension is found and use that above so that it can
 751    //        be tested against ""
 752    else if(compExt == flNotDir(URI(Desc.URI).Path))
 753       decompProg = "copy";
 754    else {
 755       _error->Error("Unsupported extension: %s", compExt.c_str());
 756       return;
 757    }
 758
 759    Decompression = true;
 760    DestFile += ".decomp";
 761    Desc.URI = decompProg + ":" + FileName;
 762    QueueURI(Desc);
 763    Mode = decompProg.c_str();
 764 }
 765                                                                         /*}}}*/
 766 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 767 // ---------------------------------------------------------------------
 768 /* The Translation file is added to the queue */
 769 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 770                             string URI,string URIDesc,string ShortDesc)
 771   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 772 {
 773 }
 774                                                                         /*}}}*/
 775 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 776 // ---------------------------------------------------------------------
 777 /* */
 778 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 779 {
 780    if (Cnf->LocalOnly == true ||
 781        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 782    {
 783       // Ignore this
 784       Status = StatDone;
 785       Complete = false;
 786       Dequeue();
 787       return;
 788    }
 789
 790    Item::Failed(Message,Cnf);
 791 }
 792                                                                         /*}}}*/
 793 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 794                              string URI,string URIDesc,string ShortDesc,
 795                              string MetaIndexURI, string MetaIndexURIDesc,
 796                              string MetaIndexShortDesc,
 797                              const vector<IndexTarget*>* IndexTargets,
 798                              indexRecords* MetaIndexParser) :
 799    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 800    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 801    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 802 {
 803    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 804    DestFile += URItoFileName(URI);
 805
 806    // remove any partial downloaded sig-file in partial/.
 807    // it may confuse proxies and is too small to warrant a
 808    // partial download anyway
 809    unlink(DestFile.c_str());
 810
 811    // Create the item
 812    Desc.Description = URIDesc;
 813    Desc.Owner = this;
 814    Desc.ShortDesc = ShortDesc;
 815    Desc.URI = URI;
 816
 817    string Final = _config->FindDir("Dir::State::lists");
 818    Final += URItoFileName(RealURI);
 819    struct stat Buf;
 820    if (stat(Final.c_str(),&Buf) == 0)
 821    {
 822       // File was already in place.  It needs to be re-downloaded/verified
 823       // because Release might have changed, we do give it a differnt
 824       // name than DestFile because otherwise the http method will
 825       // send If-Range requests and there are too many broken servers
 826       // out there that do not understand them
 827       LastGoodSig = DestFile+".reverify";
 828       Rename(Final,LastGoodSig);
 829    }
 830
 831    QueueURI(Desc);
 832 }
 833                                                                         /*}}}*/
 834 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 835 // ---------------------------------------------------------------------
 836 /* The only header we use is the last-modified header. */
 837 string pkgAcqMetaSig::Custom600Headers()
 838 {
 839    struct stat Buf;
 840    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 841       return "\nIndex-File: true";
 842
 843    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 844 }
 845
 846 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 847                          pkgAcquire::MethodConfig *Cfg)
 848 {
 849    Item::Done(Message,Size,MD5,Cfg);
 850
 851    string FileName = LookupTag(Message,"Filename");
 852    if (FileName.empty() == true)
 853    {
 854       Status = StatError;
 855       ErrorText = "Method gave a blank filename";
 856       return;
 857    }
 858
 859    if (FileName != DestFile)
 860    {
 861       // We have to copy it into place
 862       Local = true;
 863       Desc.URI = "copy:" + FileName;
 864       QueueURI(Desc);
 865       return;
 866    }
 867
 868    Complete = true;
 869
 870    // put the last known good file back on i-m-s hit (it will
 871    // be re-verified again)
 872    // Else do nothing, we have the new file in DestFile then
 873    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 874       Rename(LastGoodSig, DestFile);
 875
 876    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 877    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 878                        DestFile, IndexTargets, MetaIndexParser);
 879
 880 }
 881                                                                         /*}}}*/
 882 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 883 {
 884    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 885
 886    // if we get a network error we fail gracefully
 887    if(Status == StatTransientNetworkError)
 888    {
 889       Item::Failed(Message,Cnf);
 890       // move the sigfile back on transient network failures
 891       if(FileExists(DestFile))
 892          Rename(LastGoodSig,Final);
 893
 894       // set the status back to , Item::Failed likes to reset it
 895       Status = pkgAcquire::Item::StatTransientNetworkError;
 896       return;
 897    }
 898
 899    // Delete any existing sigfile when the acquire failed
 900    unlink(Final.c_str());
 901
 902    // queue a pkgAcqMetaIndex with no sigfile
 903    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 904                        "", IndexTargets, MetaIndexParser);
 905
 906    if (Cnf->LocalOnly == true ||
 907        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 908    {
 909       // Ignore this
 910       Status = StatDone;
 911       Complete = false;
 912       Dequeue();
 913       return;
 914    }
 915
 916    Item::Failed(Message,Cnf);
 917 }
 918                                                                         /*}}}*/
 919 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 920                                  string URI,string URIDesc,string ShortDesc,
 921                                  string SigFile,
 922                                  const vector<struct IndexTarget*>* IndexTargets,
 923                                  indexRecords* MetaIndexParser) :
 924    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 925    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 926 {
 927    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 928    DestFile += URItoFileName(URI);
 929
 930    // Create the item
 931    Desc.Description = URIDesc;
 932    Desc.Owner = this;
 933    Desc.ShortDesc = ShortDesc;
 934    Desc.URI = URI;
 935
 936    QueueURI(Desc);
 937 }
 938                                                                         /*}}}*/
 939 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* The only header we use is the last-modified header. */
 942 string pkgAcqMetaIndex::Custom600Headers()
 943 {
 944    string Final = _config->FindDir("Dir::State::lists");
 945    Final += URItoFileName(RealURI);
 946
 947    struct stat Buf;
 948    if (stat(Final.c_str(),&Buf) != 0)
 949       return "\nIndex-File: true";
 950
 951    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 952 }
 953                                                                         /*}}}*/
 954 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 955                            pkgAcquire::MethodConfig *Cfg)
 956 {
 957    Item::Done(Message,Size,Hash,Cfg);
 958
 959    // MetaIndexes are done in two passes: one to download the
 960    // metaindex with an appropriate method, and a second to verify it
 961    // with the gpgv method
 962
 963    if (AuthPass == true)
 964    {
 965       AuthDone(Message);
 966    }
 967    else
 968    {
 969       RetrievalDone(Message);
 970       if (!Complete)
 971          // Still more retrieving to do
 972          return;
 973
 974       if (SigFile == "")
 975       {
 976          // There was no signature file, so we are finished.  Download
 977          // the indexes without verification.
 978          QueueIndexes(false);
 979       }
 980       else
 981       {
 982          // There was a signature file, so pass it to gpgv for
 983          // verification
 984
 985          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 986             std::cerr << "Metaindex acquired, queueing gpg verification ("
 987                       << SigFile << "," << DestFile << ")\n";
 988          AuthPass = true;
 989          Desc.URI = "gpgv:" + SigFile;
 990          QueueURI(Desc);
 991          Mode = "gpgv";
 992       }
 993    }
 994 }
 995                                                                         /*}}}*/
 996 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
 997 {
 998    // We have just finished downloading a Release file (it is not
 999    // verified yet)
1000
1001    string FileName = LookupTag(Message,"Filename");
1002    if (FileName.empty() == true)
1003    {
1004       Status = StatError;
1005       ErrorText = "Method gave a blank filename";
1006       return;
1007    }
1008
1009    if (FileName != DestFile)
1010    {
1011       Local = true;
1012       Desc.URI = "copy:" + FileName;
1013       QueueURI(Desc);
1014       return;
1015    }
1016
1017    // see if the download was a IMSHit
1018    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1019    Complete = true;
1020
1021    string FinalFile = _config->FindDir("Dir::State::lists");
1022    FinalFile += URItoFileName(RealURI);
1023
1024    // If we get a IMS hit we can remove the empty file in partial
1025    // othersie we move the file in place
1026    if (IMSHit)
1027       unlink(DestFile.c_str());
1028    else
1029       Rename(DestFile,FinalFile);
1030
1031    chmod(FinalFile.c_str(),0644);
1032    DestFile = FinalFile;
1033 }
1034                                                                         /*}}}*/
1035 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1036 {
1037    // At this point, the gpgv method has succeeded, so there is a
1038    // valid signature from a key in the trusted keyring.  We
1039    // perform additional verification of its contents, and use them
1040    // to verify the indexes we are about to download
1041
1042    if (!MetaIndexParser->Load(DestFile))
1043    {
1044       Status = StatAuthError;
1045       ErrorText = MetaIndexParser->ErrorText;
1046       return;
1047    }
1048
1049    if (!VerifyVendor(Message))
1050    {
1051       return;
1052    }
1053
1054    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1055       std::cerr << "Signature verification succeeded: "
1056                 << DestFile << std::endl;
1057
1058    // Download further indexes with verification
1059    QueueIndexes(true);
1060
1061    // Done, move signature file into position
1062
1063    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1064       URItoFileName(RealURI) + ".gpg";
1065    Rename(SigFile,VerifiedSigFile);
1066    chmod(VerifiedSigFile.c_str(),0644);
1067 }
1068                                                                         /*}}}*/
1069 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1070 {
1071    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1072         Target != IndexTargets->end();
1073         Target++)
1074    {
1075       HashString ExpectedIndexHash;
1076       if (verify)
1077       {
1078          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1079          if (!Record)
1080          {
1081             Status = StatAuthError;
1082             ErrorText = "Unable to find expected entry  "
1083                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1084             return;
1085          }
1086          ExpectedIndexHash = Record->Hash;
1087          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1088          {
1089             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1090             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1091          }
1092          if (ExpectedIndexHash.empty())
1093          {
1094             Status = StatAuthError;
1095             ErrorText = "Unable to find hash sum for "
1096                + (*Target)->MetaKey + " in Meta-index file";
1097             return;
1098          }
1099       }
1100
1101       // Queue Packages file (either diff or full packages files, depending
1102       // on the users option)
1103       if(_config->FindB("Acquire::PDiffs",true) == true)
1104          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1105                              (*Target)->ShortDesc, ExpectedIndexHash);
1106       else
1107          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1108                             (*Target)->ShortDesc, ExpectedIndexHash);
1109    }
1110 }
1111                                                                         /*}}}*/
1112 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1113 {
1114 //    // Maybe this should be made available from above so we don't have
1115 //    // to read and parse it every time?
1116 //    pkgVendorList List;
1117 //    List.ReadMainList();
1118
1119 //    const Vendor* Vndr = NULL;
1120 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1121 //    {
1122 //       string::size_type pos = (*I).find("VALIDSIG ");
1123 //       if (_config->FindB("Debug::Vendor", false))
1124 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1125 //                    << std::endl;
1126 //       if (pos != std::string::npos)
1127 //       {
1128 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1129 //          if (_config->FindB("Debug::Vendor", false))
1130 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1131 //                std::endl;
1132 //          Vndr = List.FindVendor(Fingerprint) != "";
1133 //          if (Vndr != NULL);
1134 //          break;
1135 //       }
1136 //    }
1137    string::size_type pos;
1138
1139    // check for missing sigs (that where not fatal because otherwise we had
1140    // bombed earlier)
1141    string missingkeys;
1142    string msg = _("There is no public key available for the "
1143                   "following key IDs:\n");
1144    pos = Message.find("NO_PUBKEY ");
1145    if (pos != std::string::npos)
1146    {
1147       string::size_type start = pos+strlen("NO_PUBKEY ");
1148       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1149       missingkeys += (Fingerprint);
1150    }
1151    if(!missingkeys.empty())
1152       _error->Warning("%s", string(msg+missingkeys).c_str());
1153
1154    string Transformed = MetaIndexParser->GetExpectedDist();
1155
1156    if (Transformed == "../project/experimental")
1157    {
1158       Transformed = "experimental";
1159    }
1160
1161    pos = Transformed.rfind('/');
1162    if (pos != string::npos)
1163    {
1164       Transformed = Transformed.substr(0, pos);
1165    }
1166
1167    if (Transformed == ".")
1168    {
1169       Transformed = "";
1170    }
1171
1172    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1173    {
1174       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1175       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1176       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1177    }
1178
1179    if (MetaIndexParser->CheckDist(Transformed) == false)
1180    {
1181       // This might become fatal one day
1182 //       Status = StatAuthError;
1183 //       ErrorText = "Conflicting distribution; expected "
1184 //          + MetaIndexParser->GetExpectedDist() + " but got "
1185 //          + MetaIndexParser->GetDist();
1186 //       return false;
1187       if (!Transformed.empty())
1188       {
1189          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1190                          Desc.Description.c_str(),
1191                          Transformed.c_str(),
1192                          MetaIndexParser->GetDist().c_str());
1193       }
1194    }
1195
1196    return true;
1197 }
1198                                                                         /*}}}*/
1199 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1200 // ---------------------------------------------------------------------
1201 /* */
1202 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1203 {
1204    if (AuthPass == true)
1205    {
1206       // if we fail the authentication but got the file via a IMS-Hit
1207       // this means that the file wasn't downloaded and that it might be
1208       // just stale (server problem, proxy etc). we delete what we have
1209       // queue it again without i-m-s
1210       // alternatively we could just unlink the file and let the user try again
1211       if (IMSHit)
1212       {
1213          Complete = false;
1214          Local = false;
1215          AuthPass = false;
1216          unlink(DestFile.c_str());
1217
1218          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1219          DestFile += URItoFileName(RealURI);
1220          Desc.URI = RealURI;
1221          QueueURI(Desc);
1222          return;
1223       }
1224
1225       // gpgv method failed
1226       _error->Warning("GPG error: %s: %s",
1227                       Desc.Description.c_str(),
1228                       LookupTag(Message,"Message").c_str());
1229
1230    }
1231
1232    // No Release file was present, or verification failed, so fall
1233    // back to queueing Packages files without verification
1234    QueueIndexes(false);
1235 }
1236                                                                         /*}}}*/
1237 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1238 // ---------------------------------------------------------------------
1239 /* This just sets up the initial fetch environment and queues the first
1240    possibilitiy */
1241 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1242                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1243                              string &StoreFilename) :
1244                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1245                StoreFilename(StoreFilename), Vf(Version.FileList()),
1246                Trusted(false)
1247 {
1248    Retries = _config->FindI("Acquire::Retries",0);
1249
1250    if (Version.Arch() == 0)
1251    {
1252       _error->Error(_("I wasn't able to locate a file for the %s package. "
1253                       "This might mean you need to manually fix this package. "
1254                       "(due to missing arch)"),
1255                     Version.ParentPkg().Name());
1256       return;
1257    }
1258
1259    /* We need to find a filename to determine the extension. We make the
1260       assumption here that all the available sources for this version share
1261       the same extension.. */
1262    // Skip not source sources, they do not have file fields.
1263    for (; Vf.end() == false; Vf++)
1264    {
1265       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1266          continue;
1267       break;
1268    }
1269
1270    // Does not really matter here.. we are going to fail out below
1271    if (Vf.end() != true)
1272    {
1273       // If this fails to get a file name we will bomb out below.
1274       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1275       if (_error->PendingError() == true)
1276          return;
1277
1278       // Generate the final file name as: package_version_arch.foo
1279       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1280                       QuoteString(Version.VerStr(),"_:") + '_' +
1281                       QuoteString(Version.Arch(),"_:.") +
1282                       "." + flExtension(Parse.FileName());
1283    }
1284
1285    // check if we have one trusted source for the package. if so, switch
1286    // to "TrustedOnly" mode
1287    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1288    {
1289       pkgIndexFile *Index;
1290       if (Sources->FindIndex(i.File(),Index) == false)
1291          continue;
1292       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1293       {
1294          std::cerr << "Checking index: " << Index->Describe()
1295                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1296       }
1297       if (Index->IsTrusted()) {
1298          Trusted = true;
1299          break;
1300       }
1301    }
1302
1303    // "allow-unauthenticated" restores apts old fetching behaviour
1304    // that means that e.g. unauthenticated file:// uris are higher
1305    // priority than authenticated http:// uris
1306    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1307       Trusted = false;
1308
1309    // Select a source
1310    if (QueueNext() == false && _error->PendingError() == false)
1311       _error->Error(_("I wasn't able to locate file for the %s package. "
1312                     "This might mean you need to manually fix this package."),
1313                     Version.ParentPkg().Name());
1314 }
1315                                                                         /*}}}*/
1316 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1317 // ---------------------------------------------------------------------
1318 /* This queues the next available file version for download. It checks if
1319    the archive is already available in the cache and stashs the MD5 for
1320    checking later. */
1321 bool pkgAcqArchive::QueueNext()
1322 {
1323    for (; Vf.end() == false; Vf++)
1324    {
1325       // Ignore not source sources
1326       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1327          continue;
1328
1329       // Try to cross match against the source list
1330       pkgIndexFile *Index;
1331       if (Sources->FindIndex(Vf.File(),Index) == false)
1332             continue;
1333
1334       // only try to get a trusted package from another source if that source
1335       // is also trusted
1336       if(Trusted && !Index->IsTrusted())
1337          continue;
1338
1339       // Grab the text package record
1340       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1341       if (_error->PendingError() == true)
1342          return false;
1343
1344       string PkgFile = Parse.FileName();
1345       if(Parse.SHA256Hash() != "")
1346          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1347       else if (Parse.SHA1Hash() != "")
1348          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1349       else
1350          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1351       if (PkgFile.empty() == true)
1352          return _error->Error(_("The package index files are corrupted. No Filename: "
1353                               "field for package %s."),
1354                               Version.ParentPkg().Name());
1355
1356       Desc.URI = Index->ArchiveURI(PkgFile);
1357       Desc.Description = Index->ArchiveInfo(Version);
1358       Desc.Owner = this;
1359       Desc.ShortDesc = Version.ParentPkg().Name();
1360
1361       // See if we already have the file. (Legacy filenames)
1362       FileSize = Version->Size;
1363       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1364       struct stat Buf;
1365       if (stat(FinalFile.c_str(),&Buf) == 0)
1366       {
1367          // Make sure the size matches
1368          if ((unsigned)Buf.st_size == Version->Size)
1369          {
1370             Complete = true;
1371             Local = true;
1372             Status = StatDone;
1373             StoreFilename = DestFile = FinalFile;
1374             return true;
1375          }
1376
1377          /* Hmm, we have a file and its size does not match, this means it is
1378             an old style mismatched arch */
1379          unlink(FinalFile.c_str());
1380       }
1381
1382       // Check it again using the new style output filenames
1383       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1384       if (stat(FinalFile.c_str(),&Buf) == 0)
1385       {
1386          // Make sure the size matches
1387          if ((unsigned)Buf.st_size == Version->Size)
1388          {
1389             Complete = true;
1390             Local = true;
1391             Status = StatDone;
1392             StoreFilename = DestFile = FinalFile;
1393             return true;
1394          }
1395
1396          /* Hmm, we have a file and its size does not match, this shouldnt
1397             happen.. */
1398          unlink(FinalFile.c_str());
1399       }
1400
1401       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1402
1403       // Check the destination file
1404       if (stat(DestFile.c_str(),&Buf) == 0)
1405       {
1406          // Hmm, the partial file is too big, erase it
1407          if ((unsigned)Buf.st_size > Version->Size)
1408             unlink(DestFile.c_str());
1409          else
1410             PartialSize = Buf.st_size;
1411       }
1412
1413       // Create the item
1414       Local = false;
1415       Desc.URI = Index->ArchiveURI(PkgFile);
1416       Desc.Description = Index->ArchiveInfo(Version);
1417       Desc.Owner = this;
1418       Desc.ShortDesc = Version.ParentPkg().Name();
1419       QueueURI(Desc);
1420
1421       Vf++;
1422       return true;
1423    }
1424    return false;
1425 }
1426                                                                         /*}}}*/
1427 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1428 // ---------------------------------------------------------------------
1429 /* */
1430 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1431                          pkgAcquire::MethodConfig *Cfg)
1432 {
1433    Item::Done(Message,Size,CalcHash,Cfg);
1434
1435    // Check the size
1436    if (Size != Version->Size)
1437    {
1438       Status = StatError;
1439       ErrorText = _("Size mismatch");
1440       return;
1441    }
1442
1443    // Check the hash
1444    if(ExpectedHash.toStr() != CalcHash)
1445    {
1446       Status = StatError;
1447       ErrorText = _("Hash Sum mismatch");
1448       if(FileExists(DestFile))
1449          Rename(DestFile,DestFile + ".FAILED");
1450       return;
1451    }
1452
1453    // Grab the output filename
1454    string FileName = LookupTag(Message,"Filename");
1455    if (FileName.empty() == true)
1456    {
1457       Status = StatError;
1458       ErrorText = "Method gave a blank filename";
1459       return;
1460    }
1461
1462    Complete = true;
1463
1464    // Reference filename
1465    if (FileName != DestFile)
1466    {
1467       StoreFilename = DestFile = FileName;
1468       Local = true;
1469       return;
1470    }
1471
1472    // Done, move it into position
1473    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1474    FinalFile += flNotDir(StoreFilename);
1475    Rename(DestFile,FinalFile);
1476
1477    StoreFilename = DestFile = FinalFile;
1478    Complete = true;
1479 }
1480                                                                         /*}}}*/
1481 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1482 // ---------------------------------------------------------------------
1483 /* Here we try other sources */
1484 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1485 {
1486    ErrorText = LookupTag(Message,"Message");
1487
1488    /* We don't really want to retry on failed media swaps, this prevents
1489       that. An interesting observation is that permanent failures are not
1490       recorded. */
1491    if (Cnf->Removable == true &&
1492        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1493    {
1494       // Vf = Version.FileList();
1495       while (Vf.end() == false) Vf++;
1496       StoreFilename = string();
1497       Item::Failed(Message,Cnf);
1498       return;
1499    }
1500
1501    if (QueueNext() == false)
1502    {
1503       // This is the retry counter
1504       if (Retries != 0 &&
1505           Cnf->LocalOnly == false &&
1506           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1507       {
1508          Retries--;
1509          Vf = Version.FileList();
1510          if (QueueNext() == true)
1511             return;
1512       }
1513
1514       StoreFilename = string();
1515       Item::Failed(Message,Cnf);
1516    }
1517 }
1518                                                                         /*}}}*/
1519 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1520 // ---------------------------------------------------------------------
1521 bool pkgAcqArchive::IsTrusted()
1522 {
1523    return Trusted;
1524 }
1525                                                                         /*}}}*/
1526 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1527 // ---------------------------------------------------------------------
1528 /* */
1529 void pkgAcqArchive::Finished()
1530 {
1531    if (Status == pkgAcquire::Item::StatDone &&
1532        Complete == true)
1533       return;
1534    StoreFilename = string();
1535 }
1536                                                                         /*}}}*/
1537 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1538 // ---------------------------------------------------------------------
1539 /* The file is added to the queue */
1540 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1541                        unsigned long Size,string Dsc,string ShortDesc,
1542                        const string &DestDir, const string &DestFilename,
1543                        bool IsIndexFile) :
1544                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1545 {
1546    Retries = _config->FindI("Acquire::Retries",0);
1547
1548    if(!DestFilename.empty())
1549       DestFile = DestFilename;
1550    else if(!DestDir.empty())
1551       DestFile = DestDir + "/" + flNotDir(URI);
1552    else
1553       DestFile = flNotDir(URI);
1554
1555    // Create the item
1556    Desc.URI = URI;
1557    Desc.Description = Dsc;
1558    Desc.Owner = this;
1559
1560    // Set the short description to the archive component
1561    Desc.ShortDesc = ShortDesc;
1562
1563    // Get the transfer sizes
1564    FileSize = Size;
1565    struct stat Buf;
1566    if (stat(DestFile.c_str(),&Buf) == 0)
1567    {
1568       // Hmm, the partial file is too big, erase it
1569       if ((unsigned)Buf.st_size > Size)
1570          unlink(DestFile.c_str());
1571       else
1572          PartialSize = Buf.st_size;
1573    }
1574
1575    QueueURI(Desc);
1576 }
1577                                                                         /*}}}*/
1578 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1579 // ---------------------------------------------------------------------
1580 /* */
1581 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1582                       pkgAcquire::MethodConfig *Cnf)
1583 {
1584    Item::Done(Message,Size,CalcHash,Cnf);
1585
1586    // Check the hash
1587    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1588    {
1589       Status = StatError;
1590       ErrorText = "Hash Sum mismatch";
1591       Rename(DestFile,DestFile + ".FAILED");
1592       return;
1593    }
1594
1595    string FileName = LookupTag(Message,"Filename");
1596    if (FileName.empty() == true)
1597    {
1598       Status = StatError;
1599       ErrorText = "Method gave a blank filename";
1600       return;
1601    }
1602
1603    Complete = true;
1604
1605    // The files timestamp matches
1606    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1607       return;
1608
1609    // We have to copy it into place
1610    if (FileName != DestFile)
1611    {
1612       Local = true;
1613       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1614           Cnf->Removable == true)
1615       {
1616          Desc.URI = "copy:" + FileName;
1617          QueueURI(Desc);
1618          return;
1619       }
1620
1621       // Erase the file if it is a symlink so we can overwrite it
1622       struct stat St;
1623       if (lstat(DestFile.c_str(),&St) == 0)
1624       {
1625          if (S_ISLNK(St.st_mode) != 0)
1626             unlink(DestFile.c_str());
1627       }
1628
1629       // Symlink the file
1630       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1631       {
1632          ErrorText = "Link to " + DestFile + " failure ";
1633          Status = StatError;
1634          Complete = false;
1635       }
1636    }
1637 }
1638                                                                         /*}}}*/
1639 // AcqFile::Failed - Failure handler                                    /*{{{*/
1640 // ---------------------------------------------------------------------
1641 /* Here we try other sources */
1642 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1643 {
1644    ErrorText = LookupTag(Message,"Message");
1645
1646    // This is the retry counter
1647    if (Retries != 0 &&
1648        Cnf->LocalOnly == false &&
1649        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1650    {
1651       Retries--;
1652       QueueURI(Desc);
1653       return;
1654    }
1655
1656    Item::Failed(Message,Cnf);
1657 }
1658                                                                         /*}}}*/
1659 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1660 // ---------------------------------------------------------------------
1661 /* The only header we use is the last-modified header. */
1662 string pkgAcqFile::Custom600Headers()
1663 {
1664    if (IsIndexFile)
1665       return "\nIndex-File: true";
1666 }
1667                                                                         /*}}}*/