apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Acquire::Item::Item - Constructor                                    /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  44                        PartialSize(0), Mode(0), ID(0), Complete(false),
  45                        Local(false), QueueCounter(0)
  46 {
  47    Owner->Add(this);
  48    Status = StatIdle;
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* */
  54 pkgAcquire::Item::~Item()
  55 {
  56    Owner->Remove(this);
  57 }
  58                                                                         /*}}}*/
  59 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* We return to an idle state if there are still other queues that could
  62    fetch this object */
  63 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  64 {
  65    Status = StatIdle;
  66    ErrorText = LookupTag(Message,"Message");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83 }
  84                                                                         /*}}}*/
  85 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Stash status and the file size. Note that setting Complete means
  88    sub-phases of the acquire process such as decompresion are operating */
  89 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  90 {
  91    Status = StatFetching;
  92    if (FileSize == 0 && Complete == false)
  93       FileSize = Size;
  94 }
  95                                                                         /*}}}*/
  96 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 100                             pkgAcquire::MethodConfig *Cnf)
 101 {
 102    // We just downloaded something..
 103    string FileName = LookupTag(Message,"Filename");
 104    // we only inform the Log class if it was actually not a local thing
 105    if (Complete == false && !Local && FileName == DestFile)
 106    {
 107       if (Owner->Log != 0)
 108          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 109    }
 110
 111    if (FileSize == 0)
 112       FileSize= Size;
 113
 114    Status = StatDone;
 115    ErrorText = string();
 116    Owner->Dequeue(this);
 117 }
 118                                                                         /*}}}*/
 119 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This helper function is used by alot of item methods as thier final
 122    step */
 123 void pkgAcquire::Item::Rename(string From,string To)
 124 {
 125    if (rename(From.c_str(),To.c_str()) != 0)
 126    {
 127       char S[300];
 128       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 129               From.c_str(),To.c_str());
 130       Status = StatError;
 131       ErrorText = S;
 132    }
 133 }
 134                                                                         /*}}}*/
 135 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* Get the DiffIndex file first and see if there are patches availabe
 138  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 139  * patches. If anything goes wrong in that process, it will fall back to
 140  * the original packages file
 141  */
 142 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 143                                  string URI,string URIDesc,string ShortDesc,
 144                                  HashString ExpectedHash)
 145    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 146      Description(URIDesc)
 147 {
 148
 149    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 150
 151    Desc.Description = URIDesc + "/DiffIndex";
 152    Desc.Owner = this;
 153    Desc.ShortDesc = ShortDesc;
 154    Desc.URI = URI + ".diff/Index";
 155
 156    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 157    DestFile += URItoFileName(URI) + string(".DiffIndex");
 158
 159    if(Debug)
 160       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 161
 162    // look for the current package file
 163    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 164    CurrentPackagesFile += URItoFileName(RealURI);
 165
 166    // FIXME: this file:/ check is a hack to prevent fetching
 167    //        from local sources. this is really silly, and
 168    //        should be fixed cleanly as soon as possible
 169    if(!FileExists(CurrentPackagesFile) ||
 170       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 171    {
 172       // we don't have a pkg file or we don't want to queue
 173       if(Debug)
 174          std::clog << "No index file, local or canceld by user" << std::endl;
 175       Failed("", NULL);
 176       return;
 177    }
 178
 179    if(Debug)
 180       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 181                 << CurrentPackagesFile << std::endl;
 182
 183    QueueURI(Desc);
 184
 185 }
 186                                                                         /*}}}*/
 187 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* The only header we use is the last-modified header. */
 190 string pkgAcqDiffIndex::Custom600Headers()
 191 {
 192    string Final = _config->FindDir("Dir::State::lists");
 193    Final += URItoFileName(RealURI) + string(".IndexDiff");
 194
 195    if(Debug)
 196       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 197
 198    struct stat Buf;
 199    if (stat(Final.c_str(),&Buf) != 0)
 200       return "\nIndex-File: true";
 201
 202    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 203 }
 204                                                                         /*}}}*/
 205 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 206 {
 207    if(Debug)
 208       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 209                 << std::endl;
 210
 211    pkgTagSection Tags;
 212    string ServerSha1;
 213    vector<DiffInfo> available_patches;
 214
 215    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 216    pkgTagFile TF(&Fd);
 217    if (_error->PendingError() == true)
 218       return false;
 219
 220    if(TF.Step(Tags) == true)
 221    {
 222       bool found = false;
 223       DiffInfo d;
 224       string size;
 225
 226       string const tmp = Tags.FindS("SHA1-Current");
 227       std::stringstream ss(tmp);
 228       ss >> ServerSha1 >> size;
 229       unsigned long const ServerSize = atol(size.c_str());
 230
 231       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 232       SHA1Summation SHA1;
 233       SHA1.AddFD(fd.Fd(), fd.Size());
 234       string const local_sha1 = SHA1.Result();
 235
 236       if(local_sha1 == ServerSha1)
 237       {
 238          // we have the same sha1 as the server
 239          if(Debug)
 240             std::clog << "Package file is up-to-date" << std::endl;
 241          // set found to true, this will queue a pkgAcqIndexDiffs with
 242          // a empty availabe_patches
 243          found = true;
 244       }
 245       else
 246       {
 247          if(Debug)
 248             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 249
 250          // check the historie and see what patches we need
 251          string const history = Tags.FindS("SHA1-History");
 252          std::stringstream hist(history);
 253          while(hist >> d.sha1 >> size >> d.file)
 254          {
 255             // read until the first match is found
 256             // from that point on, we probably need all diffs
 257             if(d.sha1 == local_sha1)
 258                found=true;
 259             else if (found == false)
 260                continue;
 261
 262             if(Debug)
 263                std::clog << "Need to get diff: " << d.file << std::endl;
 264             available_patches.push_back(d);
 265          }
 266
 267          if (available_patches.empty() == false)
 268          {
 269             // patching with too many files is rather slow compared to a fast download
 270             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 271             if (fileLimit != 0 && fileLimit < available_patches.size())
 272             {
 273                if (Debug)
 274                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 275                         << ") so fallback to complete download" << std::endl;
 276                return false;
 277             }
 278
 279             // see if the patches are too big
 280             found = false; // it was true and it will be true again at the end
 281             d = *available_patches.begin();
 282             string const firstPatch = d.file;
 283             unsigned long patchesSize = 0;
 284             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 285             while(patches >> d.sha1 >> size >> d.file)
 286             {
 287                if (firstPatch == d.file)
 288                   found = true;
 289                else if (found == false)
 290                   continue;
 291
 292                patchesSize += atol(size.c_str());
 293             }
 294             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 295             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 296             {
 297                if (Debug)
 298                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 299                         << ") so fallback to complete download" << std::endl;
 300                return false;
 301             }
 302          }
 303       }
 304
 305       // we have something, queue the next diff
 306       if(found)
 307       {
 308          // queue the diffs
 309          string::size_type const last_space = Description.rfind(" ");
 310          if(last_space != string::npos)
 311             Description.erase(last_space, Description.size()-last_space);
 312          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 313                               ExpectedHash, ServerSha1, available_patches);
 314          Complete = false;
 315          Status = StatDone;
 316          Dequeue();
 317          return true;
 318       }
 319    }
 320
 321    // Nothing found, report and return false
 322    // Failing here is ok, if we return false later, the full
 323    // IndexFile is queued
 324    if(Debug)
 325       std::clog << "Can't find a patch in the index file" << std::endl;
 326    return false;
 327 }
 328                                                                         /*}}}*/
 329 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 330 {
 331    if(Debug)
 332       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 333                 << "Falling back to normal index file aquire" << std::endl;
 334
 335    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 336                    ExpectedHash);
 337
 338    Complete = false;
 339    Status = StatDone;
 340    Dequeue();
 341 }
 342                                                                         /*}}}*/
 343 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 344                            pkgAcquire::MethodConfig *Cnf)
 345 {
 346    if(Debug)
 347       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 348
 349    Item::Done(Message,Size,Md5Hash,Cnf);
 350
 351    string FinalFile;
 352    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 353
 354    // sucess in downloading the index
 355    // rename the index
 356    FinalFile += string(".IndexDiff");
 357    if(Debug)
 358       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 359                 << std::endl;
 360    Rename(DestFile,FinalFile);
 361    chmod(FinalFile.c_str(),0644);
 362    DestFile = FinalFile;
 363
 364    if(!ParseDiffIndex(DestFile))
 365       return Failed("", NULL);
 366
 367    Complete = true;
 368    Status = StatDone;
 369    Dequeue();
 370    return;
 371 }
 372                                                                         /*}}}*/
 373 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* The package diff is added to the queue. one object is constructed
 376  * for each diff and the index
 377  */
 378 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 379                                    string URI,string URIDesc,string ShortDesc,
 380                                    HashString ExpectedHash,
 381                                    string ServerSha1,
 382                                    vector<DiffInfo> diffs)
 383    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 384      available_patches(diffs), ServerSha1(ServerSha1)
 385 {
 386
 387    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 388    DestFile += URItoFileName(URI);
 389
 390    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 391
 392    Description = URIDesc;
 393    Desc.Owner = this;
 394    Desc.ShortDesc = ShortDesc;
 395
 396    if(available_patches.size() == 0)
 397    {
 398       // we are done (yeah!)
 399       Finish(true);
 400    }
 401    else
 402    {
 403       // get the next diff
 404       State = StateFetchDiff;
 405       QueueNextDiff();
 406    }
 407 }
 408                                                                         /*}}}*/
 409 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 410 {
 411    if(Debug)
 412       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 413                 << "Falling back to normal index file aquire" << std::endl;
 414    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 415                    ExpectedHash);
 416    Finish();
 417 }
 418                                                                         /*}}}*/
 419 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 420 void pkgAcqIndexDiffs::Finish(bool allDone)
 421 {
 422    // we restore the original name, this is required, otherwise
 423    // the file will be cleaned
 424    if(allDone)
 425    {
 426       DestFile = _config->FindDir("Dir::State::lists");
 427       DestFile += URItoFileName(RealURI);
 428
 429       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 430       {
 431          Status = StatAuthError;
 432          ErrorText = _("MD5Sum mismatch");
 433          Rename(DestFile,DestFile + ".FAILED");
 434          Dequeue();
 435          return;
 436       }
 437
 438       // this is for the "real" finish
 439       Complete = true;
 440       Status = StatDone;
 441       Dequeue();
 442       if(Debug)
 443          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 444       return;
 445    }
 446
 447    if(Debug)
 448       std::clog << "Finishing: " << Desc.URI << std::endl;
 449    Complete = false;
 450    Status = StatDone;
 451    Dequeue();
 452    return;
 453 }
 454                                                                         /*}}}*/
 455 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 456 {
 457
 458    // calc sha1 of the just patched file
 459    string FinalFile = _config->FindDir("Dir::State::lists");
 460    FinalFile += URItoFileName(RealURI);
 461
 462    FileFd fd(FinalFile, FileFd::ReadOnly);
 463    SHA1Summation SHA1;
 464    SHA1.AddFD(fd.Fd(), fd.Size());
 465    string local_sha1 = string(SHA1.Result());
 466    if(Debug)
 467       std::clog << "QueueNextDiff: "
 468                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 469
 470    // final file reached before all patches are applied
 471    if(local_sha1 == ServerSha1)
 472    {
 473       Finish(true);
 474       return true;
 475    }
 476
 477    // remove all patches until the next matching patch is found
 478    // this requires the Index file to be ordered
 479    for(vector<DiffInfo>::iterator I=available_patches.begin();
 480        available_patches.size() > 0 &&
 481           I != available_patches.end() &&
 482           (*I).sha1 != local_sha1;
 483        I++)
 484    {
 485       available_patches.erase(I);
 486    }
 487
 488    // error checking and falling back if no patch was found
 489    if(available_patches.size() == 0)
 490    {
 491       Failed("", NULL);
 492       return false;
 493    }
 494
 495    // queue the right diff
 496    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 497    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 498    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 499    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 500
 501    if(Debug)
 502       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 503
 504    QueueURI(Desc);
 505
 506    return true;
 507 }
 508                                                                         /*}}}*/
 509 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 510                             pkgAcquire::MethodConfig *Cnf)
 511 {
 512    if(Debug)
 513       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 514
 515    Item::Done(Message,Size,Md5Hash,Cnf);
 516
 517    string FinalFile;
 518    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 519
 520    // sucess in downloading a diff, enter ApplyDiff state
 521    if(State == StateFetchDiff)
 522    {
 523
 524       if(Debug)
 525          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 526
 527       string FileName = LookupTag(Message,"Filename");
 528       State = StateUnzipDiff;
 529       Local = true;
 530       Desc.URI = "gzip:" + FileName;
 531       DestFile += ".decomp";
 532       QueueURI(Desc);
 533       Mode = "gzip";
 534       return;
 535    }
 536
 537    // sucess in downloading a diff, enter ApplyDiff state
 538    if(State == StateUnzipDiff)
 539    {
 540
 541       // rred excepts the patch as $FinalFile.ed
 542       Rename(DestFile,FinalFile+".ed");
 543
 544       if(Debug)
 545          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 546
 547       State = StateApplyDiff;
 548       Local = true;
 549       Desc.URI = "rred:" + FinalFile;
 550       QueueURI(Desc);
 551       Mode = "rred";
 552       return;
 553    }
 554
 555
 556    // success in download/apply a diff, queue next (if needed)
 557    if(State == StateApplyDiff)
 558    {
 559       // remove the just applied patch
 560       available_patches.erase(available_patches.begin());
 561
 562       // move into place
 563       if(Debug)
 564       {
 565          std::clog << "Moving patched file in place: " << std::endl
 566                    << DestFile << " -> " << FinalFile << std::endl;
 567       }
 568       Rename(DestFile,FinalFile);
 569       chmod(FinalFile.c_str(),0644);
 570
 571       // see if there is more to download
 572       if(available_patches.size() > 0) {
 573          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 574                               ExpectedHash, ServerSha1, available_patches);
 575          return Finish();
 576       } else
 577          return Finish(true);
 578    }
 579 }
 580                                                                         /*}}}*/
 581 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* The package file is added to the queue and a second class is
 584    instantiated to fetch the revision file */
 585 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 586                          string URI,string URIDesc,string ShortDesc,
 587                          HashString ExpectedHash, string comprExt)
 588    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 589 {
 590    Decompression = false;
 591    Erase = false;
 592
 593    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 594    DestFile += URItoFileName(URI);
 595
 596    if(comprExt.empty())
 597    {
 598       // autoselect the compression method
 599       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 600       if (types.empty() == true)
 601          comprExt = "plain";
 602       else
 603          comprExt = "." + types[0];
 604    }
 605    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 606
 607    Desc.URI = URI + CompressionExtension;
 608
 609    Desc.Description = URIDesc;
 610    Desc.Owner = this;
 611    Desc.ShortDesc = ShortDesc;
 612
 613    QueueURI(Desc);
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The only header we use is the last-modified header. */
 619 string pkgAcqIndex::Custom600Headers()
 620 {
 621    string Final = _config->FindDir("Dir::State::lists");
 622    Final += URItoFileName(RealURI);
 623
 624    struct stat Buf;
 625    if (stat(Final.c_str(),&Buf) != 0)
 626       return "\nIndex-File: true";
 627
 628    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 629 }
 630                                                                         /*}}}*/
 631 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 632 {
 633    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 634
 635    for (std::vector<std::string>::const_iterator t = types.begin();
 636         t != types.end(); t++)
 637    {
 638       // jump over all already tried compression types
 639       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 640       if(Desc.URI.substr(nameLen) != *t)
 641          continue;
 642
 643       // we want to try it with the next extension (and make sure to
 644       // not skip over the end)
 645       t++;
 646       if (t == types.end())
 647          break;
 648
 649       // queue new download
 650       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 651       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 652       ExpectedHash, string(".").append(*t));
 653
 654       Status = StatDone;
 655       Complete = false;
 656       Dequeue();
 657       return;
 658    }
 659
 660    // on decompression failure, remove bad versions in partial/
 661    if(Decompression && Erase) {
 662       string s = _config->FindDir("Dir::State::lists") + "partial/";
 663       s += URItoFileName(RealURI);
 664       unlink(s.c_str());
 665    }
 666
 667    Item::Failed(Message,Cnf);
 668 }
 669                                                                         /*}}}*/
 670 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* This goes through a number of states.. On the initial fetch the
 673    method could possibly return an alternate filename which points
 674    to the uncompressed version of the file. If this is so the file
 675    is copied into the partial directory. In all other cases the file
 676    is decompressed with a gzip uri. */
 677 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 678                        pkgAcquire::MethodConfig *Cfg)
 679 {
 680    Item::Done(Message,Size,Hash,Cfg);
 681
 682    if (Decompression == true)
 683    {
 684       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 685       {
 686          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 687          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 688       }
 689
 690       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 691       {
 692          Status = StatAuthError;
 693          ErrorText = _("Hash Sum mismatch");
 694          Rename(DestFile,DestFile + ".FAILED");
 695          return;
 696       }
 697       // Done, move it into position
 698       string FinalFile = _config->FindDir("Dir::State::lists");
 699       FinalFile += URItoFileName(RealURI);
 700       Rename(DestFile,FinalFile);
 701       chmod(FinalFile.c_str(),0644);
 702
 703       /* We restore the original name to DestFile so that the clean operation
 704          will work OK */
 705       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 706       DestFile += URItoFileName(RealURI);
 707
 708       // Remove the compressed version.
 709       if (Erase == true)
 710          unlink(DestFile.c_str());
 711       return;
 712    }
 713
 714    Erase = false;
 715    Complete = true;
 716
 717    // Handle the unzipd case
 718    string FileName = LookupTag(Message,"Alt-Filename");
 719    if (FileName.empty() == false)
 720    {
 721       // The files timestamp matches
 722       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 723          return;
 724       Decompression = true;
 725       Local = true;
 726       DestFile += ".decomp";
 727       Desc.URI = "copy:" + FileName;
 728       QueueURI(Desc);
 729       Mode = "copy";
 730       return;
 731    }
 732
 733    FileName = LookupTag(Message,"Filename");
 734    if (FileName.empty() == true)
 735    {
 736       Status = StatError;
 737       ErrorText = "Method gave a blank filename";
 738    }
 739
 740    // The files timestamp matches
 741    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 742       return;
 743
 744    if (FileName == DestFile)
 745       Erase = true;
 746    else
 747       Local = true;
 748
 749    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 750    string decompProg;
 751
 752    // get the binary name for your used compression type
 753    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 754    if(decompProg.empty() == false);
 755    // flExtensions returns the full name if no extension is found
 756    // this is why we have this complicated compare operation here
 757    // FIMXE: add a new flJustExtension() that return "" if no
 758    //        extension is found and use that above so that it can
 759    //        be tested against ""
 760    else if(compExt == flNotDir(URI(Desc.URI).Path))
 761       decompProg = "copy";
 762    else {
 763       _error->Error("Unsupported extension: %s", compExt.c_str());
 764       return;
 765    }
 766
 767    Decompression = true;
 768    DestFile += ".decomp";
 769    Desc.URI = decompProg + ":" + FileName;
 770    QueueURI(Desc);
 771    Mode = decompProg.c_str();
 772 }
 773                                                                         /*}}}*/
 774 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* The Translation file is added to the queue */
 777 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 778                             string URI,string URIDesc,string ShortDesc)
 779   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 780 {
 781 }
 782                                                                         /*}}}*/
 783 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 784 // ---------------------------------------------------------------------
 785 /* */
 786 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 787 {
 788    if (Cnf->LocalOnly == true ||
 789        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 790    {
 791       // Ignore this
 792       Status = StatDone;
 793       Complete = false;
 794       Dequeue();
 795       return;
 796    }
 797
 798    Item::Failed(Message,Cnf);
 799 }
 800                                                                         /*}}}*/
 801 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 802                              string URI,string URIDesc,string ShortDesc,
 803                              string MetaIndexURI, string MetaIndexURIDesc,
 804                              string MetaIndexShortDesc,
 805                              const vector<IndexTarget*>* IndexTargets,
 806                              indexRecords* MetaIndexParser) :
 807    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 808    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 809    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 810 {
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(URI);
 813
 814    // remove any partial downloaded sig-file in partial/.
 815    // it may confuse proxies and is too small to warrant a
 816    // partial download anyway
 817    unlink(DestFile.c_str());
 818
 819    // Create the item
 820    Desc.Description = URIDesc;
 821    Desc.Owner = this;
 822    Desc.ShortDesc = ShortDesc;
 823    Desc.URI = URI;
 824
 825    string Final = _config->FindDir("Dir::State::lists");
 826    Final += URItoFileName(RealURI);
 827    struct stat Buf;
 828    if (stat(Final.c_str(),&Buf) == 0)
 829    {
 830       // File was already in place.  It needs to be re-downloaded/verified
 831       // because Release might have changed, we do give it a differnt
 832       // name than DestFile because otherwise the http method will
 833       // send If-Range requests and there are too many broken servers
 834       // out there that do not understand them
 835       LastGoodSig = DestFile+".reverify";
 836       Rename(Final,LastGoodSig);
 837    }
 838
 839    QueueURI(Desc);
 840 }
 841                                                                         /*}}}*/
 842 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 843 // ---------------------------------------------------------------------
 844 /* The only header we use is the last-modified header. */
 845 string pkgAcqMetaSig::Custom600Headers()
 846 {
 847    struct stat Buf;
 848    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 849       return "\nIndex-File: true";
 850
 851    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 852 }
 853
 854 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 855                          pkgAcquire::MethodConfig *Cfg)
 856 {
 857    Item::Done(Message,Size,MD5,Cfg);
 858
 859    string FileName = LookupTag(Message,"Filename");
 860    if (FileName.empty() == true)
 861    {
 862       Status = StatError;
 863       ErrorText = "Method gave a blank filename";
 864       return;
 865    }
 866
 867    if (FileName != DestFile)
 868    {
 869       // We have to copy it into place
 870       Local = true;
 871       Desc.URI = "copy:" + FileName;
 872       QueueURI(Desc);
 873       return;
 874    }
 875
 876    Complete = true;
 877
 878    // put the last known good file back on i-m-s hit (it will
 879    // be re-verified again)
 880    // Else do nothing, we have the new file in DestFile then
 881    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 882       Rename(LastGoodSig, DestFile);
 883
 884    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 885    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 886                        DestFile, IndexTargets, MetaIndexParser);
 887
 888 }
 889                                                                         /*}}}*/
 890 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 891 {
 892    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 893
 894    // if we get a network error we fail gracefully
 895    if(Status == StatTransientNetworkError)
 896    {
 897       Item::Failed(Message,Cnf);
 898       // move the sigfile back on transient network failures
 899       if(FileExists(DestFile))
 900          Rename(LastGoodSig,Final);
 901
 902       // set the status back to , Item::Failed likes to reset it
 903       Status = pkgAcquire::Item::StatTransientNetworkError;
 904       return;
 905    }
 906
 907    // Delete any existing sigfile when the acquire failed
 908    unlink(Final.c_str());
 909
 910    // queue a pkgAcqMetaIndex with no sigfile
 911    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 912                        "", IndexTargets, MetaIndexParser);
 913
 914    if (Cnf->LocalOnly == true ||
 915        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 916    {
 917       // Ignore this
 918       Status = StatDone;
 919       Complete = false;
 920       Dequeue();
 921       return;
 922    }
 923
 924    Item::Failed(Message,Cnf);
 925 }
 926                                                                         /*}}}*/
 927 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 928                                  string URI,string URIDesc,string ShortDesc,
 929                                  string SigFile,
 930                                  const vector<struct IndexTarget*>* IndexTargets,
 931                                  indexRecords* MetaIndexParser) :
 932    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 933    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 934 {
 935    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 936    DestFile += URItoFileName(URI);
 937
 938    // Create the item
 939    Desc.Description = URIDesc;
 940    Desc.Owner = this;
 941    Desc.ShortDesc = ShortDesc;
 942    Desc.URI = URI;
 943
 944    QueueURI(Desc);
 945 }
 946                                                                         /*}}}*/
 947 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 948 // ---------------------------------------------------------------------
 949 /* The only header we use is the last-modified header. */
 950 string pkgAcqMetaIndex::Custom600Headers()
 951 {
 952    string Final = _config->FindDir("Dir::State::lists");
 953    Final += URItoFileName(RealURI);
 954
 955    struct stat Buf;
 956    if (stat(Final.c_str(),&Buf) != 0)
 957       return "\nIndex-File: true";
 958
 959    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 960 }
 961                                                                         /*}}}*/
 962 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 963                            pkgAcquire::MethodConfig *Cfg)
 964 {
 965    Item::Done(Message,Size,Hash,Cfg);
 966
 967    // MetaIndexes are done in two passes: one to download the
 968    // metaindex with an appropriate method, and a second to verify it
 969    // with the gpgv method
 970
 971    if (AuthPass == true)
 972    {
 973       AuthDone(Message);
 974    }
 975    else
 976    {
 977       RetrievalDone(Message);
 978       if (!Complete)
 979          // Still more retrieving to do
 980          return;
 981
 982       if (SigFile == "")
 983       {
 984          // There was no signature file, so we are finished.  Download
 985          // the indexes without verification.
 986          QueueIndexes(false);
 987       }
 988       else
 989       {
 990          // There was a signature file, so pass it to gpgv for
 991          // verification
 992
 993          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 994             std::cerr << "Metaindex acquired, queueing gpg verification ("
 995                       << SigFile << "," << DestFile << ")\n";
 996          AuthPass = true;
 997          Desc.URI = "gpgv:" + SigFile;
 998          QueueURI(Desc);
 999          Mode = "gpgv";
1000       }
1001    }
1002 }
1003                                                                         /*}}}*/
1004 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1005 {
1006    // We have just finished downloading a Release file (it is not
1007    // verified yet)
1008
1009    string FileName = LookupTag(Message,"Filename");
1010    if (FileName.empty() == true)
1011    {
1012       Status = StatError;
1013       ErrorText = "Method gave a blank filename";
1014       return;
1015    }
1016
1017    if (FileName != DestFile)
1018    {
1019       Local = true;
1020       Desc.URI = "copy:" + FileName;
1021       QueueURI(Desc);
1022       return;
1023    }
1024
1025    // see if the download was a IMSHit
1026    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1027    Complete = true;
1028
1029    string FinalFile = _config->FindDir("Dir::State::lists");
1030    FinalFile += URItoFileName(RealURI);
1031
1032    // If we get a IMS hit we can remove the empty file in partial
1033    // othersie we move the file in place
1034    if (IMSHit)
1035       unlink(DestFile.c_str());
1036    else
1037       Rename(DestFile,FinalFile);
1038
1039    chmod(FinalFile.c_str(),0644);
1040    DestFile = FinalFile;
1041 }
1042                                                                         /*}}}*/
1043 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1044 {
1045    // At this point, the gpgv method has succeeded, so there is a
1046    // valid signature from a key in the trusted keyring.  We
1047    // perform additional verification of its contents, and use them
1048    // to verify the indexes we are about to download
1049
1050    if (!MetaIndexParser->Load(DestFile))
1051    {
1052       Status = StatAuthError;
1053       ErrorText = MetaIndexParser->ErrorText;
1054       return;
1055    }
1056
1057    if (!VerifyVendor(Message))
1058    {
1059       return;
1060    }
1061
1062    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1063       std::cerr << "Signature verification succeeded: "
1064                 << DestFile << std::endl;
1065
1066    // Download further indexes with verification
1067    QueueIndexes(true);
1068
1069    // Done, move signature file into position
1070
1071    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1072       URItoFileName(RealURI) + ".gpg";
1073    Rename(SigFile,VerifiedSigFile);
1074    chmod(VerifiedSigFile.c_str(),0644);
1075 }
1076                                                                         /*}}}*/
1077 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1078 {
1079    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1080         Target != IndexTargets->end();
1081         Target++)
1082    {
1083       HashString ExpectedIndexHash;
1084       if (verify)
1085       {
1086          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1087          if (!Record)
1088          {
1089             Status = StatAuthError;
1090             ErrorText = "Unable to find expected entry  "
1091                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1092             return;
1093          }
1094          ExpectedIndexHash = Record->Hash;
1095          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1096          {
1097             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1098             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1099          }
1100          if (ExpectedIndexHash.empty())
1101          {
1102             Status = StatAuthError;
1103             ErrorText = "Unable to find hash sum for "
1104                + (*Target)->MetaKey + " in Meta-index file";
1105             return;
1106          }
1107       }
1108
1109       /* Queue Packages file (either diff or full packages files, depending
1110          on the users option) - we also check if the PDiff Index file is listed
1111          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1112          instead, but passing the required info to it is to much hassle */
1113       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1114           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1115          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1116                              (*Target)->ShortDesc, ExpectedIndexHash);
1117       else
1118          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1119                             (*Target)->ShortDesc, ExpectedIndexHash);
1120    }
1121 }
1122                                                                         /*}}}*/
1123 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1124 {
1125 //    // Maybe this should be made available from above so we don't have
1126 //    // to read and parse it every time?
1127 //    pkgVendorList List;
1128 //    List.ReadMainList();
1129
1130 //    const Vendor* Vndr = NULL;
1131 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1132 //    {
1133 //       string::size_type pos = (*I).find("VALIDSIG ");
1134 //       if (_config->FindB("Debug::Vendor", false))
1135 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1136 //                    << std::endl;
1137 //       if (pos != std::string::npos)
1138 //       {
1139 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1140 //          if (_config->FindB("Debug::Vendor", false))
1141 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1142 //                std::endl;
1143 //          Vndr = List.FindVendor(Fingerprint) != "";
1144 //          if (Vndr != NULL);
1145 //          break;
1146 //       }
1147 //    }
1148    string::size_type pos;
1149
1150    // check for missing sigs (that where not fatal because otherwise we had
1151    // bombed earlier)
1152    string missingkeys;
1153    string msg = _("There is no public key available for the "
1154                   "following key IDs:\n");
1155    pos = Message.find("NO_PUBKEY ");
1156    if (pos != std::string::npos)
1157    {
1158       string::size_type start = pos+strlen("NO_PUBKEY ");
1159       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1160       missingkeys += (Fingerprint);
1161    }
1162    if(!missingkeys.empty())
1163       _error->Warning("%s", string(msg+missingkeys).c_str());
1164
1165    string Transformed = MetaIndexParser->GetExpectedDist();
1166
1167    if (Transformed == "../project/experimental")
1168    {
1169       Transformed = "experimental";
1170    }
1171
1172    pos = Transformed.rfind('/');
1173    if (pos != string::npos)
1174    {
1175       Transformed = Transformed.substr(0, pos);
1176    }
1177
1178    if (Transformed == ".")
1179    {
1180       Transformed = "";
1181    }
1182
1183    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1184    {
1185       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1186       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1187       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1188    }
1189
1190    if (MetaIndexParser->CheckDist(Transformed) == false)
1191    {
1192       // This might become fatal one day
1193 //       Status = StatAuthError;
1194 //       ErrorText = "Conflicting distribution; expected "
1195 //          + MetaIndexParser->GetExpectedDist() + " but got "
1196 //          + MetaIndexParser->GetDist();
1197 //       return false;
1198       if (!Transformed.empty())
1199       {
1200          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1201                          Desc.Description.c_str(),
1202                          Transformed.c_str(),
1203                          MetaIndexParser->GetDist().c_str());
1204       }
1205    }
1206
1207    return true;
1208 }
1209                                                                         /*}}}*/
1210 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1211 // ---------------------------------------------------------------------
1212 /* */
1213 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1214 {
1215    if (AuthPass == true)
1216    {
1217       // if we fail the authentication but got the file via a IMS-Hit
1218       // this means that the file wasn't downloaded and that it might be
1219       // just stale (server problem, proxy etc). we delete what we have
1220       // queue it again without i-m-s
1221       // alternatively we could just unlink the file and let the user try again
1222       if (IMSHit)
1223       {
1224          Complete = false;
1225          Local = false;
1226          AuthPass = false;
1227          unlink(DestFile.c_str());
1228
1229          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1230          DestFile += URItoFileName(RealURI);
1231          Desc.URI = RealURI;
1232          QueueURI(Desc);
1233          return;
1234       }
1235
1236       // gpgv method failed
1237       _error->Warning("GPG error: %s: %s",
1238                       Desc.Description.c_str(),
1239                       LookupTag(Message,"Message").c_str());
1240
1241    }
1242
1243    // No Release file was present, or verification failed, so fall
1244    // back to queueing Packages files without verification
1245    QueueIndexes(false);
1246 }
1247                                                                         /*}}}*/
1248 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1249 // ---------------------------------------------------------------------
1250 /* This just sets up the initial fetch environment and queues the first
1251    possibilitiy */
1252 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1253                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1254                              string &StoreFilename) :
1255                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1256                StoreFilename(StoreFilename), Vf(Version.FileList()),
1257                Trusted(false)
1258 {
1259    Retries = _config->FindI("Acquire::Retries",0);
1260
1261    if (Version.Arch() == 0)
1262    {
1263       _error->Error(_("I wasn't able to locate a file for the %s package. "
1264                       "This might mean you need to manually fix this package. "
1265                       "(due to missing arch)"),
1266                     Version.ParentPkg().Name());
1267       return;
1268    }
1269
1270    /* We need to find a filename to determine the extension. We make the
1271       assumption here that all the available sources for this version share
1272       the same extension.. */
1273    // Skip not source sources, they do not have file fields.
1274    for (; Vf.end() == false; Vf++)
1275    {
1276       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1277          continue;
1278       break;
1279    }
1280
1281    // Does not really matter here.. we are going to fail out below
1282    if (Vf.end() != true)
1283    {
1284       // If this fails to get a file name we will bomb out below.
1285       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1286       if (_error->PendingError() == true)
1287          return;
1288
1289       // Generate the final file name as: package_version_arch.foo
1290       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1291                       QuoteString(Version.VerStr(),"_:") + '_' +
1292                       QuoteString(Version.Arch(),"_:.") +
1293                       "." + flExtension(Parse.FileName());
1294    }
1295
1296    // check if we have one trusted source for the package. if so, switch
1297    // to "TrustedOnly" mode
1298    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1299    {
1300       pkgIndexFile *Index;
1301       if (Sources->FindIndex(i.File(),Index) == false)
1302          continue;
1303       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1304       {
1305          std::cerr << "Checking index: " << Index->Describe()
1306                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1307       }
1308       if (Index->IsTrusted()) {
1309          Trusted = true;
1310          break;
1311       }
1312    }
1313
1314    // "allow-unauthenticated" restores apts old fetching behaviour
1315    // that means that e.g. unauthenticated file:// uris are higher
1316    // priority than authenticated http:// uris
1317    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1318       Trusted = false;
1319
1320    // Select a source
1321    if (QueueNext() == false && _error->PendingError() == false)
1322       _error->Error(_("I wasn't able to locate file for the %s package. "
1323                     "This might mean you need to manually fix this package."),
1324                     Version.ParentPkg().Name());
1325 }
1326                                                                         /*}}}*/
1327 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1328 // ---------------------------------------------------------------------
1329 /* This queues the next available file version for download. It checks if
1330    the archive is already available in the cache and stashs the MD5 for
1331    checking later. */
1332 bool pkgAcqArchive::QueueNext()
1333 {
1334    string const ForceHash = _config->Find("Acquire::ForceHash");
1335    for (; Vf.end() == false; Vf++)
1336    {
1337       // Ignore not source sources
1338       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1339          continue;
1340
1341       // Try to cross match against the source list
1342       pkgIndexFile *Index;
1343       if (Sources->FindIndex(Vf.File(),Index) == false)
1344             continue;
1345
1346       // only try to get a trusted package from another source if that source
1347       // is also trusted
1348       if(Trusted && !Index->IsTrusted())
1349          continue;
1350
1351       // Grab the text package record
1352       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1353       if (_error->PendingError() == true)
1354          return false;
1355
1356       string PkgFile = Parse.FileName();
1357       if (ForceHash.empty() == false)
1358       {
1359          if(stringcasecmp(ForceHash, "sha256") == 0)
1360             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1361          else if (stringcasecmp(ForceHash, "sha1") == 0)
1362             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1363          else
1364             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1365       }
1366       else
1367       {
1368          string Hash;
1369          if ((Hash = Parse.SHA256Hash()).empty() == false)
1370             ExpectedHash = HashString("SHA256", Hash);
1371          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1372             ExpectedHash = HashString("SHA1", Hash);
1373          else
1374             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1375       }
1376       if (PkgFile.empty() == true)
1377          return _error->Error(_("The package index files are corrupted. No Filename: "
1378                               "field for package %s."),
1379                               Version.ParentPkg().Name());
1380
1381       Desc.URI = Index->ArchiveURI(PkgFile);
1382       Desc.Description = Index->ArchiveInfo(Version);
1383       Desc.Owner = this;
1384       Desc.ShortDesc = Version.ParentPkg().Name();
1385
1386       // See if we already have the file. (Legacy filenames)
1387       FileSize = Version->Size;
1388       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1389       struct stat Buf;
1390       if (stat(FinalFile.c_str(),&Buf) == 0)
1391       {
1392          // Make sure the size matches
1393          if ((unsigned)Buf.st_size == Version->Size)
1394          {
1395             Complete = true;
1396             Local = true;
1397             Status = StatDone;
1398             StoreFilename = DestFile = FinalFile;
1399             return true;
1400          }
1401
1402          /* Hmm, we have a file and its size does not match, this means it is
1403             an old style mismatched arch */
1404          unlink(FinalFile.c_str());
1405       }
1406
1407       // Check it again using the new style output filenames
1408       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1409       if (stat(FinalFile.c_str(),&Buf) == 0)
1410       {
1411          // Make sure the size matches
1412          if ((unsigned)Buf.st_size == Version->Size)
1413          {
1414             Complete = true;
1415             Local = true;
1416             Status = StatDone;
1417             StoreFilename = DestFile = FinalFile;
1418             return true;
1419          }
1420
1421          /* Hmm, we have a file and its size does not match, this shouldnt
1422             happen.. */
1423          unlink(FinalFile.c_str());
1424       }
1425
1426       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1427
1428       // Check the destination file
1429       if (stat(DestFile.c_str(),&Buf) == 0)
1430       {
1431          // Hmm, the partial file is too big, erase it
1432          if ((unsigned)Buf.st_size > Version->Size)
1433             unlink(DestFile.c_str());
1434          else
1435             PartialSize = Buf.st_size;
1436       }
1437
1438       // Create the item
1439       Local = false;
1440       Desc.URI = Index->ArchiveURI(PkgFile);
1441       Desc.Description = Index->ArchiveInfo(Version);
1442       Desc.Owner = this;
1443       Desc.ShortDesc = Version.ParentPkg().Name();
1444       QueueURI(Desc);
1445
1446       Vf++;
1447       return true;
1448    }
1449    return false;
1450 }
1451                                                                         /*}}}*/
1452 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1453 // ---------------------------------------------------------------------
1454 /* */
1455 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1456                          pkgAcquire::MethodConfig *Cfg)
1457 {
1458    Item::Done(Message,Size,CalcHash,Cfg);
1459
1460    // Check the size
1461    if (Size != Version->Size)
1462    {
1463       Status = StatError;
1464       ErrorText = _("Size mismatch");
1465       return;
1466    }
1467
1468    // Check the hash
1469    if(ExpectedHash.toStr() != CalcHash)
1470    {
1471       Status = StatError;
1472       ErrorText = _("Hash Sum mismatch");
1473       if(FileExists(DestFile))
1474          Rename(DestFile,DestFile + ".FAILED");
1475       return;
1476    }
1477
1478    // Grab the output filename
1479    string FileName = LookupTag(Message,"Filename");
1480    if (FileName.empty() == true)
1481    {
1482       Status = StatError;
1483       ErrorText = "Method gave a blank filename";
1484       return;
1485    }
1486
1487    Complete = true;
1488
1489    // Reference filename
1490    if (FileName != DestFile)
1491    {
1492       StoreFilename = DestFile = FileName;
1493       Local = true;
1494       return;
1495    }
1496
1497    // Done, move it into position
1498    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1499    FinalFile += flNotDir(StoreFilename);
1500    Rename(DestFile,FinalFile);
1501
1502    StoreFilename = DestFile = FinalFile;
1503    Complete = true;
1504 }
1505                                                                         /*}}}*/
1506 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1507 // ---------------------------------------------------------------------
1508 /* Here we try other sources */
1509 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1510 {
1511    ErrorText = LookupTag(Message,"Message");
1512
1513    /* We don't really want to retry on failed media swaps, this prevents
1514       that. An interesting observation is that permanent failures are not
1515       recorded. */
1516    if (Cnf->Removable == true &&
1517        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1518    {
1519       // Vf = Version.FileList();
1520       while (Vf.end() == false) Vf++;
1521       StoreFilename = string();
1522       Item::Failed(Message,Cnf);
1523       return;
1524    }
1525
1526    if (QueueNext() == false)
1527    {
1528       // This is the retry counter
1529       if (Retries != 0 &&
1530           Cnf->LocalOnly == false &&
1531           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1532       {
1533          Retries--;
1534          Vf = Version.FileList();
1535          if (QueueNext() == true)
1536             return;
1537       }
1538
1539       StoreFilename = string();
1540       Item::Failed(Message,Cnf);
1541    }
1542 }
1543                                                                         /*}}}*/
1544 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1545 // ---------------------------------------------------------------------
1546 bool pkgAcqArchive::IsTrusted()
1547 {
1548    return Trusted;
1549 }
1550                                                                         /*}}}*/
1551 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1552 // ---------------------------------------------------------------------
1553 /* */
1554 void pkgAcqArchive::Finished()
1555 {
1556    if (Status == pkgAcquire::Item::StatDone &&
1557        Complete == true)
1558       return;
1559    StoreFilename = string();
1560 }
1561                                                                         /*}}}*/
1562 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1563 // ---------------------------------------------------------------------
1564 /* The file is added to the queue */
1565 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1566                        unsigned long Size,string Dsc,string ShortDesc,
1567                        const string &DestDir, const string &DestFilename,
1568                        bool IsIndexFile) :
1569                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1570 {
1571    Retries = _config->FindI("Acquire::Retries",0);
1572
1573    if(!DestFilename.empty())
1574       DestFile = DestFilename;
1575    else if(!DestDir.empty())
1576       DestFile = DestDir + "/" + flNotDir(URI);
1577    else
1578       DestFile = flNotDir(URI);
1579
1580    // Create the item
1581    Desc.URI = URI;
1582    Desc.Description = Dsc;
1583    Desc.Owner = this;
1584
1585    // Set the short description to the archive component
1586    Desc.ShortDesc = ShortDesc;
1587
1588    // Get the transfer sizes
1589    FileSize = Size;
1590    struct stat Buf;
1591    if (stat(DestFile.c_str(),&Buf) == 0)
1592    {
1593       // Hmm, the partial file is too big, erase it
1594       if ((unsigned)Buf.st_size > Size)
1595          unlink(DestFile.c_str());
1596       else
1597          PartialSize = Buf.st_size;
1598    }
1599
1600    QueueURI(Desc);
1601 }
1602                                                                         /*}}}*/
1603 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1604 // ---------------------------------------------------------------------
1605 /* */
1606 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1607                       pkgAcquire::MethodConfig *Cnf)
1608 {
1609    Item::Done(Message,Size,CalcHash,Cnf);
1610
1611    // Check the hash
1612    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1613    {
1614       Status = StatError;
1615       ErrorText = "Hash Sum mismatch";
1616       Rename(DestFile,DestFile + ".FAILED");
1617       return;
1618    }
1619
1620    string FileName = LookupTag(Message,"Filename");
1621    if (FileName.empty() == true)
1622    {
1623       Status = StatError;
1624       ErrorText = "Method gave a blank filename";
1625       return;
1626    }
1627
1628    Complete = true;
1629
1630    // The files timestamp matches
1631    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1632       return;
1633
1634    // We have to copy it into place
1635    if (FileName != DestFile)
1636    {
1637       Local = true;
1638       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1639           Cnf->Removable == true)
1640       {
1641          Desc.URI = "copy:" + FileName;
1642          QueueURI(Desc);
1643          return;
1644       }
1645
1646       // Erase the file if it is a symlink so we can overwrite it
1647       struct stat St;
1648       if (lstat(DestFile.c_str(),&St) == 0)
1649       {
1650          if (S_ISLNK(St.st_mode) != 0)
1651             unlink(DestFile.c_str());
1652       }
1653
1654       // Symlink the file
1655       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1656       {
1657          ErrorText = "Link to " + DestFile + " failure ";
1658          Status = StatError;
1659          Complete = false;
1660       }
1661    }
1662 }
1663                                                                         /*}}}*/
1664 // AcqFile::Failed - Failure handler                                    /*{{{*/
1665 // ---------------------------------------------------------------------
1666 /* Here we try other sources */
1667 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1668 {
1669    ErrorText = LookupTag(Message,"Message");
1670
1671    // This is the retry counter
1672    if (Retries != 0 &&
1673        Cnf->LocalOnly == false &&
1674        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1675    {
1676       Retries--;
1677       QueueURI(Desc);
1678       return;
1679    }
1680
1681    Item::Failed(Message,Cnf);
1682 }
1683                                                                         /*}}}*/
1684 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1685 // ---------------------------------------------------------------------
1686 /* The only header we use is the last-modified header. */
1687 string pkgAcqFile::Custom600Headers()
1688 {
1689    if (IsIndexFile)
1690       return "\nIndex-File: true";
1691    return "";
1692 }
1693                                                                         /*}}}*/