apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Acquire::Item::Item - Constructor                                    /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  44                        PartialSize(0), Mode(0), ID(0), Complete(false),
  45                        Local(false), QueueCounter(0)
  46 {
  47    Owner->Add(this);
  48    Status = StatIdle;
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* */
  54 pkgAcquire::Item::~Item()
  55 {
  56    Owner->Remove(this);
  57 }
  58                                                                         /*}}}*/
  59 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* We return to an idle state if there are still other queues that could
  62    fetch this object */
  63 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  64 {
  65    Status = StatIdle;
  66    ErrorText = LookupTag(Message,"Message");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83 }
  84                                                                         /*}}}*/
  85 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Stash status and the file size. Note that setting Complete means
  88    sub-phases of the acquire process such as decompresion are operating */
  89 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  90 {
  91    Status = StatFetching;
  92    if (FileSize == 0 && Complete == false)
  93       FileSize = Size;
  94 }
  95                                                                         /*}}}*/
  96 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 100                             pkgAcquire::MethodConfig *Cnf)
 101 {
 102    // We just downloaded something..
 103    string FileName = LookupTag(Message,"Filename");
 104    // we only inform the Log class if it was actually not a local thing
 105    if (Complete == false && !Local && FileName == DestFile)
 106    {
 107       if (Owner->Log != 0)
 108          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 109    }
 110
 111    if (FileSize == 0)
 112       FileSize= Size;
 113
 114    Status = StatDone;
 115    ErrorText = string();
 116    Owner->Dequeue(this);
 117 }
 118                                                                         /*}}}*/
 119 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This helper function is used by alot of item methods as thier final
 122    step */
 123 void pkgAcquire::Item::Rename(string From,string To)
 124 {
 125    if (rename(From.c_str(),To.c_str()) != 0)
 126    {
 127       char S[300];
 128       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 129               From.c_str(),To.c_str());
 130       Status = StatError;
 131       ErrorText = S;
 132    }
 133 }
 134                                                                         /*}}}*/
 135 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* Get the DiffIndex file first and see if there are patches availabe
 138  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 139  * patches. If anything goes wrong in that process, it will fall back to
 140  * the original packages file
 141  */
 142 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 143                                  string URI,string URIDesc,string ShortDesc,
 144                                  HashString ExpectedHash)
 145    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 146      Description(URIDesc)
 147 {
 148
 149    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 150
 151    Desc.Description = URIDesc + "/DiffIndex";
 152    Desc.Owner = this;
 153    Desc.ShortDesc = ShortDesc;
 154    Desc.URI = URI + ".diff/Index";
 155
 156    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 157    DestFile += URItoFileName(URI) + string(".DiffIndex");
 158
 159    if(Debug)
 160       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 161
 162    // look for the current package file
 163    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 164    CurrentPackagesFile += URItoFileName(RealURI);
 165
 166    // FIXME: this file:/ check is a hack to prevent fetching
 167    //        from local sources. this is really silly, and
 168    //        should be fixed cleanly as soon as possible
 169    if(!FileExists(CurrentPackagesFile) ||
 170       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 171    {
 172       // we don't have a pkg file or we don't want to queue
 173       if(Debug)
 174          std::clog << "No index file, local or canceld by user" << std::endl;
 175       Failed("", NULL);
 176       return;
 177    }
 178
 179    if(Debug)
 180       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 181                 << CurrentPackagesFile << std::endl;
 182
 183    QueueURI(Desc);
 184
 185 }
 186                                                                         /*}}}*/
 187 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* The only header we use is the last-modified header. */
 190 string pkgAcqDiffIndex::Custom600Headers()
 191 {
 192    string Final = _config->FindDir("Dir::State::lists");
 193    Final += URItoFileName(RealURI) + string(".IndexDiff");
 194
 195    if(Debug)
 196       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 197
 198    struct stat Buf;
 199    if (stat(Final.c_str(),&Buf) != 0)
 200       return "\nIndex-File: true";
 201
 202    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 203 }
 204                                                                         /*}}}*/
 205 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 206 {
 207    if(Debug)
 208       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 209                 << std::endl;
 210
 211    pkgTagSection Tags;
 212    string ServerSha1;
 213    vector<DiffInfo> available_patches;
 214
 215    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 216    pkgTagFile TF(&Fd);
 217    if (_error->PendingError() == true)
 218       return false;
 219
 220    if(TF.Step(Tags) == true)
 221    {
 222       bool found = false;
 223       DiffInfo d;
 224       string size;
 225
 226       string const tmp = Tags.FindS("SHA1-Current");
 227       std::stringstream ss(tmp);
 228       ss >> ServerSha1 >> size;
 229       unsigned long const ServerSize = atol(size.c_str());
 230
 231       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 232       SHA1Summation SHA1;
 233       SHA1.AddFD(fd.Fd(), fd.Size());
 234       string const local_sha1 = SHA1.Result();
 235
 236       if(local_sha1 == ServerSha1)
 237       {
 238          // we have the same sha1 as the server
 239          if(Debug)
 240             std::clog << "Package file is up-to-date" << std::endl;
 241          // set found to true, this will queue a pkgAcqIndexDiffs with
 242          // a empty availabe_patches
 243          found = true;
 244       }
 245       else
 246       {
 247          if(Debug)
 248             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 249
 250          // check the historie and see what patches we need
 251          string const history = Tags.FindS("SHA1-History");
 252          std::stringstream hist(history);
 253          while(hist >> d.sha1 >> size >> d.file)
 254          {
 255             // read until the first match is found
 256             // from that point on, we probably need all diffs
 257             if(d.sha1 == local_sha1)
 258                found=true;
 259             else if (found == false)
 260                continue;
 261
 262             if(Debug)
 263                std::clog << "Need to get diff: " << d.file << std::endl;
 264             available_patches.push_back(d);
 265          }
 266
 267          if (available_patches.empty() == false)
 268          {
 269             // patching with too many files is rather slow compared to a fast download
 270             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 271             if (fileLimit != 0 && fileLimit < available_patches.size())
 272             {
 273                if (Debug)
 274                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 275                         << ") so fallback to complete download" << std::endl;
 276                return false;
 277             }
 278
 279             // see if the patches are too big
 280             found = false; // it was true and it will be true again at the end
 281             d = *available_patches.begin();
 282             string const firstPatch = d.file;
 283             unsigned long patchesSize = 0;
 284             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 285             while(patches >> d.sha1 >> size >> d.file)
 286             {
 287                if (firstPatch == d.file)
 288                   found = true;
 289                else if (found == false)
 290                   continue;
 291
 292                patchesSize += atol(size.c_str());
 293             }
 294             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 295             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 296             {
 297                if (Debug)
 298                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 299                         << ") so fallback to complete download" << std::endl;
 300                return false;
 301             }
 302          }
 303       }
 304
 305       // we have something, queue the next diff
 306       if(found)
 307       {
 308          // queue the diffs
 309          string::size_type const last_space = Description.rfind(" ");
 310          if(last_space != string::npos)
 311             Description.erase(last_space, Description.size()-last_space);
 312          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 313                               ExpectedHash, ServerSha1, available_patches);
 314          Complete = false;
 315          Status = StatDone;
 316          Dequeue();
 317          return true;
 318       }
 319    }
 320
 321    // Nothing found, report and return false
 322    // Failing here is ok, if we return false later, the full
 323    // IndexFile is queued
 324    if(Debug)
 325       std::clog << "Can't find a patch in the index file" << std::endl;
 326    return false;
 327 }
 328                                                                         /*}}}*/
 329 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 330 {
 331    if(Debug)
 332       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 333                 << "Falling back to normal index file aquire" << std::endl;
 334
 335    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 336                    ExpectedHash);
 337
 338    Complete = false;
 339    Status = StatDone;
 340    Dequeue();
 341 }
 342                                                                         /*}}}*/
 343 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 344                            pkgAcquire::MethodConfig *Cnf)
 345 {
 346    if(Debug)
 347       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 348
 349    Item::Done(Message,Size,Md5Hash,Cnf);
 350
 351    string FinalFile;
 352    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 353
 354    // sucess in downloading the index
 355    // rename the index
 356    FinalFile += string(".IndexDiff");
 357    if(Debug)
 358       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 359                 << std::endl;
 360    Rename(DestFile,FinalFile);
 361    chmod(FinalFile.c_str(),0644);
 362    DestFile = FinalFile;
 363
 364    if(!ParseDiffIndex(DestFile))
 365       return Failed("", NULL);
 366
 367    Complete = true;
 368    Status = StatDone;
 369    Dequeue();
 370    return;
 371 }
 372                                                                         /*}}}*/
 373 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* The package diff is added to the queue. one object is constructed
 376  * for each diff and the index
 377  */
 378 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 379                                    string URI,string URIDesc,string ShortDesc,
 380                                    HashString ExpectedHash,
 381                                    string ServerSha1,
 382                                    vector<DiffInfo> diffs)
 383    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 384      available_patches(diffs), ServerSha1(ServerSha1)
 385 {
 386
 387    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 388    DestFile += URItoFileName(URI);
 389
 390    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 391
 392    Description = URIDesc;
 393    Desc.Owner = this;
 394    Desc.ShortDesc = ShortDesc;
 395
 396    if(available_patches.size() == 0)
 397    {
 398       // we are done (yeah!)
 399       Finish(true);
 400    }
 401    else
 402    {
 403       // get the next diff
 404       State = StateFetchDiff;
 405       QueueNextDiff();
 406    }
 407 }
 408                                                                         /*}}}*/
 409 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 410 {
 411    if(Debug)
 412       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 413                 << "Falling back to normal index file aquire" << std::endl;
 414    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 415                    ExpectedHash);
 416    Finish();
 417 }
 418                                                                         /*}}}*/
 419 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 420 void pkgAcqIndexDiffs::Finish(bool allDone)
 421 {
 422    // we restore the original name, this is required, otherwise
 423    // the file will be cleaned
 424    if(allDone)
 425    {
 426       DestFile = _config->FindDir("Dir::State::lists");
 427       DestFile += URItoFileName(RealURI);
 428
 429       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 430       {
 431          Status = StatAuthError;
 432          ErrorText = _("MD5Sum mismatch");
 433          Rename(DestFile,DestFile + ".FAILED");
 434          Dequeue();
 435          return;
 436       }
 437
 438       // this is for the "real" finish
 439       Complete = true;
 440       Status = StatDone;
 441       Dequeue();
 442       if(Debug)
 443          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 444       return;
 445    }
 446
 447    if(Debug)
 448       std::clog << "Finishing: " << Desc.URI << std::endl;
 449    Complete = false;
 450    Status = StatDone;
 451    Dequeue();
 452    return;
 453 }
 454                                                                         /*}}}*/
 455 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 456 {
 457
 458    // calc sha1 of the just patched file
 459    string FinalFile = _config->FindDir("Dir::State::lists");
 460    FinalFile += URItoFileName(RealURI);
 461
 462    FileFd fd(FinalFile, FileFd::ReadOnly);
 463    SHA1Summation SHA1;
 464    SHA1.AddFD(fd.Fd(), fd.Size());
 465    string local_sha1 = string(SHA1.Result());
 466    if(Debug)
 467       std::clog << "QueueNextDiff: "
 468                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 469
 470    // final file reached before all patches are applied
 471    if(local_sha1 == ServerSha1)
 472    {
 473       Finish(true);
 474       return true;
 475    }
 476
 477    // remove all patches until the next matching patch is found
 478    // this requires the Index file to be ordered
 479    for(vector<DiffInfo>::iterator I=available_patches.begin();
 480        available_patches.size() > 0 &&
 481           I != available_patches.end() &&
 482           (*I).sha1 != local_sha1;
 483        I++)
 484    {
 485       available_patches.erase(I);
 486    }
 487
 488    // error checking and falling back if no patch was found
 489    if(available_patches.size() == 0)
 490    {
 491       Failed("", NULL);
 492       return false;
 493    }
 494
 495    // queue the right diff
 496    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 497    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 498    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 499    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 500
 501    if(Debug)
 502       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 503
 504    QueueURI(Desc);
 505
 506    return true;
 507 }
 508                                                                         /*}}}*/
 509 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 510                             pkgAcquire::MethodConfig *Cnf)
 511 {
 512    if(Debug)
 513       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 514
 515    Item::Done(Message,Size,Md5Hash,Cnf);
 516
 517    string FinalFile;
 518    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 519
 520    // sucess in downloading a diff, enter ApplyDiff state
 521    if(State == StateFetchDiff)
 522    {
 523
 524       if(Debug)
 525          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 526
 527       string FileName = LookupTag(Message,"Filename");
 528       State = StateUnzipDiff;
 529       Local = true;
 530       Desc.URI = "gzip:" + FileName;
 531       DestFile += ".decomp";
 532       QueueURI(Desc);
 533       Mode = "gzip";
 534       return;
 535    }
 536
 537    // sucess in downloading a diff, enter ApplyDiff state
 538    if(State == StateUnzipDiff)
 539    {
 540
 541       // rred excepts the patch as $FinalFile.ed
 542       Rename(DestFile,FinalFile+".ed");
 543
 544       if(Debug)
 545          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 546
 547       State = StateApplyDiff;
 548       Local = true;
 549       Desc.URI = "rred:" + FinalFile;
 550       QueueURI(Desc);
 551       Mode = "rred";
 552       return;
 553    }
 554
 555
 556    // success in download/apply a diff, queue next (if needed)
 557    if(State == StateApplyDiff)
 558    {
 559       // remove the just applied patch
 560       available_patches.erase(available_patches.begin());
 561
 562       // move into place
 563       if(Debug)
 564       {
 565          std::clog << "Moving patched file in place: " << std::endl
 566                    << DestFile << " -> " << FinalFile << std::endl;
 567       }
 568       Rename(DestFile,FinalFile);
 569       chmod(FinalFile.c_str(),0644);
 570
 571       // see if there is more to download
 572       if(available_patches.size() > 0) {
 573          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 574                               ExpectedHash, ServerSha1, available_patches);
 575          return Finish();
 576       } else
 577          return Finish(true);
 578    }
 579 }
 580                                                                         /*}}}*/
 581 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* The package file is added to the queue and a second class is
 584    instantiated to fetch the revision file */
 585 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 586                          string URI,string URIDesc,string ShortDesc,
 587                          HashString ExpectedHash, string comprExt)
 588    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 589 {
 590    Decompression = false;
 591    Erase = false;
 592
 593    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 594    DestFile += URItoFileName(URI);
 595
 596    if(comprExt.empty())
 597    {
 598       // autoselect the compression method
 599       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 600       if (types.empty() == true)
 601          comprExt = "plain";
 602       else
 603          comprExt = "." + types[0];
 604    }
 605    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 606
 607    Desc.URI = URI + CompressionExtension;
 608
 609    Desc.Description = URIDesc;
 610    Desc.Owner = this;
 611    Desc.ShortDesc = ShortDesc;
 612
 613    QueueURI(Desc);
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The only header we use is the last-modified header. */
 619 string pkgAcqIndex::Custom600Headers()
 620 {
 621    string Final = _config->FindDir("Dir::State::lists");
 622    Final += URItoFileName(RealURI);
 623
 624    struct stat Buf;
 625    if (stat(Final.c_str(),&Buf) != 0)
 626       return "\nIndex-File: true";
 627
 628    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 629 }
 630                                                                         /*}}}*/
 631 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 632 {
 633    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 634
 635    for (std::vector<std::string>::const_iterator t = types.begin();
 636         t != types.end(); t++)
 637    {
 638       // jump over all already tried compression types
 639       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 640       if(Desc.URI.substr(nameLen) != *t)
 641          continue;
 642
 643       // we want to try it with the next extension (and make sure to
 644       // not skip over the end)
 645       t++;
 646       if (t == types.end())
 647          break;
 648
 649       // queue new download
 650       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 651       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 652       ExpectedHash, string(".").append(*t));
 653
 654       Status = StatDone;
 655       Complete = false;
 656       Dequeue();
 657       return;
 658    }
 659
 660    // on decompression failure, remove bad versions in partial/
 661    if(Decompression && Erase) {
 662       string s = _config->FindDir("Dir::State::lists") + "partial/";
 663       s += URItoFileName(RealURI);
 664       unlink(s.c_str());
 665    }
 666
 667    Item::Failed(Message,Cnf);
 668 }
 669                                                                         /*}}}*/
 670 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* This goes through a number of states.. On the initial fetch the
 673    method could possibly return an alternate filename which points
 674    to the uncompressed version of the file. If this is so the file
 675    is copied into the partial directory. In all other cases the file
 676    is decompressed with a gzip uri. */
 677 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 678                        pkgAcquire::MethodConfig *Cfg)
 679 {
 680    Item::Done(Message,Size,Hash,Cfg);
 681
 682    if (Decompression == true)
 683    {
 684       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 685       {
 686          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 687          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 688       }
 689
 690       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 691       {
 692          Status = StatAuthError;
 693          ErrorText = _("Hash Sum mismatch");
 694          Rename(DestFile,DestFile + ".FAILED");
 695          return;
 696       }
 697       // Done, move it into position
 698       string FinalFile = _config->FindDir("Dir::State::lists");
 699       FinalFile += URItoFileName(RealURI);
 700       Rename(DestFile,FinalFile);
 701       chmod(FinalFile.c_str(),0644);
 702
 703       /* We restore the original name to DestFile so that the clean operation
 704          will work OK */
 705       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 706       DestFile += URItoFileName(RealURI);
 707
 708       // Remove the compressed version.
 709       if (Erase == true)
 710          unlink(DestFile.c_str());
 711       return;
 712    }
 713
 714    Erase = false;
 715    Complete = true;
 716
 717    // Handle the unzipd case
 718    string FileName = LookupTag(Message,"Alt-Filename");
 719    if (FileName.empty() == false)
 720    {
 721       // The files timestamp matches
 722       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 723          return;
 724       Decompression = true;
 725       Local = true;
 726       DestFile += ".decomp";
 727       Desc.URI = "copy:" + FileName;
 728       QueueURI(Desc);
 729       Mode = "copy";
 730       return;
 731    }
 732
 733    FileName = LookupTag(Message,"Filename");
 734    if (FileName.empty() == true)
 735    {
 736       Status = StatError;
 737       ErrorText = "Method gave a blank filename";
 738    }
 739
 740    // The files timestamp matches
 741    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 742       return;
 743
 744    if (FileName == DestFile)
 745       Erase = true;
 746    else
 747       Local = true;
 748
 749    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 750    string decompProg;
 751
 752    // get the binary name for your used compression type
 753    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 754    if(decompProg.empty() == false);
 755    // flExtensions returns the full name if no extension is found
 756    // this is why we have this complicated compare operation here
 757    // FIMXE: add a new flJustExtension() that return "" if no
 758    //        extension is found and use that above so that it can
 759    //        be tested against ""
 760    else if(compExt == flNotDir(URI(Desc.URI).Path))
 761       decompProg = "copy";
 762    else {
 763       _error->Error("Unsupported extension: %s", compExt.c_str());
 764       return;
 765    }
 766
 767    Decompression = true;
 768    DestFile += ".decomp";
 769    Desc.URI = decompProg + ":" + FileName;
 770    QueueURI(Desc);
 771    Mode = decompProg.c_str();
 772 }
 773                                                                         /*}}}*/
 774 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* The Translation file is added to the queue */
 777 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 778                             string URI,string URIDesc,string ShortDesc)
 779   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 780 {
 781 }
 782                                                                         /*}}}*/
 783 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 784 // ---------------------------------------------------------------------
 785 /* */
 786 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 787 {
 788    if (Cnf->LocalOnly == true ||
 789        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 790    {
 791       // Ignore this
 792       Status = StatDone;
 793       Complete = false;
 794       Dequeue();
 795       return;
 796    }
 797
 798    Item::Failed(Message,Cnf);
 799 }
 800                                                                         /*}}}*/
 801 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 802                              string URI,string URIDesc,string ShortDesc,
 803                              string MetaIndexURI, string MetaIndexURIDesc,
 804                              string MetaIndexShortDesc,
 805                              const vector<IndexTarget*>* IndexTargets,
 806                              indexRecords* MetaIndexParser) :
 807    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 808    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 809    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 810 {
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(URI);
 813
 814    // remove any partial downloaded sig-file in partial/.
 815    // it may confuse proxies and is too small to warrant a
 816    // partial download anyway
 817    unlink(DestFile.c_str());
 818
 819    // Create the item
 820    Desc.Description = URIDesc;
 821    Desc.Owner = this;
 822    Desc.ShortDesc = ShortDesc;
 823    Desc.URI = URI;
 824
 825    string Final = _config->FindDir("Dir::State::lists");
 826    Final += URItoFileName(RealURI);
 827    struct stat Buf;
 828    if (stat(Final.c_str(),&Buf) == 0)
 829    {
 830       // File was already in place.  It needs to be re-downloaded/verified
 831       // because Release might have changed, we do give it a differnt
 832       // name than DestFile because otherwise the http method will
 833       // send If-Range requests and there are too many broken servers
 834       // out there that do not understand them
 835       LastGoodSig = DestFile+".reverify";
 836       Rename(Final,LastGoodSig);
 837    }
 838
 839    QueueURI(Desc);
 840 }
 841                                                                         /*}}}*/
 842 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 843 // ---------------------------------------------------------------------
 844 /* The only header we use is the last-modified header. */
 845 string pkgAcqMetaSig::Custom600Headers()
 846 {
 847    struct stat Buf;
 848    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 849       return "\nIndex-File: true";
 850
 851    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 852 }
 853
 854 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 855                          pkgAcquire::MethodConfig *Cfg)
 856 {
 857    Item::Done(Message,Size,MD5,Cfg);
 858
 859    string FileName = LookupTag(Message,"Filename");
 860    if (FileName.empty() == true)
 861    {
 862       Status = StatError;
 863       ErrorText = "Method gave a blank filename";
 864       return;
 865    }
 866
 867    if (FileName != DestFile)
 868    {
 869       // We have to copy it into place
 870       Local = true;
 871       Desc.URI = "copy:" + FileName;
 872       QueueURI(Desc);
 873       return;
 874    }
 875
 876    Complete = true;
 877
 878    // put the last known good file back on i-m-s hit (it will
 879    // be re-verified again)
 880    // Else do nothing, we have the new file in DestFile then
 881    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 882       Rename(LastGoodSig, DestFile);
 883
 884    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 885    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 886                        DestFile, IndexTargets, MetaIndexParser);
 887
 888 }
 889                                                                         /*}}}*/
 890 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 891 {
 892    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 893
 894    // if we get a network error we fail gracefully
 895    if(Status == StatTransientNetworkError)
 896    {
 897       Item::Failed(Message,Cnf);
 898       // move the sigfile back on transient network failures
 899       if(FileExists(DestFile))
 900          Rename(LastGoodSig,Final);
 901
 902       // set the status back to , Item::Failed likes to reset it
 903       Status = pkgAcquire::Item::StatTransientNetworkError;
 904       return;
 905    }
 906
 907    // Delete any existing sigfile when the acquire failed
 908    unlink(Final.c_str());
 909
 910    // queue a pkgAcqMetaIndex with no sigfile
 911    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 912                        "", IndexTargets, MetaIndexParser);
 913
 914    if (Cnf->LocalOnly == true ||
 915        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 916    {
 917       // Ignore this
 918       Status = StatDone;
 919       Complete = false;
 920       Dequeue();
 921       return;
 922    }
 923
 924    Item::Failed(Message,Cnf);
 925 }
 926                                                                         /*}}}*/
 927 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 928                                  string URI,string URIDesc,string ShortDesc,
 929                                  string SigFile,
 930                                  const vector<struct IndexTarget*>* IndexTargets,
 931                                  indexRecords* MetaIndexParser) :
 932    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 933    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 934 {
 935    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 936    DestFile += URItoFileName(URI);
 937
 938    // Create the item
 939    Desc.Description = URIDesc;
 940    Desc.Owner = this;
 941    Desc.ShortDesc = ShortDesc;
 942    Desc.URI = URI;
 943
 944    QueueURI(Desc);
 945 }
 946                                                                         /*}}}*/
 947 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 948 // ---------------------------------------------------------------------
 949 /* The only header we use is the last-modified header. */
 950 string pkgAcqMetaIndex::Custom600Headers()
 951 {
 952    string Final = _config->FindDir("Dir::State::lists");
 953    Final += URItoFileName(RealURI);
 954
 955    struct stat Buf;
 956    if (stat(Final.c_str(),&Buf) != 0)
 957       return "\nIndex-File: true";
 958
 959    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 960 }
 961                                                                         /*}}}*/
 962 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 963                            pkgAcquire::MethodConfig *Cfg)
 964 {
 965    Item::Done(Message,Size,Hash,Cfg);
 966
 967    // MetaIndexes are done in two passes: one to download the
 968    // metaindex with an appropriate method, and a second to verify it
 969    // with the gpgv method
 970
 971    if (AuthPass == true)
 972    {
 973       AuthDone(Message);
 974    }
 975    else
 976    {
 977       RetrievalDone(Message);
 978       if (!Complete)
 979          // Still more retrieving to do
 980          return;
 981
 982       if (SigFile == "")
 983       {
 984          // There was no signature file, so we are finished.  Download
 985          // the indexes without verification.
 986          QueueIndexes(false);
 987       }
 988       else
 989       {
 990          // There was a signature file, so pass it to gpgv for
 991          // verification
 992
 993          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 994             std::cerr << "Metaindex acquired, queueing gpg verification ("
 995                       << SigFile << "," << DestFile << ")\n";
 996          AuthPass = true;
 997          Desc.URI = "gpgv:" + SigFile;
 998          QueueURI(Desc);
 999          Mode = "gpgv";
1000       }
1001    }
1002 }
1003                                                                         /*}}}*/
1004 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1005 {
1006    // We have just finished downloading a Release file (it is not
1007    // verified yet)
1008
1009    string FileName = LookupTag(Message,"Filename");
1010    if (FileName.empty() == true)
1011    {
1012       Status = StatError;
1013       ErrorText = "Method gave a blank filename";
1014       return;
1015    }
1016
1017    if (FileName != DestFile)
1018    {
1019       Local = true;
1020       Desc.URI = "copy:" + FileName;
1021       QueueURI(Desc);
1022       return;
1023    }
1024
1025    // see if the download was a IMSHit
1026    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1027    Complete = true;
1028
1029    string FinalFile = _config->FindDir("Dir::State::lists");
1030    FinalFile += URItoFileName(RealURI);
1031
1032    // If we get a IMS hit we can remove the empty file in partial
1033    // othersie we move the file in place
1034    if (IMSHit)
1035       unlink(DestFile.c_str());
1036    else
1037       Rename(DestFile,FinalFile);
1038
1039    chmod(FinalFile.c_str(),0644);
1040    DestFile = FinalFile;
1041 }
1042                                                                         /*}}}*/
1043 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1044 {
1045    // At this point, the gpgv method has succeeded, so there is a
1046    // valid signature from a key in the trusted keyring.  We
1047    // perform additional verification of its contents, and use them
1048    // to verify the indexes we are about to download
1049
1050    if (!MetaIndexParser->Load(DestFile))
1051    {
1052       Status = StatAuthError;
1053       ErrorText = MetaIndexParser->ErrorText;
1054       return;
1055    }
1056
1057    if (!VerifyVendor(Message))
1058    {
1059       return;
1060    }
1061
1062    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1063       std::cerr << "Signature verification succeeded: "
1064                 << DestFile << std::endl;
1065
1066    // Download further indexes with verification
1067    QueueIndexes(true);
1068
1069    // Done, move signature file into position
1070
1071    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1072       URItoFileName(RealURI) + ".gpg";
1073    Rename(SigFile,VerifiedSigFile);
1074    chmod(VerifiedSigFile.c_str(),0644);
1075 }
1076                                                                         /*}}}*/
1077 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1078 {
1079    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1080         Target != IndexTargets->end();
1081         Target++)
1082    {
1083       HashString ExpectedIndexHash;
1084       if (verify)
1085       {
1086          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1087          if (!Record)
1088          {
1089             Status = StatAuthError;
1090             ErrorText = "Unable to find expected entry  "
1091                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1092             return;
1093          }
1094          ExpectedIndexHash = Record->Hash;
1095          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1096          {
1097             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1098             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1099          }
1100          if (ExpectedIndexHash.empty())
1101          {
1102             Status = StatAuthError;
1103             ErrorText = "Unable to find hash sum for "
1104                + (*Target)->MetaKey + " in Meta-index file";
1105             return;
1106          }
1107       }
1108
1109       // Queue Packages file (either diff or full packages files, depending
1110       // on the users option)
1111       if(_config->FindB("Acquire::PDiffs",true) == true)
1112          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1113                              (*Target)->ShortDesc, ExpectedIndexHash);
1114       else
1115          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1116                             (*Target)->ShortDesc, ExpectedIndexHash);
1117    }
1118 }
1119                                                                         /*}}}*/
1120 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1121 {
1122 //    // Maybe this should be made available from above so we don't have
1123 //    // to read and parse it every time?
1124 //    pkgVendorList List;
1125 //    List.ReadMainList();
1126
1127 //    const Vendor* Vndr = NULL;
1128 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1129 //    {
1130 //       string::size_type pos = (*I).find("VALIDSIG ");
1131 //       if (_config->FindB("Debug::Vendor", false))
1132 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1133 //                    << std::endl;
1134 //       if (pos != std::string::npos)
1135 //       {
1136 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1137 //          if (_config->FindB("Debug::Vendor", false))
1138 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1139 //                std::endl;
1140 //          Vndr = List.FindVendor(Fingerprint) != "";
1141 //          if (Vndr != NULL);
1142 //          break;
1143 //       }
1144 //    }
1145    string::size_type pos;
1146
1147    // check for missing sigs (that where not fatal because otherwise we had
1148    // bombed earlier)
1149    string missingkeys;
1150    string msg = _("There is no public key available for the "
1151                   "following key IDs:\n");
1152    pos = Message.find("NO_PUBKEY ");
1153    if (pos != std::string::npos)
1154    {
1155       string::size_type start = pos+strlen("NO_PUBKEY ");
1156       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1157       missingkeys += (Fingerprint);
1158    }
1159    if(!missingkeys.empty())
1160       _error->Warning("%s", string(msg+missingkeys).c_str());
1161
1162    string Transformed = MetaIndexParser->GetExpectedDist();
1163
1164    if (Transformed == "../project/experimental")
1165    {
1166       Transformed = "experimental";
1167    }
1168
1169    pos = Transformed.rfind('/');
1170    if (pos != string::npos)
1171    {
1172       Transformed = Transformed.substr(0, pos);
1173    }
1174
1175    if (Transformed == ".")
1176    {
1177       Transformed = "";
1178    }
1179
1180    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1181    {
1182       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1183       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1184       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1185    }
1186
1187    if (MetaIndexParser->CheckDist(Transformed) == false)
1188    {
1189       // This might become fatal one day
1190 //       Status = StatAuthError;
1191 //       ErrorText = "Conflicting distribution; expected "
1192 //          + MetaIndexParser->GetExpectedDist() + " but got "
1193 //          + MetaIndexParser->GetDist();
1194 //       return false;
1195       if (!Transformed.empty())
1196       {
1197          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1198                          Desc.Description.c_str(),
1199                          Transformed.c_str(),
1200                          MetaIndexParser->GetDist().c_str());
1201       }
1202    }
1203
1204    return true;
1205 }
1206                                                                         /*}}}*/
1207 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1208 // ---------------------------------------------------------------------
1209 /* */
1210 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1211 {
1212    if (AuthPass == true)
1213    {
1214       // if we fail the authentication but got the file via a IMS-Hit
1215       // this means that the file wasn't downloaded and that it might be
1216       // just stale (server problem, proxy etc). we delete what we have
1217       // queue it again without i-m-s
1218       // alternatively we could just unlink the file and let the user try again
1219       if (IMSHit)
1220       {
1221          Complete = false;
1222          Local = false;
1223          AuthPass = false;
1224          unlink(DestFile.c_str());
1225
1226          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1227          DestFile += URItoFileName(RealURI);
1228          Desc.URI = RealURI;
1229          QueueURI(Desc);
1230          return;
1231       }
1232
1233       // gpgv method failed
1234       _error->Warning("GPG error: %s: %s",
1235                       Desc.Description.c_str(),
1236                       LookupTag(Message,"Message").c_str());
1237
1238    }
1239
1240    // No Release file was present, or verification failed, so fall
1241    // back to queueing Packages files without verification
1242    QueueIndexes(false);
1243 }
1244                                                                         /*}}}*/
1245 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1246 // ---------------------------------------------------------------------
1247 /* This just sets up the initial fetch environment and queues the first
1248    possibilitiy */
1249 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1250                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1251                              string &StoreFilename) :
1252                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1253                StoreFilename(StoreFilename), Vf(Version.FileList()),
1254                Trusted(false)
1255 {
1256    Retries = _config->FindI("Acquire::Retries",0);
1257
1258    if (Version.Arch() == 0)
1259    {
1260       _error->Error(_("I wasn't able to locate a file for the %s package. "
1261                       "This might mean you need to manually fix this package. "
1262                       "(due to missing arch)"),
1263                     Version.ParentPkg().Name());
1264       return;
1265    }
1266
1267    /* We need to find a filename to determine the extension. We make the
1268       assumption here that all the available sources for this version share
1269       the same extension.. */
1270    // Skip not source sources, they do not have file fields.
1271    for (; Vf.end() == false; Vf++)
1272    {
1273       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1274          continue;
1275       break;
1276    }
1277
1278    // Does not really matter here.. we are going to fail out below
1279    if (Vf.end() != true)
1280    {
1281       // If this fails to get a file name we will bomb out below.
1282       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1283       if (_error->PendingError() == true)
1284          return;
1285
1286       // Generate the final file name as: package_version_arch.foo
1287       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1288                       QuoteString(Version.VerStr(),"_:") + '_' +
1289                       QuoteString(Version.Arch(),"_:.") +
1290                       "." + flExtension(Parse.FileName());
1291    }
1292
1293    // check if we have one trusted source for the package. if so, switch
1294    // to "TrustedOnly" mode
1295    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1296    {
1297       pkgIndexFile *Index;
1298       if (Sources->FindIndex(i.File(),Index) == false)
1299          continue;
1300       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1301       {
1302          std::cerr << "Checking index: " << Index->Describe()
1303                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1304       }
1305       if (Index->IsTrusted()) {
1306          Trusted = true;
1307          break;
1308       }
1309    }
1310
1311    // "allow-unauthenticated" restores apts old fetching behaviour
1312    // that means that e.g. unauthenticated file:// uris are higher
1313    // priority than authenticated http:// uris
1314    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1315       Trusted = false;
1316
1317    // Select a source
1318    if (QueueNext() == false && _error->PendingError() == false)
1319       _error->Error(_("I wasn't able to locate file for the %s package. "
1320                     "This might mean you need to manually fix this package."),
1321                     Version.ParentPkg().Name());
1322 }
1323                                                                         /*}}}*/
1324 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1325 // ---------------------------------------------------------------------
1326 /* This queues the next available file version for download. It checks if
1327    the archive is already available in the cache and stashs the MD5 for
1328    checking later. */
1329 bool pkgAcqArchive::QueueNext()
1330 {
1331    for (; Vf.end() == false; Vf++)
1332    {
1333       // Ignore not source sources
1334       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1335          continue;
1336
1337       // Try to cross match against the source list
1338       pkgIndexFile *Index;
1339       if (Sources->FindIndex(Vf.File(),Index) == false)
1340             continue;
1341
1342       // only try to get a trusted package from another source if that source
1343       // is also trusted
1344       if(Trusted && !Index->IsTrusted())
1345          continue;
1346
1347       // Grab the text package record
1348       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1349       if (_error->PendingError() == true)
1350          return false;
1351
1352       string PkgFile = Parse.FileName();
1353       if(Parse.SHA256Hash() != "")
1354          ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1355       else if (Parse.SHA1Hash() != "")
1356          ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1357       else
1358          ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1359       if (PkgFile.empty() == true)
1360          return _error->Error(_("The package index files are corrupted. No Filename: "
1361                               "field for package %s."),
1362                               Version.ParentPkg().Name());
1363
1364       Desc.URI = Index->ArchiveURI(PkgFile);
1365       Desc.Description = Index->ArchiveInfo(Version);
1366       Desc.Owner = this;
1367       Desc.ShortDesc = Version.ParentPkg().Name();
1368
1369       // See if we already have the file. (Legacy filenames)
1370       FileSize = Version->Size;
1371       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1372       struct stat Buf;
1373       if (stat(FinalFile.c_str(),&Buf) == 0)
1374       {
1375          // Make sure the size matches
1376          if ((unsigned)Buf.st_size == Version->Size)
1377          {
1378             Complete = true;
1379             Local = true;
1380             Status = StatDone;
1381             StoreFilename = DestFile = FinalFile;
1382             return true;
1383          }
1384
1385          /* Hmm, we have a file and its size does not match, this means it is
1386             an old style mismatched arch */
1387          unlink(FinalFile.c_str());
1388       }
1389
1390       // Check it again using the new style output filenames
1391       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1392       if (stat(FinalFile.c_str(),&Buf) == 0)
1393       {
1394          // Make sure the size matches
1395          if ((unsigned)Buf.st_size == Version->Size)
1396          {
1397             Complete = true;
1398             Local = true;
1399             Status = StatDone;
1400             StoreFilename = DestFile = FinalFile;
1401             return true;
1402          }
1403
1404          /* Hmm, we have a file and its size does not match, this shouldnt
1405             happen.. */
1406          unlink(FinalFile.c_str());
1407       }
1408
1409       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1410
1411       // Check the destination file
1412       if (stat(DestFile.c_str(),&Buf) == 0)
1413       {
1414          // Hmm, the partial file is too big, erase it
1415          if ((unsigned)Buf.st_size > Version->Size)
1416             unlink(DestFile.c_str());
1417          else
1418             PartialSize = Buf.st_size;
1419       }
1420
1421       // Create the item
1422       Local = false;
1423       Desc.URI = Index->ArchiveURI(PkgFile);
1424       Desc.Description = Index->ArchiveInfo(Version);
1425       Desc.Owner = this;
1426       Desc.ShortDesc = Version.ParentPkg().Name();
1427       QueueURI(Desc);
1428
1429       Vf++;
1430       return true;
1431    }
1432    return false;
1433 }
1434                                                                         /*}}}*/
1435 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1436 // ---------------------------------------------------------------------
1437 /* */
1438 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1439                          pkgAcquire::MethodConfig *Cfg)
1440 {
1441    Item::Done(Message,Size,CalcHash,Cfg);
1442
1443    // Check the size
1444    if (Size != Version->Size)
1445    {
1446       Status = StatError;
1447       ErrorText = _("Size mismatch");
1448       return;
1449    }
1450
1451    // Check the hash
1452    if(ExpectedHash.toStr() != CalcHash)
1453    {
1454       Status = StatError;
1455       ErrorText = _("Hash Sum mismatch");
1456       if(FileExists(DestFile))
1457          Rename(DestFile,DestFile + ".FAILED");
1458       return;
1459    }
1460
1461    // Grab the output filename
1462    string FileName = LookupTag(Message,"Filename");
1463    if (FileName.empty() == true)
1464    {
1465       Status = StatError;
1466       ErrorText = "Method gave a blank filename";
1467       return;
1468    }
1469
1470    Complete = true;
1471
1472    // Reference filename
1473    if (FileName != DestFile)
1474    {
1475       StoreFilename = DestFile = FileName;
1476       Local = true;
1477       return;
1478    }
1479
1480    // Done, move it into position
1481    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1482    FinalFile += flNotDir(StoreFilename);
1483    Rename(DestFile,FinalFile);
1484
1485    StoreFilename = DestFile = FinalFile;
1486    Complete = true;
1487 }
1488                                                                         /*}}}*/
1489 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1490 // ---------------------------------------------------------------------
1491 /* Here we try other sources */
1492 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1493 {
1494    ErrorText = LookupTag(Message,"Message");
1495
1496    /* We don't really want to retry on failed media swaps, this prevents
1497       that. An interesting observation is that permanent failures are not
1498       recorded. */
1499    if (Cnf->Removable == true &&
1500        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1501    {
1502       // Vf = Version.FileList();
1503       while (Vf.end() == false) Vf++;
1504       StoreFilename = string();
1505       Item::Failed(Message,Cnf);
1506       return;
1507    }
1508
1509    if (QueueNext() == false)
1510    {
1511       // This is the retry counter
1512       if (Retries != 0 &&
1513           Cnf->LocalOnly == false &&
1514           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1515       {
1516          Retries--;
1517          Vf = Version.FileList();
1518          if (QueueNext() == true)
1519             return;
1520       }
1521
1522       StoreFilename = string();
1523       Item::Failed(Message,Cnf);
1524    }
1525 }
1526                                                                         /*}}}*/
1527 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1528 // ---------------------------------------------------------------------
1529 bool pkgAcqArchive::IsTrusted()
1530 {
1531    return Trusted;
1532 }
1533                                                                         /*}}}*/
1534 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1535 // ---------------------------------------------------------------------
1536 /* */
1537 void pkgAcqArchive::Finished()
1538 {
1539    if (Status == pkgAcquire::Item::StatDone &&
1540        Complete == true)
1541       return;
1542    StoreFilename = string();
1543 }
1544                                                                         /*}}}*/
1545 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1546 // ---------------------------------------------------------------------
1547 /* The file is added to the queue */
1548 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1549                        unsigned long Size,string Dsc,string ShortDesc,
1550                        const string &DestDir, const string &DestFilename,
1551                        bool IsIndexFile) :
1552                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1553 {
1554    Retries = _config->FindI("Acquire::Retries",0);
1555
1556    if(!DestFilename.empty())
1557       DestFile = DestFilename;
1558    else if(!DestDir.empty())
1559       DestFile = DestDir + "/" + flNotDir(URI);
1560    else
1561       DestFile = flNotDir(URI);
1562
1563    // Create the item
1564    Desc.URI = URI;
1565    Desc.Description = Dsc;
1566    Desc.Owner = this;
1567
1568    // Set the short description to the archive component
1569    Desc.ShortDesc = ShortDesc;
1570
1571    // Get the transfer sizes
1572    FileSize = Size;
1573    struct stat Buf;
1574    if (stat(DestFile.c_str(),&Buf) == 0)
1575    {
1576       // Hmm, the partial file is too big, erase it
1577       if ((unsigned)Buf.st_size > Size)
1578          unlink(DestFile.c_str());
1579       else
1580          PartialSize = Buf.st_size;
1581    }
1582
1583    QueueURI(Desc);
1584 }
1585                                                                         /*}}}*/
1586 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1587 // ---------------------------------------------------------------------
1588 /* */
1589 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1590                       pkgAcquire::MethodConfig *Cnf)
1591 {
1592    Item::Done(Message,Size,CalcHash,Cnf);
1593
1594    // Check the hash
1595    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1596    {
1597       Status = StatError;
1598       ErrorText = "Hash Sum mismatch";
1599       Rename(DestFile,DestFile + ".FAILED");
1600       return;
1601    }
1602
1603    string FileName = LookupTag(Message,"Filename");
1604    if (FileName.empty() == true)
1605    {
1606       Status = StatError;
1607       ErrorText = "Method gave a blank filename";
1608       return;
1609    }
1610
1611    Complete = true;
1612
1613    // The files timestamp matches
1614    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1615       return;
1616
1617    // We have to copy it into place
1618    if (FileName != DestFile)
1619    {
1620       Local = true;
1621       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1622           Cnf->Removable == true)
1623       {
1624          Desc.URI = "copy:" + FileName;
1625          QueueURI(Desc);
1626          return;
1627       }
1628
1629       // Erase the file if it is a symlink so we can overwrite it
1630       struct stat St;
1631       if (lstat(DestFile.c_str(),&St) == 0)
1632       {
1633          if (S_ISLNK(St.st_mode) != 0)
1634             unlink(DestFile.c_str());
1635       }
1636
1637       // Symlink the file
1638       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1639       {
1640          ErrorText = "Link to " + DestFile + " failure ";
1641          Status = StatError;
1642          Complete = false;
1643       }
1644    }
1645 }
1646                                                                         /*}}}*/
1647 // AcqFile::Failed - Failure handler                                    /*{{{*/
1648 // ---------------------------------------------------------------------
1649 /* Here we try other sources */
1650 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1651 {
1652    ErrorText = LookupTag(Message,"Message");
1653
1654    // This is the retry counter
1655    if (Retries != 0 &&
1656        Cnf->LocalOnly == false &&
1657        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1658    {
1659       Retries--;
1660       QueueURI(Desc);
1661       return;
1662    }
1663
1664    Item::Failed(Message,Cnf);
1665 }
1666                                                                         /*}}}*/
1667 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1668 // ---------------------------------------------------------------------
1669 /* The only header we use is the last-modified header. */
1670 string pkgAcqFile::Custom600Headers()
1671 {
1672    if (IsIndexFile)
1673       return "\nIndex-File: true";
1674 }
1675                                                                         /*}}}*/