apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36                                                                         /*}}}*/
  37
  38 using namespace std;
  39
  40 // Acquire::Item::Item - Constructor                                    /*{{{*/
  41 // ---------------------------------------------------------------------
  42 /* */
  43 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  44                        PartialSize(0), Mode(0), ID(0), Complete(false),
  45                        Local(false), QueueCounter(0)
  46 {
  47    Owner->Add(this);
  48    Status = StatIdle;
  49 }
  50                                                                         /*}}}*/
  51 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  52 // ---------------------------------------------------------------------
  53 /* */
  54 pkgAcquire::Item::~Item()
  55 {
  56    Owner->Remove(this);
  57 }
  58                                                                         /*}}}*/
  59 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* We return to an idle state if there are still other queues that could
  62    fetch this object */
  63 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  64 {
  65    Status = StatIdle;
  66    ErrorText = LookupTag(Message,"Message");
  67    if (QueueCounter <= 1)
  68    {
  69       /* This indicates that the file is not available right now but might
  70          be sometime later. If we do a retry cycle then this should be
  71          retried [CDROMs] */
  72       if (Cnf->LocalOnly == true &&
  73           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  74       {
  75          Status = StatIdle;
  76          Dequeue();
  77          return;
  78       }
  79
  80       Status = StatError;
  81       Dequeue();
  82    }
  83 }
  84                                                                         /*}}}*/
  85 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Stash status and the file size. Note that setting Complete means
  88    sub-phases of the acquire process such as decompresion are operating */
  89 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  90 {
  91    Status = StatFetching;
  92    if (FileSize == 0 && Complete == false)
  93       FileSize = Size;
  94 }
  95                                                                         /*}}}*/
  96 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 100                             pkgAcquire::MethodConfig *Cnf)
 101 {
 102    // We just downloaded something..
 103    string FileName = LookupTag(Message,"Filename");
 104    // we only inform the Log class if it was actually not a local thing
 105    if (Complete == false && !Local && FileName == DestFile)
 106    {
 107       if (Owner->Log != 0)
 108          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 109    }
 110
 111    if (FileSize == 0)
 112       FileSize= Size;
 113
 114    Status = StatDone;
 115    ErrorText = string();
 116    Owner->Dequeue(this);
 117 }
 118                                                                         /*}}}*/
 119 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 120 // ---------------------------------------------------------------------
 121 /* This helper function is used by alot of item methods as thier final
 122    step */
 123 void pkgAcquire::Item::Rename(string From,string To)
 124 {
 125    if (rename(From.c_str(),To.c_str()) != 0)
 126    {
 127       char S[300];
 128       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 129               From.c_str(),To.c_str());
 130       Status = StatError;
 131       ErrorText = S;
 132    }
 133 }
 134                                                                         /*}}}*/
 135 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 136 // ---------------------------------------------------------------------
 137 /* Get the DiffIndex file first and see if there are patches availabe
 138  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 139  * patches. If anything goes wrong in that process, it will fall back to
 140  * the original packages file
 141  */
 142 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 143                                  string URI,string URIDesc,string ShortDesc,
 144                                  HashString ExpectedHash)
 145    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 146      Description(URIDesc)
 147 {
 148
 149    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 150
 151    Desc.Description = URIDesc + "/DiffIndex";
 152    Desc.Owner = this;
 153    Desc.ShortDesc = ShortDesc;
 154    Desc.URI = URI + ".diff/Index";
 155
 156    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 157    DestFile += URItoFileName(URI) + string(".DiffIndex");
 158
 159    if(Debug)
 160       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 161
 162    // look for the current package file
 163    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 164    CurrentPackagesFile += URItoFileName(RealURI);
 165
 166    // FIXME: this file:/ check is a hack to prevent fetching
 167    //        from local sources. this is really silly, and
 168    //        should be fixed cleanly as soon as possible
 169    if(!FileExists(CurrentPackagesFile) ||
 170       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 171    {
 172       // we don't have a pkg file or we don't want to queue
 173       if(Debug)
 174          std::clog << "No index file, local or canceld by user" << std::endl;
 175       Failed("", NULL);
 176       return;
 177    }
 178
 179    if(Debug)
 180       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 181                 << CurrentPackagesFile << std::endl;
 182
 183    QueueURI(Desc);
 184
 185 }
 186                                                                         /*}}}*/
 187 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* The only header we use is the last-modified header. */
 190 string pkgAcqDiffIndex::Custom600Headers()
 191 {
 192    string Final = _config->FindDir("Dir::State::lists");
 193    Final += URItoFileName(RealURI) + string(".IndexDiff");
 194
 195    if(Debug)
 196       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 197
 198    struct stat Buf;
 199    if (stat(Final.c_str(),&Buf) != 0)
 200       return "\nIndex-File: true";
 201
 202    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 203 }
 204                                                                         /*}}}*/
 205 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 206 {
 207    if(Debug)
 208       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 209                 << std::endl;
 210
 211    pkgTagSection Tags;
 212    string ServerSha1;
 213    vector<DiffInfo> available_patches;
 214
 215    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 216    pkgTagFile TF(&Fd);
 217    if (_error->PendingError() == true)
 218       return false;
 219
 220    if(TF.Step(Tags) == true)
 221    {
 222       bool found = false;
 223       DiffInfo d;
 224       string size;
 225
 226       string const tmp = Tags.FindS("SHA1-Current");
 227       std::stringstream ss(tmp);
 228       ss >> ServerSha1 >> size;
 229       unsigned long const ServerSize = atol(size.c_str());
 230
 231       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 232       SHA1Summation SHA1;
 233       SHA1.AddFD(fd.Fd(), fd.Size());
 234       string const local_sha1 = SHA1.Result();
 235
 236       if(local_sha1 == ServerSha1)
 237       {
 238          // we have the same sha1 as the server
 239          if(Debug)
 240             std::clog << "Package file is up-to-date" << std::endl;
 241          // set found to true, this will queue a pkgAcqIndexDiffs with
 242          // a empty availabe_patches
 243          found = true;
 244       }
 245       else
 246       {
 247          if(Debug)
 248             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 249
 250          // check the historie and see what patches we need
 251          string const history = Tags.FindS("SHA1-History");
 252          std::stringstream hist(history);
 253          while(hist >> d.sha1 >> size >> d.file)
 254          {
 255             // read until the first match is found
 256             // from that point on, we probably need all diffs
 257             if(d.sha1 == local_sha1)
 258                found=true;
 259             else if (found == false)
 260                continue;
 261
 262             if(Debug)
 263                std::clog << "Need to get diff: " << d.file << std::endl;
 264             available_patches.push_back(d);
 265          }
 266
 267          if (available_patches.empty() == false)
 268          {
 269             // patching with too many files is rather slow compared to a fast download
 270             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 271             if (fileLimit != 0 && fileLimit < available_patches.size())
 272             {
 273                if (Debug)
 274                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 275                         << ") so fallback to complete download" << std::endl;
 276                return false;
 277             }
 278
 279             // see if the patches are too big
 280             found = false; // it was true and it will be true again at the end
 281             d = *available_patches.begin();
 282             string const firstPatch = d.file;
 283             unsigned long patchesSize = 0;
 284             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 285             while(patches >> d.sha1 >> size >> d.file)
 286             {
 287                if (firstPatch == d.file)
 288                   found = true;
 289                else if (found == false)
 290                   continue;
 291
 292                patchesSize += atol(size.c_str());
 293             }
 294             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 295             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 296             {
 297                if (Debug)
 298                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 299                         << ") so fallback to complete download" << std::endl;
 300                return false;
 301             }
 302          }
 303       }
 304
 305       // we have something, queue the next diff
 306       if(found)
 307       {
 308          // queue the diffs
 309          string::size_type const last_space = Description.rfind(" ");
 310          if(last_space != string::npos)
 311             Description.erase(last_space, Description.size()-last_space);
 312          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 313                               ExpectedHash, ServerSha1, available_patches);
 314          Complete = false;
 315          Status = StatDone;
 316          Dequeue();
 317          return true;
 318       }
 319    }
 320
 321    // Nothing found, report and return false
 322    // Failing here is ok, if we return false later, the full
 323    // IndexFile is queued
 324    if(Debug)
 325       std::clog << "Can't find a patch in the index file" << std::endl;
 326    return false;
 327 }
 328                                                                         /*}}}*/
 329 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 330 {
 331    if(Debug)
 332       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 333                 << "Falling back to normal index file aquire" << std::endl;
 334
 335    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 336                    ExpectedHash);
 337
 338    Complete = false;
 339    Status = StatDone;
 340    Dequeue();
 341 }
 342                                                                         /*}}}*/
 343 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 344                            pkgAcquire::MethodConfig *Cnf)
 345 {
 346    if(Debug)
 347       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 348
 349    Item::Done(Message,Size,Md5Hash,Cnf);
 350
 351    string FinalFile;
 352    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 353
 354    // sucess in downloading the index
 355    // rename the index
 356    FinalFile += string(".IndexDiff");
 357    if(Debug)
 358       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 359                 << std::endl;
 360    Rename(DestFile,FinalFile);
 361    chmod(FinalFile.c_str(),0644);
 362    DestFile = FinalFile;
 363
 364    if(!ParseDiffIndex(DestFile))
 365       return Failed("", NULL);
 366
 367    Complete = true;
 368    Status = StatDone;
 369    Dequeue();
 370    return;
 371 }
 372                                                                         /*}}}*/
 373 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* The package diff is added to the queue. one object is constructed
 376  * for each diff and the index
 377  */
 378 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 379                                    string URI,string URIDesc,string ShortDesc,
 380                                    HashString ExpectedHash,
 381                                    string ServerSha1,
 382                                    vector<DiffInfo> diffs)
 383    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 384      available_patches(diffs), ServerSha1(ServerSha1)
 385 {
 386
 387    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 388    DestFile += URItoFileName(URI);
 389
 390    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 391
 392    Description = URIDesc;
 393    Desc.Owner = this;
 394    Desc.ShortDesc = ShortDesc;
 395
 396    if(available_patches.size() == 0)
 397    {
 398       // we are done (yeah!)
 399       Finish(true);
 400    }
 401    else
 402    {
 403       // get the next diff
 404       State = StateFetchDiff;
 405       QueueNextDiff();
 406    }
 407 }
 408                                                                         /*}}}*/
 409 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 410 {
 411    if(Debug)
 412       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 413                 << "Falling back to normal index file aquire" << std::endl;
 414    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 415                    ExpectedHash);
 416    Finish();
 417 }
 418                                                                         /*}}}*/
 419 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 420 void pkgAcqIndexDiffs::Finish(bool allDone)
 421 {
 422    // we restore the original name, this is required, otherwise
 423    // the file will be cleaned
 424    if(allDone)
 425    {
 426       DestFile = _config->FindDir("Dir::State::lists");
 427       DestFile += URItoFileName(RealURI);
 428
 429       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 430       {
 431          Status = StatAuthError;
 432          ErrorText = _("MD5Sum mismatch");
 433          Rename(DestFile,DestFile + ".FAILED");
 434          Dequeue();
 435          return;
 436       }
 437
 438       // this is for the "real" finish
 439       Complete = true;
 440       Status = StatDone;
 441       Dequeue();
 442       if(Debug)
 443          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 444       return;
 445    }
 446
 447    if(Debug)
 448       std::clog << "Finishing: " << Desc.URI << std::endl;
 449    Complete = false;
 450    Status = StatDone;
 451    Dequeue();
 452    return;
 453 }
 454                                                                         /*}}}*/
 455 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 456 {
 457
 458    // calc sha1 of the just patched file
 459    string FinalFile = _config->FindDir("Dir::State::lists");
 460    FinalFile += URItoFileName(RealURI);
 461
 462    FileFd fd(FinalFile, FileFd::ReadOnly);
 463    SHA1Summation SHA1;
 464    SHA1.AddFD(fd.Fd(), fd.Size());
 465    string local_sha1 = string(SHA1.Result());
 466    if(Debug)
 467       std::clog << "QueueNextDiff: "
 468                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 469
 470    // final file reached before all patches are applied
 471    if(local_sha1 == ServerSha1)
 472    {
 473       Finish(true);
 474       return true;
 475    }
 476
 477    // remove all patches until the next matching patch is found
 478    // this requires the Index file to be ordered
 479    for(vector<DiffInfo>::iterator I=available_patches.begin();
 480        available_patches.size() > 0 &&
 481           I != available_patches.end() &&
 482           (*I).sha1 != local_sha1;
 483        I++)
 484    {
 485       available_patches.erase(I);
 486    }
 487
 488    // error checking and falling back if no patch was found
 489    if(available_patches.size() == 0)
 490    {
 491       Failed("", NULL);
 492       return false;
 493    }
 494
 495    // queue the right diff
 496    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 497    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 498    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 499    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 500
 501    if(Debug)
 502       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 503
 504    QueueURI(Desc);
 505
 506    return true;
 507 }
 508                                                                         /*}}}*/
 509 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 510                             pkgAcquire::MethodConfig *Cnf)
 511 {
 512    if(Debug)
 513       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 514
 515    Item::Done(Message,Size,Md5Hash,Cnf);
 516
 517    string FinalFile;
 518    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 519
 520    // sucess in downloading a diff, enter ApplyDiff state
 521    if(State == StateFetchDiff)
 522    {
 523
 524       if(Debug)
 525          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 526
 527       string FileName = LookupTag(Message,"Filename");
 528       State = StateUnzipDiff;
 529       Local = true;
 530       Desc.URI = "gzip:" + FileName;
 531       DestFile += ".decomp";
 532       QueueURI(Desc);
 533       Mode = "gzip";
 534       return;
 535    }
 536
 537    // sucess in downloading a diff, enter ApplyDiff state
 538    if(State == StateUnzipDiff)
 539    {
 540
 541       // rred excepts the patch as $FinalFile.ed
 542       Rename(DestFile,FinalFile+".ed");
 543
 544       if(Debug)
 545          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 546
 547       State = StateApplyDiff;
 548       Local = true;
 549       Desc.URI = "rred:" + FinalFile;
 550       QueueURI(Desc);
 551       Mode = "rred";
 552       return;
 553    }
 554
 555
 556    // success in download/apply a diff, queue next (if needed)
 557    if(State == StateApplyDiff)
 558    {
 559       // remove the just applied patch
 560       available_patches.erase(available_patches.begin());
 561
 562       // move into place
 563       if(Debug)
 564       {
 565          std::clog << "Moving patched file in place: " << std::endl
 566                    << DestFile << " -> " << FinalFile << std::endl;
 567       }
 568       Rename(DestFile,FinalFile);
 569       chmod(FinalFile.c_str(),0644);
 570
 571       // see if there is more to download
 572       if(available_patches.size() > 0) {
 573          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 574                               ExpectedHash, ServerSha1, available_patches);
 575          return Finish();
 576       } else
 577          return Finish(true);
 578    }
 579 }
 580                                                                         /*}}}*/
 581 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 582 // ---------------------------------------------------------------------
 583 /* The package file is added to the queue and a second class is
 584    instantiated to fetch the revision file */
 585 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 586                          string URI,string URIDesc,string ShortDesc,
 587                          HashString ExpectedHash, string comprExt)
 588    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 589 {
 590    Decompression = false;
 591    Erase = false;
 592
 593    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 594    DestFile += URItoFileName(URI);
 595
 596    if(comprExt.empty())
 597    {
 598       // autoselect the compression method
 599       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 600       if (types.empty() == true)
 601          comprExt = "plain";
 602       else
 603          comprExt = "." + types[0];
 604    }
 605    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 606
 607    Desc.URI = URI + CompressionExtension;
 608
 609    Desc.Description = URIDesc;
 610    Desc.Owner = this;
 611    Desc.ShortDesc = ShortDesc;
 612
 613    QueueURI(Desc);
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The only header we use is the last-modified header. */
 619 string pkgAcqIndex::Custom600Headers()
 620 {
 621    string Final = _config->FindDir("Dir::State::lists");
 622    Final += URItoFileName(RealURI);
 623
 624    struct stat Buf;
 625    if (stat(Final.c_str(),&Buf) != 0)
 626       return "\nIndex-File: true";
 627
 628    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 629 }
 630                                                                         /*}}}*/
 631 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 632 {
 633    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 634
 635    for (std::vector<std::string>::const_iterator t = types.begin();
 636         t != types.end(); t++)
 637    {
 638       // jump over all already tried compression types
 639       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 640       if(Desc.URI.substr(nameLen) != *t)
 641          continue;
 642
 643       // we want to try it with the next extension (and make sure to
 644       // not skip over the end)
 645       t++;
 646       if (t == types.end())
 647          break;
 648
 649       // queue new download
 650       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 651       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 652       ExpectedHash, string(".").append(*t));
 653
 654       Status = StatDone;
 655       Complete = false;
 656       Dequeue();
 657       return;
 658    }
 659
 660    // on decompression failure, remove bad versions in partial/
 661    if(Decompression && Erase) {
 662       string s = _config->FindDir("Dir::State::lists") + "partial/";
 663       s += URItoFileName(RealURI);
 664       unlink(s.c_str());
 665    }
 666
 667    Item::Failed(Message,Cnf);
 668 }
 669                                                                         /*}}}*/
 670 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 671 // ---------------------------------------------------------------------
 672 /* This goes through a number of states.. On the initial fetch the
 673    method could possibly return an alternate filename which points
 674    to the uncompressed version of the file. If this is so the file
 675    is copied into the partial directory. In all other cases the file
 676    is decompressed with a gzip uri. */
 677 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 678                        pkgAcquire::MethodConfig *Cfg)
 679 {
 680    Item::Done(Message,Size,Hash,Cfg);
 681
 682    if (Decompression == true)
 683    {
 684       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 685       {
 686          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 687          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 688       }
 689
 690       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 691       {
 692          Status = StatAuthError;
 693          ErrorText = _("Hash Sum mismatch");
 694          Rename(DestFile,DestFile + ".FAILED");
 695          return;
 696       }
 697       // Done, move it into position
 698       string FinalFile = _config->FindDir("Dir::State::lists");
 699       FinalFile += URItoFileName(RealURI);
 700       Rename(DestFile,FinalFile);
 701       chmod(FinalFile.c_str(),0644);
 702
 703       /* We restore the original name to DestFile so that the clean operation
 704          will work OK */
 705       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 706       DestFile += URItoFileName(RealURI);
 707
 708       // Remove the compressed version.
 709       if (Erase == true)
 710          unlink(DestFile.c_str());
 711       return;
 712    }
 713
 714    Erase = false;
 715    Complete = true;
 716
 717    // Handle the unzipd case
 718    string FileName = LookupTag(Message,"Alt-Filename");
 719    if (FileName.empty() == false)
 720    {
 721       // The files timestamp matches
 722       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 723          return;
 724       Decompression = true;
 725       Local = true;
 726       DestFile += ".decomp";
 727       Desc.URI = "copy:" + FileName;
 728       QueueURI(Desc);
 729       Mode = "copy";
 730       return;
 731    }
 732
 733    FileName = LookupTag(Message,"Filename");
 734    if (FileName.empty() == true)
 735    {
 736       Status = StatError;
 737       ErrorText = "Method gave a blank filename";
 738    }
 739
 740    // The files timestamp matches
 741    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 742       return;
 743
 744    if (FileName == DestFile)
 745       Erase = true;
 746    else
 747       Local = true;
 748
 749    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 750    string decompProg;
 751
 752    // get the binary name for your used compression type
 753    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 754    if(decompProg.empty() == false);
 755    // flExtensions returns the full name if no extension is found
 756    // this is why we have this complicated compare operation here
 757    // FIMXE: add a new flJustExtension() that return "" if no
 758    //        extension is found and use that above so that it can
 759    //        be tested against ""
 760    else if(compExt == flNotDir(URI(Desc.URI).Path))
 761       decompProg = "copy";
 762    else {
 763       _error->Error("Unsupported extension: %s", compExt.c_str());
 764       return;
 765    }
 766
 767    Decompression = true;
 768    DestFile += ".decomp";
 769    Desc.URI = decompProg + ":" + FileName;
 770    QueueURI(Desc);
 771    Mode = decompProg.c_str();
 772 }
 773                                                                         /*}}}*/
 774 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 775 // ---------------------------------------------------------------------
 776 /* The Translation file is added to the queue */
 777 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 778                             string URI,string URIDesc,string ShortDesc)
 779   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 780 {
 781 }
 782                                                                         /*}}}*/
 783 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 784 // ---------------------------------------------------------------------
 785 /* */
 786 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 787 {
 788    if (Cnf->LocalOnly == true ||
 789        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 790    {
 791       // Ignore this
 792       Status = StatDone;
 793       Complete = false;
 794       Dequeue();
 795       return;
 796    }
 797
 798    Item::Failed(Message,Cnf);
 799 }
 800                                                                         /*}}}*/
 801 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 802                              string URI,string URIDesc,string ShortDesc,
 803                              string MetaIndexURI, string MetaIndexURIDesc,
 804                              string MetaIndexShortDesc,
 805                              const vector<IndexTarget*>* IndexTargets,
 806                              indexRecords* MetaIndexParser) :
 807    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 808    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 809    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 810 {
 811    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 812    DestFile += URItoFileName(URI);
 813
 814    // remove any partial downloaded sig-file in partial/.
 815    // it may confuse proxies and is too small to warrant a
 816    // partial download anyway
 817    unlink(DestFile.c_str());
 818
 819    // Create the item
 820    Desc.Description = URIDesc;
 821    Desc.Owner = this;
 822    Desc.ShortDesc = ShortDesc;
 823    Desc.URI = URI;
 824
 825    string Final = _config->FindDir("Dir::State::lists");
 826    Final += URItoFileName(RealURI);
 827    struct stat Buf;
 828    if (stat(Final.c_str(),&Buf) == 0)
 829    {
 830       // File was already in place.  It needs to be re-downloaded/verified
 831       // because Release might have changed, we do give it a differnt
 832       // name than DestFile because otherwise the http method will
 833       // send If-Range requests and there are too many broken servers
 834       // out there that do not understand them
 835       LastGoodSig = DestFile+".reverify";
 836       Rename(Final,LastGoodSig);
 837    }
 838
 839    QueueURI(Desc);
 840 }
 841                                                                         /*}}}*/
 842 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 843 // ---------------------------------------------------------------------
 844 /* The only header we use is the last-modified header. */
 845 string pkgAcqMetaSig::Custom600Headers()
 846 {
 847    struct stat Buf;
 848    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 849       return "\nIndex-File: true";
 850
 851    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 852 }
 853
 854 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 855                          pkgAcquire::MethodConfig *Cfg)
 856 {
 857    Item::Done(Message,Size,MD5,Cfg);
 858
 859    string FileName = LookupTag(Message,"Filename");
 860    if (FileName.empty() == true)
 861    {
 862       Status = StatError;
 863       ErrorText = "Method gave a blank filename";
 864       return;
 865    }
 866
 867    if (FileName != DestFile)
 868    {
 869       // We have to copy it into place
 870       Local = true;
 871       Desc.URI = "copy:" + FileName;
 872       QueueURI(Desc);
 873       return;
 874    }
 875
 876    Complete = true;
 877
 878    // put the last known good file back on i-m-s hit (it will
 879    // be re-verified again)
 880    // Else do nothing, we have the new file in DestFile then
 881    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 882       Rename(LastGoodSig, DestFile);
 883
 884    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 885    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 886                        DestFile, IndexTargets, MetaIndexParser);
 887
 888 }
 889                                                                         /*}}}*/
 890 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 891 {
 892    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 893
 894    // if we get a network error we fail gracefully
 895    if(Status == StatTransientNetworkError)
 896    {
 897       Item::Failed(Message,Cnf);
 898       // move the sigfile back on transient network failures
 899       if(FileExists(DestFile))
 900          Rename(LastGoodSig,Final);
 901
 902       // set the status back to , Item::Failed likes to reset it
 903       Status = pkgAcquire::Item::StatTransientNetworkError;
 904       return;
 905    }
 906
 907    // Delete any existing sigfile when the acquire failed
 908    unlink(Final.c_str());
 909
 910    // queue a pkgAcqMetaIndex with no sigfile
 911    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 912                        "", IndexTargets, MetaIndexParser);
 913
 914    if (Cnf->LocalOnly == true ||
 915        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 916    {
 917       // Ignore this
 918       Status = StatDone;
 919       Complete = false;
 920       Dequeue();
 921       return;
 922    }
 923
 924    Item::Failed(Message,Cnf);
 925 }
 926                                                                         /*}}}*/
 927 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 928                                  string URI,string URIDesc,string ShortDesc,
 929                                  string SigFile,
 930                                  const vector<struct IndexTarget*>* IndexTargets,
 931                                  indexRecords* MetaIndexParser) :
 932    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 933    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 934 {
 935    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 936    DestFile += URItoFileName(URI);
 937
 938    // Create the item
 939    Desc.Description = URIDesc;
 940    Desc.Owner = this;
 941    Desc.ShortDesc = ShortDesc;
 942    Desc.URI = URI;
 943
 944    QueueURI(Desc);
 945 }
 946                                                                         /*}}}*/
 947 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 948 // ---------------------------------------------------------------------
 949 /* The only header we use is the last-modified header. */
 950 string pkgAcqMetaIndex::Custom600Headers()
 951 {
 952    string Final = _config->FindDir("Dir::State::lists");
 953    Final += URItoFileName(RealURI);
 954
 955    struct stat Buf;
 956    if (stat(Final.c_str(),&Buf) != 0)
 957       return "\nIndex-File: true";
 958
 959    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 960 }
 961                                                                         /*}}}*/
 962 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 963                            pkgAcquire::MethodConfig *Cfg)
 964 {
 965    Item::Done(Message,Size,Hash,Cfg);
 966
 967    // MetaIndexes are done in two passes: one to download the
 968    // metaindex with an appropriate method, and a second to verify it
 969    // with the gpgv method
 970
 971    if (AuthPass == true)
 972    {
 973       AuthDone(Message);
 974    }
 975    else
 976    {
 977       RetrievalDone(Message);
 978       if (!Complete)
 979          // Still more retrieving to do
 980          return;
 981
 982       if (SigFile == "")
 983       {
 984          // There was no signature file, so we are finished.  Download
 985          // the indexes without verification.
 986          QueueIndexes(false);
 987       }
 988       else
 989       {
 990          // There was a signature file, so pass it to gpgv for
 991          // verification
 992
 993          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 994             std::cerr << "Metaindex acquired, queueing gpg verification ("
 995                       << SigFile << "," << DestFile << ")\n";
 996          AuthPass = true;
 997          Desc.URI = "gpgv:" + SigFile;
 998          QueueURI(Desc);
 999          Mode = "gpgv";
1000       }
1001    }
1002 }
1003                                                                         /*}}}*/
1004 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1005 {
1006    // We have just finished downloading a Release file (it is not
1007    // verified yet)
1008
1009    string FileName = LookupTag(Message,"Filename");
1010    if (FileName.empty() == true)
1011    {
1012       Status = StatError;
1013       ErrorText = "Method gave a blank filename";
1014       return;
1015    }
1016
1017    if (FileName != DestFile)
1018    {
1019       Local = true;
1020       Desc.URI = "copy:" + FileName;
1021       QueueURI(Desc);
1022       return;
1023    }
1024
1025    // see if the download was a IMSHit
1026    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1027    Complete = true;
1028
1029    string FinalFile = _config->FindDir("Dir::State::lists");
1030    FinalFile += URItoFileName(RealURI);
1031
1032    // If we get a IMS hit we can remove the empty file in partial
1033    // othersie we move the file in place
1034    if (IMSHit)
1035       unlink(DestFile.c_str());
1036    else
1037       Rename(DestFile,FinalFile);
1038
1039    chmod(FinalFile.c_str(),0644);
1040    DestFile = FinalFile;
1041 }
1042                                                                         /*}}}*/
1043 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1044 {
1045    // At this point, the gpgv method has succeeded, so there is a
1046    // valid signature from a key in the trusted keyring.  We
1047    // perform additional verification of its contents, and use them
1048    // to verify the indexes we are about to download
1049
1050    if (!MetaIndexParser->Load(DestFile))
1051    {
1052       Status = StatAuthError;
1053       ErrorText = MetaIndexParser->ErrorText;
1054       return;
1055    }
1056
1057    if (!VerifyVendor(Message))
1058    {
1059       return;
1060    }
1061
1062    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1063       std::cerr << "Signature verification succeeded: "
1064                 << DestFile << std::endl;
1065
1066    // Download further indexes with verification
1067    QueueIndexes(true);
1068
1069    // Done, move signature file into position
1070
1071    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1072       URItoFileName(RealURI) + ".gpg";
1073    Rename(SigFile,VerifiedSigFile);
1074    chmod(VerifiedSigFile.c_str(),0644);
1075 }
1076                                                                         /*}}}*/
1077 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1078 {
1079    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1080         Target != IndexTargets->end();
1081         Target++)
1082    {
1083       HashString ExpectedIndexHash;
1084       if (verify)
1085       {
1086          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1087          if (!Record)
1088          {
1089             Status = StatAuthError;
1090             ErrorText = "Unable to find expected entry  "
1091                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1092             return;
1093          }
1094          ExpectedIndexHash = Record->Hash;
1095          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1096          {
1097             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1098             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1099          }
1100          if (ExpectedIndexHash.empty())
1101          {
1102             Status = StatAuthError;
1103             ErrorText = "Unable to find hash sum for "
1104                + (*Target)->MetaKey + " in Meta-index file";
1105             return;
1106          }
1107       }
1108
1109       // Queue Packages file (either diff or full packages files, depending
1110       // on the users option)
1111       if(_config->FindB("Acquire::PDiffs",true) == true)
1112          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1113                              (*Target)->ShortDesc, ExpectedIndexHash);
1114       else
1115          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1116                             (*Target)->ShortDesc, ExpectedIndexHash);
1117    }
1118 }
1119                                                                         /*}}}*/
1120 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1121 {
1122 //    // Maybe this should be made available from above so we don't have
1123 //    // to read and parse it every time?
1124 //    pkgVendorList List;
1125 //    List.ReadMainList();
1126
1127 //    const Vendor* Vndr = NULL;
1128 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1129 //    {
1130 //       string::size_type pos = (*I).find("VALIDSIG ");
1131 //       if (_config->FindB("Debug::Vendor", false))
1132 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1133 //                    << std::endl;
1134 //       if (pos != std::string::npos)
1135 //       {
1136 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1137 //          if (_config->FindB("Debug::Vendor", false))
1138 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1139 //                std::endl;
1140 //          Vndr = List.FindVendor(Fingerprint) != "";
1141 //          if (Vndr != NULL);
1142 //          break;
1143 //       }
1144 //    }
1145    string::size_type pos;
1146
1147    // check for missing sigs (that where not fatal because otherwise we had
1148    // bombed earlier)
1149    string missingkeys;
1150    string msg = _("There is no public key available for the "
1151                   "following key IDs:\n");
1152    pos = Message.find("NO_PUBKEY ");
1153    if (pos != std::string::npos)
1154    {
1155       string::size_type start = pos+strlen("NO_PUBKEY ");
1156       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1157       missingkeys += (Fingerprint);
1158    }
1159    if(!missingkeys.empty())
1160       _error->Warning("%s", string(msg+missingkeys).c_str());
1161
1162    string Transformed = MetaIndexParser->GetExpectedDist();
1163
1164    if (Transformed == "../project/experimental")
1165    {
1166       Transformed = "experimental";
1167    }
1168
1169    pos = Transformed.rfind('/');
1170    if (pos != string::npos)
1171    {
1172       Transformed = Transformed.substr(0, pos);
1173    }
1174
1175    if (Transformed == ".")
1176    {
1177       Transformed = "";
1178    }
1179
1180    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1181    {
1182       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1183       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1184       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1185    }
1186
1187    if (MetaIndexParser->CheckDist(Transformed) == false)
1188    {
1189       // This might become fatal one day
1190 //       Status = StatAuthError;
1191 //       ErrorText = "Conflicting distribution; expected "
1192 //          + MetaIndexParser->GetExpectedDist() + " but got "
1193 //          + MetaIndexParser->GetDist();
1194 //       return false;
1195       if (!Transformed.empty())
1196       {
1197          _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
1198                          Desc.Description.c_str(),
1199                          Transformed.c_str(),
1200                          MetaIndexParser->GetDist().c_str());
1201       }
1202    }
1203
1204    return true;
1205 }
1206                                                                         /*}}}*/
1207 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1208 // ---------------------------------------------------------------------
1209 /* */
1210 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1211 {
1212    if (AuthPass == true)
1213    {
1214       // if we fail the authentication but got the file via a IMS-Hit
1215       // this means that the file wasn't downloaded and that it might be
1216       // just stale (server problem, proxy etc). we delete what we have
1217       // queue it again without i-m-s
1218       // alternatively we could just unlink the file and let the user try again
1219       if (IMSHit)
1220       {
1221          Complete = false;
1222          Local = false;
1223          AuthPass = false;
1224          unlink(DestFile.c_str());
1225
1226          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1227          DestFile += URItoFileName(RealURI);
1228          Desc.URI = RealURI;
1229          QueueURI(Desc);
1230          return;
1231       }
1232
1233       // gpgv method failed
1234       _error->Warning("GPG error: %s: %s",
1235                       Desc.Description.c_str(),
1236                       LookupTag(Message,"Message").c_str());
1237
1238    }
1239
1240    // No Release file was present, or verification failed, so fall
1241    // back to queueing Packages files without verification
1242    QueueIndexes(false);
1243 }
1244                                                                         /*}}}*/
1245 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1246 // ---------------------------------------------------------------------
1247 /* This just sets up the initial fetch environment and queues the first
1248    possibilitiy */
1249 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1250                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1251                              string &StoreFilename) :
1252                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1253                StoreFilename(StoreFilename), Vf(Version.FileList()),
1254                Trusted(false)
1255 {
1256    Retries = _config->FindI("Acquire::Retries",0);
1257
1258    if (Version.Arch() == 0)
1259    {
1260       _error->Error(_("I wasn't able to locate a file for the %s package. "
1261                       "This might mean you need to manually fix this package. "
1262                       "(due to missing arch)"),
1263                     Version.ParentPkg().Name());
1264       return;
1265    }
1266
1267    /* We need to find a filename to determine the extension. We make the
1268       assumption here that all the available sources for this version share
1269       the same extension.. */
1270    // Skip not source sources, they do not have file fields.
1271    for (; Vf.end() == false; Vf++)
1272    {
1273       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1274          continue;
1275       break;
1276    }
1277
1278    // Does not really matter here.. we are going to fail out below
1279    if (Vf.end() != true)
1280    {
1281       // If this fails to get a file name we will bomb out below.
1282       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1283       if (_error->PendingError() == true)
1284          return;
1285
1286       // Generate the final file name as: package_version_arch.foo
1287       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1288                       QuoteString(Version.VerStr(),"_:") + '_' +
1289                       QuoteString(Version.Arch(),"_:.") +
1290                       "." + flExtension(Parse.FileName());
1291    }
1292
1293    // check if we have one trusted source for the package. if so, switch
1294    // to "TrustedOnly" mode
1295    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1296    {
1297       pkgIndexFile *Index;
1298       if (Sources->FindIndex(i.File(),Index) == false)
1299          continue;
1300       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1301       {
1302          std::cerr << "Checking index: " << Index->Describe()
1303                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1304       }
1305       if (Index->IsTrusted()) {
1306          Trusted = true;
1307          break;
1308       }
1309    }
1310
1311    // "allow-unauthenticated" restores apts old fetching behaviour
1312    // that means that e.g. unauthenticated file:// uris are higher
1313    // priority than authenticated http:// uris
1314    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1315       Trusted = false;
1316
1317    // Select a source
1318    if (QueueNext() == false && _error->PendingError() == false)
1319       _error->Error(_("I wasn't able to locate file for the %s package. "
1320                     "This might mean you need to manually fix this package."),
1321                     Version.ParentPkg().Name());
1322 }
1323                                                                         /*}}}*/
1324 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1325 // ---------------------------------------------------------------------
1326 /* This queues the next available file version for download. It checks if
1327    the archive is already available in the cache and stashs the MD5 for
1328    checking later. */
1329 bool pkgAcqArchive::QueueNext()
1330 {
1331    string const ForceHash = _config->Find("Acquire::ForceHash");
1332    for (; Vf.end() == false; Vf++)
1333    {
1334       // Ignore not source sources
1335       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1336          continue;
1337
1338       // Try to cross match against the source list
1339       pkgIndexFile *Index;
1340       if (Sources->FindIndex(Vf.File(),Index) == false)
1341             continue;
1342
1343       // only try to get a trusted package from another source if that source
1344       // is also trusted
1345       if(Trusted && !Index->IsTrusted())
1346          continue;
1347
1348       // Grab the text package record
1349       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1350       if (_error->PendingError() == true)
1351          return false;
1352
1353       string PkgFile = Parse.FileName();
1354       if (ForceHash.empty() == false)
1355       {
1356          if(stringcasecmp(ForceHash, "sha256") == 0)
1357             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1358          else if (stringcasecmp(ForceHash, "sha1") == 0)
1359             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1360          else
1361             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1362       }
1363       else
1364       {
1365          string Hash;
1366          if ((Hash = Parse.SHA256Hash()).empty() == false)
1367             ExpectedHash = HashString("SHA256", Hash);
1368          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1369             ExpectedHash = HashString("SHA1", Hash);
1370          else
1371             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1372       }
1373       if (PkgFile.empty() == true)
1374          return _error->Error(_("The package index files are corrupted. No Filename: "
1375                               "field for package %s."),
1376                               Version.ParentPkg().Name());
1377
1378       Desc.URI = Index->ArchiveURI(PkgFile);
1379       Desc.Description = Index->ArchiveInfo(Version);
1380       Desc.Owner = this;
1381       Desc.ShortDesc = Version.ParentPkg().Name();
1382
1383       // See if we already have the file. (Legacy filenames)
1384       FileSize = Version->Size;
1385       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1386       struct stat Buf;
1387       if (stat(FinalFile.c_str(),&Buf) == 0)
1388       {
1389          // Make sure the size matches
1390          if ((unsigned)Buf.st_size == Version->Size)
1391          {
1392             Complete = true;
1393             Local = true;
1394             Status = StatDone;
1395             StoreFilename = DestFile = FinalFile;
1396             return true;
1397          }
1398
1399          /* Hmm, we have a file and its size does not match, this means it is
1400             an old style mismatched arch */
1401          unlink(FinalFile.c_str());
1402       }
1403
1404       // Check it again using the new style output filenames
1405       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1406       if (stat(FinalFile.c_str(),&Buf) == 0)
1407       {
1408          // Make sure the size matches
1409          if ((unsigned)Buf.st_size == Version->Size)
1410          {
1411             Complete = true;
1412             Local = true;
1413             Status = StatDone;
1414             StoreFilename = DestFile = FinalFile;
1415             return true;
1416          }
1417
1418          /* Hmm, we have a file and its size does not match, this shouldnt
1419             happen.. */
1420          unlink(FinalFile.c_str());
1421       }
1422
1423       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1424
1425       // Check the destination file
1426       if (stat(DestFile.c_str(),&Buf) == 0)
1427       {
1428          // Hmm, the partial file is too big, erase it
1429          if ((unsigned)Buf.st_size > Version->Size)
1430             unlink(DestFile.c_str());
1431          else
1432             PartialSize = Buf.st_size;
1433       }
1434
1435       // Create the item
1436       Local = false;
1437       Desc.URI = Index->ArchiveURI(PkgFile);
1438       Desc.Description = Index->ArchiveInfo(Version);
1439       Desc.Owner = this;
1440       Desc.ShortDesc = Version.ParentPkg().Name();
1441       QueueURI(Desc);
1442
1443       Vf++;
1444       return true;
1445    }
1446    return false;
1447 }
1448                                                                         /*}}}*/
1449 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1450 // ---------------------------------------------------------------------
1451 /* */
1452 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1453                          pkgAcquire::MethodConfig *Cfg)
1454 {
1455    Item::Done(Message,Size,CalcHash,Cfg);
1456
1457    // Check the size
1458    if (Size != Version->Size)
1459    {
1460       Status = StatError;
1461       ErrorText = _("Size mismatch");
1462       return;
1463    }
1464
1465    // Check the hash
1466    if(ExpectedHash.toStr() != CalcHash)
1467    {
1468       Status = StatError;
1469       ErrorText = _("Hash Sum mismatch");
1470       if(FileExists(DestFile))
1471          Rename(DestFile,DestFile + ".FAILED");
1472       return;
1473    }
1474
1475    // Grab the output filename
1476    string FileName = LookupTag(Message,"Filename");
1477    if (FileName.empty() == true)
1478    {
1479       Status = StatError;
1480       ErrorText = "Method gave a blank filename";
1481       return;
1482    }
1483
1484    Complete = true;
1485
1486    // Reference filename
1487    if (FileName != DestFile)
1488    {
1489       StoreFilename = DestFile = FileName;
1490       Local = true;
1491       return;
1492    }
1493
1494    // Done, move it into position
1495    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1496    FinalFile += flNotDir(StoreFilename);
1497    Rename(DestFile,FinalFile);
1498
1499    StoreFilename = DestFile = FinalFile;
1500    Complete = true;
1501 }
1502                                                                         /*}}}*/
1503 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1504 // ---------------------------------------------------------------------
1505 /* Here we try other sources */
1506 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1507 {
1508    ErrorText = LookupTag(Message,"Message");
1509
1510    /* We don't really want to retry on failed media swaps, this prevents
1511       that. An interesting observation is that permanent failures are not
1512       recorded. */
1513    if (Cnf->Removable == true &&
1514        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1515    {
1516       // Vf = Version.FileList();
1517       while (Vf.end() == false) Vf++;
1518       StoreFilename = string();
1519       Item::Failed(Message,Cnf);
1520       return;
1521    }
1522
1523    if (QueueNext() == false)
1524    {
1525       // This is the retry counter
1526       if (Retries != 0 &&
1527           Cnf->LocalOnly == false &&
1528           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1529       {
1530          Retries--;
1531          Vf = Version.FileList();
1532          if (QueueNext() == true)
1533             return;
1534       }
1535
1536       StoreFilename = string();
1537       Item::Failed(Message,Cnf);
1538    }
1539 }
1540                                                                         /*}}}*/
1541 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1542 // ---------------------------------------------------------------------
1543 bool pkgAcqArchive::IsTrusted()
1544 {
1545    return Trusted;
1546 }
1547                                                                         /*}}}*/
1548 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1549 // ---------------------------------------------------------------------
1550 /* */
1551 void pkgAcqArchive::Finished()
1552 {
1553    if (Status == pkgAcquire::Item::StatDone &&
1554        Complete == true)
1555       return;
1556    StoreFilename = string();
1557 }
1558                                                                         /*}}}*/
1559 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1560 // ---------------------------------------------------------------------
1561 /* The file is added to the queue */
1562 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1563                        unsigned long Size,string Dsc,string ShortDesc,
1564                        const string &DestDir, const string &DestFilename,
1565                        bool IsIndexFile) :
1566                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1567 {
1568    Retries = _config->FindI("Acquire::Retries",0);
1569
1570    if(!DestFilename.empty())
1571       DestFile = DestFilename;
1572    else if(!DestDir.empty())
1573       DestFile = DestDir + "/" + flNotDir(URI);
1574    else
1575       DestFile = flNotDir(URI);
1576
1577    // Create the item
1578    Desc.URI = URI;
1579    Desc.Description = Dsc;
1580    Desc.Owner = this;
1581
1582    // Set the short description to the archive component
1583    Desc.ShortDesc = ShortDesc;
1584
1585    // Get the transfer sizes
1586    FileSize = Size;
1587    struct stat Buf;
1588    if (stat(DestFile.c_str(),&Buf) == 0)
1589    {
1590       // Hmm, the partial file is too big, erase it
1591       if ((unsigned)Buf.st_size > Size)
1592          unlink(DestFile.c_str());
1593       else
1594          PartialSize = Buf.st_size;
1595    }
1596
1597    QueueURI(Desc);
1598 }
1599                                                                         /*}}}*/
1600 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1601 // ---------------------------------------------------------------------
1602 /* */
1603 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1604                       pkgAcquire::MethodConfig *Cnf)
1605 {
1606    Item::Done(Message,Size,CalcHash,Cnf);
1607
1608    // Check the hash
1609    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1610    {
1611       Status = StatError;
1612       ErrorText = "Hash Sum mismatch";
1613       Rename(DestFile,DestFile + ".FAILED");
1614       return;
1615    }
1616
1617    string FileName = LookupTag(Message,"Filename");
1618    if (FileName.empty() == true)
1619    {
1620       Status = StatError;
1621       ErrorText = "Method gave a blank filename";
1622       return;
1623    }
1624
1625    Complete = true;
1626
1627    // The files timestamp matches
1628    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1629       return;
1630
1631    // We have to copy it into place
1632    if (FileName != DestFile)
1633    {
1634       Local = true;
1635       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1636           Cnf->Removable == true)
1637       {
1638          Desc.URI = "copy:" + FileName;
1639          QueueURI(Desc);
1640          return;
1641       }
1642
1643       // Erase the file if it is a symlink so we can overwrite it
1644       struct stat St;
1645       if (lstat(DestFile.c_str(),&St) == 0)
1646       {
1647          if (S_ISLNK(St.st_mode) != 0)
1648             unlink(DestFile.c_str());
1649       }
1650
1651       // Symlink the file
1652       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1653       {
1654          ErrorText = "Link to " + DestFile + " failure ";
1655          Status = StatError;
1656          Complete = false;
1657       }
1658    }
1659 }
1660                                                                         /*}}}*/
1661 // AcqFile::Failed - Failure handler                                    /*{{{*/
1662 // ---------------------------------------------------------------------
1663 /* Here we try other sources */
1664 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1665 {
1666    ErrorText = LookupTag(Message,"Message");
1667
1668    // This is the retry counter
1669    if (Retries != 0 &&
1670        Cnf->LocalOnly == false &&
1671        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1672    {
1673       Retries--;
1674       QueueURI(Desc);
1675       return;
1676    }
1677
1678    Item::Failed(Message,Cnf);
1679 }
1680                                                                         /*}}}*/
1681 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1682 // ---------------------------------------------------------------------
1683 /* The only header we use is the last-modified header. */
1684 string pkgAcqFile::Custom600Headers()
1685 {
1686    if (IsIndexFile)
1687       return "\nIndex-File: true";
1688    return "";
1689 }
1690                                                                         /*}}}*/