apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    if (QueueCounter <= 1)
  69    {
  70       /* This indicates that the file is not available right now but might
  71          be sometime later. If we do a retry cycle then this should be
  72          retried [CDROMs] */
  73       if (Cnf->LocalOnly == true &&
  74           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  75       {
  76          Status = StatIdle;
  77          Dequeue();
  78          return;
  79       }
  80
  81       Status = StatError;
  82       Dequeue();
  83    }
  84 }
  85                                                                         /*}}}*/
  86 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* Stash status and the file size. Note that setting Complete means
  89    sub-phases of the acquire process such as decompresion are operating */
  90 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  91 {
  92    Status = StatFetching;
  93    if (FileSize == 0 && Complete == false)
  94       FileSize = Size;
  95 }
  96                                                                         /*}}}*/
  97 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* */
 100 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 101                             pkgAcquire::MethodConfig *Cnf)
 102 {
 103    // We just downloaded something..
 104    string FileName = LookupTag(Message,"Filename");
 105    // we only inform the Log class if it was actually not a local thing
 106    if (Complete == false && !Local && FileName == DestFile)
 107    {
 108       if (Owner->Log != 0)
 109          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 110    }
 111
 112    if (FileSize == 0)
 113       FileSize= Size;
 114
 115    Status = StatDone;
 116    ErrorText = string();
 117    Owner->Dequeue(this);
 118 }
 119                                                                         /*}}}*/
 120 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 121 // ---------------------------------------------------------------------
 122 /* This helper function is used by alot of item methods as thier final
 123    step */
 124 void pkgAcquire::Item::Rename(string From,string To)
 125 {
 126    if (rename(From.c_str(),To.c_str()) != 0)
 127    {
 128       char S[300];
 129       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 130               From.c_str(),To.c_str());
 131       Status = StatError;
 132       ErrorText = S;
 133    }
 134 }
 135                                                                         /*}}}*/
 136 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 137 // ---------------------------------------------------------------------
 138 /* Get the DiffIndex file first and see if there are patches availabe
 139  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 140  * patches. If anything goes wrong in that process, it will fall back to
 141  * the original packages file
 142  */
 143 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 144                                  string URI,string URIDesc,string ShortDesc,
 145                                  HashString ExpectedHash)
 146    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 147      Description(URIDesc)
 148 {
 149
 150    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 151
 152    Desc.Description = URIDesc + "/DiffIndex";
 153    Desc.Owner = this;
 154    Desc.ShortDesc = ShortDesc;
 155    Desc.URI = URI + ".diff/Index";
 156
 157    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 158    DestFile += URItoFileName(URI) + string(".DiffIndex");
 159
 160    if(Debug)
 161       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 162
 163    // look for the current package file
 164    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 165    CurrentPackagesFile += URItoFileName(RealURI);
 166
 167    // FIXME: this file:/ check is a hack to prevent fetching
 168    //        from local sources. this is really silly, and
 169    //        should be fixed cleanly as soon as possible
 170    if(!FileExists(CurrentPackagesFile) ||
 171       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 172    {
 173       // we don't have a pkg file or we don't want to queue
 174       if(Debug)
 175          std::clog << "No index file, local or canceld by user" << std::endl;
 176       Failed("", NULL);
 177       return;
 178    }
 179
 180    if(Debug)
 181       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 182                 << CurrentPackagesFile << std::endl;
 183
 184    QueueURI(Desc);
 185
 186 }
 187                                                                         /*}}}*/
 188 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 189 // ---------------------------------------------------------------------
 190 /* The only header we use is the last-modified header. */
 191 string pkgAcqDiffIndex::Custom600Headers()
 192 {
 193    string Final = _config->FindDir("Dir::State::lists");
 194    Final += URItoFileName(RealURI) + string(".IndexDiff");
 195
 196    if(Debug)
 197       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 198
 199    struct stat Buf;
 200    if (stat(Final.c_str(),&Buf) != 0)
 201       return "\nIndex-File: true";
 202
 203    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 204 }
 205                                                                         /*}}}*/
 206 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 207 {
 208    if(Debug)
 209       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 210                 << std::endl;
 211
 212    pkgTagSection Tags;
 213    string ServerSha1;
 214    vector<DiffInfo> available_patches;
 215
 216    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 217    pkgTagFile TF(&Fd);
 218    if (_error->PendingError() == true)
 219       return false;
 220
 221    if(TF.Step(Tags) == true)
 222    {
 223       bool found = false;
 224       DiffInfo d;
 225       string size;
 226
 227       string const tmp = Tags.FindS("SHA1-Current");
 228       std::stringstream ss(tmp);
 229       ss >> ServerSha1 >> size;
 230       unsigned long const ServerSize = atol(size.c_str());
 231
 232       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 233       SHA1Summation SHA1;
 234       SHA1.AddFD(fd.Fd(), fd.Size());
 235       string const local_sha1 = SHA1.Result();
 236
 237       if(local_sha1 == ServerSha1)
 238       {
 239          // we have the same sha1 as the server
 240          if(Debug)
 241             std::clog << "Package file is up-to-date" << std::endl;
 242          // set found to true, this will queue a pkgAcqIndexDiffs with
 243          // a empty availabe_patches
 244          found = true;
 245       }
 246       else
 247       {
 248          if(Debug)
 249             std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 250
 251          // check the historie and see what patches we need
 252          string const history = Tags.FindS("SHA1-History");
 253          std::stringstream hist(history);
 254          while(hist >> d.sha1 >> size >> d.file)
 255          {
 256             // read until the first match is found
 257             // from that point on, we probably need all diffs
 258             if(d.sha1 == local_sha1)
 259                found=true;
 260             else if (found == false)
 261                continue;
 262
 263             if(Debug)
 264                std::clog << "Need to get diff: " << d.file << std::endl;
 265             available_patches.push_back(d);
 266          }
 267
 268          if (available_patches.empty() == false)
 269          {
 270             // patching with too many files is rather slow compared to a fast download
 271             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 272             if (fileLimit != 0 && fileLimit < available_patches.size())
 273             {
 274                if (Debug)
 275                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 276                         << ") so fallback to complete download" << std::endl;
 277                return false;
 278             }
 279
 280             // see if the patches are too big
 281             found = false; // it was true and it will be true again at the end
 282             d = *available_patches.begin();
 283             string const firstPatch = d.file;
 284             unsigned long patchesSize = 0;
 285             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 286             while(patches >> d.sha1 >> size >> d.file)
 287             {
 288                if (firstPatch == d.file)
 289                   found = true;
 290                else if (found == false)
 291                   continue;
 292
 293                patchesSize += atol(size.c_str());
 294             }
 295             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 296             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 297             {
 298                if (Debug)
 299                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 300                         << ") so fallback to complete download" << std::endl;
 301                return false;
 302             }
 303          }
 304       }
 305
 306       // we have something, queue the next diff
 307       if(found)
 308       {
 309          // queue the diffs
 310          string::size_type const last_space = Description.rfind(" ");
 311          if(last_space != string::npos)
 312             Description.erase(last_space, Description.size()-last_space);
 313          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 314                               ExpectedHash, ServerSha1, available_patches);
 315          Complete = false;
 316          Status = StatDone;
 317          Dequeue();
 318          return true;
 319       }
 320    }
 321
 322    // Nothing found, report and return false
 323    // Failing here is ok, if we return false later, the full
 324    // IndexFile is queued
 325    if(Debug)
 326       std::clog << "Can't find a patch in the index file" << std::endl;
 327    return false;
 328 }
 329                                                                         /*}}}*/
 330 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 331 {
 332    if(Debug)
 333       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 334                 << "Falling back to normal index file aquire" << std::endl;
 335
 336    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 337                    ExpectedHash);
 338
 339    Complete = false;
 340    Status = StatDone;
 341    Dequeue();
 342 }
 343                                                                         /*}}}*/
 344 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 345                            pkgAcquire::MethodConfig *Cnf)
 346 {
 347    if(Debug)
 348       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 349
 350    Item::Done(Message,Size,Md5Hash,Cnf);
 351
 352    string FinalFile;
 353    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 354
 355    // sucess in downloading the index
 356    // rename the index
 357    FinalFile += string(".IndexDiff");
 358    if(Debug)
 359       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 360                 << std::endl;
 361    Rename(DestFile,FinalFile);
 362    chmod(FinalFile.c_str(),0644);
 363    DestFile = FinalFile;
 364
 365    if(!ParseDiffIndex(DestFile))
 366       return Failed("", NULL);
 367
 368    Complete = true;
 369    Status = StatDone;
 370    Dequeue();
 371    return;
 372 }
 373                                                                         /*}}}*/
 374 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* The package diff is added to the queue. one object is constructed
 377  * for each diff and the index
 378  */
 379 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 380                                    string URI,string URIDesc,string ShortDesc,
 381                                    HashString ExpectedHash,
 382                                    string ServerSha1,
 383                                    vector<DiffInfo> diffs)
 384    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 385      available_patches(diffs), ServerSha1(ServerSha1)
 386 {
 387
 388    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 389    DestFile += URItoFileName(URI);
 390
 391    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 392
 393    Description = URIDesc;
 394    Desc.Owner = this;
 395    Desc.ShortDesc = ShortDesc;
 396
 397    if(available_patches.size() == 0)
 398    {
 399       // we are done (yeah!)
 400       Finish(true);
 401    }
 402    else
 403    {
 404       // get the next diff
 405       State = StateFetchDiff;
 406       QueueNextDiff();
 407    }
 408 }
 409                                                                         /*}}}*/
 410 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 411 {
 412    if(Debug)
 413       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 414                 << "Falling back to normal index file aquire" << std::endl;
 415    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 416                    ExpectedHash);
 417    Finish();
 418 }
 419                                                                         /*}}}*/
 420 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 421 void pkgAcqIndexDiffs::Finish(bool allDone)
 422 {
 423    // we restore the original name, this is required, otherwise
 424    // the file will be cleaned
 425    if(allDone)
 426    {
 427       DestFile = _config->FindDir("Dir::State::lists");
 428       DestFile += URItoFileName(RealURI);
 429
 430       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 431       {
 432          Status = StatAuthError;
 433          ErrorText = _("MD5Sum mismatch");
 434          Rename(DestFile,DestFile + ".FAILED");
 435          Dequeue();
 436          return;
 437       }
 438
 439       // this is for the "real" finish
 440       Complete = true;
 441       Status = StatDone;
 442       Dequeue();
 443       if(Debug)
 444          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 445       return;
 446    }
 447
 448    if(Debug)
 449       std::clog << "Finishing: " << Desc.URI << std::endl;
 450    Complete = false;
 451    Status = StatDone;
 452    Dequeue();
 453    return;
 454 }
 455                                                                         /*}}}*/
 456 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 457 {
 458
 459    // calc sha1 of the just patched file
 460    string FinalFile = _config->FindDir("Dir::State::lists");
 461    FinalFile += URItoFileName(RealURI);
 462
 463    FileFd fd(FinalFile, FileFd::ReadOnly);
 464    SHA1Summation SHA1;
 465    SHA1.AddFD(fd.Fd(), fd.Size());
 466    string local_sha1 = string(SHA1.Result());
 467    if(Debug)
 468       std::clog << "QueueNextDiff: "
 469                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 470
 471    // final file reached before all patches are applied
 472    if(local_sha1 == ServerSha1)
 473    {
 474       Finish(true);
 475       return true;
 476    }
 477
 478    // remove all patches until the next matching patch is found
 479    // this requires the Index file to be ordered
 480    for(vector<DiffInfo>::iterator I=available_patches.begin();
 481        available_patches.size() > 0 &&
 482           I != available_patches.end() &&
 483           (*I).sha1 != local_sha1;
 484        I++)
 485    {
 486       available_patches.erase(I);
 487    }
 488
 489    // error checking and falling back if no patch was found
 490    if(available_patches.size() == 0)
 491    {
 492       Failed("", NULL);
 493       return false;
 494    }
 495
 496    // queue the right diff
 497    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 498    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 499    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 500    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 501
 502    if(Debug)
 503       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 504
 505    QueueURI(Desc);
 506
 507    return true;
 508 }
 509                                                                         /*}}}*/
 510 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 511                             pkgAcquire::MethodConfig *Cnf)
 512 {
 513    if(Debug)
 514       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 515
 516    Item::Done(Message,Size,Md5Hash,Cnf);
 517
 518    string FinalFile;
 519    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 520
 521    // sucess in downloading a diff, enter ApplyDiff state
 522    if(State == StateFetchDiff)
 523    {
 524
 525       if(Debug)
 526          std::clog << "Sending to gzip method: " << FinalFile << std::endl;
 527
 528       string FileName = LookupTag(Message,"Filename");
 529       State = StateUnzipDiff;
 530       Local = true;
 531       Desc.URI = "gzip:" + FileName;
 532       DestFile += ".decomp";
 533       QueueURI(Desc);
 534       Mode = "gzip";
 535       return;
 536    }
 537
 538    // sucess in downloading a diff, enter ApplyDiff state
 539    if(State == StateUnzipDiff)
 540    {
 541
 542       // rred excepts the patch as $FinalFile.ed
 543       Rename(DestFile,FinalFile+".ed");
 544
 545       if(Debug)
 546          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 547
 548       State = StateApplyDiff;
 549       Local = true;
 550       Desc.URI = "rred:" + FinalFile;
 551       QueueURI(Desc);
 552       Mode = "rred";
 553       return;
 554    }
 555
 556
 557    // success in download/apply a diff, queue next (if needed)
 558    if(State == StateApplyDiff)
 559    {
 560       // remove the just applied patch
 561       available_patches.erase(available_patches.begin());
 562
 563       // move into place
 564       if(Debug)
 565       {
 566          std::clog << "Moving patched file in place: " << std::endl
 567                    << DestFile << " -> " << FinalFile << std::endl;
 568       }
 569       Rename(DestFile,FinalFile);
 570       chmod(FinalFile.c_str(),0644);
 571
 572       // see if there is more to download
 573       if(available_patches.size() > 0) {
 574          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 575                               ExpectedHash, ServerSha1, available_patches);
 576          return Finish();
 577       } else
 578          return Finish(true);
 579    }
 580 }
 581                                                                         /*}}}*/
 582 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 583 // ---------------------------------------------------------------------
 584 /* The package file is added to the queue and a second class is
 585    instantiated to fetch the revision file */
 586 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 587                          string URI,string URIDesc,string ShortDesc,
 588                          HashString ExpectedHash, string comprExt)
 589    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 590 {
 591    Decompression = false;
 592    Erase = false;
 593
 594    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 595    DestFile += URItoFileName(URI);
 596
 597    if(comprExt.empty())
 598    {
 599       // autoselect the compression method
 600       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 601       if (types.empty() == true)
 602          comprExt = "plain";
 603       else
 604          comprExt = "." + types[0];
 605    }
 606    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 607
 608    Desc.URI = URI + CompressionExtension;
 609
 610    Desc.Description = URIDesc;
 611    Desc.Owner = this;
 612    Desc.ShortDesc = ShortDesc;
 613
 614    QueueURI(Desc);
 615 }
 616                                                                         /*}}}*/
 617 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 618 // ---------------------------------------------------------------------
 619 /* The only header we use is the last-modified header. */
 620 string pkgAcqIndex::Custom600Headers()
 621 {
 622    string Final = _config->FindDir("Dir::State::lists");
 623    Final += URItoFileName(RealURI);
 624
 625    struct stat Buf;
 626    if (stat(Final.c_str(),&Buf) != 0)
 627       return "\nIndex-File: true";
 628
 629    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 630 }
 631                                                                         /*}}}*/
 632 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 633 {
 634    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 635
 636    for (std::vector<std::string>::const_iterator t = types.begin();
 637         t != types.end(); t++)
 638    {
 639       // jump over all already tried compression types
 640       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 641       if(Desc.URI.substr(nameLen) != *t)
 642          continue;
 643
 644       // we want to try it with the next extension (and make sure to
 645       // not skip over the end)
 646       t++;
 647       if (t == types.end())
 648          break;
 649
 650       // queue new download
 651       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 652       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 653       ExpectedHash, string(".").append(*t));
 654
 655       Status = StatDone;
 656       Complete = false;
 657       Dequeue();
 658       return;
 659    }
 660
 661    // on decompression failure, remove bad versions in partial/
 662    if(Decompression && Erase) {
 663       string s = _config->FindDir("Dir::State::lists") + "partial/";
 664       s += URItoFileName(RealURI);
 665       unlink(s.c_str());
 666    }
 667
 668    Item::Failed(Message,Cnf);
 669 }
 670                                                                         /*}}}*/
 671 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 672 // ---------------------------------------------------------------------
 673 /* This goes through a number of states.. On the initial fetch the
 674    method could possibly return an alternate filename which points
 675    to the uncompressed version of the file. If this is so the file
 676    is copied into the partial directory. In all other cases the file
 677    is decompressed with a gzip uri. */
 678 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 679                        pkgAcquire::MethodConfig *Cfg)
 680 {
 681    Item::Done(Message,Size,Hash,Cfg);
 682
 683    if (Decompression == true)
 684    {
 685       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 686       {
 687          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 688          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 689       }
 690
 691       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 692       {
 693          Status = StatAuthError;
 694          ErrorText = _("Hash Sum mismatch");
 695          Rename(DestFile,DestFile + ".FAILED");
 696          return;
 697       }
 698       // Done, move it into position
 699       string FinalFile = _config->FindDir("Dir::State::lists");
 700       FinalFile += URItoFileName(RealURI);
 701       Rename(DestFile,FinalFile);
 702       chmod(FinalFile.c_str(),0644);
 703
 704       /* We restore the original name to DestFile so that the clean operation
 705          will work OK */
 706       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 707       DestFile += URItoFileName(RealURI);
 708
 709       // Remove the compressed version.
 710       if (Erase == true)
 711          unlink(DestFile.c_str());
 712       return;
 713    }
 714
 715    Erase = false;
 716    Complete = true;
 717
 718    // Handle the unzipd case
 719    string FileName = LookupTag(Message,"Alt-Filename");
 720    if (FileName.empty() == false)
 721    {
 722       // The files timestamp matches
 723       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 724          return;
 725       Decompression = true;
 726       Local = true;
 727       DestFile += ".decomp";
 728       Desc.URI = "copy:" + FileName;
 729       QueueURI(Desc);
 730       Mode = "copy";
 731       return;
 732    }
 733
 734    FileName = LookupTag(Message,"Filename");
 735    if (FileName.empty() == true)
 736    {
 737       Status = StatError;
 738       ErrorText = "Method gave a blank filename";
 739    }
 740
 741    // The files timestamp matches
 742    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 743       return;
 744
 745    if (FileName == DestFile)
 746       Erase = true;
 747    else
 748       Local = true;
 749
 750    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 751    string decompProg;
 752
 753    // get the binary name for your used compression type
 754    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 755    if(decompProg.empty() == false);
 756    // flExtensions returns the full name if no extension is found
 757    // this is why we have this complicated compare operation here
 758    // FIMXE: add a new flJustExtension() that return "" if no
 759    //        extension is found and use that above so that it can
 760    //        be tested against ""
 761    else if(compExt == flNotDir(URI(Desc.URI).Path))
 762       decompProg = "copy";
 763    else {
 764       _error->Error("Unsupported extension: %s", compExt.c_str());
 765       return;
 766    }
 767
 768    Decompression = true;
 769    DestFile += ".decomp";
 770    Desc.URI = decompProg + ":" + FileName;
 771    QueueURI(Desc);
 772    Mode = decompProg.c_str();
 773 }
 774                                                                         /*}}}*/
 775 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 776 // ---------------------------------------------------------------------
 777 /* The Translation file is added to the queue */
 778 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 779                             string URI,string URIDesc,string ShortDesc)
 780   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 781 {
 782 }
 783                                                                         /*}}}*/
 784 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 785 // ---------------------------------------------------------------------
 786 /* */
 787 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 788 {
 789    if (Cnf->LocalOnly == true ||
 790        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 791    {
 792       // Ignore this
 793       Status = StatDone;
 794       Complete = false;
 795       Dequeue();
 796       return;
 797    }
 798
 799    Item::Failed(Message,Cnf);
 800 }
 801                                                                         /*}}}*/
 802 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 803                              string URI,string URIDesc,string ShortDesc,
 804                              string MetaIndexURI, string MetaIndexURIDesc,
 805                              string MetaIndexShortDesc,
 806                              const vector<IndexTarget*>* IndexTargets,
 807                              indexRecords* MetaIndexParser) :
 808    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 809    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 810    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 811 {
 812    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 813    DestFile += URItoFileName(URI);
 814
 815    // remove any partial downloaded sig-file in partial/.
 816    // it may confuse proxies and is too small to warrant a
 817    // partial download anyway
 818    unlink(DestFile.c_str());
 819
 820    // Create the item
 821    Desc.Description = URIDesc;
 822    Desc.Owner = this;
 823    Desc.ShortDesc = ShortDesc;
 824    Desc.URI = URI;
 825
 826    string Final = _config->FindDir("Dir::State::lists");
 827    Final += URItoFileName(RealURI);
 828    struct stat Buf;
 829    if (stat(Final.c_str(),&Buf) == 0)
 830    {
 831       // File was already in place.  It needs to be re-downloaded/verified
 832       // because Release might have changed, we do give it a differnt
 833       // name than DestFile because otherwise the http method will
 834       // send If-Range requests and there are too many broken servers
 835       // out there that do not understand them
 836       LastGoodSig = DestFile+".reverify";
 837       Rename(Final,LastGoodSig);
 838    }
 839
 840    QueueURI(Desc);
 841 }
 842                                                                         /*}}}*/
 843 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 844 // ---------------------------------------------------------------------
 845 /* The only header we use is the last-modified header. */
 846 string pkgAcqMetaSig::Custom600Headers()
 847 {
 848    struct stat Buf;
 849    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 850       return "\nIndex-File: true";
 851
 852    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 853 }
 854
 855 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 856                          pkgAcquire::MethodConfig *Cfg)
 857 {
 858    Item::Done(Message,Size,MD5,Cfg);
 859
 860    string FileName = LookupTag(Message,"Filename");
 861    if (FileName.empty() == true)
 862    {
 863       Status = StatError;
 864       ErrorText = "Method gave a blank filename";
 865       return;
 866    }
 867
 868    if (FileName != DestFile)
 869    {
 870       // We have to copy it into place
 871       Local = true;
 872       Desc.URI = "copy:" + FileName;
 873       QueueURI(Desc);
 874       return;
 875    }
 876
 877    Complete = true;
 878
 879    // put the last known good file back on i-m-s hit (it will
 880    // be re-verified again)
 881    // Else do nothing, we have the new file in DestFile then
 882    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 883       Rename(LastGoodSig, DestFile);
 884
 885    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 886    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 887                        DestFile, IndexTargets, MetaIndexParser);
 888
 889 }
 890                                                                         /*}}}*/
 891 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 892 {
 893    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 894
 895    // if we get a network error we fail gracefully
 896    if(Status == StatTransientNetworkError)
 897    {
 898       Item::Failed(Message,Cnf);
 899       // move the sigfile back on transient network failures
 900       if(FileExists(DestFile))
 901          Rename(LastGoodSig,Final);
 902
 903       // set the status back to , Item::Failed likes to reset it
 904       Status = pkgAcquire::Item::StatTransientNetworkError;
 905       return;
 906    }
 907
 908    // Delete any existing sigfile when the acquire failed
 909    unlink(Final.c_str());
 910
 911    // queue a pkgAcqMetaIndex with no sigfile
 912    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 913                        "", IndexTargets, MetaIndexParser);
 914
 915    if (Cnf->LocalOnly == true ||
 916        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 917    {
 918       // Ignore this
 919       Status = StatDone;
 920       Complete = false;
 921       Dequeue();
 922       return;
 923    }
 924
 925    Item::Failed(Message,Cnf);
 926 }
 927                                                                         /*}}}*/
 928 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 929                                  string URI,string URIDesc,string ShortDesc,
 930                                  string SigFile,
 931                                  const vector<struct IndexTarget*>* IndexTargets,
 932                                  indexRecords* MetaIndexParser) :
 933    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
 934    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
 935 {
 936    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 937    DestFile += URItoFileName(URI);
 938
 939    // Create the item
 940    Desc.Description = URIDesc;
 941    Desc.Owner = this;
 942    Desc.ShortDesc = ShortDesc;
 943    Desc.URI = URI;
 944
 945    QueueURI(Desc);
 946 }
 947                                                                         /*}}}*/
 948 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
 949 // ---------------------------------------------------------------------
 950 /* The only header we use is the last-modified header. */
 951 string pkgAcqMetaIndex::Custom600Headers()
 952 {
 953    string Final = _config->FindDir("Dir::State::lists");
 954    Final += URItoFileName(RealURI);
 955
 956    struct stat Buf;
 957    if (stat(Final.c_str(),&Buf) != 0)
 958       return "\nIndex-File: true";
 959
 960    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 961 }
 962                                                                         /*}}}*/
 963 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
 964                            pkgAcquire::MethodConfig *Cfg)
 965 {
 966    Item::Done(Message,Size,Hash,Cfg);
 967
 968    // MetaIndexes are done in two passes: one to download the
 969    // metaindex with an appropriate method, and a second to verify it
 970    // with the gpgv method
 971
 972    if (AuthPass == true)
 973    {
 974       AuthDone(Message);
 975    }
 976    else
 977    {
 978       RetrievalDone(Message);
 979       if (!Complete)
 980          // Still more retrieving to do
 981          return;
 982
 983       if (SigFile == "")
 984       {
 985          // There was no signature file, so we are finished.  Download
 986          // the indexes without verification.
 987          QueueIndexes(false);
 988       }
 989       else
 990       {
 991          // There was a signature file, so pass it to gpgv for
 992          // verification
 993
 994          if (_config->FindB("Debug::pkgAcquire::Auth", false))
 995             std::cerr << "Metaindex acquired, queueing gpg verification ("
 996                       << SigFile << "," << DestFile << ")\n";
 997          AuthPass = true;
 998          Desc.URI = "gpgv:" + SigFile;
 999          QueueURI(Desc);
1000          Mode = "gpgv";
1001       }
1002    }
1003 }
1004                                                                         /*}}}*/
1005 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1006 {
1007    // We have just finished downloading a Release file (it is not
1008    // verified yet)
1009
1010    string FileName = LookupTag(Message,"Filename");
1011    if (FileName.empty() == true)
1012    {
1013       Status = StatError;
1014       ErrorText = "Method gave a blank filename";
1015       return;
1016    }
1017
1018    if (FileName != DestFile)
1019    {
1020       Local = true;
1021       Desc.URI = "copy:" + FileName;
1022       QueueURI(Desc);
1023       return;
1024    }
1025
1026    // see if the download was a IMSHit
1027    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1028    Complete = true;
1029
1030    string FinalFile = _config->FindDir("Dir::State::lists");
1031    FinalFile += URItoFileName(RealURI);
1032
1033    // If we get a IMS hit we can remove the empty file in partial
1034    // othersie we move the file in place
1035    if (IMSHit)
1036       unlink(DestFile.c_str());
1037    else
1038       Rename(DestFile,FinalFile);
1039
1040    chmod(FinalFile.c_str(),0644);
1041    DestFile = FinalFile;
1042 }
1043                                                                         /*}}}*/
1044 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1045 {
1046    // At this point, the gpgv method has succeeded, so there is a
1047    // valid signature from a key in the trusted keyring.  We
1048    // perform additional verification of its contents, and use them
1049    // to verify the indexes we are about to download
1050
1051    if (!MetaIndexParser->Load(DestFile))
1052    {
1053       Status = StatAuthError;
1054       ErrorText = MetaIndexParser->ErrorText;
1055       return;
1056    }
1057
1058    if (!VerifyVendor(Message))
1059    {
1060       return;
1061    }
1062
1063    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1064       std::cerr << "Signature verification succeeded: "
1065                 << DestFile << std::endl;
1066
1067    // Download further indexes with verification
1068    QueueIndexes(true);
1069
1070    // Done, move signature file into position
1071
1072    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1073       URItoFileName(RealURI) + ".gpg";
1074    Rename(SigFile,VerifiedSigFile);
1075    chmod(VerifiedSigFile.c_str(),0644);
1076 }
1077                                                                         /*}}}*/
1078 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1079 {
1080    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1081         Target != IndexTargets->end();
1082         Target++)
1083    {
1084       HashString ExpectedIndexHash;
1085       if (verify)
1086       {
1087          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1088          if (!Record)
1089          {
1090             Status = StatAuthError;
1091             ErrorText = "Unable to find expected entry  "
1092                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1093             return;
1094          }
1095          ExpectedIndexHash = Record->Hash;
1096          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1097          {
1098             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1099             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1100          }
1101          if (ExpectedIndexHash.empty())
1102          {
1103             Status = StatAuthError;
1104             ErrorText = "Unable to find hash sum for "
1105                + (*Target)->MetaKey + " in Meta-index file";
1106             return;
1107          }
1108       }
1109
1110       /* Queue Packages file (either diff or full packages files, depending
1111          on the users option) - we also check if the PDiff Index file is listed
1112          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1113          instead, but passing the required info to it is to much hassle */
1114       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1115           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1116          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1117                              (*Target)->ShortDesc, ExpectedIndexHash);
1118       else
1119          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1120                             (*Target)->ShortDesc, ExpectedIndexHash);
1121    }
1122 }
1123                                                                         /*}}}*/
1124 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1125 {
1126 //    // Maybe this should be made available from above so we don't have
1127 //    // to read and parse it every time?
1128 //    pkgVendorList List;
1129 //    List.ReadMainList();
1130
1131 //    const Vendor* Vndr = NULL;
1132 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1133 //    {
1134 //       string::size_type pos = (*I).find("VALIDSIG ");
1135 //       if (_config->FindB("Debug::Vendor", false))
1136 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1137 //                    << std::endl;
1138 //       if (pos != std::string::npos)
1139 //       {
1140 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1141 //          if (_config->FindB("Debug::Vendor", false))
1142 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1143 //                std::endl;
1144 //          Vndr = List.FindVendor(Fingerprint) != "";
1145 //          if (Vndr != NULL);
1146 //          break;
1147 //       }
1148 //    }
1149    string::size_type pos;
1150
1151    // check for missing sigs (that where not fatal because otherwise we had
1152    // bombed earlier)
1153    string missingkeys;
1154    string msg = _("There is no public key available for the "
1155                   "following key IDs:\n");
1156    pos = Message.find("NO_PUBKEY ");
1157    if (pos != std::string::npos)
1158    {
1159       string::size_type start = pos+strlen("NO_PUBKEY ");
1160       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1161       missingkeys += (Fingerprint);
1162    }
1163    if(!missingkeys.empty())
1164       _error->Warning("%s", string(msg+missingkeys).c_str());
1165
1166    string Transformed = MetaIndexParser->GetExpectedDist();
1167
1168    if (Transformed == "../project/experimental")
1169    {
1170       Transformed = "experimental";
1171    }
1172
1173    pos = Transformed.rfind('/');
1174    if (pos != string::npos)
1175    {
1176       Transformed = Transformed.substr(0, pos);
1177    }
1178
1179    if (Transformed == ".")
1180    {
1181       Transformed = "";
1182    }
1183
1184    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1185        MetaIndexParser->GetValidUntil() > 0) {
1186       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1187       if (invalid_since > 0)
1188          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1189          // the time since then the file is invalid - formated in the same way as in
1190          // the download progress display (e.g. 7d 3h 42min 1s)
1191          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1192                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1193    }
1194
1195    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1196    {
1197       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1198       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1199       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1200    }
1201
1202    if (MetaIndexParser->CheckDist(Transformed) == false)
1203    {
1204       // This might become fatal one day
1205 //       Status = StatAuthError;
1206 //       ErrorText = "Conflicting distribution; expected "
1207 //          + MetaIndexParser->GetExpectedDist() + " but got "
1208 //          + MetaIndexParser->GetDist();
1209 //       return false;
1210       if (!Transformed.empty())
1211       {
1212          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1213                          Desc.Description.c_str(),
1214                          Transformed.c_str(),
1215                          MetaIndexParser->GetDist().c_str());
1216       }
1217    }
1218
1219    return true;
1220 }
1221                                                                         /*}}}*/
1222 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1223 // ---------------------------------------------------------------------
1224 /* */
1225 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1226 {
1227    if (AuthPass == true)
1228    {
1229       // if we fail the authentication but got the file via a IMS-Hit
1230       // this means that the file wasn't downloaded and that it might be
1231       // just stale (server problem, proxy etc). we delete what we have
1232       // queue it again without i-m-s
1233       // alternatively we could just unlink the file and let the user try again
1234       if (IMSHit)
1235       {
1236          Complete = false;
1237          Local = false;
1238          AuthPass = false;
1239          unlink(DestFile.c_str());
1240
1241          DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1242          DestFile += URItoFileName(RealURI);
1243          Desc.URI = RealURI;
1244          QueueURI(Desc);
1245          return;
1246       }
1247
1248       // gpgv method failed
1249       _error->Warning("GPG error: %s: %s",
1250                       Desc.Description.c_str(),
1251                       LookupTag(Message,"Message").c_str());
1252
1253    }
1254
1255    // No Release file was present, or verification failed, so fall
1256    // back to queueing Packages files without verification
1257    QueueIndexes(false);
1258 }
1259                                                                         /*}}}*/
1260 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1261 // ---------------------------------------------------------------------
1262 /* This just sets up the initial fetch environment and queues the first
1263    possibilitiy */
1264 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1265                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1266                              string &StoreFilename) :
1267                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1268                StoreFilename(StoreFilename), Vf(Version.FileList()),
1269                Trusted(false)
1270 {
1271    Retries = _config->FindI("Acquire::Retries",0);
1272
1273    if (Version.Arch() == 0)
1274    {
1275       _error->Error(_("I wasn't able to locate a file for the %s package. "
1276                       "This might mean you need to manually fix this package. "
1277                       "(due to missing arch)"),
1278                     Version.ParentPkg().Name());
1279       return;
1280    }
1281
1282    /* We need to find a filename to determine the extension. We make the
1283       assumption here that all the available sources for this version share
1284       the same extension.. */
1285    // Skip not source sources, they do not have file fields.
1286    for (; Vf.end() == false; Vf++)
1287    {
1288       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1289          continue;
1290       break;
1291    }
1292
1293    // Does not really matter here.. we are going to fail out below
1294    if (Vf.end() != true)
1295    {
1296       // If this fails to get a file name we will bomb out below.
1297       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1298       if (_error->PendingError() == true)
1299          return;
1300
1301       // Generate the final file name as: package_version_arch.foo
1302       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1303                       QuoteString(Version.VerStr(),"_:") + '_' +
1304                       QuoteString(Version.Arch(),"_:.") +
1305                       "." + flExtension(Parse.FileName());
1306    }
1307
1308    // check if we have one trusted source for the package. if so, switch
1309    // to "TrustedOnly" mode
1310    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1311    {
1312       pkgIndexFile *Index;
1313       if (Sources->FindIndex(i.File(),Index) == false)
1314          continue;
1315       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1316       {
1317          std::cerr << "Checking index: " << Index->Describe()
1318                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1319       }
1320       if (Index->IsTrusted()) {
1321          Trusted = true;
1322          break;
1323       }
1324    }
1325
1326    // "allow-unauthenticated" restores apts old fetching behaviour
1327    // that means that e.g. unauthenticated file:// uris are higher
1328    // priority than authenticated http:// uris
1329    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1330       Trusted = false;
1331
1332    // Select a source
1333    if (QueueNext() == false && _error->PendingError() == false)
1334       _error->Error(_("I wasn't able to locate file for the %s package. "
1335                     "This might mean you need to manually fix this package."),
1336                     Version.ParentPkg().Name());
1337 }
1338                                                                         /*}}}*/
1339 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1340 // ---------------------------------------------------------------------
1341 /* This queues the next available file version for download. It checks if
1342    the archive is already available in the cache and stashs the MD5 for
1343    checking later. */
1344 bool pkgAcqArchive::QueueNext()
1345 {
1346    string const ForceHash = _config->Find("Acquire::ForceHash");
1347    for (; Vf.end() == false; Vf++)
1348    {
1349       // Ignore not source sources
1350       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1351          continue;
1352
1353       // Try to cross match against the source list
1354       pkgIndexFile *Index;
1355       if (Sources->FindIndex(Vf.File(),Index) == false)
1356             continue;
1357
1358       // only try to get a trusted package from another source if that source
1359       // is also trusted
1360       if(Trusted && !Index->IsTrusted())
1361          continue;
1362
1363       // Grab the text package record
1364       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1365       if (_error->PendingError() == true)
1366          return false;
1367
1368       string PkgFile = Parse.FileName();
1369       if (ForceHash.empty() == false)
1370       {
1371          if(stringcasecmp(ForceHash, "sha256") == 0)
1372             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1373          else if (stringcasecmp(ForceHash, "sha1") == 0)
1374             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1375          else
1376             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1377       }
1378       else
1379       {
1380          string Hash;
1381          if ((Hash = Parse.SHA256Hash()).empty() == false)
1382             ExpectedHash = HashString("SHA256", Hash);
1383          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1384             ExpectedHash = HashString("SHA1", Hash);
1385          else
1386             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1387       }
1388       if (PkgFile.empty() == true)
1389          return _error->Error(_("The package index files are corrupted. No Filename: "
1390                               "field for package %s."),
1391                               Version.ParentPkg().Name());
1392
1393       Desc.URI = Index->ArchiveURI(PkgFile);
1394       Desc.Description = Index->ArchiveInfo(Version);
1395       Desc.Owner = this;
1396       Desc.ShortDesc = Version.ParentPkg().Name();
1397
1398       // See if we already have the file. (Legacy filenames)
1399       FileSize = Version->Size;
1400       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1401       struct stat Buf;
1402       if (stat(FinalFile.c_str(),&Buf) == 0)
1403       {
1404          // Make sure the size matches
1405          if ((unsigned)Buf.st_size == Version->Size)
1406          {
1407             Complete = true;
1408             Local = true;
1409             Status = StatDone;
1410             StoreFilename = DestFile = FinalFile;
1411             return true;
1412          }
1413
1414          /* Hmm, we have a file and its size does not match, this means it is
1415             an old style mismatched arch */
1416          unlink(FinalFile.c_str());
1417       }
1418
1419       // Check it again using the new style output filenames
1420       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1421       if (stat(FinalFile.c_str(),&Buf) == 0)
1422       {
1423          // Make sure the size matches
1424          if ((unsigned)Buf.st_size == Version->Size)
1425          {
1426             Complete = true;
1427             Local = true;
1428             Status = StatDone;
1429             StoreFilename = DestFile = FinalFile;
1430             return true;
1431          }
1432
1433          /* Hmm, we have a file and its size does not match, this shouldnt
1434             happen.. */
1435          unlink(FinalFile.c_str());
1436       }
1437
1438       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1439
1440       // Check the destination file
1441       if (stat(DestFile.c_str(),&Buf) == 0)
1442       {
1443          // Hmm, the partial file is too big, erase it
1444          if ((unsigned)Buf.st_size > Version->Size)
1445             unlink(DestFile.c_str());
1446          else
1447             PartialSize = Buf.st_size;
1448       }
1449
1450       // Create the item
1451       Local = false;
1452       Desc.URI = Index->ArchiveURI(PkgFile);
1453       Desc.Description = Index->ArchiveInfo(Version);
1454       Desc.Owner = this;
1455       Desc.ShortDesc = Version.ParentPkg().Name();
1456       QueueURI(Desc);
1457
1458       Vf++;
1459       return true;
1460    }
1461    return false;
1462 }
1463                                                                         /*}}}*/
1464 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1465 // ---------------------------------------------------------------------
1466 /* */
1467 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1468                          pkgAcquire::MethodConfig *Cfg)
1469 {
1470    Item::Done(Message,Size,CalcHash,Cfg);
1471
1472    // Check the size
1473    if (Size != Version->Size)
1474    {
1475       Status = StatError;
1476       ErrorText = _("Size mismatch");
1477       return;
1478    }
1479
1480    // Check the hash
1481    if(ExpectedHash.toStr() != CalcHash)
1482    {
1483       Status = StatError;
1484       ErrorText = _("Hash Sum mismatch");
1485       if(FileExists(DestFile))
1486          Rename(DestFile,DestFile + ".FAILED");
1487       return;
1488    }
1489
1490    // Grab the output filename
1491    string FileName = LookupTag(Message,"Filename");
1492    if (FileName.empty() == true)
1493    {
1494       Status = StatError;
1495       ErrorText = "Method gave a blank filename";
1496       return;
1497    }
1498
1499    Complete = true;
1500
1501    // Reference filename
1502    if (FileName != DestFile)
1503    {
1504       StoreFilename = DestFile = FileName;
1505       Local = true;
1506       return;
1507    }
1508
1509    // Done, move it into position
1510    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1511    FinalFile += flNotDir(StoreFilename);
1512    Rename(DestFile,FinalFile);
1513
1514    StoreFilename = DestFile = FinalFile;
1515    Complete = true;
1516 }
1517                                                                         /*}}}*/
1518 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1519 // ---------------------------------------------------------------------
1520 /* Here we try other sources */
1521 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1522 {
1523    ErrorText = LookupTag(Message,"Message");
1524
1525    /* We don't really want to retry on failed media swaps, this prevents
1526       that. An interesting observation is that permanent failures are not
1527       recorded. */
1528    if (Cnf->Removable == true &&
1529        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1530    {
1531       // Vf = Version.FileList();
1532       while (Vf.end() == false) Vf++;
1533       StoreFilename = string();
1534       Item::Failed(Message,Cnf);
1535       return;
1536    }
1537
1538    if (QueueNext() == false)
1539    {
1540       // This is the retry counter
1541       if (Retries != 0 &&
1542           Cnf->LocalOnly == false &&
1543           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1544       {
1545          Retries--;
1546          Vf = Version.FileList();
1547          if (QueueNext() == true)
1548             return;
1549       }
1550
1551       StoreFilename = string();
1552       Item::Failed(Message,Cnf);
1553    }
1554 }
1555                                                                         /*}}}*/
1556 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1557 // ---------------------------------------------------------------------
1558 bool pkgAcqArchive::IsTrusted()
1559 {
1560    return Trusted;
1561 }
1562                                                                         /*}}}*/
1563 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1564 // ---------------------------------------------------------------------
1565 /* */
1566 void pkgAcqArchive::Finished()
1567 {
1568    if (Status == pkgAcquire::Item::StatDone &&
1569        Complete == true)
1570       return;
1571    StoreFilename = string();
1572 }
1573                                                                         /*}}}*/
1574 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1575 // ---------------------------------------------------------------------
1576 /* The file is added to the queue */
1577 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1578                        unsigned long Size,string Dsc,string ShortDesc,
1579                        const string &DestDir, const string &DestFilename,
1580                        bool IsIndexFile) :
1581                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1582 {
1583    Retries = _config->FindI("Acquire::Retries",0);
1584
1585    if(!DestFilename.empty())
1586       DestFile = DestFilename;
1587    else if(!DestDir.empty())
1588       DestFile = DestDir + "/" + flNotDir(URI);
1589    else
1590       DestFile = flNotDir(URI);
1591
1592    // Create the item
1593    Desc.URI = URI;
1594    Desc.Description = Dsc;
1595    Desc.Owner = this;
1596
1597    // Set the short description to the archive component
1598    Desc.ShortDesc = ShortDesc;
1599
1600    // Get the transfer sizes
1601    FileSize = Size;
1602    struct stat Buf;
1603    if (stat(DestFile.c_str(),&Buf) == 0)
1604    {
1605       // Hmm, the partial file is too big, erase it
1606       if ((unsigned)Buf.st_size > Size)
1607          unlink(DestFile.c_str());
1608       else
1609          PartialSize = Buf.st_size;
1610    }
1611
1612    QueueURI(Desc);
1613 }
1614                                                                         /*}}}*/
1615 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1616 // ---------------------------------------------------------------------
1617 /* */
1618 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1619                       pkgAcquire::MethodConfig *Cnf)
1620 {
1621    Item::Done(Message,Size,CalcHash,Cnf);
1622
1623    // Check the hash
1624    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1625    {
1626       Status = StatError;
1627       ErrorText = "Hash Sum mismatch";
1628       Rename(DestFile,DestFile + ".FAILED");
1629       return;
1630    }
1631
1632    string FileName = LookupTag(Message,"Filename");
1633    if (FileName.empty() == true)
1634    {
1635       Status = StatError;
1636       ErrorText = "Method gave a blank filename";
1637       return;
1638    }
1639
1640    Complete = true;
1641
1642    // The files timestamp matches
1643    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1644       return;
1645
1646    // We have to copy it into place
1647    if (FileName != DestFile)
1648    {
1649       Local = true;
1650       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1651           Cnf->Removable == true)
1652       {
1653          Desc.URI = "copy:" + FileName;
1654          QueueURI(Desc);
1655          return;
1656       }
1657
1658       // Erase the file if it is a symlink so we can overwrite it
1659       struct stat St;
1660       if (lstat(DestFile.c_str(),&St) == 0)
1661       {
1662          if (S_ISLNK(St.st_mode) != 0)
1663             unlink(DestFile.c_str());
1664       }
1665
1666       // Symlink the file
1667       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1668       {
1669          ErrorText = "Link to " + DestFile + " failure ";
1670          Status = StatError;
1671          Complete = false;
1672       }
1673    }
1674 }
1675                                                                         /*}}}*/
1676 // AcqFile::Failed - Failure handler                                    /*{{{*/
1677 // ---------------------------------------------------------------------
1678 /* Here we try other sources */
1679 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1680 {
1681    ErrorText = LookupTag(Message,"Message");
1682
1683    // This is the retry counter
1684    if (Retries != 0 &&
1685        Cnf->LocalOnly == false &&
1686        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1687    {
1688       Retries--;
1689       QueueURI(Desc);
1690       return;
1691    }
1692
1693    Item::Failed(Message,Cnf);
1694 }
1695                                                                         /*}}}*/
1696 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1697 // ---------------------------------------------------------------------
1698 /* The only header we use is the last-modified header. */
1699 string pkgAcqFile::Custom600Headers()
1700 {
1701    if (IsIndexFile)
1702       return "\nIndex-File: true";
1703    return "";
1704 }
1705                                                                         /*}}}*/