apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the DiffIndex file first and see if there are patches availabe
 190  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 191  * patches. If anything goes wrong in that process, it will fall back to
 192  * the original packages file
 193  */
 194 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 195                                  string URI,string URIDesc,string ShortDesc,
 196                                  HashString ExpectedHash)
 197    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 198      Description(URIDesc)
 199 {
 200
 201    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 202
 203    Desc.Description = URIDesc + "/DiffIndex";
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206    Desc.URI = URI + ".diff/Index";
 207
 208    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 209    DestFile += URItoFileName(URI) + string(".DiffIndex");
 210
 211    if(Debug)
 212       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 213
 214    // look for the current package file
 215    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 216    CurrentPackagesFile += URItoFileName(RealURI);
 217
 218    // FIXME: this file:/ check is a hack to prevent fetching
 219    //        from local sources. this is really silly, and
 220    //        should be fixed cleanly as soon as possible
 221    if(!FileExists(CurrentPackagesFile) ||
 222       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 223    {
 224       // we don't have a pkg file or we don't want to queue
 225       if(Debug)
 226          std::clog << "No index file, local or canceld by user" << std::endl;
 227       Failed("", NULL);
 228       return;
 229    }
 230
 231    if(Debug)
 232       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 233                 << CurrentPackagesFile << std::endl;
 234
 235    QueueURI(Desc);
 236
 237 }
 238                                                                         /*}}}*/
 239 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* The only header we use is the last-modified header. */
 242 string pkgAcqDiffIndex::Custom600Headers()
 243 {
 244    string Final = _config->FindDir("Dir::State::lists");
 245    Final += URItoFileName(RealURI) + string(".IndexDiff");
 246
 247    if(Debug)
 248       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true";
 253
 254    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 255 }
 256                                                                         /*}}}*/
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       bool found = false;
 275       DiffInfo d;
 276       string size;
 277
 278       string const tmp = Tags.FindS("SHA1-Current");
 279       std::stringstream ss(tmp);
 280       ss >> ServerSha1 >> size;
 281       unsigned long const ServerSize = atol(size.c_str());
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       string const local_sha1 = SHA1.Result();
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string const history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             // read until the first match is found
 308             // from that point on, we probably need all diffs
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             else if (found == false)
 312                continue;
 313
 314             if(Debug)
 315                std::clog << "Need to get diff: " << d.file << std::endl;
 316             available_patches.push_back(d);
 317          }
 318
 319          if (available_patches.empty() == false)
 320          {
 321             // patching with too many files is rather slow compared to a fast download
 322             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 323             if (fileLimit != 0 && fileLimit < available_patches.size())
 324             {
 325                if (Debug)
 326                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 327                         << ") so fallback to complete download" << std::endl;
 328                return false;
 329             }
 330
 331             // see if the patches are too big
 332             found = false; // it was true and it will be true again at the end
 333             d = *available_patches.begin();
 334             string const firstPatch = d.file;
 335             unsigned long patchesSize = 0;
 336             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 337             while(patches >> d.sha1 >> size >> d.file)
 338             {
 339                if (firstPatch == d.file)
 340                   found = true;
 341                else if (found == false)
 342                   continue;
 343
 344                patchesSize += atol(size.c_str());
 345             }
 346             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 347             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 348             {
 349                if (Debug)
 350                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 351                         << ") so fallback to complete download" << std::endl;
 352                return false;
 353             }
 354          }
 355       }
 356
 357       // we have something, queue the next diff
 358       if(found)
 359       {
 360          // queue the diffs
 361          string::size_type const last_space = Description.rfind(" ");
 362          if(last_space != string::npos)
 363             Description.erase(last_space, Description.size()-last_space);
 364          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 365                               ExpectedHash, ServerSha1, available_patches);
 366          Complete = false;
 367          Status = StatDone;
 368          Dequeue();
 369          return true;
 370       }
 371    }
 372
 373    // Nothing found, report and return false
 374    // Failing here is ok, if we return false later, the full
 375    // IndexFile is queued
 376    if(Debug)
 377       std::clog << "Can't find a patch in the index file" << std::endl;
 378    return false;
 379 }
 380                                                                         /*}}}*/
 381 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 382 {
 383    if(Debug)
 384       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 385                 << "Falling back to normal index file aquire" << std::endl;
 386
 387    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 388                    ExpectedHash);
 389
 390    Complete = false;
 391    Status = StatDone;
 392    Dequeue();
 393 }
 394                                                                         /*}}}*/
 395 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 396                            pkgAcquire::MethodConfig *Cnf)
 397 {
 398    if(Debug)
 399       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 400
 401    Item::Done(Message,Size,Md5Hash,Cnf);
 402
 403    string FinalFile;
 404    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 405
 406    // sucess in downloading the index
 407    // rename the index
 408    FinalFile += string(".IndexDiff");
 409    if(Debug)
 410       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 411                 << std::endl;
 412    Rename(DestFile,FinalFile);
 413    chmod(FinalFile.c_str(),0644);
 414    DestFile = FinalFile;
 415
 416    if(!ParseDiffIndex(DestFile))
 417       return Failed("", NULL);
 418
 419    Complete = true;
 420    Status = StatDone;
 421    Dequeue();
 422    return;
 423 }
 424                                                                         /*}}}*/
 425 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* The package diff is added to the queue. one object is constructed
 428  * for each diff and the index
 429  */
 430 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 431                                    string URI,string URIDesc,string ShortDesc,
 432                                    HashString ExpectedHash,
 433                                    string ServerSha1,
 434                                    vector<DiffInfo> diffs)
 435    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 436      available_patches(diffs), ServerSha1(ServerSha1)
 437 {
 438
 439    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 440    DestFile += URItoFileName(URI);
 441
 442    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 443
 444    Description = URIDesc;
 445    Desc.Owner = this;
 446    Desc.ShortDesc = ShortDesc;
 447
 448    if(available_patches.size() == 0)
 449    {
 450       // we are done (yeah!)
 451       Finish(true);
 452    }
 453    else
 454    {
 455       // get the next diff
 456       State = StateFetchDiff;
 457       QueueNextDiff();
 458    }
 459 }
 460                                                                         /*}}}*/
 461 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 462 {
 463    if(Debug)
 464       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 465                 << "Falling back to normal index file aquire" << std::endl;
 466    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 467                    ExpectedHash);
 468    Finish();
 469 }
 470                                                                         /*}}}*/
 471 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 472 void pkgAcqIndexDiffs::Finish(bool allDone)
 473 {
 474    // we restore the original name, this is required, otherwise
 475    // the file will be cleaned
 476    if(allDone)
 477    {
 478       DestFile = _config->FindDir("Dir::State::lists");
 479       DestFile += URItoFileName(RealURI);
 480
 481       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 482       {
 483          Status = StatAuthError;
 484          ErrorText = _("MD5Sum mismatch");
 485          Rename(DestFile,DestFile + ".FAILED");
 486          Dequeue();
 487          return;
 488       }
 489
 490       // this is for the "real" finish
 491       Complete = true;
 492       Status = StatDone;
 493       Dequeue();
 494       if(Debug)
 495          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 496       return;
 497    }
 498
 499    if(Debug)
 500       std::clog << "Finishing: " << Desc.URI << std::endl;
 501    Complete = false;
 502    Status = StatDone;
 503    Dequeue();
 504    return;
 505 }
 506                                                                         /*}}}*/
 507 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 508 {
 509
 510    // calc sha1 of the just patched file
 511    string FinalFile = _config->FindDir("Dir::State::lists");
 512    FinalFile += URItoFileName(RealURI);
 513
 514    FileFd fd(FinalFile, FileFd::ReadOnly);
 515    SHA1Summation SHA1;
 516    SHA1.AddFD(fd.Fd(), fd.Size());
 517    string local_sha1 = string(SHA1.Result());
 518    if(Debug)
 519       std::clog << "QueueNextDiff: "
 520                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 521
 522    // final file reached before all patches are applied
 523    if(local_sha1 == ServerSha1)
 524    {
 525       Finish(true);
 526       return true;
 527    }
 528
 529    // remove all patches until the next matching patch is found
 530    // this requires the Index file to be ordered
 531    for(vector<DiffInfo>::iterator I=available_patches.begin();
 532        available_patches.size() > 0 &&
 533           I != available_patches.end() &&
 534           (*I).sha1 != local_sha1;
 535        I++)
 536    {
 537       available_patches.erase(I);
 538    }
 539
 540    // error checking and falling back if no patch was found
 541    if(available_patches.size() == 0)
 542    {
 543       Failed("", NULL);
 544       return false;
 545    }
 546
 547    // queue the right diff
 548    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 549    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 550    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 551    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 552
 553    if(Debug)
 554       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 555
 556    QueueURI(Desc);
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 562                             pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading a diff, enter ApplyDiff state
 573    if(State == StateFetchDiff)
 574    {
 575
 576       // rred excepts the patch as $FinalFile.ed
 577       Rename(DestFile,FinalFile+".ed");
 578
 579       if(Debug)
 580          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 581
 582       State = StateApplyDiff;
 583       Local = true;
 584       Desc.URI = "rred:" + FinalFile;
 585       QueueURI(Desc);
 586       Mode = "rred";
 587       return;
 588    }
 589
 590
 591    // success in download/apply a diff, queue next (if needed)
 592    if(State == StateApplyDiff)
 593    {
 594       // remove the just applied patch
 595       available_patches.erase(available_patches.begin());
 596
 597       // move into place
 598       if(Debug)
 599       {
 600          std::clog << "Moving patched file in place: " << std::endl
 601                    << DestFile << " -> " << FinalFile << std::endl;
 602       }
 603       Rename(DestFile,FinalFile);
 604       chmod(FinalFile.c_str(),0644);
 605
 606       // see if there is more to download
 607       if(available_patches.size() > 0) {
 608          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 609                               ExpectedHash, ServerSha1, available_patches);
 610          return Finish();
 611       } else
 612          return Finish(true);
 613    }
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The package file is added to the queue and a second class is
 619    instantiated to fetch the revision file */
 620 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 621                          string URI,string URIDesc,string ShortDesc,
 622                          HashString ExpectedHash, string comprExt)
 623    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 624 {
 625    Decompression = false;
 626    Erase = false;
 627
 628    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 629    DestFile += URItoFileName(URI);
 630
 631    if(comprExt.empty())
 632    {
 633       // autoselect the compression method
 634       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 635       if (types.empty() == true)
 636          comprExt = "plain";
 637       else
 638          comprExt = "." + types[0];
 639    }
 640    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 641
 642    Desc.URI = URI + CompressionExtension;
 643
 644    Desc.Description = URIDesc;
 645    Desc.Owner = this;
 646    Desc.ShortDesc = ShortDesc;
 647
 648    QueueURI(Desc);
 649 }
 650                                                                         /*}}}*/
 651 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 652 // ---------------------------------------------------------------------
 653 /* The only header we use is the last-modified header. */
 654 string pkgAcqIndex::Custom600Headers()
 655 {
 656    string Final = _config->FindDir("Dir::State::lists");
 657    Final += URItoFileName(RealURI);
 658    if (_config->FindB("Acquire::GzipIndexes",false))
 659       Final += ".gz";
 660
 661    struct stat Buf;
 662    if (stat(Final.c_str(),&Buf) != 0)
 663       return "\nIndex-File: true";
 664    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 665 }
 666                                                                         /*}}}*/
 667 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 668 {
 669    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 670
 671    for (std::vector<std::string>::const_iterator t = types.begin();
 672         t != types.end(); t++)
 673    {
 674       // jump over all already tried compression types
 675       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 676       if(Desc.URI.substr(nameLen) != *t)
 677          continue;
 678
 679       // we want to try it with the next extension (and make sure to
 680       // not skip over the end)
 681       t++;
 682       if (t == types.end())
 683          break;
 684
 685       // queue new download
 686       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 687       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 688       ExpectedHash, string(".").append(*t));
 689
 690       Status = StatDone;
 691       Complete = false;
 692       Dequeue();
 693       return;
 694    }
 695
 696    // on decompression failure, remove bad versions in partial/
 697    if(Decompression && Erase) {
 698       string s = _config->FindDir("Dir::State::lists") + "partial/";
 699       s += URItoFileName(RealURI);
 700       unlink(s.c_str());
 701    }
 702
 703    Item::Failed(Message,Cnf);
 704 }
 705                                                                         /*}}}*/
 706 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 707 // ---------------------------------------------------------------------
 708 /* This goes through a number of states.. On the initial fetch the
 709    method could possibly return an alternate filename which points
 710    to the uncompressed version of the file. If this is so the file
 711    is copied into the partial directory. In all other cases the file
 712    is decompressed with a gzip uri. */
 713 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 714                        pkgAcquire::MethodConfig *Cfg)
 715 {
 716    Item::Done(Message,Size,Hash,Cfg);
 717
 718    if (Decompression == true)
 719    {
 720       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 721       {
 722          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 723          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 724       }
 725
 726       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 727       {
 728          Status = StatAuthError;
 729          ErrorText = _("Hash Sum mismatch");
 730          Rename(DestFile,DestFile + ".FAILED");
 731          ReportMirrorFailure("HashChecksumFailure");
 732          return;
 733       }
 734       // Done, move it into position
 735       string FinalFile = _config->FindDir("Dir::State::lists");
 736       FinalFile += URItoFileName(RealURI);
 737       Rename(DestFile,FinalFile);
 738       chmod(FinalFile.c_str(),0644);
 739
 740       /* We restore the original name to DestFile so that the clean operation
 741          will work OK */
 742       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 743       DestFile += URItoFileName(RealURI);
 744
 745       // Remove the compressed version.
 746       if (Erase == true)
 747          unlink(DestFile.c_str());
 748       return;
 749    }
 750
 751    Erase = false;
 752    Complete = true;
 753
 754    // Handle the unzipd case
 755    string FileName = LookupTag(Message,"Alt-Filename");
 756    if (FileName.empty() == false)
 757    {
 758       // The files timestamp matches
 759       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 760          return;
 761       Decompression = true;
 762       Local = true;
 763       DestFile += ".decomp";
 764       Desc.URI = "copy:" + FileName;
 765       QueueURI(Desc);
 766       Mode = "copy";
 767       return;
 768    }
 769
 770    FileName = LookupTag(Message,"Filename");
 771    if (FileName.empty() == true)
 772    {
 773       Status = StatError;
 774       ErrorText = "Method gave a blank filename";
 775    }
 776
 777    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 778
 779    // The files timestamp matches
 780    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 781        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 782           // Update DestFile for .gz suffix so that the clean operation keeps it
 783           DestFile += ".gz";
 784       return;
 785     }
 786
 787    if (FileName == DestFile)
 788       Erase = true;
 789    else
 790       Local = true;
 791
 792    string decompProg;
 793
 794    // If we enable compressed indexes and already have gzip, keep it
 795    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 796       string FinalFile = _config->FindDir("Dir::State::lists");
 797       FinalFile += URItoFileName(RealURI) + ".gz";
 798       Rename(DestFile,FinalFile);
 799       chmod(FinalFile.c_str(),0644);
 800
 801       // Update DestFile for .gz suffix so that the clean operation keeps it
 802       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 803       DestFile += URItoFileName(RealURI) + ".gz";
 804       return;
 805     }
 806
 807    // get the binary name for your used compression type
 808    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 809    if(decompProg.empty() == false);
 810    // flExtensions returns the full name if no extension is found
 811    // this is why we have this complicated compare operation here
 812    // FIMXE: add a new flJustExtension() that return "" if no
 813    //        extension is found and use that above so that it can
 814    //        be tested against ""
 815    else if(compExt == flNotDir(URI(Desc.URI).Path))
 816       decompProg = "copy";
 817    else {
 818       _error->Error("Unsupported extension: %s", compExt.c_str());
 819       return;
 820    }
 821
 822    Decompression = true;
 823    DestFile += ".decomp";
 824    Desc.URI = decompProg + ":" + FileName;
 825    QueueURI(Desc);
 826    Mode = decompProg.c_str();
 827 }
 828                                                                         /*}}}*/
 829 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 830 // ---------------------------------------------------------------------
 831 /* The Translation file is added to the queue */
 832 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 833                             string URI,string URIDesc,string ShortDesc)
 834   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 835 {
 836 }
 837                                                                         /*}}}*/
 838 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 839 // ---------------------------------------------------------------------
 840 string pkgAcqIndexTrans::Custom600Headers()
 841 {
 842    string Final = _config->FindDir("Dir::State::lists");
 843    Final += URItoFileName(RealURI);
 844
 845    struct stat Buf;
 846    if (stat(Final.c_str(),&Buf) != 0)
 847       return "\nFail-Ignore: true";
 848    return "\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 849 }
 850                                                                         /*}}}*/
 851 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 852 // ---------------------------------------------------------------------
 853 /* */
 854 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 855 {
 856    if (Cnf->LocalOnly == true ||
 857        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 858    {
 859       // Ignore this
 860       Status = StatDone;
 861       Complete = false;
 862       Dequeue();
 863       return;
 864    }
 865
 866    Item::Failed(Message,Cnf);
 867 }
 868                                                                         /*}}}*/
 869 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 870                              string URI,string URIDesc,string ShortDesc,
 871                              string MetaIndexURI, string MetaIndexURIDesc,
 872                              string MetaIndexShortDesc,
 873                              const vector<IndexTarget*>* IndexTargets,
 874                              indexRecords* MetaIndexParser) :
 875    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 876    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 877    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 878 {
 879    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 880    DestFile += URItoFileName(URI);
 881
 882    // remove any partial downloaded sig-file in partial/.
 883    // it may confuse proxies and is too small to warrant a
 884    // partial download anyway
 885    unlink(DestFile.c_str());
 886
 887    // Create the item
 888    Desc.Description = URIDesc;
 889    Desc.Owner = this;
 890    Desc.ShortDesc = ShortDesc;
 891    Desc.URI = URI;
 892
 893    string Final = _config->FindDir("Dir::State::lists");
 894    Final += URItoFileName(RealURI);
 895    struct stat Buf;
 896    if (stat(Final.c_str(),&Buf) == 0)
 897    {
 898       // File was already in place.  It needs to be re-downloaded/verified
 899       // because Release might have changed, we do give it a differnt
 900       // name than DestFile because otherwise the http method will
 901       // send If-Range requests and there are too many broken servers
 902       // out there that do not understand them
 903       LastGoodSig = DestFile+".reverify";
 904       Rename(Final,LastGoodSig);
 905    }
 906
 907    QueueURI(Desc);
 908 }
 909                                                                         /*}}}*/
 910 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 911 // ---------------------------------------------------------------------
 912 /* The only header we use is the last-modified header. */
 913 string pkgAcqMetaSig::Custom600Headers()
 914 {
 915    struct stat Buf;
 916    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 917       return "\nIndex-File: true";
 918
 919    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 920 }
 921
 922 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 923                          pkgAcquire::MethodConfig *Cfg)
 924 {
 925    Item::Done(Message,Size,MD5,Cfg);
 926
 927    string FileName = LookupTag(Message,"Filename");
 928    if (FileName.empty() == true)
 929    {
 930       Status = StatError;
 931       ErrorText = "Method gave a blank filename";
 932       return;
 933    }
 934
 935    if (FileName != DestFile)
 936    {
 937       // We have to copy it into place
 938       Local = true;
 939       Desc.URI = "copy:" + FileName;
 940       QueueURI(Desc);
 941       return;
 942    }
 943
 944    Complete = true;
 945
 946    // put the last known good file back on i-m-s hit (it will
 947    // be re-verified again)
 948    // Else do nothing, we have the new file in DestFile then
 949    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 950       Rename(LastGoodSig, DestFile);
 951
 952    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 953    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 954                        MetaIndexShortDesc,  DestFile, IndexTargets,
 955                        MetaIndexParser);
 956
 957 }
 958                                                                         /*}}}*/
 959 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 960 {
 961    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 962
 963    // if we get a network error we fail gracefully
 964    if(Status == StatTransientNetworkError)
 965    {
 966       Item::Failed(Message,Cnf);
 967       // move the sigfile back on transient network failures
 968       if(FileExists(LastGoodSig))
 969          Rename(LastGoodSig,Final);
 970
 971       // set the status back to , Item::Failed likes to reset it
 972       Status = pkgAcquire::Item::StatTransientNetworkError;
 973       return;
 974    }
 975
 976    // Delete any existing sigfile when the acquire failed
 977    unlink(Final.c_str());
 978
 979    // queue a pkgAcqMetaIndex with no sigfile
 980    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 981                        "", IndexTargets, MetaIndexParser);
 982
 983    if (Cnf->LocalOnly == true ||
 984        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 985    {
 986       // Ignore this
 987       Status = StatDone;
 988       Complete = false;
 989       Dequeue();
 990       return;
 991    }
 992
 993    Item::Failed(Message,Cnf);
 994 }
 995                                                                         /*}}}*/
 996 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 997                                  string URI,string URIDesc,string ShortDesc,
 998                                  string SigFile,
 999                                  const vector<struct IndexTarget*>* IndexTargets,
1000                                  indexRecords* MetaIndexParser) :
1001    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1002    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1003 {
1004    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1005    DestFile += URItoFileName(URI);
1006
1007    // Create the item
1008    Desc.Description = URIDesc;
1009    Desc.Owner = this;
1010    Desc.ShortDesc = ShortDesc;
1011    Desc.URI = URI;
1012
1013    QueueURI(Desc);
1014 }
1015                                                                         /*}}}*/
1016 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1017 // ---------------------------------------------------------------------
1018 /* The only header we use is the last-modified header. */
1019 string pkgAcqMetaIndex::Custom600Headers()
1020 {
1021    string Final = _config->FindDir("Dir::State::lists");
1022    Final += URItoFileName(RealURI);
1023
1024    struct stat Buf;
1025    if (stat(Final.c_str(),&Buf) != 0)
1026       return "\nIndex-File: true";
1027
1028    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1029 }
1030                                                                         /*}}}*/
1031 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1032                            pkgAcquire::MethodConfig *Cfg)
1033 {
1034    Item::Done(Message,Size,Hash,Cfg);
1035
1036    // MetaIndexes are done in two passes: one to download the
1037    // metaindex with an appropriate method, and a second to verify it
1038    // with the gpgv method
1039
1040    if (AuthPass == true)
1041    {
1042       AuthDone(Message);
1043
1044       // all cool, move Release file into place
1045       Complete = true;
1046    }
1047    else
1048    {
1049       RetrievalDone(Message);
1050       if (!Complete)
1051          // Still more retrieving to do
1052          return;
1053
1054       if (SigFile == "")
1055       {
1056          // There was no signature file, so we are finished.  Download
1057          // the indexes without verification.
1058          QueueIndexes(false);
1059       }
1060       else
1061       {
1062          // There was a signature file, so pass it to gpgv for
1063          // verification
1064
1065          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1066             std::cerr << "Metaindex acquired, queueing gpg verification ("
1067                       << SigFile << "," << DestFile << ")\n";
1068          AuthPass = true;
1069          Desc.URI = "gpgv:" + SigFile;
1070          QueueURI(Desc);
1071          Mode = "gpgv";
1072          return;
1073       }
1074    }
1075
1076    if (Complete == true)
1077    {
1078       string FinalFile = _config->FindDir("Dir::State::lists");
1079       FinalFile += URItoFileName(RealURI);
1080       Rename(DestFile,FinalFile);
1081       chmod(FinalFile.c_str(),0644);
1082       DestFile = FinalFile;
1083    }
1084 }
1085                                                                         /*}}}*/
1086 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1087 {
1088    // We have just finished downloading a Release file (it is not
1089    // verified yet)
1090
1091    string FileName = LookupTag(Message,"Filename");
1092    if (FileName.empty() == true)
1093    {
1094       Status = StatError;
1095       ErrorText = "Method gave a blank filename";
1096       return;
1097    }
1098
1099    if (FileName != DestFile)
1100    {
1101       Local = true;
1102       Desc.URI = "copy:" + FileName;
1103       QueueURI(Desc);
1104       return;
1105    }
1106
1107    // make sure to verify against the right file on I-M-S hit
1108    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1109    if(IMSHit)
1110    {
1111       string FinalFile = _config->FindDir("Dir::State::lists");
1112       FinalFile += URItoFileName(RealURI);
1113       DestFile = FinalFile;
1114    }
1115    Complete = true;
1116 }
1117                                                                         /*}}}*/
1118 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1119 {
1120    // At this point, the gpgv method has succeeded, so there is a
1121    // valid signature from a key in the trusted keyring.  We
1122    // perform additional verification of its contents, and use them
1123    // to verify the indexes we are about to download
1124
1125    if (!MetaIndexParser->Load(DestFile))
1126    {
1127       Status = StatAuthError;
1128       ErrorText = MetaIndexParser->ErrorText;
1129       return;
1130    }
1131
1132    if (!VerifyVendor(Message))
1133    {
1134       return;
1135    }
1136
1137    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1138       std::cerr << "Signature verification succeeded: "
1139                 << DestFile << std::endl;
1140
1141    // Download further indexes with verification
1142    QueueIndexes(true);
1143
1144    // Done, move signature file into position
1145    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1146       URItoFileName(RealURI) + ".gpg";
1147    Rename(SigFile,VerifiedSigFile);
1148    chmod(VerifiedSigFile.c_str(),0644);
1149 }
1150                                                                         /*}}}*/
1151 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1152 {
1153    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1154         Target != IndexTargets->end();
1155         Target++)
1156    {
1157       HashString ExpectedIndexHash;
1158       if (verify)
1159       {
1160          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1161          if (!Record)
1162          {
1163             Status = StatAuthError;
1164             ErrorText = "Unable to find expected entry  "
1165                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1166             return;
1167          }
1168          ExpectedIndexHash = Record->Hash;
1169          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1170          {
1171             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1172             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1173          }
1174          if (ExpectedIndexHash.empty())
1175          {
1176             Status = StatAuthError;
1177             ErrorText = "Unable to find hash sum for "
1178                + (*Target)->MetaKey + " in Meta-index file";
1179             return;
1180          }
1181       }
1182
1183       /* Queue Packages file (either diff or full packages files, depending
1184          on the users option) - we also check if the PDiff Index file is listed
1185          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1186          instead, but passing the required info to it is to much hassle */
1187       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1188           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1189          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1190                              (*Target)->ShortDesc, ExpectedIndexHash);
1191       else
1192          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1193                             (*Target)->ShortDesc, ExpectedIndexHash);
1194    }
1195 }
1196                                                                         /*}}}*/
1197 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1198 {
1199 //    // Maybe this should be made available from above so we don't have
1200 //    // to read and parse it every time?
1201 //    pkgVendorList List;
1202 //    List.ReadMainList();
1203
1204 //    const Vendor* Vndr = NULL;
1205 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1206 //    {
1207 //       string::size_type pos = (*I).find("VALIDSIG ");
1208 //       if (_config->FindB("Debug::Vendor", false))
1209 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1210 //                    << std::endl;
1211 //       if (pos != std::string::npos)
1212 //       {
1213 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1214 //          if (_config->FindB("Debug::Vendor", false))
1215 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1216 //                std::endl;
1217 //          Vndr = List.FindVendor(Fingerprint) != "";
1218 //          if (Vndr != NULL);
1219 //          break;
1220 //       }
1221 //    }
1222    string::size_type pos;
1223
1224    // check for missing sigs (that where not fatal because otherwise we had
1225    // bombed earlier)
1226    string missingkeys;
1227    string msg = _("There is no public key available for the "
1228                   "following key IDs:\n");
1229    pos = Message.find("NO_PUBKEY ");
1230    if (pos != std::string::npos)
1231    {
1232       string::size_type start = pos+strlen("NO_PUBKEY ");
1233       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1234       missingkeys += (Fingerprint);
1235    }
1236    if(!missingkeys.empty())
1237       _error->Warning("%s", string(msg+missingkeys).c_str());
1238
1239    string Transformed = MetaIndexParser->GetExpectedDist();
1240
1241    if (Transformed == "../project/experimental")
1242    {
1243       Transformed = "experimental";
1244    }
1245
1246    pos = Transformed.rfind('/');
1247    if (pos != string::npos)
1248    {
1249       Transformed = Transformed.substr(0, pos);
1250    }
1251
1252    if (Transformed == ".")
1253    {
1254       Transformed = "";
1255    }
1256
1257    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1258        MetaIndexParser->GetValidUntil() > 0) {
1259       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1260       if (invalid_since > 0)
1261          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1262          // the time since then the file is invalid - formated in the same way as in
1263          // the download progress display (e.g. 7d 3h 42min 1s)
1264          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1265                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1266    }
1267
1268    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1269    {
1270       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1271       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1272       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1273    }
1274
1275    if (MetaIndexParser->CheckDist(Transformed) == false)
1276    {
1277       // This might become fatal one day
1278 //       Status = StatAuthError;
1279 //       ErrorText = "Conflicting distribution; expected "
1280 //          + MetaIndexParser->GetExpectedDist() + " but got "
1281 //          + MetaIndexParser->GetDist();
1282 //       return false;
1283       if (!Transformed.empty())
1284       {
1285          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1286                          Desc.Description.c_str(),
1287                          Transformed.c_str(),
1288                          MetaIndexParser->GetDist().c_str());
1289       }
1290    }
1291
1292    return true;
1293 }
1294                                                                         /*}}}*/
1295 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1296 // ---------------------------------------------------------------------
1297 /* */
1298 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1299 {
1300    if (AuthPass == true)
1301    {
1302       // gpgv method failed, if we have a good signature
1303       string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
1304          "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
1305       if(FileExists(LastGoodSigFile))
1306       {
1307          string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1308             URItoFileName(RealURI) + ".gpg";
1309          Rename(LastGoodSigFile,VerifiedSigFile);
1310          Status = StatTransientNetworkError;
1311          _error->Warning(_("A error occurred during the signature "
1312                            "verification. The repository is not updated "
1313                            "and the previous index files will be used. "
1314                            "GPG error: %s: %s\n"),
1315                          Desc.Description.c_str(),
1316                          LookupTag(Message,"Message").c_str());
1317          RunScripts("APT::Update::Auth-Failure");
1318          return;
1319       } else {
1320          _error->Warning(_("GPG error: %s: %s"),
1321                          Desc.Description.c_str(),
1322                          LookupTag(Message,"Message").c_str());
1323       }
1324       // gpgv method failed
1325       ReportMirrorFailure("GPGFailure");
1326    }
1327
1328    // No Release file was present, or verification failed, so fall
1329    // back to queueing Packages files without verification
1330    QueueIndexes(false);
1331 }
1332                                                                         /*}}}*/
1333 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1334 // ---------------------------------------------------------------------
1335 /* This just sets up the initial fetch environment and queues the first
1336    possibilitiy */
1337 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1338                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1339                              string &StoreFilename) :
1340                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1341                StoreFilename(StoreFilename), Vf(Version.FileList()),
1342                Trusted(false)
1343 {
1344    Retries = _config->FindI("Acquire::Retries",0);
1345
1346    if (Version.Arch() == 0)
1347    {
1348       _error->Error(_("I wasn't able to locate a file for the %s package. "
1349                       "This might mean you need to manually fix this package. "
1350                       "(due to missing arch)"),
1351                     Version.ParentPkg().Name());
1352       return;
1353    }
1354
1355    /* We need to find a filename to determine the extension. We make the
1356       assumption here that all the available sources for this version share
1357       the same extension.. */
1358    // Skip not source sources, they do not have file fields.
1359    for (; Vf.end() == false; Vf++)
1360    {
1361       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1362          continue;
1363       break;
1364    }
1365
1366    // Does not really matter here.. we are going to fail out below
1367    if (Vf.end() != true)
1368    {
1369       // If this fails to get a file name we will bomb out below.
1370       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1371       if (_error->PendingError() == true)
1372          return;
1373
1374       // Generate the final file name as: package_version_arch.foo
1375       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1376                       QuoteString(Version.VerStr(),"_:") + '_' +
1377                       QuoteString(Version.Arch(),"_:.") +
1378                       "." + flExtension(Parse.FileName());
1379    }
1380
1381    // check if we have one trusted source for the package. if so, switch
1382    // to "TrustedOnly" mode
1383    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1384    {
1385       pkgIndexFile *Index;
1386       if (Sources->FindIndex(i.File(),Index) == false)
1387          continue;
1388       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1389       {
1390          std::cerr << "Checking index: " << Index->Describe()
1391                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1392       }
1393       if (Index->IsTrusted()) {
1394          Trusted = true;
1395          break;
1396       }
1397    }
1398
1399    // "allow-unauthenticated" restores apts old fetching behaviour
1400    // that means that e.g. unauthenticated file:// uris are higher
1401    // priority than authenticated http:// uris
1402    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1403       Trusted = false;
1404
1405    // Select a source
1406    if (QueueNext() == false && _error->PendingError() == false)
1407       _error->Error(_("I wasn't able to locate file for the %s package. "
1408                     "This might mean you need to manually fix this package."),
1409                     Version.ParentPkg().Name());
1410 }
1411                                                                         /*}}}*/
1412 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1413 // ---------------------------------------------------------------------
1414 /* This queues the next available file version for download. It checks if
1415    the archive is already available in the cache and stashs the MD5 for
1416    checking later. */
1417 bool pkgAcqArchive::QueueNext()
1418 {
1419    string const ForceHash = _config->Find("Acquire::ForceHash");
1420    for (; Vf.end() == false; Vf++)
1421    {
1422       // Ignore not source sources
1423       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1424          continue;
1425
1426       // Try to cross match against the source list
1427       pkgIndexFile *Index;
1428       if (Sources->FindIndex(Vf.File(),Index) == false)
1429             continue;
1430
1431       // only try to get a trusted package from another source if that source
1432       // is also trusted
1433       if(Trusted && !Index->IsTrusted())
1434          continue;
1435
1436       // Grab the text package record
1437       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1438       if (_error->PendingError() == true)
1439          return false;
1440
1441       string PkgFile = Parse.FileName();
1442       if (ForceHash.empty() == false)
1443       {
1444          if(stringcasecmp(ForceHash, "sha256") == 0)
1445             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1446          else if (stringcasecmp(ForceHash, "sha1") == 0)
1447             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1448          else
1449             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1450       }
1451       else
1452       {
1453          string Hash;
1454          if ((Hash = Parse.SHA256Hash()).empty() == false)
1455             ExpectedHash = HashString("SHA256", Hash);
1456          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1457             ExpectedHash = HashString("SHA1", Hash);
1458          else
1459             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1460       }
1461       if (PkgFile.empty() == true)
1462          return _error->Error(_("The package index files are corrupted. No Filename: "
1463                               "field for package %s."),
1464                               Version.ParentPkg().Name());
1465
1466       Desc.URI = Index->ArchiveURI(PkgFile);
1467       Desc.Description = Index->ArchiveInfo(Version);
1468       Desc.Owner = this;
1469       Desc.ShortDesc = Version.ParentPkg().Name();
1470
1471       // See if we already have the file. (Legacy filenames)
1472       FileSize = Version->Size;
1473       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1474       struct stat Buf;
1475       if (stat(FinalFile.c_str(),&Buf) == 0)
1476       {
1477          // Make sure the size matches
1478          if ((unsigned)Buf.st_size == Version->Size)
1479          {
1480             Complete = true;
1481             Local = true;
1482             Status = StatDone;
1483             StoreFilename = DestFile = FinalFile;
1484             return true;
1485          }
1486
1487          /* Hmm, we have a file and its size does not match, this means it is
1488             an old style mismatched arch */
1489          unlink(FinalFile.c_str());
1490       }
1491
1492       // Check it again using the new style output filenames
1493       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1494       if (stat(FinalFile.c_str(),&Buf) == 0)
1495       {
1496          // Make sure the size matches
1497          if ((unsigned)Buf.st_size == Version->Size)
1498          {
1499             Complete = true;
1500             Local = true;
1501             Status = StatDone;
1502             StoreFilename = DestFile = FinalFile;
1503             return true;
1504          }
1505
1506          /* Hmm, we have a file and its size does not match, this shouldnt
1507             happen.. */
1508          unlink(FinalFile.c_str());
1509       }
1510
1511       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1512
1513       // Check the destination file
1514       if (stat(DestFile.c_str(),&Buf) == 0)
1515       {
1516          // Hmm, the partial file is too big, erase it
1517          if ((unsigned)Buf.st_size > Version->Size)
1518             unlink(DestFile.c_str());
1519          else
1520             PartialSize = Buf.st_size;
1521       }
1522
1523       // Create the item
1524       Local = false;
1525       Desc.URI = Index->ArchiveURI(PkgFile);
1526       Desc.Description = Index->ArchiveInfo(Version);
1527       Desc.Owner = this;
1528       Desc.ShortDesc = Version.ParentPkg().Name();
1529       QueueURI(Desc);
1530
1531       Vf++;
1532       return true;
1533    }
1534    return false;
1535 }
1536                                                                         /*}}}*/
1537 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1538 // ---------------------------------------------------------------------
1539 /* */
1540 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1541                          pkgAcquire::MethodConfig *Cfg)
1542 {
1543    Item::Done(Message,Size,CalcHash,Cfg);
1544
1545    // Check the size
1546    if (Size != Version->Size)
1547    {
1548       Status = StatError;
1549       ErrorText = _("Size mismatch");
1550       return;
1551    }
1552
1553    // Check the hash
1554    if(ExpectedHash.toStr() != CalcHash)
1555    {
1556       Status = StatError;
1557       ErrorText = _("Hash Sum mismatch");
1558       if(FileExists(DestFile))
1559          Rename(DestFile,DestFile + ".FAILED");
1560       return;
1561    }
1562
1563    // Grab the output filename
1564    string FileName = LookupTag(Message,"Filename");
1565    if (FileName.empty() == true)
1566    {
1567       Status = StatError;
1568       ErrorText = "Method gave a blank filename";
1569       return;
1570    }
1571
1572    Complete = true;
1573
1574    // Reference filename
1575    if (FileName != DestFile)
1576    {
1577       StoreFilename = DestFile = FileName;
1578       Local = true;
1579       return;
1580    }
1581
1582    // Done, move it into position
1583    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1584    FinalFile += flNotDir(StoreFilename);
1585    Rename(DestFile,FinalFile);
1586
1587    StoreFilename = DestFile = FinalFile;
1588    Complete = true;
1589 }
1590                                                                         /*}}}*/
1591 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1592 // ---------------------------------------------------------------------
1593 /* Here we try other sources */
1594 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1595 {
1596    ErrorText = LookupTag(Message,"Message");
1597
1598    /* We don't really want to retry on failed media swaps, this prevents
1599       that. An interesting observation is that permanent failures are not
1600       recorded. */
1601    if (Cnf->Removable == true &&
1602        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1603    {
1604       // Vf = Version.FileList();
1605       while (Vf.end() == false) Vf++;
1606       StoreFilename = string();
1607       Item::Failed(Message,Cnf);
1608       return;
1609    }
1610
1611    if (QueueNext() == false)
1612    {
1613       // This is the retry counter
1614       if (Retries != 0 &&
1615           Cnf->LocalOnly == false &&
1616           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1617       {
1618          Retries--;
1619          Vf = Version.FileList();
1620          if (QueueNext() == true)
1621             return;
1622       }
1623
1624       StoreFilename = string();
1625       Item::Failed(Message,Cnf);
1626    }
1627 }
1628                                                                         /*}}}*/
1629 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1630 // ---------------------------------------------------------------------
1631 bool pkgAcqArchive::IsTrusted()
1632 {
1633    return Trusted;
1634 }
1635                                                                         /*}}}*/
1636 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1637 // ---------------------------------------------------------------------
1638 /* */
1639 void pkgAcqArchive::Finished()
1640 {
1641    if (Status == pkgAcquire::Item::StatDone &&
1642        Complete == true)
1643       return;
1644    StoreFilename = string();
1645 }
1646                                                                         /*}}}*/
1647 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1648 // ---------------------------------------------------------------------
1649 /* The file is added to the queue */
1650 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1651                        unsigned long Size,string Dsc,string ShortDesc,
1652                        const string &DestDir, const string &DestFilename,
1653                        bool IsIndexFile) :
1654                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1655 {
1656    Retries = _config->FindI("Acquire::Retries",0);
1657
1658    if(!DestFilename.empty())
1659       DestFile = DestFilename;
1660    else if(!DestDir.empty())
1661       DestFile = DestDir + "/" + flNotDir(URI);
1662    else
1663       DestFile = flNotDir(URI);
1664
1665    // Create the item
1666    Desc.URI = URI;
1667    Desc.Description = Dsc;
1668    Desc.Owner = this;
1669
1670    // Set the short description to the archive component
1671    Desc.ShortDesc = ShortDesc;
1672
1673    // Get the transfer sizes
1674    FileSize = Size;
1675    struct stat Buf;
1676    if (stat(DestFile.c_str(),&Buf) == 0)
1677    {
1678       // Hmm, the partial file is too big, erase it
1679       if ((unsigned)Buf.st_size > Size)
1680          unlink(DestFile.c_str());
1681       else
1682          PartialSize = Buf.st_size;
1683    }
1684
1685    QueueURI(Desc);
1686 }
1687                                                                         /*}}}*/
1688 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1689 // ---------------------------------------------------------------------
1690 /* */
1691 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1692                       pkgAcquire::MethodConfig *Cnf)
1693 {
1694    Item::Done(Message,Size,CalcHash,Cnf);
1695
1696    // Check the hash
1697    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1698    {
1699       Status = StatError;
1700       ErrorText = _("Hash Sum mismatch");
1701       Rename(DestFile,DestFile + ".FAILED");
1702       return;
1703    }
1704
1705    string FileName = LookupTag(Message,"Filename");
1706    if (FileName.empty() == true)
1707    {
1708       Status = StatError;
1709       ErrorText = "Method gave a blank filename";
1710       return;
1711    }
1712
1713    Complete = true;
1714
1715    // The files timestamp matches
1716    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1717       return;
1718
1719    // We have to copy it into place
1720    if (FileName != DestFile)
1721    {
1722       Local = true;
1723       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1724           Cnf->Removable == true)
1725       {
1726          Desc.URI = "copy:" + FileName;
1727          QueueURI(Desc);
1728          return;
1729       }
1730
1731       // Erase the file if it is a symlink so we can overwrite it
1732       struct stat St;
1733       if (lstat(DestFile.c_str(),&St) == 0)
1734       {
1735          if (S_ISLNK(St.st_mode) != 0)
1736             unlink(DestFile.c_str());
1737       }
1738
1739       // Symlink the file
1740       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1741       {
1742          ErrorText = "Link to " + DestFile + " failure ";
1743          Status = StatError;
1744          Complete = false;
1745       }
1746    }
1747 }
1748                                                                         /*}}}*/
1749 // AcqFile::Failed - Failure handler                                    /*{{{*/
1750 // ---------------------------------------------------------------------
1751 /* Here we try other sources */
1752 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1753 {
1754    ErrorText = LookupTag(Message,"Message");
1755
1756    // This is the retry counter
1757    if (Retries != 0 &&
1758        Cnf->LocalOnly == false &&
1759        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1760    {
1761       Retries--;
1762       QueueURI(Desc);
1763       return;
1764    }
1765
1766    Item::Failed(Message,Cnf);
1767 }
1768                                                                         /*}}}*/
1769 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1770 // ---------------------------------------------------------------------
1771 /* The only header we use is the last-modified header. */
1772 string pkgAcqFile::Custom600Headers()
1773 {
1774    if (IsIndexFile)
1775       return "\nIndex-File: true";
1776    return "";
1777 }
1778                                                                         /*}}}*/