apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the DiffIndex file first and see if there are patches availabe
 190  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 191  * patches. If anything goes wrong in that process, it will fall back to
 192  * the original packages file
 193  */
 194 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 195                                  string URI,string URIDesc,string ShortDesc,
 196                                  HashString ExpectedHash)
 197    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 198      Description(URIDesc)
 199 {
 200
 201    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 202
 203    Desc.Description = URIDesc + "/DiffIndex";
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206    Desc.URI = URI + ".diff/Index";
 207
 208    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 209    DestFile += URItoFileName(URI) + string(".DiffIndex");
 210
 211    if(Debug)
 212       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 213
 214    // look for the current package file
 215    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 216    CurrentPackagesFile += URItoFileName(RealURI);
 217
 218    // FIXME: this file:/ check is a hack to prevent fetching
 219    //        from local sources. this is really silly, and
 220    //        should be fixed cleanly as soon as possible
 221    if(!FileExists(CurrentPackagesFile) ||
 222       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 223    {
 224       // we don't have a pkg file or we don't want to queue
 225       if(Debug)
 226          std::clog << "No index file, local or canceld by user" << std::endl;
 227       Failed("", NULL);
 228       return;
 229    }
 230
 231    if(Debug)
 232       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 233                 << CurrentPackagesFile << std::endl;
 234
 235    QueueURI(Desc);
 236
 237 }
 238                                                                         /*}}}*/
 239 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* The only header we use is the last-modified header. */
 242 string pkgAcqDiffIndex::Custom600Headers()
 243 {
 244    string Final = _config->FindDir("Dir::State::lists");
 245    Final += URItoFileName(RealURI) + string(".IndexDiff");
 246
 247    if(Debug)
 248       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true";
 253
 254    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 255 }
 256                                                                         /*}}}*/
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       bool found = false;
 275       DiffInfo d;
 276       string size;
 277
 278       string const tmp = Tags.FindS("SHA1-Current");
 279       std::stringstream ss(tmp);
 280       ss >> ServerSha1 >> size;
 281       unsigned long const ServerSize = atol(size.c_str());
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       string const local_sha1 = SHA1.Result();
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string const history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             // read until the first match is found
 308             // from that point on, we probably need all diffs
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             else if (found == false)
 312                continue;
 313
 314             if(Debug)
 315                std::clog << "Need to get diff: " << d.file << std::endl;
 316             available_patches.push_back(d);
 317          }
 318
 319          if (available_patches.empty() == false)
 320          {
 321             // patching with too many files is rather slow compared to a fast download
 322             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 323             if (fileLimit != 0 && fileLimit < available_patches.size())
 324             {
 325                if (Debug)
 326                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 327                         << ") so fallback to complete download" << std::endl;
 328                return false;
 329             }
 330
 331             // see if the patches are too big
 332             found = false; // it was true and it will be true again at the end
 333             d = *available_patches.begin();
 334             string const firstPatch = d.file;
 335             unsigned long patchesSize = 0;
 336             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 337             while(patches >> d.sha1 >> size >> d.file)
 338             {
 339                if (firstPatch == d.file)
 340                   found = true;
 341                else if (found == false)
 342                   continue;
 343
 344                patchesSize += atol(size.c_str());
 345             }
 346             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 347             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 348             {
 349                if (Debug)
 350                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 351                         << ") so fallback to complete download" << std::endl;
 352                return false;
 353             }
 354          }
 355       }
 356
 357       // we have something, queue the next diff
 358       if(found)
 359       {
 360          // queue the diffs
 361          string::size_type const last_space = Description.rfind(" ");
 362          if(last_space != string::npos)
 363             Description.erase(last_space, Description.size()-last_space);
 364          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 365                               ExpectedHash, ServerSha1, available_patches);
 366          Complete = false;
 367          Status = StatDone;
 368          Dequeue();
 369          return true;
 370       }
 371    }
 372
 373    // Nothing found, report and return false
 374    // Failing here is ok, if we return false later, the full
 375    // IndexFile is queued
 376    if(Debug)
 377       std::clog << "Can't find a patch in the index file" << std::endl;
 378    return false;
 379 }
 380                                                                         /*}}}*/
 381 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 382 {
 383    if(Debug)
 384       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 385                 << "Falling back to normal index file aquire" << std::endl;
 386
 387    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 388                    ExpectedHash);
 389
 390    Complete = false;
 391    Status = StatDone;
 392    Dequeue();
 393 }
 394                                                                         /*}}}*/
 395 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 396                            pkgAcquire::MethodConfig *Cnf)
 397 {
 398    if(Debug)
 399       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 400
 401    Item::Done(Message,Size,Md5Hash,Cnf);
 402
 403    string FinalFile;
 404    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 405
 406    // sucess in downloading the index
 407    // rename the index
 408    FinalFile += string(".IndexDiff");
 409    if(Debug)
 410       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 411                 << std::endl;
 412    Rename(DestFile,FinalFile);
 413    chmod(FinalFile.c_str(),0644);
 414    DestFile = FinalFile;
 415
 416    if(!ParseDiffIndex(DestFile))
 417       return Failed("", NULL);
 418
 419    Complete = true;
 420    Status = StatDone;
 421    Dequeue();
 422    return;
 423 }
 424                                                                         /*}}}*/
 425 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* The package diff is added to the queue. one object is constructed
 428  * for each diff and the index
 429  */
 430 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 431                                    string URI,string URIDesc,string ShortDesc,
 432                                    HashString ExpectedHash,
 433                                    string ServerSha1,
 434                                    vector<DiffInfo> diffs)
 435    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 436      available_patches(diffs), ServerSha1(ServerSha1)
 437 {
 438
 439    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 440    DestFile += URItoFileName(URI);
 441
 442    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 443
 444    Description = URIDesc;
 445    Desc.Owner = this;
 446    Desc.ShortDesc = ShortDesc;
 447
 448    if(available_patches.size() == 0)
 449    {
 450       // we are done (yeah!)
 451       Finish(true);
 452    }
 453    else
 454    {
 455       // get the next diff
 456       State = StateFetchDiff;
 457       QueueNextDiff();
 458    }
 459 }
 460                                                                         /*}}}*/
 461 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 462 {
 463    if(Debug)
 464       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 465                 << "Falling back to normal index file aquire" << std::endl;
 466    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 467                    ExpectedHash);
 468    Finish();
 469 }
 470                                                                         /*}}}*/
 471 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 472 void pkgAcqIndexDiffs::Finish(bool allDone)
 473 {
 474    // we restore the original name, this is required, otherwise
 475    // the file will be cleaned
 476    if(allDone)
 477    {
 478       DestFile = _config->FindDir("Dir::State::lists");
 479       DestFile += URItoFileName(RealURI);
 480
 481       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 482       {
 483          Status = StatAuthError;
 484          ErrorText = _("MD5Sum mismatch");
 485          Rename(DestFile,DestFile + ".FAILED");
 486          Dequeue();
 487          return;
 488       }
 489
 490       // this is for the "real" finish
 491       Complete = true;
 492       Status = StatDone;
 493       Dequeue();
 494       if(Debug)
 495          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 496       return;
 497    }
 498
 499    if(Debug)
 500       std::clog << "Finishing: " << Desc.URI << std::endl;
 501    Complete = false;
 502    Status = StatDone;
 503    Dequeue();
 504    return;
 505 }
 506                                                                         /*}}}*/
 507 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 508 {
 509
 510    // calc sha1 of the just patched file
 511    string FinalFile = _config->FindDir("Dir::State::lists");
 512    FinalFile += URItoFileName(RealURI);
 513
 514    FileFd fd(FinalFile, FileFd::ReadOnly);
 515    SHA1Summation SHA1;
 516    SHA1.AddFD(fd.Fd(), fd.Size());
 517    string local_sha1 = string(SHA1.Result());
 518    if(Debug)
 519       std::clog << "QueueNextDiff: "
 520                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 521
 522    // final file reached before all patches are applied
 523    if(local_sha1 == ServerSha1)
 524    {
 525       Finish(true);
 526       return true;
 527    }
 528
 529    // remove all patches until the next matching patch is found
 530    // this requires the Index file to be ordered
 531    for(vector<DiffInfo>::iterator I=available_patches.begin();
 532        available_patches.size() > 0 &&
 533           I != available_patches.end() &&
 534           (*I).sha1 != local_sha1;
 535        I++)
 536    {
 537       available_patches.erase(I);
 538    }
 539
 540    // error checking and falling back if no patch was found
 541    if(available_patches.size() == 0)
 542    {
 543       Failed("", NULL);
 544       return false;
 545    }
 546
 547    // queue the right diff
 548    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 549    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 550    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 551    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 552
 553    if(Debug)
 554       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 555
 556    QueueURI(Desc);
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 562                             pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading a diff, enter ApplyDiff state
 573    if(State == StateFetchDiff)
 574    {
 575
 576       // rred excepts the patch as $FinalFile.ed
 577       Rename(DestFile,FinalFile+".ed");
 578
 579       if(Debug)
 580          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 581
 582       State = StateApplyDiff;
 583       Local = true;
 584       Desc.URI = "rred:" + FinalFile;
 585       QueueURI(Desc);
 586       Mode = "rred";
 587       return;
 588    }
 589
 590
 591    // success in download/apply a diff, queue next (if needed)
 592    if(State == StateApplyDiff)
 593    {
 594       // remove the just applied patch
 595       available_patches.erase(available_patches.begin());
 596
 597       // move into place
 598       if(Debug)
 599       {
 600          std::clog << "Moving patched file in place: " << std::endl
 601                    << DestFile << " -> " << FinalFile << std::endl;
 602       }
 603       Rename(DestFile,FinalFile);
 604       chmod(FinalFile.c_str(),0644);
 605
 606       // see if there is more to download
 607       if(available_patches.size() > 0) {
 608          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 609                               ExpectedHash, ServerSha1, available_patches);
 610          return Finish();
 611       } else
 612          return Finish(true);
 613    }
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The package file is added to the queue and a second class is
 619    instantiated to fetch the revision file */
 620 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 621                          string URI,string URIDesc,string ShortDesc,
 622                          HashString ExpectedHash, string comprExt)
 623    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 624 {
 625    Decompression = false;
 626    Erase = false;
 627
 628    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 629    DestFile += URItoFileName(URI);
 630
 631    if(comprExt.empty())
 632    {
 633       // autoselect the compression method
 634       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 635       if (types.empty() == true)
 636          comprExt = "plain";
 637       else
 638          comprExt = "." + types[0];
 639    }
 640    CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
 641
 642    Desc.URI = URI + CompressionExtension;
 643
 644    Desc.Description = URIDesc;
 645    Desc.Owner = this;
 646    Desc.ShortDesc = ShortDesc;
 647
 648    QueueURI(Desc);
 649 }
 650                                                                         /*}}}*/
 651 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 652 // ---------------------------------------------------------------------
 653 /* The only header we use is the last-modified header. */
 654 string pkgAcqIndex::Custom600Headers()
 655 {
 656    string Final = _config->FindDir("Dir::State::lists");
 657    Final += URItoFileName(RealURI);
 658    if (_config->FindB("Acquire::GzipIndexes",false))
 659       Final += ".gz";
 660
 661    struct stat Buf;
 662    if (stat(Final.c_str(),&Buf) != 0)
 663       return "\nIndex-File: true";
 664    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 665 }
 666                                                                         /*}}}*/
 667 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 668 {
 669    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 670
 671    for (std::vector<std::string>::const_iterator t = types.begin();
 672         t != types.end(); t++)
 673    {
 674       // jump over all already tried compression types
 675       const unsigned int nameLen = Desc.URI.size() - (*t).size();
 676       if(Desc.URI.substr(nameLen) != *t)
 677          continue;
 678
 679       // we want to try it with the next extension (and make sure to
 680       // not skip over the end)
 681       t++;
 682       if (t == types.end())
 683          break;
 684
 685       // queue new download
 686       Desc.URI = Desc.URI.substr(0, nameLen) + *t;
 687       new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
 688       ExpectedHash, string(".").append(*t));
 689
 690       Status = StatDone;
 691       Complete = false;
 692       Dequeue();
 693       return;
 694    }
 695
 696    // on decompression failure, remove bad versions in partial/
 697    if(Decompression && Erase) {
 698       string s = _config->FindDir("Dir::State::lists") + "partial/";
 699       s += URItoFileName(RealURI);
 700       unlink(s.c_str());
 701    }
 702
 703    Item::Failed(Message,Cnf);
 704 }
 705                                                                         /*}}}*/
 706 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 707 // ---------------------------------------------------------------------
 708 /* This goes through a number of states.. On the initial fetch the
 709    method could possibly return an alternate filename which points
 710    to the uncompressed version of the file. If this is so the file
 711    is copied into the partial directory. In all other cases the file
 712    is decompressed with a gzip uri. */
 713 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 714                        pkgAcquire::MethodConfig *Cfg)
 715 {
 716    Item::Done(Message,Size,Hash,Cfg);
 717
 718    if (Decompression == true)
 719    {
 720       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 721       {
 722          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 723          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 724       }
 725
 726       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 727       {
 728          Status = StatAuthError;
 729          ErrorText = _("Hash Sum mismatch");
 730          Rename(DestFile,DestFile + ".FAILED");
 731          ReportMirrorFailure("HashChecksumFailure");
 732          return;
 733       }
 734       // Done, move it into position
 735       string FinalFile = _config->FindDir("Dir::State::lists");
 736       FinalFile += URItoFileName(RealURI);
 737       Rename(DestFile,FinalFile);
 738       chmod(FinalFile.c_str(),0644);
 739
 740       /* We restore the original name to DestFile so that the clean operation
 741          will work OK */
 742       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 743       DestFile += URItoFileName(RealURI);
 744
 745       // Remove the compressed version.
 746       if (Erase == true)
 747          unlink(DestFile.c_str());
 748       return;
 749    }
 750
 751    Erase = false;
 752    Complete = true;
 753
 754    // Handle the unzipd case
 755    string FileName = LookupTag(Message,"Alt-Filename");
 756    if (FileName.empty() == false)
 757    {
 758       // The files timestamp matches
 759       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 760          return;
 761       Decompression = true;
 762       Local = true;
 763       DestFile += ".decomp";
 764       Desc.URI = "copy:" + FileName;
 765       QueueURI(Desc);
 766       Mode = "copy";
 767       return;
 768    }
 769
 770    FileName = LookupTag(Message,"Filename");
 771    if (FileName.empty() == true)
 772    {
 773       Status = StatError;
 774       ErrorText = "Method gave a blank filename";
 775    }
 776
 777    string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
 778
 779    // The files timestamp matches
 780    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 781        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 782           // Update DestFile for .gz suffix so that the clean operation keeps it
 783           DestFile += ".gz";
 784       return;
 785     }
 786
 787    if (FileName == DestFile)
 788       Erase = true;
 789    else
 790       Local = true;
 791
 792    string decompProg;
 793
 794    // If we enable compressed indexes and already have gzip, keep it
 795    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 796       string FinalFile = _config->FindDir("Dir::State::lists");
 797       FinalFile += URItoFileName(RealURI) + ".gz";
 798       Rename(DestFile,FinalFile);
 799       chmod(FinalFile.c_str(),0644);
 800
 801       // Update DestFile for .gz suffix so that the clean operation keeps it
 802       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 803       DestFile += URItoFileName(RealURI) + ".gz";
 804       return;
 805     }
 806
 807    // get the binary name for your used compression type
 808    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 809    if(decompProg.empty() == false);
 810    // flExtensions returns the full name if no extension is found
 811    // this is why we have this complicated compare operation here
 812    // FIMXE: add a new flJustExtension() that return "" if no
 813    //        extension is found and use that above so that it can
 814    //        be tested against ""
 815    else if(compExt == flNotDir(URI(Desc.URI).Path))
 816       decompProg = "copy";
 817    else {
 818       _error->Error("Unsupported extension: %s", compExt.c_str());
 819       return;
 820    }
 821
 822    Decompression = true;
 823    DestFile += ".decomp";
 824    Desc.URI = decompProg + ":" + FileName;
 825    QueueURI(Desc);
 826    Mode = decompProg.c_str();
 827 }
 828                                                                         /*}}}*/
 829 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 830 // ---------------------------------------------------------------------
 831 /* The Translation file is added to the queue */
 832 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 833                             string URI,string URIDesc,string ShortDesc)
 834   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 835 {
 836 }
 837                                                                         /*}}}*/
 838 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 839 // ---------------------------------------------------------------------
 840 string pkgAcqIndexTrans::Custom600Headers()
 841 {
 842    string Final = _config->FindDir("Dir::State::lists");
 843    Final += URItoFileName(RealURI);
 844
 845    struct stat Buf;
 846    if (stat(Final.c_str(),&Buf) != 0)
 847       return "\nFail-Ignore: true";
 848    return "\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 849 }
 850                                                                         /*}}}*/
 851 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 852 // ---------------------------------------------------------------------
 853 /* */
 854 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 855 {
 856    if (Cnf->LocalOnly == true ||
 857        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 858    {
 859       // Ignore this
 860       Status = StatDone;
 861       Complete = false;
 862       Dequeue();
 863       return;
 864    }
 865
 866    Item::Failed(Message,Cnf);
 867 }
 868                                                                         /*}}}*/
 869 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 870                              string URI,string URIDesc,string ShortDesc,
 871                              string MetaIndexURI, string MetaIndexURIDesc,
 872                              string MetaIndexShortDesc,
 873                              const vector<IndexTarget*>* IndexTargets,
 874                              indexRecords* MetaIndexParser) :
 875    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 876    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 877    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 878 {
 879    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 880    DestFile += URItoFileName(URI);
 881
 882    // remove any partial downloaded sig-file in partial/.
 883    // it may confuse proxies and is too small to warrant a
 884    // partial download anyway
 885    unlink(DestFile.c_str());
 886
 887    // Create the item
 888    Desc.Description = URIDesc;
 889    Desc.Owner = this;
 890    Desc.ShortDesc = ShortDesc;
 891    Desc.URI = URI;
 892
 893    string Final = _config->FindDir("Dir::State::lists");
 894    Final += URItoFileName(RealURI);
 895    struct stat Buf;
 896    if (stat(Final.c_str(),&Buf) == 0)
 897    {
 898       // File was already in place.  It needs to be re-downloaded/verified
 899       // because Release might have changed, we do give it a differnt
 900       // name than DestFile because otherwise the http method will
 901       // send If-Range requests and there are too many broken servers
 902       // out there that do not understand them
 903       LastGoodSig = DestFile+".reverify";
 904       Rename(Final,LastGoodSig);
 905    }
 906
 907    QueueURI(Desc);
 908 }
 909                                                                         /*}}}*/
 910 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 911 // ---------------------------------------------------------------------
 912 /* The only header we use is the last-modified header. */
 913 string pkgAcqMetaSig::Custom600Headers()
 914 {
 915    struct stat Buf;
 916    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 917       return "\nIndex-File: true";
 918
 919    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 920 }
 921
 922 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 923                          pkgAcquire::MethodConfig *Cfg)
 924 {
 925    Item::Done(Message,Size,MD5,Cfg);
 926
 927    string FileName = LookupTag(Message,"Filename");
 928    if (FileName.empty() == true)
 929    {
 930       Status = StatError;
 931       ErrorText = "Method gave a blank filename";
 932       return;
 933    }
 934
 935    if (FileName != DestFile)
 936    {
 937       // We have to copy it into place
 938       Local = true;
 939       Desc.URI = "copy:" + FileName;
 940       QueueURI(Desc);
 941       return;
 942    }
 943
 944    Complete = true;
 945
 946    // put the last known good file back on i-m-s hit (it will
 947    // be re-verified again)
 948    // Else do nothing, we have the new file in DestFile then
 949    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 950       Rename(LastGoodSig, DestFile);
 951
 952    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 953    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 954                        MetaIndexShortDesc,  DestFile, IndexTargets,
 955                        MetaIndexParser);
 956
 957 }
 958                                                                         /*}}}*/
 959 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 960 {
 961    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 962
 963    // if we get a network error we fail gracefully
 964    if(Status == StatTransientNetworkError)
 965    {
 966       Item::Failed(Message,Cnf);
 967       // move the sigfile back on transient network failures
 968       if(FileExists(LastGoodSig))
 969          Rename(LastGoodSig,Final);
 970
 971       // set the status back to , Item::Failed likes to reset it
 972       Status = pkgAcquire::Item::StatTransientNetworkError;
 973       return;
 974    }
 975
 976    // Delete any existing sigfile when the acquire failed
 977    unlink(Final.c_str());
 978
 979    // queue a pkgAcqMetaIndex with no sigfile
 980    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 981                        "", IndexTargets, MetaIndexParser);
 982
 983    if (Cnf->LocalOnly == true ||
 984        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 985    {
 986       // Ignore this
 987       Status = StatDone;
 988       Complete = false;
 989       Dequeue();
 990       return;
 991    }
 992
 993    Item::Failed(Message,Cnf);
 994 }
 995                                                                         /*}}}*/
 996 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
 997                                  string URI,string URIDesc,string ShortDesc,
 998                                  string SigFile,
 999                                  const vector<struct IndexTarget*>* IndexTargets,
1000                                  indexRecords* MetaIndexParser) :
1001    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1002    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1003 {
1004    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1005    DestFile += URItoFileName(URI);
1006
1007    // Create the item
1008    Desc.Description = URIDesc;
1009    Desc.Owner = this;
1010    Desc.ShortDesc = ShortDesc;
1011    Desc.URI = URI;
1012
1013    QueueURI(Desc);
1014 }
1015                                                                         /*}}}*/
1016 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1017 // ---------------------------------------------------------------------
1018 /* The only header we use is the last-modified header. */
1019 string pkgAcqMetaIndex::Custom600Headers()
1020 {
1021    string Final = _config->FindDir("Dir::State::lists");
1022    Final += URItoFileName(RealURI);
1023
1024    struct stat Buf;
1025    if (stat(Final.c_str(),&Buf) != 0)
1026       return "\nIndex-File: true";
1027
1028    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1029 }
1030                                                                         /*}}}*/
1031 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1032                            pkgAcquire::MethodConfig *Cfg)
1033 {
1034    Item::Done(Message,Size,Hash,Cfg);
1035
1036    // MetaIndexes are done in two passes: one to download the
1037    // metaindex with an appropriate method, and a second to verify it
1038    // with the gpgv method
1039
1040    if (AuthPass == true)
1041    {
1042       AuthDone(Message);
1043
1044       // all cool, move Release file into place
1045       Complete = true;
1046    }
1047    else
1048    {
1049       RetrievalDone(Message);
1050       if (!Complete)
1051          // Still more retrieving to do
1052          return;
1053
1054       if (SigFile == "")
1055       {
1056          // There was no signature file, so we are finished.  Download
1057          // the indexes without verification.
1058          QueueIndexes(false);
1059       }
1060       else
1061       {
1062          // There was a signature file, so pass it to gpgv for
1063          // verification
1064
1065          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1066             std::cerr << "Metaindex acquired, queueing gpg verification ("
1067                       << SigFile << "," << DestFile << ")\n";
1068          AuthPass = true;
1069          Desc.URI = "gpgv:" + SigFile;
1070          QueueURI(Desc);
1071          Mode = "gpgv";
1072          return;
1073       }
1074    }
1075
1076    if (Complete == true)
1077    {
1078       string FinalFile = _config->FindDir("Dir::State::lists");
1079       FinalFile += URItoFileName(RealURI);
1080       if (SigFile == DestFile)
1081          SigFile = FinalFile;
1082       Rename(DestFile,FinalFile);
1083       chmod(FinalFile.c_str(),0644);
1084       DestFile = FinalFile;
1085    }
1086 }
1087                                                                         /*}}}*/
1088 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1089 {
1090    // We have just finished downloading a Release file (it is not
1091    // verified yet)
1092
1093    string FileName = LookupTag(Message,"Filename");
1094    if (FileName.empty() == true)
1095    {
1096       Status = StatError;
1097       ErrorText = "Method gave a blank filename";
1098       return;
1099    }
1100
1101    if (FileName != DestFile)
1102    {
1103       Local = true;
1104       Desc.URI = "copy:" + FileName;
1105       QueueURI(Desc);
1106       return;
1107    }
1108
1109    // make sure to verify against the right file on I-M-S hit
1110    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1111    if(IMSHit)
1112    {
1113       string FinalFile = _config->FindDir("Dir::State::lists");
1114       FinalFile += URItoFileName(RealURI);
1115       if (SigFile == DestFile)
1116          SigFile = FinalFile;
1117       DestFile = FinalFile;
1118    }
1119    Complete = true;
1120 }
1121                                                                         /*}}}*/
1122 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1123 {
1124    // At this point, the gpgv method has succeeded, so there is a
1125    // valid signature from a key in the trusted keyring.  We
1126    // perform additional verification of its contents, and use them
1127    // to verify the indexes we are about to download
1128
1129    if (!MetaIndexParser->Load(DestFile))
1130    {
1131       Status = StatAuthError;
1132       ErrorText = MetaIndexParser->ErrorText;
1133       return;
1134    }
1135
1136    if (!VerifyVendor(Message))
1137    {
1138       return;
1139    }
1140
1141    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1142       std::cerr << "Signature verification succeeded: "
1143                 << DestFile << std::endl;
1144
1145    // Download further indexes with verification
1146    QueueIndexes(true);
1147
1148    // is it a clearsigned MetaIndex file?
1149    if (DestFile == SigFile)
1150       return;
1151
1152    // Done, move signature file into position
1153    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1154       URItoFileName(RealURI) + ".gpg";
1155    Rename(SigFile,VerifiedSigFile);
1156    chmod(VerifiedSigFile.c_str(),0644);
1157 }
1158                                                                         /*}}}*/
1159 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1160 {
1161    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1162         Target != IndexTargets->end();
1163         Target++)
1164    {
1165       HashString ExpectedIndexHash;
1166       if (verify)
1167       {
1168          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1169          if (!Record)
1170          {
1171             Status = StatAuthError;
1172             ErrorText = "Unable to find expected entry  "
1173                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1174             return;
1175          }
1176          ExpectedIndexHash = Record->Hash;
1177          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1178          {
1179             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1180             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1181          }
1182          if (ExpectedIndexHash.empty())
1183          {
1184             Status = StatAuthError;
1185             ErrorText = "Unable to find hash sum for "
1186                + (*Target)->MetaKey + " in Meta-index file";
1187             return;
1188          }
1189       }
1190
1191       /* Queue Packages file (either diff or full packages files, depending
1192          on the users option) - we also check if the PDiff Index file is listed
1193          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1194          instead, but passing the required info to it is to much hassle */
1195       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1196           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1197          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1198                              (*Target)->ShortDesc, ExpectedIndexHash);
1199       else
1200          new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
1201                             (*Target)->ShortDesc, ExpectedIndexHash);
1202    }
1203 }
1204                                                                         /*}}}*/
1205 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1206 {
1207 //    // Maybe this should be made available from above so we don't have
1208 //    // to read and parse it every time?
1209 //    pkgVendorList List;
1210 //    List.ReadMainList();
1211
1212 //    const Vendor* Vndr = NULL;
1213 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1214 //    {
1215 //       string::size_type pos = (*I).find("VALIDSIG ");
1216 //       if (_config->FindB("Debug::Vendor", false))
1217 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1218 //                    << std::endl;
1219 //       if (pos != std::string::npos)
1220 //       {
1221 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1222 //          if (_config->FindB("Debug::Vendor", false))
1223 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1224 //                std::endl;
1225 //          Vndr = List.FindVendor(Fingerprint) != "";
1226 //          if (Vndr != NULL);
1227 //          break;
1228 //       }
1229 //    }
1230    string::size_type pos;
1231
1232    // check for missing sigs (that where not fatal because otherwise we had
1233    // bombed earlier)
1234    string missingkeys;
1235    string msg = _("There is no public key available for the "
1236                   "following key IDs:\n");
1237    pos = Message.find("NO_PUBKEY ");
1238    if (pos != std::string::npos)
1239    {
1240       string::size_type start = pos+strlen("NO_PUBKEY ");
1241       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1242       missingkeys += (Fingerprint);
1243    }
1244    if(!missingkeys.empty())
1245       _error->Warning("%s", string(msg+missingkeys).c_str());
1246
1247    string Transformed = MetaIndexParser->GetExpectedDist();
1248
1249    if (Transformed == "../project/experimental")
1250    {
1251       Transformed = "experimental";
1252    }
1253
1254    pos = Transformed.rfind('/');
1255    if (pos != string::npos)
1256    {
1257       Transformed = Transformed.substr(0, pos);
1258    }
1259
1260    if (Transformed == ".")
1261    {
1262       Transformed = "";
1263    }
1264
1265    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1266        MetaIndexParser->GetValidUntil() > 0) {
1267       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1268       if (invalid_since > 0)
1269          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1270          // the time since then the file is invalid - formated in the same way as in
1271          // the download progress display (e.g. 7d 3h 42min 1s)
1272          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1273                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1274    }
1275
1276    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1277    {
1278       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1279       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1280       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1281    }
1282
1283    if (MetaIndexParser->CheckDist(Transformed) == false)
1284    {
1285       // This might become fatal one day
1286 //       Status = StatAuthError;
1287 //       ErrorText = "Conflicting distribution; expected "
1288 //          + MetaIndexParser->GetExpectedDist() + " but got "
1289 //          + MetaIndexParser->GetDist();
1290 //       return false;
1291       if (!Transformed.empty())
1292       {
1293          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1294                          Desc.Description.c_str(),
1295                          Transformed.c_str(),
1296                          MetaIndexParser->GetDist().c_str());
1297       }
1298    }
1299
1300    return true;
1301 }
1302                                                                         /*}}}*/
1303 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1304 // ---------------------------------------------------------------------
1305 /* */
1306 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1307 {
1308    if (AuthPass == true)
1309    {
1310       // gpgv method failed, if we have a good signature
1311       string LastGoodSigFile = _config->FindDir("Dir::State::lists");
1312       if (DestFile == SigFile)
1313          LastGoodSigFile.append(URItoFileName(RealURI));
1314       else
1315          LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify");
1316
1317       if(FileExists(LastGoodSigFile))
1318       {
1319          if (DestFile != SigFile)
1320          {
1321             string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1322                                         URItoFileName(RealURI) + ".gpg";
1323             Rename(LastGoodSigFile,VerifiedSigFile);
1324          }
1325          Status = StatTransientNetworkError;
1326          _error->Warning(_("A error occurred during the signature "
1327                            "verification. The repository is not updated "
1328                            "and the previous index files will be used. "
1329                            "GPG error: %s: %s\n"),
1330                          Desc.Description.c_str(),
1331                          LookupTag(Message,"Message").c_str());
1332          RunScripts("APT::Update::Auth-Failure");
1333          return;
1334       } else {
1335          _error->Warning(_("GPG error: %s: %s"),
1336                          Desc.Description.c_str(),
1337                          LookupTag(Message,"Message").c_str());
1338       }
1339       // gpgv method failed
1340       ReportMirrorFailure("GPGFailure");
1341    }
1342
1343    // No Release file was present, or verification failed, so fall
1344    // back to queueing Packages files without verification
1345    QueueIndexes(false);
1346 }
1347                                                                         /*}}}*/
1348 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1349                 string const &URI, string const &URIDesc, string const &ShortDesc,
1350                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1351                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1352                 const vector<struct IndexTarget*>* IndexTargets,
1353                 indexRecords* MetaIndexParser) :
1354         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1355         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1356         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1357 {
1358    SigFile = DestFile;
1359 }
1360                                                                         /*}}}*/
1361 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1362 {
1363    if (AuthPass == false)
1364    {
1365       new pkgAcqMetaSig(Owner,
1366                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1367                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1368                         IndexTargets, MetaIndexParser);
1369       if (Cnf->LocalOnly == true ||
1370           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1371          Dequeue();
1372    }
1373    else
1374       pkgAcqMetaIndex::Failed(Message, Cnf);
1375 }
1376                                                                         /*}}}*/
1377 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1378 // ---------------------------------------------------------------------
1379 /* This just sets up the initial fetch environment and queues the first
1380    possibilitiy */
1381 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1382                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1383                              string &StoreFilename) :
1384                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1385                StoreFilename(StoreFilename), Vf(Version.FileList()),
1386                Trusted(false)
1387 {
1388    Retries = _config->FindI("Acquire::Retries",0);
1389
1390    if (Version.Arch() == 0)
1391    {
1392       _error->Error(_("I wasn't able to locate a file for the %s package. "
1393                       "This might mean you need to manually fix this package. "
1394                       "(due to missing arch)"),
1395                     Version.ParentPkg().Name());
1396       return;
1397    }
1398
1399    /* We need to find a filename to determine the extension. We make the
1400       assumption here that all the available sources for this version share
1401       the same extension.. */
1402    // Skip not source sources, they do not have file fields.
1403    for (; Vf.end() == false; Vf++)
1404    {
1405       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1406          continue;
1407       break;
1408    }
1409
1410    // Does not really matter here.. we are going to fail out below
1411    if (Vf.end() != true)
1412    {
1413       // If this fails to get a file name we will bomb out below.
1414       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1415       if (_error->PendingError() == true)
1416          return;
1417
1418       // Generate the final file name as: package_version_arch.foo
1419       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1420                       QuoteString(Version.VerStr(),"_:") + '_' +
1421                       QuoteString(Version.Arch(),"_:.") +
1422                       "." + flExtension(Parse.FileName());
1423    }
1424
1425    // check if we have one trusted source for the package. if so, switch
1426    // to "TrustedOnly" mode
1427    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1428    {
1429       pkgIndexFile *Index;
1430       if (Sources->FindIndex(i.File(),Index) == false)
1431          continue;
1432       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1433       {
1434          std::cerr << "Checking index: " << Index->Describe()
1435                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1436       }
1437       if (Index->IsTrusted()) {
1438          Trusted = true;
1439          break;
1440       }
1441    }
1442
1443    // "allow-unauthenticated" restores apts old fetching behaviour
1444    // that means that e.g. unauthenticated file:// uris are higher
1445    // priority than authenticated http:// uris
1446    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1447       Trusted = false;
1448
1449    // Select a source
1450    if (QueueNext() == false && _error->PendingError() == false)
1451       _error->Error(_("I wasn't able to locate file for the %s package. "
1452                     "This might mean you need to manually fix this package."),
1453                     Version.ParentPkg().Name());
1454 }
1455                                                                         /*}}}*/
1456 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1457 // ---------------------------------------------------------------------
1458 /* This queues the next available file version for download. It checks if
1459    the archive is already available in the cache and stashs the MD5 for
1460    checking later. */
1461 bool pkgAcqArchive::QueueNext()
1462 {
1463    string const ForceHash = _config->Find("Acquire::ForceHash");
1464    for (; Vf.end() == false; Vf++)
1465    {
1466       // Ignore not source sources
1467       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1468          continue;
1469
1470       // Try to cross match against the source list
1471       pkgIndexFile *Index;
1472       if (Sources->FindIndex(Vf.File(),Index) == false)
1473             continue;
1474
1475       // only try to get a trusted package from another source if that source
1476       // is also trusted
1477       if(Trusted && !Index->IsTrusted())
1478          continue;
1479
1480       // Grab the text package record
1481       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1482       if (_error->PendingError() == true)
1483          return false;
1484
1485       string PkgFile = Parse.FileName();
1486       if (ForceHash.empty() == false)
1487       {
1488          if(stringcasecmp(ForceHash, "sha256") == 0)
1489             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1490          else if (stringcasecmp(ForceHash, "sha1") == 0)
1491             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1492          else
1493             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1494       }
1495       else
1496       {
1497          string Hash;
1498          if ((Hash = Parse.SHA256Hash()).empty() == false)
1499             ExpectedHash = HashString("SHA256", Hash);
1500          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1501             ExpectedHash = HashString("SHA1", Hash);
1502          else
1503             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1504       }
1505       if (PkgFile.empty() == true)
1506          return _error->Error(_("The package index files are corrupted. No Filename: "
1507                               "field for package %s."),
1508                               Version.ParentPkg().Name());
1509
1510       Desc.URI = Index->ArchiveURI(PkgFile);
1511       Desc.Description = Index->ArchiveInfo(Version);
1512       Desc.Owner = this;
1513       Desc.ShortDesc = Version.ParentPkg().Name();
1514
1515       // See if we already have the file. (Legacy filenames)
1516       FileSize = Version->Size;
1517       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1518       struct stat Buf;
1519       if (stat(FinalFile.c_str(),&Buf) == 0)
1520       {
1521          // Make sure the size matches
1522          if ((unsigned)Buf.st_size == Version->Size)
1523          {
1524             Complete = true;
1525             Local = true;
1526             Status = StatDone;
1527             StoreFilename = DestFile = FinalFile;
1528             return true;
1529          }
1530
1531          /* Hmm, we have a file and its size does not match, this means it is
1532             an old style mismatched arch */
1533          unlink(FinalFile.c_str());
1534       }
1535
1536       // Check it again using the new style output filenames
1537       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1538       if (stat(FinalFile.c_str(),&Buf) == 0)
1539       {
1540          // Make sure the size matches
1541          if ((unsigned)Buf.st_size == Version->Size)
1542          {
1543             Complete = true;
1544             Local = true;
1545             Status = StatDone;
1546             StoreFilename = DestFile = FinalFile;
1547             return true;
1548          }
1549
1550          /* Hmm, we have a file and its size does not match, this shouldnt
1551             happen.. */
1552          unlink(FinalFile.c_str());
1553       }
1554
1555       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1556
1557       // Check the destination file
1558       if (stat(DestFile.c_str(),&Buf) == 0)
1559       {
1560          // Hmm, the partial file is too big, erase it
1561          if ((unsigned)Buf.st_size > Version->Size)
1562             unlink(DestFile.c_str());
1563          else
1564             PartialSize = Buf.st_size;
1565       }
1566
1567       // Create the item
1568       Local = false;
1569       Desc.URI = Index->ArchiveURI(PkgFile);
1570       Desc.Description = Index->ArchiveInfo(Version);
1571       Desc.Owner = this;
1572       Desc.ShortDesc = Version.ParentPkg().Name();
1573       QueueURI(Desc);
1574
1575       Vf++;
1576       return true;
1577    }
1578    return false;
1579 }
1580                                                                         /*}}}*/
1581 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1582 // ---------------------------------------------------------------------
1583 /* */
1584 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1585                          pkgAcquire::MethodConfig *Cfg)
1586 {
1587    Item::Done(Message,Size,CalcHash,Cfg);
1588
1589    // Check the size
1590    if (Size != Version->Size)
1591    {
1592       Status = StatError;
1593       ErrorText = _("Size mismatch");
1594       return;
1595    }
1596
1597    // Check the hash
1598    if(ExpectedHash.toStr() != CalcHash)
1599    {
1600       Status = StatError;
1601       ErrorText = _("Hash Sum mismatch");
1602       if(FileExists(DestFile))
1603          Rename(DestFile,DestFile + ".FAILED");
1604       return;
1605    }
1606
1607    // Grab the output filename
1608    string FileName = LookupTag(Message,"Filename");
1609    if (FileName.empty() == true)
1610    {
1611       Status = StatError;
1612       ErrorText = "Method gave a blank filename";
1613       return;
1614    }
1615
1616    Complete = true;
1617
1618    // Reference filename
1619    if (FileName != DestFile)
1620    {
1621       StoreFilename = DestFile = FileName;
1622       Local = true;
1623       return;
1624    }
1625
1626    // Done, move it into position
1627    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1628    FinalFile += flNotDir(StoreFilename);
1629    Rename(DestFile,FinalFile);
1630
1631    StoreFilename = DestFile = FinalFile;
1632    Complete = true;
1633 }
1634                                                                         /*}}}*/
1635 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1636 // ---------------------------------------------------------------------
1637 /* Here we try other sources */
1638 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1639 {
1640    ErrorText = LookupTag(Message,"Message");
1641
1642    /* We don't really want to retry on failed media swaps, this prevents
1643       that. An interesting observation is that permanent failures are not
1644       recorded. */
1645    if (Cnf->Removable == true &&
1646        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1647    {
1648       // Vf = Version.FileList();
1649       while (Vf.end() == false) Vf++;
1650       StoreFilename = string();
1651       Item::Failed(Message,Cnf);
1652       return;
1653    }
1654
1655    if (QueueNext() == false)
1656    {
1657       // This is the retry counter
1658       if (Retries != 0 &&
1659           Cnf->LocalOnly == false &&
1660           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1661       {
1662          Retries--;
1663          Vf = Version.FileList();
1664          if (QueueNext() == true)
1665             return;
1666       }
1667
1668       StoreFilename = string();
1669       Item::Failed(Message,Cnf);
1670    }
1671 }
1672                                                                         /*}}}*/
1673 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1674 // ---------------------------------------------------------------------
1675 bool pkgAcqArchive::IsTrusted()
1676 {
1677    return Trusted;
1678 }
1679                                                                         /*}}}*/
1680 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1681 // ---------------------------------------------------------------------
1682 /* */
1683 void pkgAcqArchive::Finished()
1684 {
1685    if (Status == pkgAcquire::Item::StatDone &&
1686        Complete == true)
1687       return;
1688    StoreFilename = string();
1689 }
1690                                                                         /*}}}*/
1691 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1692 // ---------------------------------------------------------------------
1693 /* The file is added to the queue */
1694 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1695                        unsigned long Size,string Dsc,string ShortDesc,
1696                        const string &DestDir, const string &DestFilename,
1697                        bool IsIndexFile) :
1698                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1699 {
1700    Retries = _config->FindI("Acquire::Retries",0);
1701
1702    if(!DestFilename.empty())
1703       DestFile = DestFilename;
1704    else if(!DestDir.empty())
1705       DestFile = DestDir + "/" + flNotDir(URI);
1706    else
1707       DestFile = flNotDir(URI);
1708
1709    // Create the item
1710    Desc.URI = URI;
1711    Desc.Description = Dsc;
1712    Desc.Owner = this;
1713
1714    // Set the short description to the archive component
1715    Desc.ShortDesc = ShortDesc;
1716
1717    // Get the transfer sizes
1718    FileSize = Size;
1719    struct stat Buf;
1720    if (stat(DestFile.c_str(),&Buf) == 0)
1721    {
1722       // Hmm, the partial file is too big, erase it
1723       if ((unsigned)Buf.st_size > Size)
1724          unlink(DestFile.c_str());
1725       else
1726          PartialSize = Buf.st_size;
1727    }
1728
1729    QueueURI(Desc);
1730 }
1731                                                                         /*}}}*/
1732 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1733 // ---------------------------------------------------------------------
1734 /* */
1735 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1736                       pkgAcquire::MethodConfig *Cnf)
1737 {
1738    Item::Done(Message,Size,CalcHash,Cnf);
1739
1740    // Check the hash
1741    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1742    {
1743       Status = StatError;
1744       ErrorText = _("Hash Sum mismatch");
1745       Rename(DestFile,DestFile + ".FAILED");
1746       return;
1747    }
1748
1749    string FileName = LookupTag(Message,"Filename");
1750    if (FileName.empty() == true)
1751    {
1752       Status = StatError;
1753       ErrorText = "Method gave a blank filename";
1754       return;
1755    }
1756
1757    Complete = true;
1758
1759    // The files timestamp matches
1760    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1761       return;
1762
1763    // We have to copy it into place
1764    if (FileName != DestFile)
1765    {
1766       Local = true;
1767       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1768           Cnf->Removable == true)
1769       {
1770          Desc.URI = "copy:" + FileName;
1771          QueueURI(Desc);
1772          return;
1773       }
1774
1775       // Erase the file if it is a symlink so we can overwrite it
1776       struct stat St;
1777       if (lstat(DestFile.c_str(),&St) == 0)
1778       {
1779          if (S_ISLNK(St.st_mode) != 0)
1780             unlink(DestFile.c_str());
1781       }
1782
1783       // Symlink the file
1784       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1785       {
1786          ErrorText = "Link to " + DestFile + " failure ";
1787          Status = StatError;
1788          Complete = false;
1789       }
1790    }
1791 }
1792                                                                         /*}}}*/
1793 // AcqFile::Failed - Failure handler                                    /*{{{*/
1794 // ---------------------------------------------------------------------
1795 /* Here we try other sources */
1796 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1797 {
1798    ErrorText = LookupTag(Message,"Message");
1799
1800    // This is the retry counter
1801    if (Retries != 0 &&
1802        Cnf->LocalOnly == false &&
1803        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1804    {
1805       Retries--;
1806       QueueURI(Desc);
1807       return;
1808    }
1809
1810    Item::Failed(Message,Cnf);
1811 }
1812                                                                         /*}}}*/
1813 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1814 // ---------------------------------------------------------------------
1815 /* The only header we use is the last-modified header. */
1816 string pkgAcqFile::Custom600Headers()
1817 {
1818    if (IsIndexFile)
1819       return "\nIndex-File: true";
1820    return "";
1821 }
1822                                                                         /*}}}*/