apt-pkg/acquire-item.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
   4 /* ######################################################################
   5
   6    Acquire Item - Item to acquire
   7
   8    Each item can download to exactly one file at a time. This means you
   9    cannot create an item that fetches two uri's to two files at the same
  10    time. The pkgAcqIndex class creates a second class upon instantiation
  11    to fetch the other index files because of this.
  12
  13    ##################################################################### */
  14                                                                         /*}}}*/
  15 // Include Files                                                        /*{{{*/
  16 #include <apt-pkg/acquire-item.h>
  17 #include <apt-pkg/configuration.h>
  18 #include <apt-pkg/aptconfiguration.h>
  19 #include <apt-pkg/sourcelist.h>
  20 #include <apt-pkg/vendorlist.h>
  21 #include <apt-pkg/error.h>
  22 #include <apt-pkg/strutl.h>
  23 #include <apt-pkg/fileutl.h>
  24 #include <apt-pkg/md5.h>
  25 #include <apt-pkg/sha1.h>
  26 #include <apt-pkg/tagfile.h>
  27
  28 #include <apti18n.h>
  29
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <errno.h>
  33 #include <string>
  34 #include <sstream>
  35 #include <stdio.h>
  36 #include <ctime>
  37                                                                         /*}}}*/
  38
  39 using namespace std;
  40
  41 // Acquire::Item::Item - Constructor                                    /*{{{*/
  42 // ---------------------------------------------------------------------
  43 /* */
  44 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
  45                        PartialSize(0), Mode(0), ID(0), Complete(false),
  46                        Local(false), QueueCounter(0)
  47 {
  48    Owner->Add(this);
  49    Status = StatIdle;
  50 }
  51                                                                         /*}}}*/
  52 // Acquire::Item::~Item - Destructor                                    /*{{{*/
  53 // ---------------------------------------------------------------------
  54 /* */
  55 pkgAcquire::Item::~Item()
  56 {
  57    Owner->Remove(this);
  58 }
  59                                                                         /*}}}*/
  60 // Acquire::Item::Failed - Item failed to download                      /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* We return to an idle state if there are still other queues that could
  63    fetch this object */
  64 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
  65 {
  66    Status = StatIdle;
  67    ErrorText = LookupTag(Message,"Message");
  68    UsedMirror =  LookupTag(Message,"UsedMirror");
  69    if (QueueCounter <= 1)
  70    {
  71       /* This indicates that the file is not available right now but might
  72          be sometime later. If we do a retry cycle then this should be
  73          retried [CDROMs] */
  74       if (Cnf->LocalOnly == true &&
  75           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
  76       {
  77          Status = StatIdle;
  78          Dequeue();
  79          return;
  80       }
  81
  82       Status = StatError;
  83       Dequeue();
  84    }
  85
  86    // report mirror failure back to LP if we actually use a mirror
  87    string FailReason = LookupTag(Message, "FailReason");
  88    if(FailReason.size() != 0)
  89       ReportMirrorFailure(FailReason);
  90    else
  91       ReportMirrorFailure(ErrorText);
  92 }
  93                                                                         /*}}}*/
  94 // Acquire::Item::Start - Item has begun to download                    /*{{{*/
  95 // ---------------------------------------------------------------------
  96 /* Stash status and the file size. Note that setting Complete means
  97    sub-phases of the acquire process such as decompresion are operating */
  98 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
  99 {
 100    Status = StatFetching;
 101    if (FileSize == 0 && Complete == false)
 102       FileSize = Size;
 103 }
 104                                                                         /*}}}*/
 105 // Acquire::Item::Done - Item downloaded OK                             /*{{{*/
 106 // ---------------------------------------------------------------------
 107 /* */
 108 void pkgAcquire::Item::Done(string Message,unsigned long Size,string Hash,
 109                             pkgAcquire::MethodConfig *Cnf)
 110 {
 111    // We just downloaded something..
 112    string FileName = LookupTag(Message,"Filename");
 113    UsedMirror =  LookupTag(Message,"UsedMirror");
 114    if (Complete == false && !Local && FileName == DestFile)
 115    {
 116       if (Owner->Log != 0)
 117          Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 118    }
 119
 120    if (FileSize == 0)
 121       FileSize= Size;
 122    Status = StatDone;
 123    ErrorText = string();
 124    Owner->Dequeue(this);
 125 }
 126                                                                         /*}}}*/
 127 // Acquire::Item::Rename - Rename a file                                /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* This helper function is used by alot of item methods as thier final
 130    step */
 131 void pkgAcquire::Item::Rename(string From,string To)
 132 {
 133    if (rename(From.c_str(),To.c_str()) != 0)
 134    {
 135       char S[300];
 136       snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
 137               From.c_str(),To.c_str());
 138       Status = StatError;
 139       ErrorText = S;
 140    }
 141 }
 142                                                                         /*}}}*/
 143 // Acquire::Item::ReportMirrorFailure                                   /*{{{*/
 144 // ---------------------------------------------------------------------
 145 void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
 146 {
 147    // we only act if a mirror was used at all
 148    if(UsedMirror.empty())
 149       return;
 150 #if 0
 151    std::cerr << "\nReportMirrorFailure: "
 152              << UsedMirror
 153              << " Uri: " << DescURI()
 154              << " FailCode: "
 155              << FailCode << std::endl;
 156 #endif
 157    const char *Args[40];
 158    unsigned int i = 0;
 159    string report = _config->Find("Methods::Mirror::ProblemReporting",
 160                                  "/usr/lib/apt/apt-report-mirror-failure");
 161    if(!FileExists(report))
 162       return;
 163    Args[i++] = report.c_str();
 164    Args[i++] = UsedMirror.c_str();
 165    Args[i++] = DescURI().c_str();
 166    Args[i++] = FailCode.c_str();
 167    Args[i++] = NULL;
 168    pid_t pid = ExecFork();
 169    if(pid < 0)
 170    {
 171       _error->Error("ReportMirrorFailure Fork failed");
 172       return;
 173    }
 174    else if(pid == 0)
 175    {
 176       execvp(Args[0], (char**)Args);
 177       std::cerr << "Could not exec " << Args[0] << std::endl;
 178       _exit(100);
 179    }
 180    if(!ExecWait(pid, "report-mirror-failure"))
 181    {
 182       _error->Warning("Couldn't report problem to '%s'",
 183                       _config->Find("Methods::Mirror::ProblemReporting").c_str());
 184    }
 185 }
 186                                                                         /*}}}*/
 187 // AcqDiffIndex::AcqDiffIndex - Constructor                             /*{{{*/
 188 // ---------------------------------------------------------------------
 189 /* Get the DiffIndex file first and see if there are patches availabe
 190  * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
 191  * patches. If anything goes wrong in that process, it will fall back to
 192  * the original packages file
 193  */
 194 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 195                                  string URI,string URIDesc,string ShortDesc,
 196                                  HashString ExpectedHash)
 197    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 198      Description(URIDesc)
 199 {
 200
 201    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 202
 203    Desc.Description = URIDesc + "/DiffIndex";
 204    Desc.Owner = this;
 205    Desc.ShortDesc = ShortDesc;
 206    Desc.URI = URI + ".diff/Index";
 207
 208    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 209    DestFile += URItoFileName(URI) + string(".DiffIndex");
 210
 211    if(Debug)
 212       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 213
 214    // look for the current package file
 215    CurrentPackagesFile = _config->FindDir("Dir::State::lists");
 216    CurrentPackagesFile += URItoFileName(RealURI);
 217
 218    // FIXME: this file:/ check is a hack to prevent fetching
 219    //        from local sources. this is really silly, and
 220    //        should be fixed cleanly as soon as possible
 221    if(!FileExists(CurrentPackagesFile) ||
 222       Desc.URI.substr(0,strlen("file:/")) == "file:/")
 223    {
 224       // we don't have a pkg file or we don't want to queue
 225       if(Debug)
 226          std::clog << "No index file, local or canceld by user" << std::endl;
 227       Failed("", NULL);
 228       return;
 229    }
 230
 231    if(Debug)
 232       std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
 233                 << CurrentPackagesFile << std::endl;
 234
 235    QueueURI(Desc);
 236
 237 }
 238                                                                         /*}}}*/
 239 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* The only header we use is the last-modified header. */
 242 string pkgAcqDiffIndex::Custom600Headers()
 243 {
 244    string Final = _config->FindDir("Dir::State::lists");
 245    Final += URItoFileName(RealURI) + string(".IndexDiff");
 246
 247    if(Debug)
 248       std::clog << "Custom600Header-IMS: " << Final << std::endl;
 249
 250    struct stat Buf;
 251    if (stat(Final.c_str(),&Buf) != 0)
 252       return "\nIndex-File: true";
 253
 254    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 255 }
 256                                                                         /*}}}*/
 257 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)              /*{{{*/
 258 {
 259    if(Debug)
 260       std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
 261                 << std::endl;
 262
 263    pkgTagSection Tags;
 264    string ServerSha1;
 265    vector<DiffInfo> available_patches;
 266
 267    FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
 268    pkgTagFile TF(&Fd);
 269    if (_error->PendingError() == true)
 270       return false;
 271
 272    if(TF.Step(Tags) == true)
 273    {
 274       bool found = false;
 275       DiffInfo d;
 276       string size;
 277
 278       string const tmp = Tags.FindS("SHA1-Current");
 279       std::stringstream ss(tmp);
 280       ss >> ServerSha1 >> size;
 281       unsigned long const ServerSize = atol(size.c_str());
 282
 283       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
 284       SHA1Summation SHA1;
 285       SHA1.AddFD(fd.Fd(), fd.Size());
 286       string const local_sha1 = SHA1.Result();
 287
 288       if(local_sha1 == ServerSha1)
 289       {
 290          // we have the same sha1 as the server
 291          if(Debug)
 292             std::clog << "Package file is up-to-date" << std::endl;
 293          // set found to true, this will queue a pkgAcqIndexDiffs with
 294          // a empty availabe_patches
 295          found = true;
 296       }
 297       else
 298       {
 299          if(Debug)
 300             std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 301
 302          // check the historie and see what patches we need
 303          string const history = Tags.FindS("SHA1-History");
 304          std::stringstream hist(history);
 305          while(hist >> d.sha1 >> size >> d.file)
 306          {
 307             // read until the first match is found
 308             // from that point on, we probably need all diffs
 309             if(d.sha1 == local_sha1)
 310                found=true;
 311             else if (found == false)
 312                continue;
 313
 314             if(Debug)
 315                std::clog << "Need to get diff: " << d.file << std::endl;
 316             available_patches.push_back(d);
 317          }
 318
 319          if (available_patches.empty() == false)
 320          {
 321             // patching with too many files is rather slow compared to a fast download
 322             unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
 323             if (fileLimit != 0 && fileLimit < available_patches.size())
 324             {
 325                if (Debug)
 326                   std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
 327                         << ") so fallback to complete download" << std::endl;
 328                return false;
 329             }
 330
 331             // see if the patches are too big
 332             found = false; // it was true and it will be true again at the end
 333             d = *available_patches.begin();
 334             string const firstPatch = d.file;
 335             unsigned long patchesSize = 0;
 336             std::stringstream patches(Tags.FindS("SHA1-Patches"));
 337             while(patches >> d.sha1 >> size >> d.file)
 338             {
 339                if (firstPatch == d.file)
 340                   found = true;
 341                else if (found == false)
 342                   continue;
 343
 344                patchesSize += atol(size.c_str());
 345             }
 346             unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
 347             if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 348             {
 349                if (Debug)
 350                   std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
 351                         << ") so fallback to complete download" << std::endl;
 352                return false;
 353             }
 354          }
 355       }
 356
 357       // we have something, queue the next diff
 358       if(found)
 359       {
 360          // queue the diffs
 361          string::size_type const last_space = Description.rfind(" ");
 362          if(last_space != string::npos)
 363             Description.erase(last_space, Description.size()-last_space);
 364          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 365                               ExpectedHash, ServerSha1, available_patches);
 366          Complete = false;
 367          Status = StatDone;
 368          Dequeue();
 369          return true;
 370       }
 371    }
 372
 373    // Nothing found, report and return false
 374    // Failing here is ok, if we return false later, the full
 375    // IndexFile is queued
 376    if(Debug)
 377       std::clog << "Can't find a patch in the index file" << std::endl;
 378    return false;
 379 }
 380                                                                         /*}}}*/
 381 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)      /*{{{*/
 382 {
 383    if(Debug)
 384       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << std::endl
 385                 << "Falling back to normal index file aquire" << std::endl;
 386
 387    new pkgAcqIndex(Owner, RealURI, Description, Desc.ShortDesc,
 388                    ExpectedHash);
 389
 390    Complete = false;
 391    Status = StatDone;
 392    Dequeue();
 393 }
 394                                                                         /*}}}*/
 395 void pkgAcqDiffIndex::Done(string Message,unsigned long Size,string Md5Hash,    /*{{{*/
 396                            pkgAcquire::MethodConfig *Cnf)
 397 {
 398    if(Debug)
 399       std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 400
 401    Item::Done(Message,Size,Md5Hash,Cnf);
 402
 403    string FinalFile;
 404    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 405
 406    // sucess in downloading the index
 407    // rename the index
 408    FinalFile += string(".IndexDiff");
 409    if(Debug)
 410       std::clog << "Renaming: " << DestFile << " -> " << FinalFile
 411                 << std::endl;
 412    Rename(DestFile,FinalFile);
 413    chmod(FinalFile.c_str(),0644);
 414    DestFile = FinalFile;
 415
 416    if(!ParseDiffIndex(DestFile))
 417       return Failed("", NULL);
 418
 419    Complete = true;
 420    Status = StatDone;
 421    Dequeue();
 422    return;
 423 }
 424                                                                         /*}}}*/
 425 // AcqIndexDiffs::AcqIndexDiffs - Constructor                           /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* The package diff is added to the queue. one object is constructed
 428  * for each diff and the index
 429  */
 430 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
 431                                    string URI,string URIDesc,string ShortDesc,
 432                                    HashString ExpectedHash,
 433                                    string ServerSha1,
 434                                    vector<DiffInfo> diffs)
 435    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash),
 436      available_patches(diffs), ServerSha1(ServerSha1)
 437 {
 438
 439    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 440    DestFile += URItoFileName(URI);
 441
 442    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 443
 444    Description = URIDesc;
 445    Desc.Owner = this;
 446    Desc.ShortDesc = ShortDesc;
 447
 448    if(available_patches.size() == 0)
 449    {
 450       // we are done (yeah!)
 451       Finish(true);
 452    }
 453    else
 454    {
 455       // get the next diff
 456       State = StateFetchDiff;
 457       QueueNextDiff();
 458    }
 459 }
 460                                                                         /*}}}*/
 461 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)     /*{{{*/
 462 {
 463    if(Debug)
 464       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << std::endl
 465                 << "Falling back to normal index file aquire" << std::endl;
 466    new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc,
 467                    ExpectedHash);
 468    Finish();
 469 }
 470                                                                         /*}}}*/
 471 // Finish - helper that cleans the item out of the fetcher queue        /*{{{*/
 472 void pkgAcqIndexDiffs::Finish(bool allDone)
 473 {
 474    // we restore the original name, this is required, otherwise
 475    // the file will be cleaned
 476    if(allDone)
 477    {
 478       DestFile = _config->FindDir("Dir::State::lists");
 479       DestFile += URItoFileName(RealURI);
 480
 481       if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile))
 482       {
 483          Status = StatAuthError;
 484          ErrorText = _("MD5Sum mismatch");
 485          Rename(DestFile,DestFile + ".FAILED");
 486          Dequeue();
 487          return;
 488       }
 489
 490       // this is for the "real" finish
 491       Complete = true;
 492       Status = StatDone;
 493       Dequeue();
 494       if(Debug)
 495          std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
 496       return;
 497    }
 498
 499    if(Debug)
 500       std::clog << "Finishing: " << Desc.URI << std::endl;
 501    Complete = false;
 502    Status = StatDone;
 503    Dequeue();
 504    return;
 505 }
 506                                                                         /*}}}*/
 507 bool pkgAcqIndexDiffs::QueueNextDiff()                                  /*{{{*/
 508 {
 509
 510    // calc sha1 of the just patched file
 511    string FinalFile = _config->FindDir("Dir::State::lists");
 512    FinalFile += URItoFileName(RealURI);
 513
 514    FileFd fd(FinalFile, FileFd::ReadOnly);
 515    SHA1Summation SHA1;
 516    SHA1.AddFD(fd.Fd(), fd.Size());
 517    string local_sha1 = string(SHA1.Result());
 518    if(Debug)
 519       std::clog << "QueueNextDiff: "
 520                 << FinalFile << " (" << local_sha1 << ")"<<std::endl;
 521
 522    // final file reached before all patches are applied
 523    if(local_sha1 == ServerSha1)
 524    {
 525       Finish(true);
 526       return true;
 527    }
 528
 529    // remove all patches until the next matching patch is found
 530    // this requires the Index file to be ordered
 531    for(vector<DiffInfo>::iterator I=available_patches.begin();
 532        available_patches.size() > 0 &&
 533           I != available_patches.end() &&
 534           (*I).sha1 != local_sha1;
 535        I++)
 536    {
 537       available_patches.erase(I);
 538    }
 539
 540    // error checking and falling back if no patch was found
 541    if(available_patches.size() == 0)
 542    {
 543       Failed("", NULL);
 544       return false;
 545    }
 546
 547    // queue the right diff
 548    Desc.URI = string(RealURI) + ".diff/" + available_patches[0].file + ".gz";
 549    Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
 550    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 551    DestFile += URItoFileName(RealURI + ".diff/" + available_patches[0].file);
 552
 553    if(Debug)
 554       std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
 555
 556    QueueURI(Desc);
 557
 558    return true;
 559 }
 560                                                                         /*}}}*/
 561 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,   /*{{{*/
 562                             pkgAcquire::MethodConfig *Cnf)
 563 {
 564    if(Debug)
 565       std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 566
 567    Item::Done(Message,Size,Md5Hash,Cnf);
 568
 569    string FinalFile;
 570    FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI);
 571
 572    // sucess in downloading a diff, enter ApplyDiff state
 573    if(State == StateFetchDiff)
 574    {
 575
 576       // rred excepts the patch as $FinalFile.ed
 577       Rename(DestFile,FinalFile+".ed");
 578
 579       if(Debug)
 580          std::clog << "Sending to rred method: " << FinalFile << std::endl;
 581
 582       State = StateApplyDiff;
 583       Local = true;
 584       Desc.URI = "rred:" + FinalFile;
 585       QueueURI(Desc);
 586       Mode = "rred";
 587       return;
 588    }
 589
 590
 591    // success in download/apply a diff, queue next (if needed)
 592    if(State == StateApplyDiff)
 593    {
 594       // remove the just applied patch
 595       available_patches.erase(available_patches.begin());
 596
 597       // move into place
 598       if(Debug)
 599       {
 600          std::clog << "Moving patched file in place: " << std::endl
 601                    << DestFile << " -> " << FinalFile << std::endl;
 602       }
 603       Rename(DestFile,FinalFile);
 604       chmod(FinalFile.c_str(),0644);
 605
 606       // see if there is more to download
 607       if(available_patches.size() > 0) {
 608          new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
 609                               ExpectedHash, ServerSha1, available_patches);
 610          return Finish();
 611       } else
 612          return Finish(true);
 613    }
 614 }
 615                                                                         /*}}}*/
 616 // AcqIndex::AcqIndex - Constructor                                     /*{{{*/
 617 // ---------------------------------------------------------------------
 618 /* The package file is added to the queue and a second class is
 619    instantiated to fetch the revision file */
 620 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 621                          string URI,string URIDesc,string ShortDesc,
 622                          HashString ExpectedHash, string comprExt)
 623    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 624 {
 625    if(comprExt.empty() == true)
 626    {
 627       // autoselect the compression method
 628       std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 629       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 630          comprExt.append(*t).append(" ");
 631       if (comprExt.empty() == false)
 632          comprExt.erase(comprExt.end()-1);
 633    }
 634    CompressionExtension = comprExt;
 635
 636    Init(URI, URIDesc, ShortDesc);
 637 }
 638 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
 639                          HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
 640    : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
 641 {
 642    // autoselect the compression method
 643    std::vector<std::string> types = APT::Configuration::getCompressionTypes();
 644    CompressionExtension = "";
 645    if (ExpectedHash.empty() == false)
 646    {
 647       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 648          if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
 649             CompressionExtension.append(*t).append(" ");
 650    }
 651    else
 652    {
 653       for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
 654          CompressionExtension.append(*t).append(" ");
 655    }
 656    if (CompressionExtension.empty() == false)
 657       CompressionExtension.erase(CompressionExtension.end()-1);
 658
 659    Init(Target->URI, Target->Description, Target->ShortDesc);
 660 }
 661                                                                         /*}}}*/
 662 // AcqIndex::Init - defered Constructor                                 /*{{{*/
 663 void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
 664    Decompression = false;
 665    Erase = false;
 666
 667    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 668    DestFile += URItoFileName(URI);
 669
 670    std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 671    if (comprExt == "uncompressed")
 672       Desc.URI = URI;
 673    else
 674       Desc.URI = URI + '.' + comprExt;
 675
 676    Desc.Description = URIDesc;
 677    Desc.Owner = this;
 678    Desc.ShortDesc = ShortDesc;
 679
 680    QueueURI(Desc);
 681 }
 682                                                                         /*}}}*/
 683 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
 684 // ---------------------------------------------------------------------
 685 /* The only header we use is the last-modified header. */
 686 string pkgAcqIndex::Custom600Headers()
 687 {
 688    string Final = _config->FindDir("Dir::State::lists");
 689    Final += URItoFileName(RealURI);
 690    if (_config->FindB("Acquire::GzipIndexes",false))
 691       Final += ".gz";
 692
 693    struct stat Buf;
 694    if (stat(Final.c_str(),&Buf) != 0)
 695       return "\nIndex-File: true";
 696    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 697 }
 698                                                                         /*}}}*/
 699 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)  /*{{{*/
 700 {
 701    size_t const nextExt = CompressionExtension.find(' ');
 702    if (nextExt != std::string::npos)
 703    {
 704       CompressionExtension = CompressionExtension.substr(nextExt+1);
 705       Init(RealURI, Desc.Description, Desc.ShortDesc);
 706       return;
 707    }
 708
 709    // on decompression failure, remove bad versions in partial/
 710    if (Decompression && Erase) {
 711       string s = _config->FindDir("Dir::State::lists") + "partial/";
 712       s.append(URItoFileName(RealURI));
 713       unlink(s.c_str());
 714    }
 715
 716    Item::Failed(Message,Cnf);
 717 }
 718                                                                         /*}}}*/
 719 // AcqIndex::Done - Finished a fetch                                    /*{{{*/
 720 // ---------------------------------------------------------------------
 721 /* This goes through a number of states.. On the initial fetch the
 722    method could possibly return an alternate filename which points
 723    to the uncompressed version of the file. If this is so the file
 724    is copied into the partial directory. In all other cases the file
 725    is decompressed with a gzip uri. */
 726 void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
 727                        pkgAcquire::MethodConfig *Cfg)
 728 {
 729    Item::Done(Message,Size,Hash,Cfg);
 730
 731    if (Decompression == true)
 732    {
 733       if (_config->FindB("Debug::pkgAcquire::Auth", false))
 734       {
 735          std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash;
 736          std::cerr << "  Expected Hash: " << ExpectedHash.toStr() << std::endl;
 737       }
 738
 739       if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash)
 740       {
 741          Status = StatAuthError;
 742          ErrorText = _("Hash Sum mismatch");
 743          Rename(DestFile,DestFile + ".FAILED");
 744          ReportMirrorFailure("HashChecksumFailure");
 745          return;
 746       }
 747       // Done, move it into position
 748       string FinalFile = _config->FindDir("Dir::State::lists");
 749       FinalFile += URItoFileName(RealURI);
 750       Rename(DestFile,FinalFile);
 751       chmod(FinalFile.c_str(),0644);
 752
 753       /* We restore the original name to DestFile so that the clean operation
 754          will work OK */
 755       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 756       DestFile += URItoFileName(RealURI);
 757
 758       // Remove the compressed version.
 759       if (Erase == true)
 760          unlink(DestFile.c_str());
 761       return;
 762    }
 763
 764    Erase = false;
 765    Complete = true;
 766
 767    // Handle the unzipd case
 768    string FileName = LookupTag(Message,"Alt-Filename");
 769    if (FileName.empty() == false)
 770    {
 771       // The files timestamp matches
 772       if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
 773          return;
 774       Decompression = true;
 775       Local = true;
 776       DestFile += ".decomp";
 777       Desc.URI = "copy:" + FileName;
 778       QueueURI(Desc);
 779       Mode = "copy";
 780       return;
 781    }
 782
 783    FileName = LookupTag(Message,"Filename");
 784    if (FileName.empty() == true)
 785    {
 786       Status = StatError;
 787       ErrorText = "Method gave a blank filename";
 788    }
 789
 790    std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 791
 792    // The files timestamp matches
 793    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
 794        if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz")
 795           // Update DestFile for .gz suffix so that the clean operation keeps it
 796           DestFile += ".gz";
 797       return;
 798     }
 799
 800    if (FileName == DestFile)
 801       Erase = true;
 802    else
 803       Local = true;
 804
 805    string decompProg;
 806
 807    // If we enable compressed indexes and already have gzip, keep it
 808    if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) {
 809       string FinalFile = _config->FindDir("Dir::State::lists");
 810       FinalFile += URItoFileName(RealURI) + ".gz";
 811       Rename(DestFile,FinalFile);
 812       chmod(FinalFile.c_str(),0644);
 813
 814       // Update DestFile for .gz suffix so that the clean operation keeps it
 815       DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 816       DestFile += URItoFileName(RealURI) + ".gz";
 817       return;
 818     }
 819
 820    // get the binary name for your used compression type
 821    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
 822    if(decompProg.empty() == false);
 823    else if(compExt == "uncompressed")
 824       decompProg = "copy";
 825    else {
 826       _error->Error("Unsupported extension: %s", compExt.c_str());
 827       return;
 828    }
 829
 830    Decompression = true;
 831    DestFile += ".decomp";
 832    Desc.URI = decompProg + ":" + FileName;
 833    QueueURI(Desc);
 834    Mode = decompProg.c_str();
 835 }
 836                                                                         /*}}}*/
 837 // AcqIndexTrans::pkgAcqIndexTrans - Constructor                        /*{{{*/
 838 // ---------------------------------------------------------------------
 839 /* The Translation file is added to the queue */
 840 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 841                             string URI,string URIDesc,string ShortDesc)
 842   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 843 {
 844 }
 845                                                                         /*}}}*/
 846 // AcqIndexTrans::Custom600Headers - Insert custom request headers      /*{{{*/
 847 // ---------------------------------------------------------------------
 848 string pkgAcqIndexTrans::Custom600Headers()
 849 {
 850    string Final = _config->FindDir("Dir::State::lists");
 851    Final += URItoFileName(RealURI);
 852
 853    struct stat Buf;
 854    if (stat(Final.c_str(),&Buf) != 0)
 855       return "\nFail-Ignore: true";
 856    return "\nFail-Ignore: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 857 }
 858                                                                         /*}}}*/
 859 // AcqIndexTrans::Failed - Silence failure messages for missing files   /*{{{*/
 860 // ---------------------------------------------------------------------
 861 /* */
 862 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 863 {
 864    size_t const nextExt = CompressionExtension.find(' ');
 865    if (nextExt != std::string::npos)
 866    {
 867       CompressionExtension = CompressionExtension.substr(nextExt+1);
 868       Init(RealURI, Desc.Description, Desc.ShortDesc);
 869       Status = StatIdle;
 870       return;
 871    }
 872
 873    if (Cnf->LocalOnly == true ||
 874        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
 875    {
 876       // Ignore this
 877       Status = StatDone;
 878       Complete = false;
 879       Dequeue();
 880       return;
 881    }
 882
 883    Item::Failed(Message,Cnf);
 884 }
 885                                                                         /*}}}*/
 886 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,                         /*{{{*/
 887                              string URI,string URIDesc,string ShortDesc,
 888                              string MetaIndexURI, string MetaIndexURIDesc,
 889                              string MetaIndexShortDesc,
 890                              const vector<IndexTarget*>* IndexTargets,
 891                              indexRecords* MetaIndexParser) :
 892    Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
 893    MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
 894    MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets)
 895 {
 896    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
 897    DestFile += URItoFileName(URI);
 898
 899    // remove any partial downloaded sig-file in partial/.
 900    // it may confuse proxies and is too small to warrant a
 901    // partial download anyway
 902    unlink(DestFile.c_str());
 903
 904    // Create the item
 905    Desc.Description = URIDesc;
 906    Desc.Owner = this;
 907    Desc.ShortDesc = ShortDesc;
 908    Desc.URI = URI;
 909
 910    string Final = _config->FindDir("Dir::State::lists");
 911    Final += URItoFileName(RealURI);
 912    struct stat Buf;
 913    if (stat(Final.c_str(),&Buf) == 0)
 914    {
 915       // File was already in place.  It needs to be re-downloaded/verified
 916       // because Release might have changed, we do give it a differnt
 917       // name than DestFile because otherwise the http method will
 918       // send If-Range requests and there are too many broken servers
 919       // out there that do not understand them
 920       LastGoodSig = DestFile+".reverify";
 921       Rename(Final,LastGoodSig);
 922    }
 923
 924    QueueURI(Desc);
 925 }
 926                                                                         /*}}}*/
 927 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers      /*{{{*/
 928 // ---------------------------------------------------------------------
 929 /* The only header we use is the last-modified header. */
 930 string pkgAcqMetaSig::Custom600Headers()
 931 {
 932    struct stat Buf;
 933    if (stat(LastGoodSig.c_str(),&Buf) != 0)
 934       return "\nIndex-File: true";
 935
 936    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 937 }
 938
 939 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
 940                          pkgAcquire::MethodConfig *Cfg)
 941 {
 942    Item::Done(Message,Size,MD5,Cfg);
 943
 944    string FileName = LookupTag(Message,"Filename");
 945    if (FileName.empty() == true)
 946    {
 947       Status = StatError;
 948       ErrorText = "Method gave a blank filename";
 949       return;
 950    }
 951
 952    if (FileName != DestFile)
 953    {
 954       // We have to copy it into place
 955       Local = true;
 956       Desc.URI = "copy:" + FileName;
 957       QueueURI(Desc);
 958       return;
 959    }
 960
 961    Complete = true;
 962
 963    // put the last known good file back on i-m-s hit (it will
 964    // be re-verified again)
 965    // Else do nothing, we have the new file in DestFile then
 966    if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
 967       Rename(LastGoodSig, DestFile);
 968
 969    // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
 970    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc,
 971                        MetaIndexShortDesc,  DestFile, IndexTargets,
 972                        MetaIndexParser);
 973
 974 }
 975                                                                         /*}}}*/
 976 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 977 {
 978    string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
 979
 980    // if we get a network error we fail gracefully
 981    if(Status == StatTransientNetworkError)
 982    {
 983       Item::Failed(Message,Cnf);
 984       // move the sigfile back on transient network failures
 985       if(FileExists(LastGoodSig))
 986          Rename(LastGoodSig,Final);
 987
 988       // set the status back to , Item::Failed likes to reset it
 989       Status = pkgAcquire::Item::StatTransientNetworkError;
 990       return;
 991    }
 992
 993    // Delete any existing sigfile when the acquire failed
 994    unlink(Final.c_str());
 995
 996    // queue a pkgAcqMetaIndex with no sigfile
 997    new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
 998                        "", IndexTargets, MetaIndexParser);
 999
1000    if (Cnf->LocalOnly == true ||
1001        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
1002    {
1003       // Ignore this
1004       Status = StatDone;
1005       Complete = false;
1006       Dequeue();
1007       return;
1008    }
1009
1010    Item::Failed(Message,Cnf);
1011 }
1012                                                                         /*}}}*/
1013 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,                     /*{{{*/
1014                                  string URI,string URIDesc,string ShortDesc,
1015                                  string SigFile,
1016                                  const vector<struct IndexTarget*>* IndexTargets,
1017                                  indexRecords* MetaIndexParser) :
1018    Item(Owner), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets),
1019    MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false)
1020 {
1021    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
1022    DestFile += URItoFileName(URI);
1023
1024    // Create the item
1025    Desc.Description = URIDesc;
1026    Desc.Owner = this;
1027    Desc.ShortDesc = ShortDesc;
1028    Desc.URI = URI;
1029
1030    QueueURI(Desc);
1031 }
1032                                                                         /*}}}*/
1033 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers    /*{{{*/
1034 // ---------------------------------------------------------------------
1035 /* The only header we use is the last-modified header. */
1036 string pkgAcqMetaIndex::Custom600Headers()
1037 {
1038    string Final = _config->FindDir("Dir::State::lists");
1039    Final += URItoFileName(RealURI);
1040
1041    struct stat Buf;
1042    if (stat(Final.c_str(),&Buf) != 0)
1043       return "\nIndex-File: true";
1044
1045    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
1046 }
1047                                                                         /*}}}*/
1048 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,       /*{{{*/
1049                            pkgAcquire::MethodConfig *Cfg)
1050 {
1051    Item::Done(Message,Size,Hash,Cfg);
1052
1053    // MetaIndexes are done in two passes: one to download the
1054    // metaindex with an appropriate method, and a second to verify it
1055    // with the gpgv method
1056
1057    if (AuthPass == true)
1058    {
1059       AuthDone(Message);
1060
1061       // all cool, move Release file into place
1062       Complete = true;
1063    }
1064    else
1065    {
1066       RetrievalDone(Message);
1067       if (!Complete)
1068          // Still more retrieving to do
1069          return;
1070
1071       if (SigFile == "")
1072       {
1073          // There was no signature file, so we are finished.  Download
1074          // the indexes without verification.
1075          QueueIndexes(false);
1076       }
1077       else
1078       {
1079          // There was a signature file, so pass it to gpgv for
1080          // verification
1081
1082          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1083             std::cerr << "Metaindex acquired, queueing gpg verification ("
1084                       << SigFile << "," << DestFile << ")\n";
1085          AuthPass = true;
1086          Desc.URI = "gpgv:" + SigFile;
1087          QueueURI(Desc);
1088          Mode = "gpgv";
1089          return;
1090       }
1091    }
1092
1093    if (Complete == true)
1094    {
1095       string FinalFile = _config->FindDir("Dir::State::lists");
1096       FinalFile += URItoFileName(RealURI);
1097       if (SigFile == DestFile)
1098          SigFile = FinalFile;
1099       Rename(DestFile,FinalFile);
1100       chmod(FinalFile.c_str(),0644);
1101       DestFile = FinalFile;
1102    }
1103 }
1104                                                                         /*}}}*/
1105 void pkgAcqMetaIndex::RetrievalDone(string Message)                     /*{{{*/
1106 {
1107    // We have just finished downloading a Release file (it is not
1108    // verified yet)
1109
1110    string FileName = LookupTag(Message,"Filename");
1111    if (FileName.empty() == true)
1112    {
1113       Status = StatError;
1114       ErrorText = "Method gave a blank filename";
1115       return;
1116    }
1117
1118    if (FileName != DestFile)
1119    {
1120       Local = true;
1121       Desc.URI = "copy:" + FileName;
1122       QueueURI(Desc);
1123       return;
1124    }
1125
1126    // make sure to verify against the right file on I-M-S hit
1127    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
1128    if(IMSHit)
1129    {
1130       string FinalFile = _config->FindDir("Dir::State::lists");
1131       FinalFile += URItoFileName(RealURI);
1132       if (SigFile == DestFile)
1133          SigFile = FinalFile;
1134       DestFile = FinalFile;
1135    }
1136    Complete = true;
1137 }
1138                                                                         /*}}}*/
1139 void pkgAcqMetaIndex::AuthDone(string Message)                          /*{{{*/
1140 {
1141    // At this point, the gpgv method has succeeded, so there is a
1142    // valid signature from a key in the trusted keyring.  We
1143    // perform additional verification of its contents, and use them
1144    // to verify the indexes we are about to download
1145
1146    if (!MetaIndexParser->Load(DestFile))
1147    {
1148       Status = StatAuthError;
1149       ErrorText = MetaIndexParser->ErrorText;
1150       return;
1151    }
1152
1153    if (!VerifyVendor(Message))
1154    {
1155       return;
1156    }
1157
1158    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1159       std::cerr << "Signature verification succeeded: "
1160                 << DestFile << std::endl;
1161
1162    // Download further indexes with verification
1163    QueueIndexes(true);
1164
1165    // is it a clearsigned MetaIndex file?
1166    if (DestFile == SigFile)
1167       return;
1168
1169    // Done, move signature file into position
1170    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1171       URItoFileName(RealURI) + ".gpg";
1172    Rename(SigFile,VerifiedSigFile);
1173    chmod(VerifiedSigFile.c_str(),0644);
1174 }
1175                                                                         /*}}}*/
1176 void pkgAcqMetaIndex::QueueIndexes(bool verify)                         /*{{{*/
1177 {
1178    for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
1179         Target != IndexTargets->end();
1180         Target++)
1181    {
1182       HashString ExpectedIndexHash;
1183       if (verify)
1184       {
1185          const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
1186          if (!Record)
1187          {
1188             Status = StatAuthError;
1189             ErrorText = "Unable to find expected entry  "
1190                + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
1191             return;
1192          }
1193          ExpectedIndexHash = Record->Hash;
1194          if (_config->FindB("Debug::pkgAcquire::Auth", false))
1195          {
1196             std::cerr << "Queueing: " << (*Target)->URI << std::endl;
1197             std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
1198          }
1199          if (ExpectedIndexHash.empty())
1200          {
1201             Status = StatAuthError;
1202             ErrorText = "Unable to find hash sum for "
1203                + (*Target)->MetaKey + " in Meta-index file";
1204             return;
1205          }
1206       }
1207
1208       /* Queue Packages file (either diff or full packages files, depending
1209          on the users option) - we also check if the PDiff Index file is listed
1210          in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
1211          instead, but passing the required info to it is to much hassle */
1212       if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
1213           MetaIndexParser->Exists(string((*Target)->MetaKey).append(".diff/Index")) == true))
1214          new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
1215                              (*Target)->ShortDesc, ExpectedIndexHash);
1216       else
1217          new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
1218    }
1219 }
1220                                                                         /*}}}*/
1221 bool pkgAcqMetaIndex::VerifyVendor(string Message)                      /*{{{*/
1222 {
1223 //    // Maybe this should be made available from above so we don't have
1224 //    // to read and parse it every time?
1225 //    pkgVendorList List;
1226 //    List.ReadMainList();
1227
1228 //    const Vendor* Vndr = NULL;
1229 //    for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1230 //    {
1231 //       string::size_type pos = (*I).find("VALIDSIG ");
1232 //       if (_config->FindB("Debug::Vendor", false))
1233 //          std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1234 //                    << std::endl;
1235 //       if (pos != std::string::npos)
1236 //       {
1237 //          string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1238 //          if (_config->FindB("Debug::Vendor", false))
1239 //             std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1240 //                std::endl;
1241 //          Vndr = List.FindVendor(Fingerprint) != "";
1242 //          if (Vndr != NULL);
1243 //          break;
1244 //       }
1245 //    }
1246    string::size_type pos;
1247
1248    // check for missing sigs (that where not fatal because otherwise we had
1249    // bombed earlier)
1250    string missingkeys;
1251    string msg = _("There is no public key available for the "
1252                   "following key IDs:\n");
1253    pos = Message.find("NO_PUBKEY ");
1254    if (pos != std::string::npos)
1255    {
1256       string::size_type start = pos+strlen("NO_PUBKEY ");
1257       string Fingerprint = Message.substr(start, Message.find("\n")-start);
1258       missingkeys += (Fingerprint);
1259    }
1260    if(!missingkeys.empty())
1261       _error->Warning("%s", string(msg+missingkeys).c_str());
1262
1263    string Transformed = MetaIndexParser->GetExpectedDist();
1264
1265    if (Transformed == "../project/experimental")
1266    {
1267       Transformed = "experimental";
1268    }
1269
1270    pos = Transformed.rfind('/');
1271    if (pos != string::npos)
1272    {
1273       Transformed = Transformed.substr(0, pos);
1274    }
1275
1276    if (Transformed == ".")
1277    {
1278       Transformed = "";
1279    }
1280
1281    if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
1282        MetaIndexParser->GetValidUntil() > 0) {
1283       time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
1284       if (invalid_since > 0)
1285          // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
1286          // the time since then the file is invalid - formated in the same way as in
1287          // the download progress display (e.g. 7d 3h 42min 1s)
1288          return _error->Error(_("Release file expired, ignoring %s (invalid since %s)"),
1289                               RealURI.c_str(), TimeToStr(invalid_since).c_str());
1290    }
1291
1292    if (_config->FindB("Debug::pkgAcquire::Auth", false))
1293    {
1294       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
1295       std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
1296       std::cerr << "Transformed Dist: " << Transformed << std::endl;
1297    }
1298
1299    if (MetaIndexParser->CheckDist(Transformed) == false)
1300    {
1301       // This might become fatal one day
1302 //       Status = StatAuthError;
1303 //       ErrorText = "Conflicting distribution; expected "
1304 //          + MetaIndexParser->GetExpectedDist() + " but got "
1305 //          + MetaIndexParser->GetDist();
1306 //       return false;
1307       if (!Transformed.empty())
1308       {
1309          _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
1310                          Desc.Description.c_str(),
1311                          Transformed.c_str(),
1312                          MetaIndexParser->GetDist().c_str());
1313       }
1314    }
1315
1316    return true;
1317 }
1318                                                                         /*}}}*/
1319 // pkgAcqMetaIndex::Failed - no Release file present or no signature file present       /*{{{*/
1320 // ---------------------------------------------------------------------
1321 /* */
1322 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1323 {
1324    if (AuthPass == true)
1325    {
1326       // gpgv method failed, if we have a good signature
1327       string LastGoodSigFile = _config->FindDir("Dir::State::lists");
1328       if (DestFile == SigFile)
1329          LastGoodSigFile.append(URItoFileName(RealURI));
1330       else
1331          LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify");
1332
1333       if(FileExists(LastGoodSigFile))
1334       {
1335          if (DestFile != SigFile)
1336          {
1337             string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
1338                                         URItoFileName(RealURI) + ".gpg";
1339             Rename(LastGoodSigFile,VerifiedSigFile);
1340          }
1341          Status = StatTransientNetworkError;
1342          _error->Warning(_("A error occurred during the signature "
1343                            "verification. The repository is not updated "
1344                            "and the previous index files will be used. "
1345                            "GPG error: %s: %s\n"),
1346                          Desc.Description.c_str(),
1347                          LookupTag(Message,"Message").c_str());
1348          RunScripts("APT::Update::Auth-Failure");
1349          return;
1350       } else {
1351          _error->Warning(_("GPG error: %s: %s"),
1352                          Desc.Description.c_str(),
1353                          LookupTag(Message,"Message").c_str());
1354       }
1355       // gpgv method failed
1356       ReportMirrorFailure("GPGFailure");
1357    }
1358
1359    // No Release file was present, or verification failed, so fall
1360    // back to queueing Packages files without verification
1361    QueueIndexes(false);
1362 }
1363                                                                         /*}}}*/
1364 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,               /*{{{*/
1365                 string const &URI, string const &URIDesc, string const &ShortDesc,
1366                 string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
1367                 string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
1368                 const vector<struct IndexTarget*>* IndexTargets,
1369                 indexRecords* MetaIndexParser) :
1370         pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
1371         MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
1372         MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
1373 {
1374    SigFile = DestFile;
1375 }
1376                                                                         /*}}}*/
1377 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
1378 {
1379    if (AuthPass == false)
1380    {
1381       new pkgAcqMetaSig(Owner,
1382                         MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
1383                         MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
1384                         IndexTargets, MetaIndexParser);
1385       if (Cnf->LocalOnly == true ||
1386           StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
1387          Dequeue();
1388    }
1389    else
1390       pkgAcqMetaIndex::Failed(Message, Cnf);
1391 }
1392                                                                         /*}}}*/
1393 // AcqArchive::AcqArchive - Constructor                                 /*{{{*/
1394 // ---------------------------------------------------------------------
1395 /* This just sets up the initial fetch environment and queues the first
1396    possibilitiy */
1397 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
1398                              pkgRecords *Recs,pkgCache::VerIterator const &Version,
1399                              string &StoreFilename) :
1400                Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
1401                StoreFilename(StoreFilename), Vf(Version.FileList()),
1402                Trusted(false)
1403 {
1404    Retries = _config->FindI("Acquire::Retries",0);
1405
1406    if (Version.Arch() == 0)
1407    {
1408       _error->Error(_("I wasn't able to locate a file for the %s package. "
1409                       "This might mean you need to manually fix this package. "
1410                       "(due to missing arch)"),
1411                     Version.ParentPkg().Name());
1412       return;
1413    }
1414
1415    /* We need to find a filename to determine the extension. We make the
1416       assumption here that all the available sources for this version share
1417       the same extension.. */
1418    // Skip not source sources, they do not have file fields.
1419    for (; Vf.end() == false; Vf++)
1420    {
1421       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1422          continue;
1423       break;
1424    }
1425
1426    // Does not really matter here.. we are going to fail out below
1427    if (Vf.end() != true)
1428    {
1429       // If this fails to get a file name we will bomb out below.
1430       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1431       if (_error->PendingError() == true)
1432          return;
1433
1434       // Generate the final file name as: package_version_arch.foo
1435       StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1436                       QuoteString(Version.VerStr(),"_:") + '_' +
1437                       QuoteString(Version.Arch(),"_:.") +
1438                       "." + flExtension(Parse.FileName());
1439    }
1440
1441    // check if we have one trusted source for the package. if so, switch
1442    // to "TrustedOnly" mode
1443    for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1444    {
1445       pkgIndexFile *Index;
1446       if (Sources->FindIndex(i.File(),Index) == false)
1447          continue;
1448       if (_config->FindB("Debug::pkgAcquire::Auth", false))
1449       {
1450          std::cerr << "Checking index: " << Index->Describe()
1451                    << "(Trusted=" << Index->IsTrusted() << ")\n";
1452       }
1453       if (Index->IsTrusted()) {
1454          Trusted = true;
1455          break;
1456       }
1457    }
1458
1459    // "allow-unauthenticated" restores apts old fetching behaviour
1460    // that means that e.g. unauthenticated file:// uris are higher
1461    // priority than authenticated http:// uris
1462    if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true)
1463       Trusted = false;
1464
1465    // Select a source
1466    if (QueueNext() == false && _error->PendingError() == false)
1467       _error->Error(_("I wasn't able to locate file for the %s package. "
1468                     "This might mean you need to manually fix this package."),
1469                     Version.ParentPkg().Name());
1470 }
1471                                                                         /*}}}*/
1472 // AcqArchive::QueueNext - Queue the next file source                   /*{{{*/
1473 // ---------------------------------------------------------------------
1474 /* This queues the next available file version for download. It checks if
1475    the archive is already available in the cache and stashs the MD5 for
1476    checking later. */
1477 bool pkgAcqArchive::QueueNext()
1478 {
1479    string const ForceHash = _config->Find("Acquire::ForceHash");
1480    for (; Vf.end() == false; Vf++)
1481    {
1482       // Ignore not source sources
1483       if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1484          continue;
1485
1486       // Try to cross match against the source list
1487       pkgIndexFile *Index;
1488       if (Sources->FindIndex(Vf.File(),Index) == false)
1489             continue;
1490
1491       // only try to get a trusted package from another source if that source
1492       // is also trusted
1493       if(Trusted && !Index->IsTrusted())
1494          continue;
1495
1496       // Grab the text package record
1497       pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1498       if (_error->PendingError() == true)
1499          return false;
1500
1501       string PkgFile = Parse.FileName();
1502       if (ForceHash.empty() == false)
1503       {
1504          if(stringcasecmp(ForceHash, "sha256") == 0)
1505             ExpectedHash = HashString("SHA256", Parse.SHA256Hash());
1506          else if (stringcasecmp(ForceHash, "sha1") == 0)
1507             ExpectedHash = HashString("SHA1", Parse.SHA1Hash());
1508          else
1509             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1510       }
1511       else
1512       {
1513          string Hash;
1514          if ((Hash = Parse.SHA256Hash()).empty() == false)
1515             ExpectedHash = HashString("SHA256", Hash);
1516          else if ((Hash = Parse.SHA1Hash()).empty() == false)
1517             ExpectedHash = HashString("SHA1", Hash);
1518          else
1519             ExpectedHash = HashString("MD5Sum", Parse.MD5Hash());
1520       }
1521       if (PkgFile.empty() == true)
1522          return _error->Error(_("The package index files are corrupted. No Filename: "
1523                               "field for package %s."),
1524                               Version.ParentPkg().Name());
1525
1526       Desc.URI = Index->ArchiveURI(PkgFile);
1527       Desc.Description = Index->ArchiveInfo(Version);
1528       Desc.Owner = this;
1529       Desc.ShortDesc = Version.ParentPkg().Name();
1530
1531       // See if we already have the file. (Legacy filenames)
1532       FileSize = Version->Size;
1533       string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1534       struct stat Buf;
1535       if (stat(FinalFile.c_str(),&Buf) == 0)
1536       {
1537          // Make sure the size matches
1538          if ((unsigned)Buf.st_size == Version->Size)
1539          {
1540             Complete = true;
1541             Local = true;
1542             Status = StatDone;
1543             StoreFilename = DestFile = FinalFile;
1544             return true;
1545          }
1546
1547          /* Hmm, we have a file and its size does not match, this means it is
1548             an old style mismatched arch */
1549          unlink(FinalFile.c_str());
1550       }
1551
1552       // Check it again using the new style output filenames
1553       FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1554       if (stat(FinalFile.c_str(),&Buf) == 0)
1555       {
1556          // Make sure the size matches
1557          if ((unsigned)Buf.st_size == Version->Size)
1558          {
1559             Complete = true;
1560             Local = true;
1561             Status = StatDone;
1562             StoreFilename = DestFile = FinalFile;
1563             return true;
1564          }
1565
1566          /* Hmm, we have a file and its size does not match, this shouldnt
1567             happen.. */
1568          unlink(FinalFile.c_str());
1569       }
1570
1571       DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1572
1573       // Check the destination file
1574       if (stat(DestFile.c_str(),&Buf) == 0)
1575       {
1576          // Hmm, the partial file is too big, erase it
1577          if ((unsigned)Buf.st_size > Version->Size)
1578             unlink(DestFile.c_str());
1579          else
1580             PartialSize = Buf.st_size;
1581       }
1582
1583       // Create the item
1584       Local = false;
1585       Desc.URI = Index->ArchiveURI(PkgFile);
1586       Desc.Description = Index->ArchiveInfo(Version);
1587       Desc.Owner = this;
1588       Desc.ShortDesc = Version.ParentPkg().Name();
1589       QueueURI(Desc);
1590
1591       Vf++;
1592       return true;
1593    }
1594    return false;
1595 }
1596                                                                         /*}}}*/
1597 // AcqArchive::Done - Finished fetching                                 /*{{{*/
1598 // ---------------------------------------------------------------------
1599 /* */
1600 void pkgAcqArchive::Done(string Message,unsigned long Size,string CalcHash,
1601                          pkgAcquire::MethodConfig *Cfg)
1602 {
1603    Item::Done(Message,Size,CalcHash,Cfg);
1604
1605    // Check the size
1606    if (Size != Version->Size)
1607    {
1608       Status = StatError;
1609       ErrorText = _("Size mismatch");
1610       return;
1611    }
1612
1613    // Check the hash
1614    if(ExpectedHash.toStr() != CalcHash)
1615    {
1616       Status = StatError;
1617       ErrorText = _("Hash Sum mismatch");
1618       if(FileExists(DestFile))
1619          Rename(DestFile,DestFile + ".FAILED");
1620       return;
1621    }
1622
1623    // Grab the output filename
1624    string FileName = LookupTag(Message,"Filename");
1625    if (FileName.empty() == true)
1626    {
1627       Status = StatError;
1628       ErrorText = "Method gave a blank filename";
1629       return;
1630    }
1631
1632    Complete = true;
1633
1634    // Reference filename
1635    if (FileName != DestFile)
1636    {
1637       StoreFilename = DestFile = FileName;
1638       Local = true;
1639       return;
1640    }
1641
1642    // Done, move it into position
1643    string FinalFile = _config->FindDir("Dir::Cache::Archives");
1644    FinalFile += flNotDir(StoreFilename);
1645    Rename(DestFile,FinalFile);
1646
1647    StoreFilename = DestFile = FinalFile;
1648    Complete = true;
1649 }
1650                                                                         /*}}}*/
1651 // AcqArchive::Failed - Failure handler                                 /*{{{*/
1652 // ---------------------------------------------------------------------
1653 /* Here we try other sources */
1654 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1655 {
1656    ErrorText = LookupTag(Message,"Message");
1657
1658    /* We don't really want to retry on failed media swaps, this prevents
1659       that. An interesting observation is that permanent failures are not
1660       recorded. */
1661    if (Cnf->Removable == true &&
1662        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1663    {
1664       // Vf = Version.FileList();
1665       while (Vf.end() == false) Vf++;
1666       StoreFilename = string();
1667       Item::Failed(Message,Cnf);
1668       return;
1669    }
1670
1671    if (QueueNext() == false)
1672    {
1673       // This is the retry counter
1674       if (Retries != 0 &&
1675           Cnf->LocalOnly == false &&
1676           StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1677       {
1678          Retries--;
1679          Vf = Version.FileList();
1680          if (QueueNext() == true)
1681             return;
1682       }
1683
1684       StoreFilename = string();
1685       Item::Failed(Message,Cnf);
1686    }
1687 }
1688                                                                         /*}}}*/
1689 // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
1690 // ---------------------------------------------------------------------
1691 bool pkgAcqArchive::IsTrusted()
1692 {
1693    return Trusted;
1694 }
1695                                                                         /*}}}*/
1696 // AcqArchive::Finished - Fetching has finished, tidy up                /*{{{*/
1697 // ---------------------------------------------------------------------
1698 /* */
1699 void pkgAcqArchive::Finished()
1700 {
1701    if (Status == pkgAcquire::Item::StatDone &&
1702        Complete == true)
1703       return;
1704    StoreFilename = string();
1705 }
1706                                                                         /*}}}*/
1707 // AcqFile::pkgAcqFile - Constructor                                    /*{{{*/
1708 // ---------------------------------------------------------------------
1709 /* The file is added to the queue */
1710 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash,
1711                        unsigned long Size,string Dsc,string ShortDesc,
1712                        const string &DestDir, const string &DestFilename,
1713                        bool IsIndexFile) :
1714                        Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile)
1715 {
1716    Retries = _config->FindI("Acquire::Retries",0);
1717
1718    if(!DestFilename.empty())
1719       DestFile = DestFilename;
1720    else if(!DestDir.empty())
1721       DestFile = DestDir + "/" + flNotDir(URI);
1722    else
1723       DestFile = flNotDir(URI);
1724
1725    // Create the item
1726    Desc.URI = URI;
1727    Desc.Description = Dsc;
1728    Desc.Owner = this;
1729
1730    // Set the short description to the archive component
1731    Desc.ShortDesc = ShortDesc;
1732
1733    // Get the transfer sizes
1734    FileSize = Size;
1735    struct stat Buf;
1736    if (stat(DestFile.c_str(),&Buf) == 0)
1737    {
1738       // Hmm, the partial file is too big, erase it
1739       if ((unsigned)Buf.st_size > Size)
1740          unlink(DestFile.c_str());
1741       else
1742          PartialSize = Buf.st_size;
1743    }
1744
1745    QueueURI(Desc);
1746 }
1747                                                                         /*}}}*/
1748 // AcqFile::Done - Item downloaded OK                                   /*{{{*/
1749 // ---------------------------------------------------------------------
1750 /* */
1751 void pkgAcqFile::Done(string Message,unsigned long Size,string CalcHash,
1752                       pkgAcquire::MethodConfig *Cnf)
1753 {
1754    Item::Done(Message,Size,CalcHash,Cnf);
1755
1756    // Check the hash
1757    if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash)
1758    {
1759       Status = StatError;
1760       ErrorText = _("Hash Sum mismatch");
1761       Rename(DestFile,DestFile + ".FAILED");
1762       return;
1763    }
1764
1765    string FileName = LookupTag(Message,"Filename");
1766    if (FileName.empty() == true)
1767    {
1768       Status = StatError;
1769       ErrorText = "Method gave a blank filename";
1770       return;
1771    }
1772
1773    Complete = true;
1774
1775    // The files timestamp matches
1776    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1777       return;
1778
1779    // We have to copy it into place
1780    if (FileName != DestFile)
1781    {
1782       Local = true;
1783       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1784           Cnf->Removable == true)
1785       {
1786          Desc.URI = "copy:" + FileName;
1787          QueueURI(Desc);
1788          return;
1789       }
1790
1791       // Erase the file if it is a symlink so we can overwrite it
1792       struct stat St;
1793       if (lstat(DestFile.c_str(),&St) == 0)
1794       {
1795          if (S_ISLNK(St.st_mode) != 0)
1796             unlink(DestFile.c_str());
1797       }
1798
1799       // Symlink the file
1800       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1801       {
1802          ErrorText = "Link to " + DestFile + " failure ";
1803          Status = StatError;
1804          Complete = false;
1805       }
1806    }
1807 }
1808                                                                         /*}}}*/
1809 // AcqFile::Failed - Failure handler                                    /*{{{*/
1810 // ---------------------------------------------------------------------
1811 /* Here we try other sources */
1812 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1813 {
1814    ErrorText = LookupTag(Message,"Message");
1815
1816    // This is the retry counter
1817    if (Retries != 0 &&
1818        Cnf->LocalOnly == false &&
1819        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1820    {
1821       Retries--;
1822       QueueURI(Desc);
1823       return;
1824    }
1825
1826    Item::Failed(Message,Cnf);
1827 }
1828                                                                         /*}}}*/
1829 // AcqIndex::Custom600Headers - Insert custom request headers           /*{{{*/
1830 // ---------------------------------------------------------------------
1831 /* The only header we use is the last-modified header. */
1832 string pkgAcqFile::Custom600Headers()
1833 {
1834    if (IsIndexFile)
1835       return "\nIndex-File: true";
1836    return "";
1837 }
1838                                                                         /*}}}*/