1 // -*- mode: cpp; mode: fold -*-
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
6 Acquire Item - Item to acquire
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
13 ##################################################################### */
15 // Include Files /*{{{*/
17 #pragma implementation "apt-pkg/acquire-item.h"
19 #include <apt-pkg/acquire-item.h>
20 #include <apt-pkg/configuration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/vendorlist.h>
23 #include <apt-pkg/error.h>
24 #include <apt-pkg/strutl.h>
25 #include <apt-pkg/fileutl.h>
26 #include <apt-pkg/md5.h>
27 #include <apt-pkg/sha1.h>
28 #include <apt-pkg/tagfile.h>
42 // Acquire::Item::Item - Constructor /*{{{*/
43 // ---------------------------------------------------------------------
45 pkgAcquire::Item::Item(pkgAcquire
*Owner
) : Owner(Owner
), FileSize(0),
46 PartialSize(0), Mode(0), ID(0), Complete(false),
47 Local(false), QueueCounter(0)
53 // Acquire::Item::~Item - Destructor /*{{{*/
54 // ---------------------------------------------------------------------
56 pkgAcquire::Item::~Item()
61 // Acquire::Item::Failed - Item failed to download /*{{{*/
62 // ---------------------------------------------------------------------
63 /* We return to an idle state if there are still other queues that could
65 void pkgAcquire::Item::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
68 ErrorText
= LookupTag(Message
,"Message");
69 if (QueueCounter
<= 1)
71 /* This indicates that the file is not available right now but might
72 be sometime later. If we do a retry cycle then this should be
74 if (Cnf
->LocalOnly
== true &&
75 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
87 // Acquire::Item::Start - Item has begun to download /*{{{*/
88 // ---------------------------------------------------------------------
89 /* Stash status and the file size. Note that setting Complete means
90 sub-phases of the acquire process such as decompresion are operating */
91 void pkgAcquire::Item::Start(string
/*Message*/,unsigned long Size
)
93 Status
= StatFetching
;
94 if (FileSize
== 0 && Complete
== false)
98 // Acquire::Item::Done - Item downloaded OK /*{{{*/
99 // ---------------------------------------------------------------------
101 void pkgAcquire::Item::Done(string Message
,unsigned long Size
,string
,
102 pkgAcquire::MethodConfig
*Cnf
)
104 // We just downloaded something..
105 string FileName
= LookupTag(Message
,"Filename");
106 // we only inform the Log class if it was actually not a local thing
107 if (Complete
== false && !Local
&& FileName
== DestFile
)
110 Owner
->Log
->Fetched(Size
,atoi(LookupTag(Message
,"Resume-Point","0").c_str()));
117 ErrorText
= string();
118 Owner
->Dequeue(this);
121 // Acquire::Item::Rename - Rename a file /*{{{*/
122 // ---------------------------------------------------------------------
123 /* This helper function is used by alot of item methods as thier final
125 void pkgAcquire::Item::Rename(string From
,string To
)
127 if (rename(From
.c_str(),To
.c_str()) != 0)
130 snprintf(S
,sizeof(S
),_("rename failed, %s (%s -> %s)."),strerror(errno
),
131 From
.c_str(),To
.c_str());
139 // AcqDiffIndex::AcqDiffIndex - Constructor
140 // ---------------------------------------------------------------------
141 /* Get the DiffIndex file first and see if there are patches availabe
142 * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
143 * patches. If anything goes wrong in that process, it will fall back to
144 * the original packages file
146 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire
*Owner
,
147 string URI
,string URIDesc
,string ShortDesc
,
149 : Item(Owner
), RealURI(URI
), ExpectedMD5(ExpectedMD5
), Description(URIDesc
)
152 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
154 Desc
.Description
= URIDesc
+ "/DiffIndex";
156 Desc
.ShortDesc
= ShortDesc
;
157 Desc
.URI
= URI
+ ".diff/Index";
159 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
160 DestFile
+= URItoFileName(URI
) + string(".DiffIndex");
163 std::clog
<< "pkgAcqDiffIndex: " << Desc
.URI
<< std::endl
;
165 // look for the current package file
166 CurrentPackagesFile
= _config
->FindDir("Dir::State::lists");
167 CurrentPackagesFile
+= URItoFileName(RealURI
);
169 // FIXME: this file:/ check is a hack to prevent fetching
170 // from local sources. this is really silly, and
171 // should be fixed cleanly as soon as possible
172 if(!FileExists(CurrentPackagesFile
) ||
173 Desc
.URI
.substr(0,strlen("file:/")) == "file:/")
175 // we don't have a pkg file or we don't want to queue
177 std::clog
<< "No index file, local or canceld by user" << std::endl
;
183 std::clog
<< "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
184 << CurrentPackagesFile
<< std::endl
;
190 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
191 // ---------------------------------------------------------------------
192 /* The only header we use is the last-modified header. */
193 string
pkgAcqDiffIndex::Custom600Headers()
195 string Final
= _config
->FindDir("Dir::State::lists");
196 Final
+= URItoFileName(RealURI
) + string(".IndexDiff");
199 std::clog
<< "Custom600Header-IMS: " << Final
<< std::endl
;
202 if (stat(Final
.c_str(),&Buf
) != 0)
203 return "\nIndex-File: true";
205 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
209 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile
)
212 std::clog
<< "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
217 vector
<DiffInfo
> available_patches
;
219 FileFd
Fd(IndexDiffFile
,FileFd::ReadOnly
);
221 if (_error
->PendingError() == true)
224 if(TF
.Step(Tags
) == true)
231 string tmp
= Tags
.FindS("SHA1-Current");
232 std::stringstream
ss(tmp
);
235 FileFd
fd(CurrentPackagesFile
, FileFd::ReadOnly
);
237 SHA1
.AddFD(fd
.Fd(), fd
.Size());
238 local_sha1
= string(SHA1
.Result());
240 if(local_sha1
== ServerSha1
)
242 // we have the same sha1 as the server
244 std::clog
<< "Package file is up-to-date" << std::endl
;
245 // set found to true, this will queue a pkgAcqIndexDiffs with
246 // a empty availabe_patches
252 std::clog
<< "SHA1-Current: " << ServerSha1
<< std::endl
;
254 // check the historie and see what patches we need
255 string history
= Tags
.FindS("SHA1-History");
256 std::stringstream
hist(history
);
257 while(hist
>> d
.sha1
>> size
>> d
.file
)
259 d
.size
= atoi(size
.c_str());
260 // read until the first match is found
261 if(d
.sha1
== local_sha1
)
263 // from that point on, we probably need all diffs
267 std::clog
<< "Need to get diff: " << d
.file
<< std::endl
;
268 available_patches
.push_back(d
);
273 // we have something, queue the next diff
277 int last_space
= Description
.rfind(" ");
278 if(last_space
!= string::npos
)
279 Description
.erase(last_space
, Description
.size()-last_space
);
280 new pkgAcqIndexDiffs(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
281 ExpectedMD5
, available_patches
);
289 // Nothing found, report and return false
290 // Failing here is ok, if we return false later, the full
291 // IndexFile is queued
293 std::clog
<< "Can't find a patch in the index file" << std::endl
;
297 void pkgAcqDiffIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
300 std::clog
<< "pkgAcqDiffIndex failed: " << Desc
.URI
<< std::endl
301 << "Falling back to normal index file aquire" << std::endl
;
303 new pkgAcqIndex(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
311 void pkgAcqDiffIndex::Done(string Message
,unsigned long Size
,string Md5Hash
,
312 pkgAcquire::MethodConfig
*Cnf
)
315 std::clog
<< "pkgAcqDiffIndex::Done(): " << Desc
.URI
<< std::endl
;
317 Item::Done(Message
,Size
,Md5Hash
,Cnf
);
320 FinalFile
= _config
->FindDir("Dir::State::lists")+URItoFileName(RealURI
);
322 // sucess in downloading the index
324 FinalFile
+= string(".IndexDiff");
326 std::clog
<< "Renaming: " << DestFile
<< " -> " << FinalFile
328 Rename(DestFile
,FinalFile
);
329 chmod(FinalFile
.c_str(),0644);
330 DestFile
= FinalFile
;
332 if(!ParseDiffIndex(DestFile
))
333 return Failed("", NULL
);
343 // AcqIndexDiffs::AcqIndexDiffs - Constructor
344 // ---------------------------------------------------------------------
345 /* The package diff is added to the queue. one object is constructed
346 * for each diff and the index
348 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire
*Owner
,
349 string URI
,string URIDesc
,string ShortDesc
,
350 string ExpectedMD5
, vector
<DiffInfo
> diffs
)
351 : Item(Owner
), RealURI(URI
), ExpectedMD5(ExpectedMD5
),
352 available_patches(diffs
)
355 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
356 DestFile
+= URItoFileName(URI
);
358 Debug
= _config
->FindB("Debug::pkgAcquire::Diffs",false);
360 Description
= URIDesc
;
362 Desc
.ShortDesc
= ShortDesc
;
364 if(available_patches
.size() == 0)
366 // we are done (yeah!)
372 State
= StateFetchDiff
;
378 void pkgAcqIndexDiffs::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
381 std::clog
<< "pkgAcqIndexDiffs failed: " << Desc
.URI
<< std::endl
382 << "Falling back to normal index file aquire" << std::endl
;
383 new pkgAcqIndex(Owner
, RealURI
, Description
,Desc
.ShortDesc
,
389 // helper that cleans the item out of the fetcher queue
390 void pkgAcqIndexDiffs::Finish(bool allDone
)
392 // we restore the original name, this is required, otherwise
393 // the file will be cleaned
396 DestFile
= _config
->FindDir("Dir::State::lists");
397 DestFile
+= URItoFileName(RealURI
);
399 // do the final md5sum checking
401 FileFd
Fd(DestFile
, FileFd::ReadOnly
);
402 sum
.AddFD(Fd
.Fd(), Fd
.Size());
404 string MD5
= (string
)sum
.Result();
406 if (!ExpectedMD5
.empty() && MD5
!= ExpectedMD5
)
408 Status
= StatAuthError
;
409 ErrorText
= _("MD5Sum mismatch");
410 Rename(DestFile
,DestFile
+ ".FAILED");
415 // this is for the "real" finish
420 std::clog
<< "\n\nallDone: " << DestFile
<< "\n" << std::endl
;
425 std::clog
<< "Finishing: " << Desc
.URI
<< std::endl
;
434 bool pkgAcqIndexDiffs::QueueNextDiff()
437 // calc sha1 of the just patched file
438 string FinalFile
= _config
->FindDir("Dir::State::lists");
439 FinalFile
+= URItoFileName(RealURI
);
441 FileFd
fd(FinalFile
, FileFd::ReadOnly
);
443 SHA1
.AddFD(fd
.Fd(), fd
.Size());
444 string local_sha1
= string(SHA1
.Result());
446 std::clog
<< "QueueNextDiff: "
447 << FinalFile
<< " (" << local_sha1
<< ")"<<std::endl
;
449 // remove all patches until the next matching patch is found
450 // this requires the Index file to be ordered
451 for(vector
<DiffInfo
>::iterator I
=available_patches
.begin();
452 available_patches
.size() > 0 &&
453 I
!= available_patches
.end() &&
454 (*I
).sha1
!= local_sha1
;
457 available_patches
.erase(I
);
460 // error checking and falling back if no patch was found
461 if(available_patches
.size() == 0)
467 // queue the right diff
468 Desc
.URI
= string(RealURI
) + ".diff/" + available_patches
[0].file
+ ".gz";
469 Desc
.Description
= Description
+ " " + available_patches
[0].file
+ string(".pdiff");
471 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
472 DestFile
+= URItoFileName(RealURI
+ ".diff/" + available_patches
[0].file
);
475 std::clog
<< "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc
.URI
<< std::endl
;
484 void pkgAcqIndexDiffs::Done(string Message
,unsigned long Size
,string Md5Hash
,
485 pkgAcquire::MethodConfig
*Cnf
)
488 std::clog
<< "pkgAcqIndexDiffs::Done(): " << Desc
.URI
<< std::endl
;
490 Item::Done(Message
,Size
,Md5Hash
,Cnf
);
493 FinalFile
= _config
->FindDir("Dir::State::lists")+URItoFileName(RealURI
);
495 // sucess in downloading a diff, enter ApplyDiff state
496 if(State
== StateFetchDiff
)
500 std::clog
<< "Sending to gzip method: " << FinalFile
<< std::endl
;
502 string FileName
= LookupTag(Message
,"Filename");
503 State
= StateUnzipDiff
;
505 Desc
.URI
= "gzip:" + FileName
;
506 DestFile
+= ".decomp";
512 // sucess in downloading a diff, enter ApplyDiff state
513 if(State
== StateUnzipDiff
)
516 // rred excepts the patch as $FinalFile.ed
517 Rename(DestFile
,FinalFile
+".ed");
520 std::clog
<< "Sending to rred method: " << FinalFile
<< std::endl
;
522 State
= StateApplyDiff
;
524 Desc
.URI
= "rred:" + FinalFile
;
531 // success in download/apply a diff, queue next (if needed)
532 if(State
== StateApplyDiff
)
534 // remove the just applied patch
535 available_patches
.erase(available_patches
.begin());
540 std::clog
<< "Moving patched file in place: " << std::endl
541 << DestFile
<< " -> " << FinalFile
<< std::endl
;
543 Rename(DestFile
,FinalFile
);
544 chmod(FinalFile
.c_str(),0644);
546 // see if there is more to download
547 if(available_patches
.size() > 0) {
548 new pkgAcqIndexDiffs(Owner
, RealURI
, Description
, Desc
.ShortDesc
,
549 ExpectedMD5
, available_patches
);
557 // AcqIndex::AcqIndex - Constructor /*{{{*/
558 // ---------------------------------------------------------------------
559 /* The package file is added to the queue and a second class is
560 instantiated to fetch the revision file */
561 pkgAcqIndex::pkgAcqIndex(pkgAcquire
*Owner
,
562 string URI
,string URIDesc
,string ShortDesc
,
563 string ExpectedMD5
, string comprExt
)
564 : Item(Owner
), RealURI(URI
), ExpectedMD5(ExpectedMD5
)
566 Decompression
= false;
569 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
570 DestFile
+= URItoFileName(URI
);
574 // autoselect the compression method
575 if(FileExists("/bin/bzip2"))
576 CompressionExtension
= ".bz2";
578 CompressionExtension
= ".gz";
580 CompressionExtension
= comprExt
;
582 Desc
.URI
= URI
+ CompressionExtension
;
584 Desc
.Description
= URIDesc
;
586 Desc
.ShortDesc
= ShortDesc
;
591 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
592 // ---------------------------------------------------------------------
593 /* The only header we use is the last-modified header. */
594 string
pkgAcqIndex::Custom600Headers()
596 string Final
= _config
->FindDir("Dir::State::lists");
597 Final
+= URItoFileName(RealURI
);
600 if (stat(Final
.c_str(),&Buf
) != 0)
601 return "\nIndex-File: true";
603 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
607 void pkgAcqIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
609 // no .bz2 found, retry with .gz
610 if(Desc
.URI
.substr(Desc
.URI
.size()-3) == "bz2") {
611 Desc
.URI
= Desc
.URI
.substr(0,Desc
.URI
.size()-3) + "gz";
613 // retry with a gzip one
614 new pkgAcqIndex(Owner
, RealURI
, Desc
.Description
,Desc
.ShortDesc
,
615 ExpectedMD5
, string(".gz"));
623 Item::Failed(Message
,Cnf
);
627 // AcqIndex::Done - Finished a fetch /*{{{*/
628 // ---------------------------------------------------------------------
629 /* This goes through a number of states.. On the initial fetch the
630 method could possibly return an alternate filename which points
631 to the uncompressed version of the file. If this is so the file
632 is copied into the partial directory. In all other cases the file
633 is decompressed with a gzip uri. */
634 void pkgAcqIndex::Done(string Message
,unsigned long Size
,string MD5
,
635 pkgAcquire::MethodConfig
*Cfg
)
637 Item::Done(Message
,Size
,MD5
,Cfg
);
639 if (Decompression
== true)
641 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
643 std::cerr
<< std::endl
<< RealURI
<< ": Computed MD5: " << MD5
;
644 std::cerr
<< " Expected MD5: " << ExpectedMD5
<< std::endl
;
650 FileFd
Fd(DestFile
, FileFd::ReadOnly
);
651 sum
.AddFD(Fd
.Fd(), Fd
.Size());
653 MD5
= (string
)sum
.Result();
656 if (!ExpectedMD5
.empty() && MD5
!= ExpectedMD5
)
658 Status
= StatAuthError
;
659 ErrorText
= _("MD5Sum mismatch");
660 Rename(DestFile
,DestFile
+ ".FAILED");
663 // Done, move it into position
664 string FinalFile
= _config
->FindDir("Dir::State::lists");
665 FinalFile
+= URItoFileName(RealURI
);
666 Rename(DestFile
,FinalFile
);
667 chmod(FinalFile
.c_str(),0644);
669 /* We restore the original name to DestFile so that the clean operation
671 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
672 DestFile
+= URItoFileName(RealURI
);
674 // Remove the compressed version.
676 unlink(DestFile
.c_str());
683 // Handle the unzipd case
684 string FileName
= LookupTag(Message
,"Alt-Filename");
685 if (FileName
.empty() == false)
687 // The files timestamp matches
688 if (StringToBool(LookupTag(Message
,"Alt-IMS-Hit"),false) == true)
691 Decompression
= true;
693 DestFile
+= ".decomp";
694 Desc
.URI
= "copy:" + FileName
;
700 FileName
= LookupTag(Message
,"Filename");
701 if (FileName
.empty() == true)
704 ErrorText
= "Method gave a blank filename";
707 // The files timestamp matches
708 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
711 if (FileName
== DestFile
)
716 string compExt
= Desc
.URI
.substr(Desc
.URI
.size()-3);
719 decompProg
= "bzip2";
720 else if(compExt
== ".gz")
723 _error
->Error("Unsupported extension: %s", compExt
.c_str());
727 Decompression
= true;
728 DestFile
+= ".decomp";
729 Desc
.URI
= string(decompProg
) + ":" + FileName
;
734 // AcqIndexTrans::pkgAcqIndexTrans - Constructor /*{{{*/
735 // ---------------------------------------------------------------------
736 /* The Translation file is added to the queue */
737 pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire
*Owner
,
738 string URI
,string URIDesc
,string ShortDesc
) :
739 pkgAcqIndex(Owner
, URI
, URIDesc
, ShortDesc
, "", "")
744 // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/
745 // ---------------------------------------------------------------------
747 void pkgAcqIndexTrans::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
749 if (Cnf
->LocalOnly
== true ||
750 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
759 Item::Failed(Message
,Cnf
);
763 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire
*Owner
,
764 string URI
,string URIDesc
,string ShortDesc
,
765 string MetaIndexURI
, string MetaIndexURIDesc
,
766 string MetaIndexShortDesc
,
767 const vector
<IndexTarget
*>* IndexTargets
,
768 indexRecords
* MetaIndexParser
) :
769 Item(Owner
), RealURI(URI
), MetaIndexURI(MetaIndexURI
),
770 MetaIndexURIDesc(MetaIndexURIDesc
), MetaIndexShortDesc(MetaIndexShortDesc
),
771 MetaIndexParser(MetaIndexParser
), IndexTargets(IndexTargets
)
773 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
774 DestFile
+= URItoFileName(URI
);
776 // remove any partial downloaded sig-file. it may confuse proxies
777 // and is too small to warrant a partial download anyway
778 unlink(DestFile
.c_str());
781 Desc
.Description
= URIDesc
;
783 Desc
.ShortDesc
= ShortDesc
;
787 string Final
= _config
->FindDir("Dir::State::lists");
788 Final
+= URItoFileName(RealURI
);
790 if (stat(Final
.c_str(),&Buf
) == 0)
792 // File was already in place. It needs to be re-verified
793 // because Release might have changed, so Move it into partial
794 Rename(Final
,DestFile
);
800 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
801 // ---------------------------------------------------------------------
802 /* The only header we use is the last-modified header. */
803 string
pkgAcqMetaSig::Custom600Headers()
806 if (stat(DestFile
.c_str(),&Buf
) != 0)
807 return "\nIndex-File: true";
809 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
812 void pkgAcqMetaSig::Done(string Message
,unsigned long Size
,string MD5
,
813 pkgAcquire::MethodConfig
*Cfg
)
815 Item::Done(Message
,Size
,MD5
,Cfg
);
817 string FileName
= LookupTag(Message
,"Filename");
818 if (FileName
.empty() == true)
821 ErrorText
= "Method gave a blank filename";
825 if (FileName
!= DestFile
)
827 // We have to copy it into place
829 Desc
.URI
= "copy:" + FileName
;
836 // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
837 new pkgAcqMetaIndex(Owner
, MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
838 DestFile
, IndexTargets
, MetaIndexParser
);
842 void pkgAcqMetaSig::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
845 // if we get a network error we fail gracefully
846 if(LookupTag(Message
,"FailReason") == "Timeout" ||
847 LookupTag(Message
,"FailReason") == "TmpResolveFailure" ||
848 LookupTag(Message
,"FailReason") == "ConnectionRefused") {
849 Item::Failed(Message
,Cnf
);
853 // Delete any existing sigfile when the acquire failed
854 string Final
= _config
->FindDir("Dir::State::lists") + URItoFileName(RealURI
);
855 unlink(Final
.c_str());
857 // queue a pkgAcqMetaIndex with no sigfile
858 new pkgAcqMetaIndex(Owner
, MetaIndexURI
, MetaIndexURIDesc
, MetaIndexShortDesc
,
859 "", IndexTargets
, MetaIndexParser
);
861 if (Cnf
->LocalOnly
== true ||
862 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == false)
871 Item::Failed(Message
,Cnf
);
874 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire
*Owner
,
875 string URI
,string URIDesc
,string ShortDesc
,
877 const vector
<struct IndexTarget
*>* IndexTargets
,
878 indexRecords
* MetaIndexParser
) :
879 Item(Owner
), RealURI(URI
), SigFile(SigFile
), AuthPass(false),
880 MetaIndexParser(MetaIndexParser
), IndexTargets(IndexTargets
), IMSHit(false)
882 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
883 DestFile
+= URItoFileName(URI
);
886 Desc
.Description
= URIDesc
;
888 Desc
.ShortDesc
= ShortDesc
;
895 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/
896 // ---------------------------------------------------------------------
897 /* The only header we use is the last-modified header. */
898 string
pkgAcqMetaIndex::Custom600Headers()
900 string Final
= _config
->FindDir("Dir::State::lists");
901 Final
+= URItoFileName(RealURI
);
904 if (stat(Final
.c_str(),&Buf
) != 0)
905 return "\nIndex-File: true";
907 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf
.st_mtime
);
910 void pkgAcqMetaIndex::Done(string Message
,unsigned long Size
,string MD5
,
911 pkgAcquire::MethodConfig
*Cfg
)
913 Item::Done(Message
,Size
,MD5
,Cfg
);
915 // MetaIndexes are done in two passes: one to download the
916 // metaindex with an appropriate method, and a second to verify it
917 // with the gpgv method
919 if (AuthPass
== true)
925 RetrievalDone(Message
);
927 // Still more retrieving to do
932 // There was no signature file, so we are finished. Download
933 // the indexes without verification.
938 // There was a signature file, so pass it to gpgv for
941 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
942 std::cerr
<< "Metaindex acquired, queueing gpg verification ("
943 << SigFile
<< "," << DestFile
<< ")\n";
945 Desc
.URI
= "gpgv:" + SigFile
;
952 void pkgAcqMetaIndex::RetrievalDone(string Message
)
954 // We have just finished downloading a Release file (it is not
957 string FileName
= LookupTag(Message
,"Filename");
958 if (FileName
.empty() == true)
961 ErrorText
= "Method gave a blank filename";
965 if (FileName
!= DestFile
)
968 Desc
.URI
= "copy:" + FileName
;
973 // see if the download was a IMSHit
974 IMSHit
= StringToBool(LookupTag(Message
,"IMS-Hit"),false);
978 string FinalFile
= _config
->FindDir("Dir::State::lists");
979 FinalFile
+= URItoFileName(RealURI
);
981 // The files timestamp matches
982 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == false)
984 // Move it into position
985 Rename(DestFile
,FinalFile
);
987 chmod(FinalFile
.c_str(),0644);
988 DestFile
= FinalFile
;
991 void pkgAcqMetaIndex::AuthDone(string Message
)
993 // At this point, the gpgv method has succeeded, so there is a
994 // valid signature from a key in the trusted keyring. We
995 // perform additional verification of its contents, and use them
996 // to verify the indexes we are about to download
998 if (!MetaIndexParser
->Load(DestFile
))
1000 Status
= StatAuthError
;
1001 ErrorText
= MetaIndexParser
->ErrorText
;
1005 if (!VerifyVendor(Message
))
1010 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1011 std::cerr
<< "Signature verification succeeded: "
1012 << DestFile
<< std::endl
;
1014 // Download further indexes with verification
1017 // Done, move signature file into position
1019 string VerifiedSigFile
= _config
->FindDir("Dir::State::lists") +
1020 URItoFileName(RealURI
) + ".gpg";
1021 Rename(SigFile
,VerifiedSigFile
);
1022 chmod(VerifiedSigFile
.c_str(),0644);
1025 void pkgAcqMetaIndex::QueueIndexes(bool verify
)
1027 for (vector
<struct IndexTarget
*>::const_iterator Target
= IndexTargets
->begin();
1028 Target
!= IndexTargets
->end();
1031 string ExpectedIndexMD5
;
1034 const indexRecords::checkSum
*Record
= MetaIndexParser
->Lookup((*Target
)->MetaKey
);
1037 Status
= StatAuthError
;
1038 ErrorText
= "Unable to find expected entry "
1039 + (*Target
)->MetaKey
+ " in Meta-index file (malformed Release file?)";
1042 ExpectedIndexMD5
= Record
->MD5Hash
;
1043 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1045 std::cerr
<< "Queueing: " << (*Target
)->URI
<< std::endl
;
1046 std::cerr
<< "Expected MD5: " << ExpectedIndexMD5
<< std::endl
;
1048 if (ExpectedIndexMD5
.empty())
1050 Status
= StatAuthError
;
1051 ErrorText
= "Unable to find MD5 sum for "
1052 + (*Target
)->MetaKey
+ " in Meta-index file";
1057 // Queue Packages file (either diff or full packages files, depending
1058 // on the users option)
1059 if(_config
->FindB("Acquire::PDiffs",true) == true)
1060 new pkgAcqDiffIndex(Owner
, (*Target
)->URI
, (*Target
)->Description
,
1061 (*Target
)->ShortDesc
, ExpectedIndexMD5
);
1063 new pkgAcqIndex(Owner
, (*Target
)->URI
, (*Target
)->Description
,
1064 (*Target
)->ShortDesc
, ExpectedIndexMD5
);
1068 bool pkgAcqMetaIndex::VerifyVendor(string Message
)
1070 // // Maybe this should be made available from above so we don't have
1071 // // to read and parse it every time?
1072 // pkgVendorList List;
1073 // List.ReadMainList();
1075 // const Vendor* Vndr = NULL;
1076 // for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
1078 // string::size_type pos = (*I).find("VALIDSIG ");
1079 // if (_config->FindB("Debug::Vendor", false))
1080 // std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
1082 // if (pos != std::string::npos)
1084 // string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
1085 // if (_config->FindB("Debug::Vendor", false))
1086 // std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
1088 // Vndr = List.FindVendor(Fingerprint) != "";
1089 // if (Vndr != NULL);
1093 string::size_type pos
;
1095 // check for missing sigs (that where not fatal because otherwise we had
1098 string msg
= _("There is no public key available for the "
1099 "following key IDs:\n");
1100 pos
= Message
.find("NO_PUBKEY ");
1101 if (pos
!= std::string::npos
)
1103 string::size_type start
= pos
+strlen("NO_PUBKEY ");
1104 string Fingerprint
= Message
.substr(start
, Message
.find("\n")-start
);
1105 missingkeys
+= (Fingerprint
);
1107 if(!missingkeys
.empty())
1108 _error
->Warning("%s", string(msg
+missingkeys
).c_str());
1110 string Transformed
= MetaIndexParser
->GetExpectedDist();
1112 if (Transformed
== "../project/experimental")
1114 Transformed
= "experimental";
1117 pos
= Transformed
.rfind('/');
1118 if (pos
!= string::npos
)
1120 Transformed
= Transformed
.substr(0, pos
);
1123 if (Transformed
== ".")
1128 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1130 std::cerr
<< "Got Codename: " << MetaIndexParser
->GetDist() << std::endl
;
1131 std::cerr
<< "Expecting Dist: " << MetaIndexParser
->GetExpectedDist() << std::endl
;
1132 std::cerr
<< "Transformed Dist: " << Transformed
<< std::endl
;
1135 if (MetaIndexParser
->CheckDist(Transformed
) == false)
1137 // This might become fatal one day
1138 // Status = StatAuthError;
1139 // ErrorText = "Conflicting distribution; expected "
1140 // + MetaIndexParser->GetExpectedDist() + " but got "
1141 // + MetaIndexParser->GetDist();
1143 if (!Transformed
.empty())
1145 _error
->Warning("Conflicting distribution: %s (expected %s but got %s)",
1146 Desc
.Description
.c_str(),
1147 Transformed
.c_str(),
1148 MetaIndexParser
->GetDist().c_str());
1155 // pkgAcqMetaIndex::Failed - no Release file present or no signature
1156 // file present /*{{{*/
1157 // ---------------------------------------------------------------------
1159 void pkgAcqMetaIndex::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1161 if (AuthPass
== true)
1163 // if we fail the authentication but got the file via a IMS-Hit
1164 // this means that the file wasn't downloaded and that it might be
1165 // just stale (server problem, proxy etc). we delete what we have
1166 // queue it again without i-m-s
1167 // alternatively we could just unlink the file and let the user try again
1173 unlink(DestFile
.c_str());
1175 DestFile
= _config
->FindDir("Dir::State::lists") + "partial/";
1176 DestFile
+= URItoFileName(RealURI
);
1182 // gpgv method failed
1183 _error
->Warning("GPG error: %s: %s",
1184 Desc
.Description
.c_str(),
1185 LookupTag(Message
,"Message").c_str());
1189 // No Release file was present, or verification failed, so fall
1190 // back to queueing Packages files without verification
1191 QueueIndexes(false);
1196 // AcqArchive::AcqArchive - Constructor /*{{{*/
1197 // ---------------------------------------------------------------------
1198 /* This just sets up the initial fetch environment and queues the first
1200 pkgAcqArchive::pkgAcqArchive(pkgAcquire
*Owner
,pkgSourceList
*Sources
,
1201 pkgRecords
*Recs
,pkgCache::VerIterator
const &Version
,
1202 string
&StoreFilename
) :
1203 Item(Owner
), Version(Version
), Sources(Sources
), Recs(Recs
),
1204 StoreFilename(StoreFilename
), Vf(Version
.FileList()),
1207 Retries
= _config
->FindI("Acquire::Retries",0);
1209 if (Version
.Arch() == 0)
1211 _error
->Error(_("I wasn't able to locate a file for the %s package. "
1212 "This might mean you need to manually fix this package. "
1213 "(due to missing arch)"),
1214 Version
.ParentPkg().Name());
1218 /* We need to find a filename to determine the extension. We make the
1219 assumption here that all the available sources for this version share
1220 the same extension.. */
1221 // Skip not source sources, they do not have file fields.
1222 for (; Vf
.end() == false; Vf
++)
1224 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
1229 // Does not really matter here.. we are going to fail out below
1230 if (Vf
.end() != true)
1232 // If this fails to get a file name we will bomb out below.
1233 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
1234 if (_error
->PendingError() == true)
1237 // Generate the final file name as: package_version_arch.foo
1238 StoreFilename
= QuoteString(Version
.ParentPkg().Name(),"_:") + '_' +
1239 QuoteString(Version
.VerStr(),"_:") + '_' +
1240 QuoteString(Version
.Arch(),"_:.") +
1241 "." + flExtension(Parse
.FileName());
1244 // check if we have one trusted source for the package. if so, switch
1245 // to "TrustedOnly" mode
1246 for (pkgCache::VerFileIterator i
= Version
.FileList(); i
.end() == false; i
++)
1248 pkgIndexFile
*Index
;
1249 if (Sources
->FindIndex(i
.File(),Index
) == false)
1251 if (_config
->FindB("Debug::pkgAcquire::Auth", false))
1253 std::cerr
<< "Checking index: " << Index
->Describe()
1254 << "(Trusted=" << Index
->IsTrusted() << ")\n";
1256 if (Index
->IsTrusted()) {
1262 // "allow-unauthenticated" restores apts old fetching behaviour
1263 // that means that e.g. unauthenticated file:// uris are higher
1264 // priority than authenticated http:// uris
1265 if (_config
->FindB("APT::Get::AllowUnauthenticated",false) == true)
1269 if (QueueNext() == false && _error
->PendingError() == false)
1270 _error
->Error(_("I wasn't able to locate file for the %s package. "
1271 "This might mean you need to manually fix this package."),
1272 Version
.ParentPkg().Name());
1275 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
1276 // ---------------------------------------------------------------------
1277 /* This queues the next available file version for download. It checks if
1278 the archive is already available in the cache and stashs the MD5 for
1280 bool pkgAcqArchive::QueueNext()
1282 for (; Vf
.end() == false; Vf
++)
1284 // Ignore not source sources
1285 if ((Vf
.File()->Flags
& pkgCache::Flag::NotSource
) != 0)
1288 // Try to cross match against the source list
1289 pkgIndexFile
*Index
;
1290 if (Sources
->FindIndex(Vf
.File(),Index
) == false)
1293 // only try to get a trusted package from another source if that source
1295 if(Trusted
&& !Index
->IsTrusted())
1298 // Grab the text package record
1299 pkgRecords::Parser
&Parse
= Recs
->Lookup(Vf
);
1300 if (_error
->PendingError() == true)
1303 string PkgFile
= Parse
.FileName();
1304 MD5
= Parse
.MD5Hash();
1305 if (PkgFile
.empty() == true)
1306 return _error
->Error(_("The package index files are corrupted. No Filename: "
1307 "field for package %s."),
1308 Version
.ParentPkg().Name());
1310 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
1311 Desc
.Description
= Index
->ArchiveInfo(Version
);
1313 Desc
.ShortDesc
= Version
.ParentPkg().Name();
1315 // See if we already have the file. (Legacy filenames)
1316 FileSize
= Version
->Size
;
1317 string FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile
);
1319 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1321 // Make sure the size matches
1322 if ((unsigned)Buf
.st_size
== Version
->Size
)
1327 StoreFilename
= DestFile
= FinalFile
;
1331 /* Hmm, we have a file and its size does not match, this means it is
1332 an old style mismatched arch */
1333 unlink(FinalFile
.c_str());
1336 // Check it again using the new style output filenames
1337 FinalFile
= _config
->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename
);
1338 if (stat(FinalFile
.c_str(),&Buf
) == 0)
1340 // Make sure the size matches
1341 if ((unsigned)Buf
.st_size
== Version
->Size
)
1346 StoreFilename
= DestFile
= FinalFile
;
1350 /* Hmm, we have a file and its size does not match, this shouldnt
1352 unlink(FinalFile
.c_str());
1355 DestFile
= _config
->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename
);
1357 // Check the destination file
1358 if (stat(DestFile
.c_str(),&Buf
) == 0)
1360 // Hmm, the partial file is too big, erase it
1361 if ((unsigned)Buf
.st_size
> Version
->Size
)
1362 unlink(DestFile
.c_str());
1364 PartialSize
= Buf
.st_size
;
1369 Desc
.URI
= Index
->ArchiveURI(PkgFile
);
1370 Desc
.Description
= Index
->ArchiveInfo(Version
);
1372 Desc
.ShortDesc
= Version
.ParentPkg().Name();
1381 // AcqArchive::Done - Finished fetching /*{{{*/
1382 // ---------------------------------------------------------------------
1384 void pkgAcqArchive::Done(string Message
,unsigned long Size
,string Md5Hash
,
1385 pkgAcquire::MethodConfig
*Cfg
)
1387 Item::Done(Message
,Size
,Md5Hash
,Cfg
);
1390 if (Size
!= Version
->Size
)
1393 ErrorText
= _("Size mismatch");
1398 if (Md5Hash
.empty() == false && MD5
.empty() == false)
1403 ErrorText
= _("MD5Sum mismatch");
1404 if(FileExists(DestFile
))
1405 Rename(DestFile
,DestFile
+ ".FAILED");
1410 // Grab the output filename
1411 string FileName
= LookupTag(Message
,"Filename");
1412 if (FileName
.empty() == true)
1415 ErrorText
= "Method gave a blank filename";
1421 // Reference filename
1422 if (FileName
!= DestFile
)
1424 StoreFilename
= DestFile
= FileName
;
1429 // Done, move it into position
1430 string FinalFile
= _config
->FindDir("Dir::Cache::Archives");
1431 FinalFile
+= flNotDir(StoreFilename
);
1432 Rename(DestFile
,FinalFile
);
1434 StoreFilename
= DestFile
= FinalFile
;
1438 // AcqArchive::Failed - Failure handler /*{{{*/
1439 // ---------------------------------------------------------------------
1440 /* Here we try other sources */
1441 void pkgAcqArchive::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1443 ErrorText
= LookupTag(Message
,"Message");
1445 /* We don't really want to retry on failed media swaps, this prevents
1446 that. An interesting observation is that permanent failures are not
1448 if (Cnf
->Removable
== true &&
1449 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1451 // Vf = Version.FileList();
1452 while (Vf
.end() == false) Vf
++;
1453 StoreFilename
= string();
1454 Item::Failed(Message
,Cnf
);
1458 if (QueueNext() == false)
1460 // This is the retry counter
1462 Cnf
->LocalOnly
== false &&
1463 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1466 Vf
= Version
.FileList();
1467 if (QueueNext() == true)
1471 StoreFilename
= string();
1472 Item::Failed(Message
,Cnf
);
1476 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1477 // trusted source /*{{{*/
1478 // ---------------------------------------------------------------------
1479 bool pkgAcqArchive::IsTrusted()
1484 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
1485 // ---------------------------------------------------------------------
1487 void pkgAcqArchive::Finished()
1489 if (Status
== pkgAcquire::Item::StatDone
&&
1492 StoreFilename
= string();
1496 // AcqFile::pkgAcqFile - Constructor /*{{{*/
1497 // ---------------------------------------------------------------------
1498 /* The file is added to the queue */
1499 pkgAcqFile::pkgAcqFile(pkgAcquire
*Owner
,string URI
,string MD5
,
1500 unsigned long Size
,string Dsc
,string ShortDesc
,
1501 const string
&DestDir
, const string
&DestFilename
) :
1502 Item(Owner
), Md5Hash(MD5
)
1504 Retries
= _config
->FindI("Acquire::Retries",0);
1506 if(!DestFilename
.empty())
1507 DestFile
= DestFilename
;
1508 else if(!DestDir
.empty())
1509 DestFile
= DestDir
+ "/" + flNotDir(URI
);
1511 DestFile
= flNotDir(URI
);
1515 Desc
.Description
= Dsc
;
1518 // Set the short description to the archive component
1519 Desc
.ShortDesc
= ShortDesc
;
1521 // Get the transfer sizes
1524 if (stat(DestFile
.c_str(),&Buf
) == 0)
1526 // Hmm, the partial file is too big, erase it
1527 if ((unsigned)Buf
.st_size
> Size
)
1528 unlink(DestFile
.c_str());
1530 PartialSize
= Buf
.st_size
;
1536 // AcqFile::Done - Item downloaded OK /*{{{*/
1537 // ---------------------------------------------------------------------
1539 void pkgAcqFile::Done(string Message
,unsigned long Size
,string MD5
,
1540 pkgAcquire::MethodConfig
*Cnf
)
1543 if (Md5Hash
.empty() == false && MD5
.empty() == false)
1548 ErrorText
= "MD5Sum mismatch";
1549 Rename(DestFile
,DestFile
+ ".FAILED");
1554 Item::Done(Message
,Size
,MD5
,Cnf
);
1556 string FileName
= LookupTag(Message
,"Filename");
1557 if (FileName
.empty() == true)
1560 ErrorText
= "Method gave a blank filename";
1566 // The files timestamp matches
1567 if (StringToBool(LookupTag(Message
,"IMS-Hit"),false) == true)
1570 // We have to copy it into place
1571 if (FileName
!= DestFile
)
1574 if (_config
->FindB("Acquire::Source-Symlinks",true) == false ||
1575 Cnf
->Removable
== true)
1577 Desc
.URI
= "copy:" + FileName
;
1582 // Erase the file if it is a symlink so we can overwrite it
1584 if (lstat(DestFile
.c_str(),&St
) == 0)
1586 if (S_ISLNK(St
.st_mode
) != 0)
1587 unlink(DestFile
.c_str());
1591 if (symlink(FileName
.c_str(),DestFile
.c_str()) != 0)
1593 ErrorText
= "Link to " + DestFile
+ " failure ";
1600 // AcqFile::Failed - Failure handler /*{{{*/
1601 // ---------------------------------------------------------------------
1602 /* Here we try other sources */
1603 void pkgAcqFile::Failed(string Message
,pkgAcquire::MethodConfig
*Cnf
)
1605 ErrorText
= LookupTag(Message
,"Message");
1607 // This is the retry counter
1609 Cnf
->LocalOnly
== false &&
1610 StringToBool(LookupTag(Message
,"Transient-Failure"),false) == true)
1617 Item::Failed(Message
,Cnf
);