* use "red" now, print meaningfull pdiff names, use IMS for the IndexDiff
[ntk/apt.git] / apt-pkg / acquire-item.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire-item.cc,v 1.46.2.9 2004/01/16 18:51:11 mdz Exp $
4 /* ######################################################################
5
6 Acquire Item - Item to acquire
7
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #ifdef __GNUG__
17 #pragma implementation "apt-pkg/acquire-item.h"
18 #endif
19 #include <apt-pkg/acquire-item.h>
20 #include <apt-pkg/configuration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/vendorlist.h>
23 #include <apt-pkg/error.h>
24 #include <apt-pkg/strutl.h>
25 #include <apt-pkg/fileutl.h>
26 #include <apt-pkg/md5.h>
27 #include <apt-pkg/sha1.h>
28 #include <apt-pkg/tagfile.h>
29
30 #include <apti18n.h>
31
32 #include <sys/stat.h>
33 #include <unistd.h>
34 #include <errno.h>
35 #include <string>
36 #include <sstream>
37 #include <stdio.h>
38 /*}}}*/
39
40 using namespace std;
41
42 // Acquire::Item::Item - Constructor /*{{{*/
43 // ---------------------------------------------------------------------
44 /* */
45 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
46 PartialSize(0), Mode(0), ID(0), Complete(false),
47 Local(false), QueueCounter(0)
48 {
49 Owner->Add(this);
50 Status = StatIdle;
51 }
52 /*}}}*/
53 // Acquire::Item::~Item - Destructor /*{{{*/
54 // ---------------------------------------------------------------------
55 /* */
56 pkgAcquire::Item::~Item()
57 {
58 Owner->Remove(this);
59 }
60 /*}}}*/
61 // Acquire::Item::Failed - Item failed to download /*{{{*/
62 // ---------------------------------------------------------------------
63 /* We return to an idle state if there are still other queues that could
64 fetch this object */
65 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
66 {
67 Status = StatIdle;
68 ErrorText = LookupTag(Message,"Message");
69 if (QueueCounter <= 1)
70 {
71 /* This indicates that the file is not available right now but might
72 be sometime later. If we do a retry cycle then this should be
73 retried [CDROMs] */
74 if (Cnf->LocalOnly == true &&
75 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
76 {
77 Status = StatIdle;
78 Dequeue();
79 return;
80 }
81
82 Status = StatError;
83 Dequeue();
84 }
85 }
86 /*}}}*/
87 // Acquire::Item::Start - Item has begun to download /*{{{*/
88 // ---------------------------------------------------------------------
89 /* Stash status and the file size. Note that setting Complete means
90 sub-phases of the acquire process such as decompresion are operating */
91 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
92 {
93 Status = StatFetching;
94 if (FileSize == 0 && Complete == false)
95 FileSize = Size;
96 }
97 /*}}}*/
98 // Acquire::Item::Done - Item downloaded OK /*{{{*/
99 // ---------------------------------------------------------------------
100 /* */
101 void pkgAcquire::Item::Done(string Message,unsigned long Size,string,
102 pkgAcquire::MethodConfig *Cnf)
103 {
104 // We just downloaded something..
105 string FileName = LookupTag(Message,"Filename");
106 if (Complete == false && FileName == DestFile)
107 {
108 if (Owner->Log != 0)
109 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
110 }
111
112 if (FileSize == 0)
113 FileSize= Size;
114
115 Status = StatDone;
116 ErrorText = string();
117 Owner->Dequeue(this);
118 }
119 /*}}}*/
120 // Acquire::Item::Rename - Rename a file /*{{{*/
121 // ---------------------------------------------------------------------
122 /* This helper function is used by alot of item methods as thier final
123 step */
124 void pkgAcquire::Item::Rename(string From,string To)
125 {
126 if (rename(From.c_str(),To.c_str()) != 0)
127 {
128 char S[300];
129 snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
130 From.c_str(),To.c_str());
131 Status = StatError;
132 ErrorText = S;
133 }
134 }
135 /*}}}*/
136
137 // AcqIndexDiffs::AcqIndexDiffs - Constructor
138 // ---------------------------------------------------------------------
139 /* The package diff is added to the queue. one object is constructed
140 * for each diff and the index
141 */
142 pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
143 string URI,string URIDesc,string ShortDesc,
144 string ExpectedMD5, vector<DiffInfo> diffs)
145 : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5), needed_files(diffs)
146 {
147
148 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
149 DestFile += URItoFileName(URI);
150
151 Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
152
153 Desc.Description = URIDesc;
154 Desc.Owner = this;
155 Desc.ShortDesc = ShortDesc;
156
157 CurrentPackagesFile = _config->FindDir("Dir::State::lists");
158 CurrentPackagesFile += URItoFileName(RealURI);
159
160 if(Debug) {
161 std::clog << "pkgAcqIndexDiffs::pkgAcqIndexDiffs(): "
162 << CurrentPackagesFile << std::endl;
163 }
164
165 if(!FileExists(CurrentPackagesFile) ||
166 !_config->FindB("Acquire::Diffs",true)) {
167 // we don't have a pkg file or we don't want to queue
168 if(Debug)
169 std::clog << "No index file or canceld by user" << std::endl;
170 Failed("", NULL);
171 return;
172 }
173
174 if(needed_files.size() == 0)
175 QueueDiffIndex(URI);
176 else
177 QueueNextDiff();
178 }
179
180 void pkgAcqIndexDiffs::QueueDiffIndex(string URI)
181 {
182 Desc.URI = URI + ".diff/Index";
183 Desc.Description = Description + "IndexDiff";
184 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
185 DestFile += URItoFileName(URI) + string(".IndexDiff");
186
187 if(Debug)
188 std::clog << "QueueDiffIndex: " << Desc.URI << std::endl;
189
190 QueueURI(Desc);
191 }
192
193 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
194 // ---------------------------------------------------------------------
195 /* The only header we use is the last-modified header. */
196 string pkgAcqIndexDiffs::Custom600Headers()
197 {
198 if(DestFile.rfind(".IndexDiff") == string::npos)
199 return string("");
200
201 string Final = _config->FindDir("Dir::State::lists");
202 Final += URItoFileName(RealURI) + string(".IndexDiff");
203
204 if(Debug)
205 std::clog << "Custom600Header-IMS: " << Final << std::endl;
206
207 struct stat Buf;
208 if (stat(Final.c_str(),&Buf) != 0)
209 return "\nIndex-File: true";
210
211 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
212 }
213
214 void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
215 {
216 if(Debug)
217 std::clog << "Failed(): " << Desc.URI << std::endl
218 << "Falling back to big package file" << std::endl;
219 new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
220 ExpectedMD5);
221 Finish();
222 }
223
224
225 // helper that cleans the item out of the fetcher queue
226 void pkgAcqIndexDiffs::Finish(bool allDone)
227 {
228 // we restore the original name, this is required, otherwise
229 // the file will be cleaned
230 if(allDone) {
231 // this is for the "real" finish
232 DestFile = _config->FindDir("Dir::State::lists");
233 DestFile += URItoFileName(RealURI);
234 Complete = true;
235 Dequeue();
236 if(Debug)
237 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
238 return;
239
240 }
241
242 if(Debug)
243 std::clog << "Finishing: " << Desc.URI << std::endl;
244 Complete = false;
245 Status = StatDone;
246 Dequeue();
247 return;
248 }
249
250
251 // this needs to be rewriten to not depend on the external ed
252 bool pkgAcqIndexDiffs::ApplyDiff(string PatchFile)
253 {
254 char *error;
255 int res=0;
256
257 string FinalFile = _config->FindDir("Dir::State::lists");
258 FinalFile += URItoFileName(RealURI);
259
260 int Process = ExecFork();
261 if (Process == 0)
262 {
263 chdir(_config->FindDir("Dir::State::lists").c_str());
264 string cmd = "(zcat " + PatchFile + "; echo \"wq\" ) | red " + FinalFile + " >/dev/null 2>/dev/null";
265 if(Debug)
266 std::clog << "Runing: " << cmd << std::endl;
267 res = system(cmd.c_str());
268 _exit(WEXITSTATUS(res));
269 }
270 if(!ExecWait(Process, error, true)) {
271 //_error->Error("Patch failed: %s ", error);
272 return false;
273 }
274
275 return true;
276 }
277
278 bool pkgAcqIndexDiffs::QueueNextDiff()
279 {
280 // FIXME: don't use the needed_files[0] but check sha1 and search
281 // for the right patch in needed_files
282 // -> and rename needed_files to "available_patches"
283
284 // queue diff
285 Desc.URI = string(RealURI) + string(".diff/") + needed_files[0].file + string(".gz");
286 Desc.Description = needed_files[0].file + string(".pdiff");
287
288 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
289 DestFile += URItoFileName(RealURI + string(".diff/") + needed_files[0].file);
290
291 if(Debug)
292 std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
293
294 QueueURI(Desc);
295
296 return true;
297 }
298
299 bool pkgAcqIndexDiffs::ParseIndexDiff(string IndexDiffFile)
300 {
301 if(Debug)
302 std::clog << "pkgAcqIndexDiffs::ParseIndexDiff() " << IndexDiffFile
303 << std::endl;
304
305 FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
306 pkgTagFile TF(&Fd);
307 if (_error->PendingError() == true)
308 return false;
309 pkgTagSection Tags;
310 if(TF.Step(Tags) == true)
311 {
312 string local_sha1;
313 string tmp = Tags.FindS("SHA1-Current");
314 std::stringstream ss(tmp);
315 ss >> ServerSha1;
316
317 FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
318 SHA1Summation SHA1;
319 SHA1.AddFD(fd.Fd(), fd.Size());
320 local_sha1 = string(SHA1.Result());
321
322 if(local_sha1 == ServerSha1) {
323 if(Debug)
324 std::clog << "Package file is up-to-date" << std::endl;
325 Finish(true);
326 return true;
327 }
328 if(Debug)
329 std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
330
331 // check the historie and see what patches we need
332 string history = Tags.FindS("SHA1-History");
333 std::stringstream hist(history);
334 DiffInfo d;
335 string size;
336 bool found = false;
337 while(hist >> d.sha1 >> size >> d.file) {
338 d.size = atoi(size.c_str());
339 if(d.sha1 == local_sha1)
340 found=true;
341 if(found) {
342 if(Debug)
343 std::clog << "Need to get diff: " << d.file << std::endl;
344 needed_files.push_back(d);
345 }
346 }
347 // no information how to get the patches, bail out
348 if(!found) {
349 if(Debug)
350 std::clog << "Can't find a patch in the index file" << std::endl;
351 // Failed will queue a big package file
352 Failed("", NULL);
353 } else {
354 // queue the diffs
355 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
356 ExpectedMD5, needed_files);
357 Finish();
358 return true;
359 }
360 }
361
362 return false;
363 }
364
365
366 void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash,
367 pkgAcquire::MethodConfig *Cnf)
368 {
369 if(Debug)
370 std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
371
372 Item::Done(Message,Size,Md5Hash,Cnf);
373
374 int len = Desc.URI.size();
375 // sucess in downloading the index
376 if(Desc.URI.substr(len-strlen("Index"),len-1) == "Index") {
377
378 // rename
379 string FinalFile = _config->FindDir("Dir::State::lists");
380 FinalFile += URItoFileName(RealURI) + string(".IndexDiff");
381 if(Debug)
382 std::clog << "Renaming: " << DestFile << " -> " << FinalFile
383 << std::endl;
384 Rename(DestFile,FinalFile);
385 chmod(FinalFile.c_str(),0644);
386 DestFile = FinalFile;
387
388 if(!ParseIndexDiff(DestFile))
389 return Failed("", NULL);
390 else
391 return Finish();
392 }
393
394 // sucess in downloading a diff
395 if(Desc.URI.find(".diff") != string::npos) {
396 ApplyDiff(DestFile);
397 needed_files.erase(needed_files.begin());
398
399 if(needed_files.size() > 0) {
400 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
401 ExpectedMD5, needed_files);
402 } else {
403 Finish(true);
404 return;
405 }
406 }
407
408 Finish();
409 }
410
411
412 // AcqIndex::AcqIndex - Constructor /*{{{*/
413 // ---------------------------------------------------------------------
414 /* The package file is added to the queue and a second class is
415 instantiated to fetch the revision file */
416 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
417 string URI,string URIDesc,string ShortDesc,
418 string ExpectedMD5, string comprExt)
419 : Item(Owner), RealURI(URI), ExpectedMD5(ExpectedMD5)
420 {
421 Decompression = false;
422 Erase = false;
423
424 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
425 DestFile += URItoFileName(URI);
426
427 if(comprExt.empty())
428 {
429 // autoselect
430 if(FileExists("/usr/bin/bzip2"))
431 Desc.URI = URI + ".bz2";
432 else
433 Desc.URI = URI + ".gz";
434 } else {
435 Desc.URI = URI + comprExt;
436 }
437
438 Desc.Description = URIDesc;
439 Desc.Owner = this;
440 Desc.ShortDesc = ShortDesc;
441
442 QueueURI(Desc);
443 }
444 /*}}}*/
445 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
446 // ---------------------------------------------------------------------
447 /* The only header we use is the last-modified header. */
448 string pkgAcqIndex::Custom600Headers()
449 {
450 string Final = _config->FindDir("Dir::State::lists");
451 Final += URItoFileName(RealURI);
452
453 struct stat Buf;
454 if (stat(Final.c_str(),&Buf) != 0)
455 return "\nIndex-File: true";
456
457 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
458 }
459 /*}}}*/
460
461 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
462 {
463 // no .bz2 found, retry with .gz
464 if(Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1) == "bz2") {
465 Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
466
467 // retry with a gzip one
468 new pkgAcqIndex(Owner, RealURI, Desc.Description,Desc.ShortDesc,
469 ExpectedMD5, string(".gz"));
470 Status = StatDone;
471 Complete = false;
472 Dequeue();
473 return;
474 }
475
476
477 Item::Failed(Message,Cnf);
478 }
479
480
481 // AcqIndex::Done - Finished a fetch /*{{{*/
482 // ---------------------------------------------------------------------
483 /* This goes through a number of states.. On the initial fetch the
484 method could possibly return an alternate filename which points
485 to the uncompressed version of the file. If this is so the file
486 is copied into the partial directory. In all other cases the file
487 is decompressed with a gzip uri. */
488 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5,
489 pkgAcquire::MethodConfig *Cfg)
490 {
491 Item::Done(Message,Size,MD5,Cfg);
492
493 if (Decompression == true)
494 {
495 if (_config->FindB("Debug::pkgAcquire::Auth", false))
496 {
497 std::cerr << std::endl << RealURI << ": Computed MD5: " << MD5;
498 std::cerr << " Expected MD5: " << ExpectedMD5 << std::endl;
499 }
500
501 if (MD5.empty())
502 {
503 MD5Summation sum;
504 FileFd Fd(DestFile, FileFd::ReadOnly);
505 sum.AddFD(Fd.Fd(), Fd.Size());
506 Fd.Close();
507 MD5 = (string)sum.Result();
508 }
509
510 if (!ExpectedMD5.empty() && MD5 != ExpectedMD5)
511 {
512 Status = StatAuthError;
513 ErrorText = _("MD5Sum mismatch");
514 Rename(DestFile,DestFile + ".FAILED");
515 return;
516 }
517 // Done, move it into position
518 string FinalFile = _config->FindDir("Dir::State::lists");
519 FinalFile += URItoFileName(RealURI);
520 Rename(DestFile,FinalFile);
521 chmod(FinalFile.c_str(),0644);
522
523 /* We restore the original name to DestFile so that the clean operation
524 will work OK */
525 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
526 DestFile += URItoFileName(RealURI);
527
528 // Remove the compressed version.
529 if (Erase == true)
530 unlink(DestFile.c_str());
531 return;
532 }
533
534 Erase = false;
535 Complete = true;
536
537 // Handle the unzipd case
538 string FileName = LookupTag(Message,"Alt-Filename");
539 if (FileName.empty() == false)
540 {
541 // The files timestamp matches
542 if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
543 return;
544
545 Decompression = true;
546 Local = true;
547 DestFile += ".decomp";
548 Desc.URI = "copy:" + FileName;
549 QueueURI(Desc);
550 Mode = "copy";
551 return;
552 }
553
554 FileName = LookupTag(Message,"Filename");
555 if (FileName.empty() == true)
556 {
557 Status = StatError;
558 ErrorText = "Method gave a blank filename";
559 }
560
561 // The files timestamp matches
562 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
563 return;
564
565 if (FileName == DestFile)
566 Erase = true;
567 else
568 Local = true;
569
570 string compExt = Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1);
571 char *decompProg;
572 if(compExt == "bz2")
573 decompProg = "bzip2";
574 else if(compExt == ".gz")
575 decompProg = "gzip";
576 else {
577 _error->Error("Unsupported extension: %s", compExt.c_str());
578 return;
579 }
580
581 Decompression = true;
582 DestFile += ".decomp";
583 Desc.URI = string(decompProg) + ":" + FileName;
584 QueueURI(Desc);
585 Mode = decompProg;
586 }
587
588 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
589 string URI,string URIDesc,string ShortDesc,
590 string MetaIndexURI, string MetaIndexURIDesc,
591 string MetaIndexShortDesc,
592 const vector<IndexTarget*>* IndexTargets,
593 indexRecords* MetaIndexParser) :
594 Item(Owner), RealURI(URI), MetaIndexURI(MetaIndexURI),
595 MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc)
596 {
597 this->MetaIndexParser = MetaIndexParser;
598 this->IndexTargets = IndexTargets;
599 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
600 DestFile += URItoFileName(URI);
601
602 // remove any partial downloaded sig-file. it may confuse proxies
603 // and is too small to warrant a partial download anyway
604 unlink(DestFile.c_str());
605
606 // Create the item
607 Desc.Description = URIDesc;
608 Desc.Owner = this;
609 Desc.ShortDesc = ShortDesc;
610 Desc.URI = URI;
611
612
613 string Final = _config->FindDir("Dir::State::lists");
614 Final += URItoFileName(RealURI);
615 struct stat Buf;
616 if (stat(Final.c_str(),&Buf) == 0)
617 {
618 // File was already in place. It needs to be re-verified
619 // because Release might have changed, so Move it into partial
620 Rename(Final,DestFile);
621 }
622
623 QueueURI(Desc);
624 }
625 /*}}}*/
626 // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/
627 // ---------------------------------------------------------------------
628 /* The only header we use is the last-modified header. */
629 string pkgAcqMetaSig::Custom600Headers()
630 {
631 struct stat Buf;
632 if (stat(DestFile.c_str(),&Buf) != 0)
633 return "\nIndex-File: true";
634
635 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
636 }
637
638 void pkgAcqMetaSig::Done(string Message,unsigned long Size,string MD5,
639 pkgAcquire::MethodConfig *Cfg)
640 {
641 Item::Done(Message,Size,MD5,Cfg);
642
643 string FileName = LookupTag(Message,"Filename");
644 if (FileName.empty() == true)
645 {
646 Status = StatError;
647 ErrorText = "Method gave a blank filename";
648 return;
649 }
650
651 if (FileName != DestFile)
652 {
653 // We have to copy it into place
654 Local = true;
655 Desc.URI = "copy:" + FileName;
656 QueueURI(Desc);
657 return;
658 }
659
660 Complete = true;
661
662 // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved
663 new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
664 DestFile, IndexTargets, MetaIndexParser);
665
666 }
667 /*}}}*/
668 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
669 {
670 // Delete any existing sigfile, so that this source isn't
671 // mistakenly trusted
672 string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
673 unlink(Final.c_str());
674
675 // queue a pkgAcqMetaIndex with no sigfile
676 new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
677 "", IndexTargets, MetaIndexParser);
678
679 if (Cnf->LocalOnly == true ||
680 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
681 {
682 // Ignore this
683 Status = StatDone;
684 Complete = false;
685 Dequeue();
686 return;
687 }
688
689 Item::Failed(Message,Cnf);
690 }
691
692 pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,
693 string URI,string URIDesc,string ShortDesc,
694 string SigFile,
695 const vector<struct IndexTarget*>* IndexTargets,
696 indexRecords* MetaIndexParser) :
697 Item(Owner), RealURI(URI), SigFile(SigFile)
698 {
699 this->AuthPass = false;
700 this->MetaIndexParser = MetaIndexParser;
701 this->IndexTargets = IndexTargets;
702 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
703 DestFile += URItoFileName(URI);
704
705 // Create the item
706 Desc.Description = URIDesc;
707 Desc.Owner = this;
708 Desc.ShortDesc = ShortDesc;
709 Desc.URI = URI;
710
711 QueueURI(Desc);
712 }
713
714 /*}}}*/
715 // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/
716 // ---------------------------------------------------------------------
717 /* The only header we use is the last-modified header. */
718 string pkgAcqMetaIndex::Custom600Headers()
719 {
720 string Final = _config->FindDir("Dir::State::lists");
721 Final += URItoFileName(RealURI);
722
723 struct stat Buf;
724 if (stat(Final.c_str(),&Buf) != 0)
725 return "\nIndex-File: true";
726
727 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
728 }
729
730 void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string MD5,
731 pkgAcquire::MethodConfig *Cfg)
732 {
733 Item::Done(Message,Size,MD5,Cfg);
734
735 // MetaIndexes are done in two passes: one to download the
736 // metaindex with an appropriate method, and a second to verify it
737 // with the gpgv method
738
739 if (AuthPass == true)
740 {
741 AuthDone(Message);
742 }
743 else
744 {
745 RetrievalDone(Message);
746 if (!Complete)
747 // Still more retrieving to do
748 return;
749
750 if (SigFile == "")
751 {
752 // There was no signature file, so we are finished. Download
753 // the indexes without verification.
754 QueueIndexes(false);
755 }
756 else
757 {
758 // There was a signature file, so pass it to gpgv for
759 // verification
760
761 if (_config->FindB("Debug::pkgAcquire::Auth", false))
762 std::cerr << "Metaindex acquired, queueing gpg verification ("
763 << SigFile << "," << DestFile << ")\n";
764 AuthPass = true;
765 Desc.URI = "gpgv:" + SigFile;
766 QueueURI(Desc);
767 Mode = "gpgv";
768 }
769 }
770 }
771
772 void pkgAcqMetaIndex::RetrievalDone(string Message)
773 {
774 // We have just finished downloading a Release file (it is not
775 // verified yet)
776
777 string FileName = LookupTag(Message,"Filename");
778 if (FileName.empty() == true)
779 {
780 Status = StatError;
781 ErrorText = "Method gave a blank filename";
782 return;
783 }
784
785 if (FileName != DestFile)
786 {
787 Local = true;
788 Desc.URI = "copy:" + FileName;
789 QueueURI(Desc);
790 return;
791 }
792
793 Complete = true;
794
795 string FinalFile = _config->FindDir("Dir::State::lists");
796 FinalFile += URItoFileName(RealURI);
797
798 // The files timestamp matches
799 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == false)
800 {
801 // Move it into position
802 Rename(DestFile,FinalFile);
803 }
804 DestFile = FinalFile;
805 }
806
807 void pkgAcqMetaIndex::AuthDone(string Message)
808 {
809 // At this point, the gpgv method has succeeded, so there is a
810 // valid signature from a key in the trusted keyring. We
811 // perform additional verification of its contents, and use them
812 // to verify the indexes we are about to download
813
814 if (!MetaIndexParser->Load(DestFile))
815 {
816 Status = StatAuthError;
817 ErrorText = MetaIndexParser->ErrorText;
818 return;
819 }
820
821 if (!VerifyVendor())
822 {
823 return;
824 }
825
826 if (_config->FindB("Debug::pkgAcquire::Auth", false))
827 std::cerr << "Signature verification succeeded: "
828 << DestFile << std::endl;
829
830 // Download further indexes with verification
831 QueueIndexes(true);
832
833 // Done, move signature file into position
834
835 string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
836 URItoFileName(RealURI) + ".gpg";
837 Rename(SigFile,VerifiedSigFile);
838 chmod(VerifiedSigFile.c_str(),0644);
839 }
840
841 void pkgAcqMetaIndex::QueueIndexes(bool verify)
842 {
843 for (vector <struct IndexTarget*>::const_iterator Target = IndexTargets->begin();
844 Target != IndexTargets->end();
845 Target++)
846 {
847 string ExpectedIndexMD5;
848 if (verify)
849 {
850 const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
851 if (!Record)
852 {
853 Status = StatAuthError;
854 ErrorText = "Unable to find expected entry "
855 + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
856 return;
857 }
858 ExpectedIndexMD5 = Record->MD5Hash;
859 if (_config->FindB("Debug::pkgAcquire::Auth", false))
860 {
861 std::cerr << "Queueing: " << (*Target)->URI << std::endl;
862 std::cerr << "Expected MD5: " << ExpectedIndexMD5 << std::endl;
863 }
864 if (ExpectedIndexMD5.empty())
865 {
866 Status = StatAuthError;
867 ErrorText = "Unable to find MD5 sum for "
868 + (*Target)->MetaKey + " in Meta-index file";
869 return;
870 }
871 }
872
873 // Queue Packages file
874 new pkgAcqIndexDiffs(Owner, (*Target)->URI, (*Target)->Description,
875 (*Target)->ShortDesc, ExpectedIndexMD5);
876 }
877 }
878
879 bool pkgAcqMetaIndex::VerifyVendor()
880 {
881 // // Maybe this should be made available from above so we don't have
882 // // to read and parse it every time?
883 // pkgVendorList List;
884 // List.ReadMainList();
885
886 // const Vendor* Vndr = NULL;
887 // for (std::vector<string>::const_iterator I = GPGVOutput.begin(); I != GPGVOutput.end(); I++)
888 // {
889 // string::size_type pos = (*I).find("VALIDSIG ");
890 // if (_config->FindB("Debug::Vendor", false))
891 // std::cerr << "Looking for VALIDSIG in \"" << (*I) << "\": pos " << pos
892 // << std::endl;
893 // if (pos != std::string::npos)
894 // {
895 // string Fingerprint = (*I).substr(pos+sizeof("VALIDSIG"));
896 // if (_config->FindB("Debug::Vendor", false))
897 // std::cerr << "Looking for \"" << Fingerprint << "\" in vendor..." <<
898 // std::endl;
899 // Vndr = List.FindVendor(Fingerprint) != "";
900 // if (Vndr != NULL);
901 // break;
902 // }
903 // }
904
905 string Transformed = MetaIndexParser->GetExpectedDist();
906
907 if (Transformed == "../project/experimental")
908 {
909 Transformed = "experimental";
910 }
911
912 string::size_type pos = Transformed.rfind('/');
913 if (pos != string::npos)
914 {
915 Transformed = Transformed.substr(0, pos);
916 }
917
918 if (Transformed == ".")
919 {
920 Transformed = "";
921 }
922
923 if (_config->FindB("Debug::pkgAcquire::Auth", false))
924 {
925 std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
926 std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
927 std::cerr << "Transformed Dist: " << Transformed << std::endl;
928 }
929
930 if (MetaIndexParser->CheckDist(Transformed) == false)
931 {
932 // This might become fatal one day
933 // Status = StatAuthError;
934 // ErrorText = "Conflicting distribution; expected "
935 // + MetaIndexParser->GetExpectedDist() + " but got "
936 // + MetaIndexParser->GetDist();
937 // return false;
938 if (!Transformed.empty())
939 {
940 _error->Warning("Conflicting distribution: %s (expected %s but got %s)",
941 Desc.Description.c_str(),
942 Transformed.c_str(),
943 MetaIndexParser->GetDist().c_str());
944 }
945 }
946
947 return true;
948 }
949 /*}}}*/
950 // pkgAcqMetaIndex::Failed - no Release file present or no signature
951 // file present /*{{{*/
952 // ---------------------------------------------------------------------
953 /* */
954 void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
955 {
956 if (AuthPass == true)
957 {
958 // gpgv method failed
959 _error->Warning("GPG error: %s: %s",
960 Desc.Description.c_str(),
961 LookupTag(Message,"Message").c_str());
962 }
963
964 // No Release file was present, or verification failed, so fall
965 // back to queueing Packages files without verification
966 QueueIndexes(false);
967 }
968
969 /*}}}*/
970
971 // AcqArchive::AcqArchive - Constructor /*{{{*/
972 // ---------------------------------------------------------------------
973 /* This just sets up the initial fetch environment and queues the first
974 possibilitiy */
975 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
976 pkgRecords *Recs,pkgCache::VerIterator const &Version,
977 string &StoreFilename) :
978 Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
979 StoreFilename(StoreFilename), Vf(Version.FileList()),
980 Trusted(false)
981 {
982 Retries = _config->FindI("Acquire::Retries",0);
983
984 if (Version.Arch() == 0)
985 {
986 _error->Error(_("I wasn't able to locate a file for the %s package. "
987 "This might mean you need to manually fix this package. "
988 "(due to missing arch)"),
989 Version.ParentPkg().Name());
990 return;
991 }
992
993 /* We need to find a filename to determine the extension. We make the
994 assumption here that all the available sources for this version share
995 the same extension.. */
996 // Skip not source sources, they do not have file fields.
997 for (; Vf.end() == false; Vf++)
998 {
999 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1000 continue;
1001 break;
1002 }
1003
1004 // Does not really matter here.. we are going to fail out below
1005 if (Vf.end() != true)
1006 {
1007 // If this fails to get a file name we will bomb out below.
1008 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1009 if (_error->PendingError() == true)
1010 return;
1011
1012 // Generate the final file name as: package_version_arch.foo
1013 StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
1014 QuoteString(Version.VerStr(),"_:") + '_' +
1015 QuoteString(Version.Arch(),"_:.") +
1016 "." + flExtension(Parse.FileName());
1017 }
1018
1019 // check if we have one trusted source for the package. if so, switch
1020 // to "TrustedOnly" mode
1021 for (pkgCache::VerFileIterator i = Version.FileList(); i.end() == false; i++)
1022 {
1023 pkgIndexFile *Index;
1024 if (Sources->FindIndex(i.File(),Index) == false)
1025 continue;
1026 if (_config->FindB("Debug::pkgAcquire::Auth", false))
1027 {
1028 std::cerr << "Checking index: " << Index->Describe()
1029 << "(Trusted=" << Index->IsTrusted() << ")\n";
1030 }
1031 if (Index->IsTrusted()) {
1032 Trusted = true;
1033 break;
1034 }
1035 }
1036
1037 // Select a source
1038 if (QueueNext() == false && _error->PendingError() == false)
1039 _error->Error(_("I wasn't able to locate file for the %s package. "
1040 "This might mean you need to manually fix this package."),
1041 Version.ParentPkg().Name());
1042 }
1043 /*}}}*/
1044 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
1045 // ---------------------------------------------------------------------
1046 /* This queues the next available file version for download. It checks if
1047 the archive is already available in the cache and stashs the MD5 for
1048 checking later. */
1049 bool pkgAcqArchive::QueueNext()
1050 {
1051 for (; Vf.end() == false; Vf++)
1052 {
1053 // Ignore not source sources
1054 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
1055 continue;
1056
1057 // Try to cross match against the source list
1058 pkgIndexFile *Index;
1059 if (Sources->FindIndex(Vf.File(),Index) == false)
1060 continue;
1061
1062 // only try to get a trusted package from another source if that source
1063 // is also trusted
1064 if(Trusted && !Index->IsTrusted())
1065 continue;
1066
1067 // Grab the text package record
1068 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
1069 if (_error->PendingError() == true)
1070 return false;
1071
1072 string PkgFile = Parse.FileName();
1073 MD5 = Parse.MD5Hash();
1074 if (PkgFile.empty() == true)
1075 return _error->Error(_("The package index files are corrupted. No Filename: "
1076 "field for package %s."),
1077 Version.ParentPkg().Name());
1078
1079 Desc.URI = Index->ArchiveURI(PkgFile);
1080 Desc.Description = Index->ArchiveInfo(Version);
1081 Desc.Owner = this;
1082 Desc.ShortDesc = Version.ParentPkg().Name();
1083
1084 // See if we already have the file. (Legacy filenames)
1085 FileSize = Version->Size;
1086 string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
1087 struct stat Buf;
1088 if (stat(FinalFile.c_str(),&Buf) == 0)
1089 {
1090 // Make sure the size matches
1091 if ((unsigned)Buf.st_size == Version->Size)
1092 {
1093 Complete = true;
1094 Local = true;
1095 Status = StatDone;
1096 StoreFilename = DestFile = FinalFile;
1097 return true;
1098 }
1099
1100 /* Hmm, we have a file and its size does not match, this means it is
1101 an old style mismatched arch */
1102 unlink(FinalFile.c_str());
1103 }
1104
1105 // Check it again using the new style output filenames
1106 FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
1107 if (stat(FinalFile.c_str(),&Buf) == 0)
1108 {
1109 // Make sure the size matches
1110 if ((unsigned)Buf.st_size == Version->Size)
1111 {
1112 Complete = true;
1113 Local = true;
1114 Status = StatDone;
1115 StoreFilename = DestFile = FinalFile;
1116 return true;
1117 }
1118
1119 /* Hmm, we have a file and its size does not match, this shouldnt
1120 happen.. */
1121 unlink(FinalFile.c_str());
1122 }
1123
1124 DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
1125
1126 // Check the destination file
1127 if (stat(DestFile.c_str(),&Buf) == 0)
1128 {
1129 // Hmm, the partial file is too big, erase it
1130 if ((unsigned)Buf.st_size > Version->Size)
1131 unlink(DestFile.c_str());
1132 else
1133 PartialSize = Buf.st_size;
1134 }
1135
1136 // Create the item
1137 Local = false;
1138 Desc.URI = Index->ArchiveURI(PkgFile);
1139 Desc.Description = Index->ArchiveInfo(Version);
1140 Desc.Owner = this;
1141 Desc.ShortDesc = Version.ParentPkg().Name();
1142 QueueURI(Desc);
1143
1144 Vf++;
1145 return true;
1146 }
1147 return false;
1148 }
1149 /*}}}*/
1150 // AcqArchive::Done - Finished fetching /*{{{*/
1151 // ---------------------------------------------------------------------
1152 /* */
1153 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash,
1154 pkgAcquire::MethodConfig *Cfg)
1155 {
1156 Item::Done(Message,Size,Md5Hash,Cfg);
1157
1158 // Check the size
1159 if (Size != Version->Size)
1160 {
1161 Status = StatError;
1162 ErrorText = _("Size mismatch");
1163 return;
1164 }
1165
1166 // Check the md5
1167 if (Md5Hash.empty() == false && MD5.empty() == false)
1168 {
1169 if (Md5Hash != MD5)
1170 {
1171 Status = StatError;
1172 ErrorText = _("MD5Sum mismatch");
1173 Rename(DestFile,DestFile + ".FAILED");
1174 return;
1175 }
1176 }
1177
1178 // Grab the output filename
1179 string FileName = LookupTag(Message,"Filename");
1180 if (FileName.empty() == true)
1181 {
1182 Status = StatError;
1183 ErrorText = "Method gave a blank filename";
1184 return;
1185 }
1186
1187 Complete = true;
1188
1189 // Reference filename
1190 if (FileName != DestFile)
1191 {
1192 StoreFilename = DestFile = FileName;
1193 Local = true;
1194 return;
1195 }
1196
1197 // Done, move it into position
1198 string FinalFile = _config->FindDir("Dir::Cache::Archives");
1199 FinalFile += flNotDir(StoreFilename);
1200 Rename(DestFile,FinalFile);
1201
1202 StoreFilename = DestFile = FinalFile;
1203 Complete = true;
1204 }
1205 /*}}}*/
1206 // AcqArchive::Failed - Failure handler /*{{{*/
1207 // ---------------------------------------------------------------------
1208 /* Here we try other sources */
1209 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1210 {
1211 ErrorText = LookupTag(Message,"Message");
1212
1213 /* We don't really want to retry on failed media swaps, this prevents
1214 that. An interesting observation is that permanent failures are not
1215 recorded. */
1216 if (Cnf->Removable == true &&
1217 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1218 {
1219 // Vf = Version.FileList();
1220 while (Vf.end() == false) Vf++;
1221 StoreFilename = string();
1222 Item::Failed(Message,Cnf);
1223 return;
1224 }
1225
1226 if (QueueNext() == false)
1227 {
1228 // This is the retry counter
1229 if (Retries != 0 &&
1230 Cnf->LocalOnly == false &&
1231 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1232 {
1233 Retries--;
1234 Vf = Version.FileList();
1235 if (QueueNext() == true)
1236 return;
1237 }
1238
1239 StoreFilename = string();
1240 Item::Failed(Message,Cnf);
1241 }
1242 }
1243 /*}}}*/
1244 // AcqArchive::IsTrusted - Determine whether this archive comes from a
1245 // trusted source /*{{{*/
1246 // ---------------------------------------------------------------------
1247 bool pkgAcqArchive::IsTrusted()
1248 {
1249 return Trusted;
1250 }
1251
1252 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
1253 // ---------------------------------------------------------------------
1254 /* */
1255 void pkgAcqArchive::Finished()
1256 {
1257 if (Status == pkgAcquire::Item::StatDone &&
1258 Complete == true)
1259 return;
1260 StoreFilename = string();
1261 }
1262 /*}}}*/
1263
1264 // AcqFile::pkgAcqFile - Constructor /*{{{*/
1265 // ---------------------------------------------------------------------
1266 /* The file is added to the queue */
1267 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
1268 unsigned long Size,string Dsc,string ShortDesc) :
1269 Item(Owner), Md5Hash(MD5)
1270 {
1271 Retries = _config->FindI("Acquire::Retries",0);
1272
1273 DestFile = flNotDir(URI);
1274
1275 // Create the item
1276 Desc.URI = URI;
1277 Desc.Description = Dsc;
1278 Desc.Owner = this;
1279
1280 // Set the short description to the archive component
1281 Desc.ShortDesc = ShortDesc;
1282
1283 // Get the transfer sizes
1284 FileSize = Size;
1285 struct stat Buf;
1286 if (stat(DestFile.c_str(),&Buf) == 0)
1287 {
1288 // Hmm, the partial file is too big, erase it
1289 if ((unsigned)Buf.st_size > Size)
1290 unlink(DestFile.c_str());
1291 else
1292 PartialSize = Buf.st_size;
1293 }
1294
1295 QueueURI(Desc);
1296 }
1297 /*}}}*/
1298 // AcqFile::Done - Item downloaded OK /*{{{*/
1299 // ---------------------------------------------------------------------
1300 /* */
1301 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5,
1302 pkgAcquire::MethodConfig *Cnf)
1303 {
1304 // Check the md5
1305 if (Md5Hash.empty() == false && MD5.empty() == false)
1306 {
1307 if (Md5Hash != MD5)
1308 {
1309 Status = StatError;
1310 ErrorText = "MD5Sum mismatch";
1311 Rename(DestFile,DestFile + ".FAILED");
1312 return;
1313 }
1314 }
1315
1316 Item::Done(Message,Size,MD5,Cnf);
1317
1318 string FileName = LookupTag(Message,"Filename");
1319 if (FileName.empty() == true)
1320 {
1321 Status = StatError;
1322 ErrorText = "Method gave a blank filename";
1323 return;
1324 }
1325
1326 Complete = true;
1327
1328 // The files timestamp matches
1329 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
1330 return;
1331
1332 // We have to copy it into place
1333 if (FileName != DestFile)
1334 {
1335 Local = true;
1336 if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
1337 Cnf->Removable == true)
1338 {
1339 Desc.URI = "copy:" + FileName;
1340 QueueURI(Desc);
1341 return;
1342 }
1343
1344 // Erase the file if it is a symlink so we can overwrite it
1345 struct stat St;
1346 if (lstat(DestFile.c_str(),&St) == 0)
1347 {
1348 if (S_ISLNK(St.st_mode) != 0)
1349 unlink(DestFile.c_str());
1350 }
1351
1352 // Symlink the file
1353 if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
1354 {
1355 ErrorText = "Link to " + DestFile + " failure ";
1356 Status = StatError;
1357 Complete = false;
1358 }
1359 }
1360 }
1361 /*}}}*/
1362 // AcqFile::Failed - Failure handler /*{{{*/
1363 // ---------------------------------------------------------------------
1364 /* Here we try other sources */
1365 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
1366 {
1367 ErrorText = LookupTag(Message,"Message");
1368
1369 // This is the retry counter
1370 if (Retries != 0 &&
1371 Cnf->LocalOnly == false &&
1372 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
1373 {
1374 Retries--;
1375 QueueURI(Desc);
1376 return;
1377 }
1378
1379 Item::Failed(Message,Cnf);
1380 }
1381 /*}}}*/