* imported the changes from 0.5.30 to 0.5.30ubuntu2
[ntk/apt.git] / apt-pkg / acquire-item.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire-item.cc,v 1.46 2003/02/02 22:19:17 jgg Exp $
4 /* ######################################################################
5
6 Acquire Item - Item to acquire
7
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #ifdef __GNUG__
17 #pragma implementation "apt-pkg/acquire-item.h"
18 #endif
19 #include <apt-pkg/acquire-item.h>
20 #include <apt-pkg/configuration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25
26 #include <apti18n.h>
27
28 #include <sys/stat.h>
29 #include <unistd.h>
30 #include <errno.h>
31 #include <string>
32 #include <stdio.h>
33 /*}}}*/
34
35 using std::string;
36
37 // Acquire::Item::Item - Constructor /*{{{*/
38 // ---------------------------------------------------------------------
39 /* */
40 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
41 PartialSize(0), Mode(0), ID(0), Complete(false),
42 Local(false), QueueCounter(0)
43 {
44 Owner->Add(this);
45 Status = StatIdle;
46 }
47 /*}}}*/
48 // Acquire::Item::~Item - Destructor /*{{{*/
49 // ---------------------------------------------------------------------
50 /* */
51 pkgAcquire::Item::~Item()
52 {
53 Owner->Remove(this);
54 }
55 /*}}}*/
56 // Acquire::Item::Failed - Item failed to download /*{{{*/
57 // ---------------------------------------------------------------------
58 /* We return to an idle state if there are still other queues that could
59 fetch this object */
60 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
61 {
62 Status = StatIdle;
63 ErrorText = LookupTag(Message,"Message");
64 if (QueueCounter <= 1)
65 {
66 /* This indicates that the file is not available right now but might
67 be sometime later. If we do a retry cycle then this should be
68 retried [CDROMs] */
69 if (Cnf->LocalOnly == true &&
70 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
71 {
72 Status = StatIdle;
73 Dequeue();
74 return;
75 }
76
77 Status = StatError;
78 Dequeue();
79 }
80 }
81 /*}}}*/
82 // Acquire::Item::Start - Item has begun to download /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Stash status and the file size. Note that setting Complete means
85 sub-phases of the acquire process such as decompresion are operating */
86 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
87 {
88 Status = StatFetching;
89 if (FileSize == 0 && Complete == false)
90 FileSize = Size;
91 }
92 /*}}}*/
93 // Acquire::Item::Done - Item downloaded OK /*{{{*/
94 // ---------------------------------------------------------------------
95 /* */
96 void pkgAcquire::Item::Done(string Message,unsigned long Size,string,
97 pkgAcquire::MethodConfig *Cnf)
98 {
99 // We just downloaded something..
100 string FileName = LookupTag(Message,"Filename");
101 if (Complete == false && FileName == DestFile)
102 {
103 if (Owner->Log != 0)
104 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
105 }
106
107 if (FileSize == 0)
108 FileSize= Size;
109
110 Status = StatDone;
111 ErrorText = string();
112 Owner->Dequeue(this);
113 }
114 /*}}}*/
115 // Acquire::Item::Rename - Rename a file /*{{{*/
116 // ---------------------------------------------------------------------
117 /* This helper function is used by alot of item methods as thier final
118 step */
119 void pkgAcquire::Item::Rename(string From,string To)
120 {
121 if (rename(From.c_str(),To.c_str()) != 0)
122 {
123 char S[300];
124 snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
125 From.c_str(),To.c_str());
126 Status = StatError;
127 ErrorText = S;
128 }
129 }
130 /*}}}*/
131
132 // AcqIndex::AcqIndex - Constructor /*{{{*/
133 // ---------------------------------------------------------------------
134 /* The package file is added to the queue and a second class is
135 instantiated to fetch the revision file */
136 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
137 string URI,string URIDesc,string ShortDesc) :
138 Item(Owner), RealURI(URI)
139 {
140 Decompression = false;
141 Erase = false;
142
143 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
144 DestFile += URItoFileName(URI);
145
146 // Create the item
147 if(FileExists("/usr/bin/bzip2"))
148 Desc.URI = URI + ".bz2";
149 else
150 Desc.URI = URI + ".gz";
151 Desc.Description = URIDesc;
152 Desc.Owner = this;
153 Desc.ShortDesc = ShortDesc;
154
155 QueueURI(Desc);
156 }
157 /*}}}*/
158 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
159 // ---------------------------------------------------------------------
160 /* The only header we use is the last-modified header. */
161 string pkgAcqIndex::Custom600Headers()
162 {
163 string Final = _config->FindDir("Dir::State::lists");
164 Final += URItoFileName(RealURI);
165
166 struct stat Buf;
167 if (stat(Final.c_str(),&Buf) != 0)
168 return "\nIndex-File: true";
169
170 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
171 }
172 /*}}}*/
173
174 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
175 {
176 // no .bz2 found, retry with .gz
177 if(Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1) == "bz2") {
178 Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
179 QueueURI(Desc);
180 return;
181 }
182
183
184 Item::Failed(Message,Cnf);
185 }
186
187
188 // AcqIndex::Done - Finished a fetch /*{{{*/
189 // ---------------------------------------------------------------------
190 /* This goes through a number of states.. On the initial fetch the
191 method could possibly return an alternate filename which points
192 to the uncompressed version of the file. If this is so the file
193 is copied into the partial directory. In all other cases the file
194 is decompressed with a gzip uri. */
195 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5,
196 pkgAcquire::MethodConfig *Cfg)
197 {
198 Item::Done(Message,Size,MD5,Cfg);
199
200 if (Decompression == true)
201 {
202 // Done, move it into position
203 string FinalFile = _config->FindDir("Dir::State::lists");
204 FinalFile += URItoFileName(RealURI);
205 Rename(DestFile,FinalFile);
206 chmod(FinalFile.c_str(),0644);
207
208 /* We restore the original name to DestFile so that the clean operation
209 will work OK */
210 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
211 DestFile += URItoFileName(RealURI);
212
213 // Remove the compressed version.
214 if (Erase == true)
215 unlink(DestFile.c_str());
216 return;
217 }
218
219 Erase = false;
220 Complete = true;
221
222 // Handle the unzipd case
223 string FileName = LookupTag(Message,"Alt-Filename");
224 if (FileName.empty() == false)
225 {
226 // The files timestamp matches
227 if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
228 return;
229
230 Decompression = true;
231 Local = true;
232 DestFile += ".decomp";
233 Desc.URI = "copy:" + FileName;
234 QueueURI(Desc);
235 Mode = "copy";
236 return;
237 }
238
239 FileName = LookupTag(Message,"Filename");
240 if (FileName.empty() == true)
241 {
242 Status = StatError;
243 ErrorText = "Method gave a blank filename";
244 }
245
246 // The files timestamp matches
247 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
248 return;
249
250 if (FileName == DestFile)
251 Erase = true;
252 else
253 Local = true;
254
255 string compExt = Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1);
256 char *decompProg;
257 if(compExt == "bz2")
258 decompProg = "bzip2";
259 else if(compExt == ".gz")
260 decompProg = "gzip";
261 else {
262 _error->Error("Unsupported extension: %s", compExt.c_str());
263 return;
264 }
265
266 Decompression = true;
267 DestFile += ".decomp";
268 Desc.URI = string(decompProg) + ":" + FileName;
269 QueueURI(Desc);
270 Mode = decompProg;
271 }
272 /*}}}*/
273
274 // AcqIndexRel::pkgAcqIndexRel - Constructor /*{{{*/
275 // ---------------------------------------------------------------------
276 /* The Release file is added to the queue */
277 pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
278 string URI,string URIDesc,string ShortDesc) :
279 Item(Owner), RealURI(URI)
280 {
281 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
282 DestFile += URItoFileName(URI);
283
284 // Create the item
285 Desc.URI = URI;
286 Desc.Description = URIDesc;
287 Desc.ShortDesc = ShortDesc;
288 Desc.Owner = this;
289
290 QueueURI(Desc);
291 }
292 /*}}}*/
293 // AcqIndexRel::Custom600Headers - Insert custom request headers /*{{{*/
294 // ---------------------------------------------------------------------
295 /* The only header we use is the last-modified header. */
296 string pkgAcqIndexRel::Custom600Headers()
297 {
298 string Final = _config->FindDir("Dir::State::lists");
299 Final += URItoFileName(RealURI);
300
301 struct stat Buf;
302 if (stat(Final.c_str(),&Buf) != 0)
303 return "\nIndex-File: true";
304
305 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
306 }
307 /*}}}*/
308 // AcqIndexRel::Done - Item downloaded OK /*{{{*/
309 // ---------------------------------------------------------------------
310 /* The release file was not placed into the download directory then
311 a copy URI is generated and it is copied there otherwise the file
312 in the partial directory is moved into .. and the URI is finished. */
313 void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5,
314 pkgAcquire::MethodConfig *Cfg)
315 {
316 Item::Done(Message,Size,MD5,Cfg);
317
318 string FileName = LookupTag(Message,"Filename");
319 if (FileName.empty() == true)
320 {
321 Status = StatError;
322 ErrorText = "Method gave a blank filename";
323 return;
324 }
325
326 Complete = true;
327
328 // The files timestamp matches
329 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
330 return;
331
332 // We have to copy it into place
333 if (FileName != DestFile)
334 {
335 Local = true;
336 Desc.URI = "copy:" + FileName;
337 QueueURI(Desc);
338 return;
339 }
340
341 // Done, move it into position
342 string FinalFile = _config->FindDir("Dir::State::lists");
343 FinalFile += URItoFileName(RealURI);
344 Rename(DestFile,FinalFile);
345
346 chmod(FinalFile.c_str(),0644);
347 }
348 /*}}}*/
349 // AcqIndexRel::Failed - Silence failure messages for missing rel files /*{{{*/
350 // ---------------------------------------------------------------------
351 /* */
352 void pkgAcqIndexRel::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
353 {
354 if (Cnf->LocalOnly == true ||
355 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
356 {
357 // Ignore this
358 Status = StatDone;
359 Complete = false;
360 Dequeue();
361 return;
362 }
363
364 Item::Failed(Message,Cnf);
365 }
366 /*}}}*/
367
368 // AcqArchive::AcqArchive - Constructor /*{{{*/
369 // ---------------------------------------------------------------------
370 /* This just sets up the initial fetch environment and queues the first
371 possibilitiy */
372 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
373 pkgRecords *Recs,pkgCache::VerIterator const &Version,
374 string &StoreFilename) :
375 Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
376 StoreFilename(StoreFilename), Vf(Version.FileList())
377 {
378 Retries = _config->FindI("Acquire::Retries",0);
379
380 if (Version.Arch() == 0)
381 {
382 _error->Error(_("I wasn't able to locate a file for the %s package. "
383 "This might mean you need to manually fix this package. "
384 "(due to missing arch)"),
385 Version.ParentPkg().Name());
386 return;
387 }
388
389 /* We need to find a filename to determine the extension. We make the
390 assumption here that all the available sources for this version share
391 the same extension.. */
392 // Skip not source sources, they do not have file fields.
393 for (; Vf.end() == false; Vf++)
394 {
395 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
396 continue;
397 break;
398 }
399
400 // Does not really matter here.. we are going to fail out below
401 if (Vf.end() != true)
402 {
403 // If this fails to get a file name we will bomb out below.
404 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
405 if (_error->PendingError() == true)
406 return;
407
408 // Generate the final file name as: package_version_arch.foo
409 StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
410 QuoteString(Version.VerStr(),"_:") + '_' +
411 QuoteString(Version.Arch(),"_:.") +
412 "." + flExtension(Parse.FileName());
413 }
414
415 // Select a source
416 if (QueueNext() == false && _error->PendingError() == false)
417 _error->Error(_("I wasn't able to locate file for the %s package. "
418 "This might mean you need to manually fix this package."),
419 Version.ParentPkg().Name());
420 }
421 /*}}}*/
422 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
423 // ---------------------------------------------------------------------
424 /* This queues the next available file version for download. It checks if
425 the archive is already available in the cache and stashs the MD5 for
426 checking later. */
427 bool pkgAcqArchive::QueueNext()
428 {
429 for (; Vf.end() == false; Vf++)
430 {
431 // Ignore not source sources
432 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
433 continue;
434
435 // Try to cross match against the source list
436 pkgIndexFile *Index;
437 if (Sources->FindIndex(Vf.File(),Index) == false)
438 continue;
439
440 // Grab the text package record
441 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
442 if (_error->PendingError() == true)
443 return false;
444
445 string PkgFile = Parse.FileName();
446 MD5 = Parse.MD5Hash();
447 if (PkgFile.empty() == true)
448 return _error->Error(_("The package index files are corrupted. No Filename: "
449 "field for package %s."),
450 Version.ParentPkg().Name());
451
452 // See if we already have the file. (Legacy filenames)
453 FileSize = Version->Size;
454 string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
455 struct stat Buf;
456 if (stat(FinalFile.c_str(),&Buf) == 0)
457 {
458 // Make sure the size matches
459 if ((unsigned)Buf.st_size == Version->Size)
460 {
461 Complete = true;
462 Local = true;
463 Status = StatDone;
464 StoreFilename = DestFile = FinalFile;
465 return true;
466 }
467
468 /* Hmm, we have a file and its size does not match, this means it is
469 an old style mismatched arch */
470 unlink(FinalFile.c_str());
471 }
472
473 // Check it again using the new style output filenames
474 FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
475 if (stat(FinalFile.c_str(),&Buf) == 0)
476 {
477 // Make sure the size matches
478 if ((unsigned)Buf.st_size == Version->Size)
479 {
480 Complete = true;
481 Local = true;
482 Status = StatDone;
483 StoreFilename = DestFile = FinalFile;
484 return true;
485 }
486
487 /* Hmm, we have a file and its size does not match, this shouldnt
488 happen.. */
489 unlink(FinalFile.c_str());
490 }
491
492 DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
493
494 // Check the destination file
495 if (stat(DestFile.c_str(),&Buf) == 0)
496 {
497 // Hmm, the partial file is too big, erase it
498 if ((unsigned)Buf.st_size > Version->Size)
499 unlink(DestFile.c_str());
500 else
501 PartialSize = Buf.st_size;
502 }
503
504 // Create the item
505 Local = false;
506 Desc.URI = Index->ArchiveURI(PkgFile);
507 Desc.Description = Index->ArchiveInfo(Version);
508 Desc.Owner = this;
509 Desc.ShortDesc = Version.ParentPkg().Name();
510 QueueURI(Desc);
511
512 Vf++;
513 return true;
514 }
515 return false;
516 }
517 /*}}}*/
518 // AcqArchive::Done - Finished fetching /*{{{*/
519 // ---------------------------------------------------------------------
520 /* */
521 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash,
522 pkgAcquire::MethodConfig *Cfg)
523 {
524 Item::Done(Message,Size,Md5Hash,Cfg);
525
526 // Check the size
527 if (Size != Version->Size)
528 {
529 Status = StatError;
530 ErrorText = _("Size mismatch");
531 return;
532 }
533
534 // Check the md5
535 if (Md5Hash.empty() == false && MD5.empty() == false)
536 {
537 if (Md5Hash != MD5)
538 {
539 Status = StatError;
540 ErrorText = _("MD5Sum mismatch");
541 Rename(DestFile,DestFile + ".FAILED");
542 return;
543 }
544 }
545
546 // Grab the output filename
547 string FileName = LookupTag(Message,"Filename");
548 if (FileName.empty() == true)
549 {
550 Status = StatError;
551 ErrorText = "Method gave a blank filename";
552 return;
553 }
554
555 Complete = true;
556
557 // Reference filename
558 if (FileName != DestFile)
559 {
560 StoreFilename = DestFile = FileName;
561 Local = true;
562 return;
563 }
564
565 // Done, move it into position
566 string FinalFile = _config->FindDir("Dir::Cache::Archives");
567 FinalFile += flNotDir(StoreFilename);
568 Rename(DestFile,FinalFile);
569
570 StoreFilename = DestFile = FinalFile;
571 Complete = true;
572 }
573 /*}}}*/
574 // AcqArchive::Failed - Failure handler /*{{{*/
575 // ---------------------------------------------------------------------
576 /* Here we try other sources */
577 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
578 {
579 ErrorText = LookupTag(Message,"Message");
580
581 /* We don't really want to retry on failed media swaps, this prevents
582 that. An interesting observation is that permanent failures are not
583 recorded. */
584 if (Cnf->Removable == true &&
585 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
586 {
587 // Vf = Version.FileList();
588 while (Vf.end() == false) Vf++;
589 StoreFilename = string();
590 Item::Failed(Message,Cnf);
591 return;
592 }
593
594 if (QueueNext() == false)
595 {
596 // This is the retry counter
597 if (Retries != 0 &&
598 Cnf->LocalOnly == false &&
599 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
600 {
601 Retries--;
602 Vf = Version.FileList();
603 if (QueueNext() == true)
604 return;
605 }
606
607 StoreFilename = string();
608 Item::Failed(Message,Cnf);
609 }
610 }
611 /*}}}*/
612 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
613 // ---------------------------------------------------------------------
614 /* */
615 void pkgAcqArchive::Finished()
616 {
617 if (Status == pkgAcquire::Item::StatDone &&
618 Complete == true)
619 return;
620 StoreFilename = string();
621 }
622 /*}}}*/
623
624 // AcqFile::pkgAcqFile - Constructor /*{{{*/
625 // ---------------------------------------------------------------------
626 /* The file is added to the queue */
627 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
628 unsigned long Size,string Dsc,string ShortDesc) :
629 Item(Owner), Md5Hash(MD5)
630 {
631 Retries = _config->FindI("Acquire::Retries",0);
632
633 DestFile = flNotDir(URI);
634
635 // Create the item
636 Desc.URI = URI;
637 Desc.Description = Dsc;
638 Desc.Owner = this;
639
640 // Set the short description to the archive component
641 Desc.ShortDesc = ShortDesc;
642
643 // Get the transfer sizes
644 FileSize = Size;
645 struct stat Buf;
646 if (stat(DestFile.c_str(),&Buf) == 0)
647 {
648 // Hmm, the partial file is too big, erase it
649 if ((unsigned)Buf.st_size > Size)
650 unlink(DestFile.c_str());
651 else
652 PartialSize = Buf.st_size;
653 }
654
655 QueueURI(Desc);
656 }
657 /*}}}*/
658 // AcqFile::Done - Item downloaded OK /*{{{*/
659 // ---------------------------------------------------------------------
660 /* */
661 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5,
662 pkgAcquire::MethodConfig *Cnf)
663 {
664 // Check the md5
665 if (Md5Hash.empty() == false && MD5.empty() == false)
666 {
667 if (Md5Hash != MD5)
668 {
669 Status = StatError;
670 ErrorText = "MD5Sum mismatch";
671 Rename(DestFile,DestFile + ".FAILED");
672 return;
673 }
674 }
675
676 Item::Done(Message,Size,MD5,Cnf);
677
678 string FileName = LookupTag(Message,"Filename");
679 if (FileName.empty() == true)
680 {
681 Status = StatError;
682 ErrorText = "Method gave a blank filename";
683 return;
684 }
685
686 Complete = true;
687
688 // The files timestamp matches
689 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
690 return;
691
692 // We have to copy it into place
693 if (FileName != DestFile)
694 {
695 Local = true;
696 if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
697 Cnf->Removable == true)
698 {
699 Desc.URI = "copy:" + FileName;
700 QueueURI(Desc);
701 return;
702 }
703
704 // Erase the file if it is a symlink so we can overwrite it
705 struct stat St;
706 if (lstat(DestFile.c_str(),&St) == 0)
707 {
708 if (S_ISLNK(St.st_mode) != 0)
709 unlink(DestFile.c_str());
710 }
711
712 // Symlink the file
713 if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
714 {
715 ErrorText = "Link to " + DestFile + " failure ";
716 Status = StatError;
717 Complete = false;
718 }
719 }
720 }
721 /*}}}*/
722 // AcqFile::Failed - Failure handler /*{{{*/
723 // ---------------------------------------------------------------------
724 /* Here we try other sources */
725 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
726 {
727 ErrorText = LookupTag(Message,"Message");
728
729 // This is the retry counter
730 if (Retries != 0 &&
731 Cnf->LocalOnly == false &&
732 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
733 {
734 Retries--;
735 QueueURI(Desc);
736 return;
737 }
738
739 Item::Failed(Message,Cnf);
740 }
741 /*}}}*/