Packages.bz2 support from Michael Vogt
[ntk/apt.git] / apt-pkg / acquire-item.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire-item.cc,v 1.46 2003/02/02 22:19:17 jgg Exp $
4 /* ######################################################################
5
6 Acquire Item - Item to acquire
7
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #ifdef __GNUG__
17 #pragma implementation "apt-pkg/acquire-item.h"
18 #endif
19 #include <apt-pkg/acquire-item.h>
20 #include <apt-pkg/configuration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25
26 #include <apti18n.h>
27
28 #include <sys/stat.h>
29 #include <unistd.h>
30 #include <errno.h>
31 #include <string>
32 #include <stdio.h>
33 /*}}}*/
34
35 using std::string;
36
37 // Acquire::Item::Item - Constructor /*{{{*/
38 // ---------------------------------------------------------------------
39 /* */
40 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
41 PartialSize(0), Mode(0), ID(0), Complete(false),
42 Local(false), QueueCounter(0)
43 {
44 Owner->Add(this);
45 Status = StatIdle;
46 }
47 /*}}}*/
48 // Acquire::Item::~Item - Destructor /*{{{*/
49 // ---------------------------------------------------------------------
50 /* */
51 pkgAcquire::Item::~Item()
52 {
53 Owner->Remove(this);
54 }
55 /*}}}*/
56 // Acquire::Item::Failed - Item failed to download /*{{{*/
57 // ---------------------------------------------------------------------
58 /* We return to an idle state if there are still other queues that could
59 fetch this object */
60 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
61 {
62 Status = StatIdle;
63 ErrorText = LookupTag(Message,"Message");
64 if (QueueCounter <= 1)
65 {
66 /* This indicates that the file is not available right now but might
67 be sometime later. If we do a retry cycle then this should be
68 retried [CDROMs] */
69 if (Cnf->LocalOnly == true &&
70 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
71 {
72 Status = StatIdle;
73 Dequeue();
74 return;
75 }
76
77 Status = StatError;
78 Dequeue();
79 }
80 }
81 /*}}}*/
82 // Acquire::Item::Start - Item has begun to download /*{{{*/
83 // ---------------------------------------------------------------------
84 /* Stash status and the file size. Note that setting Complete means
85 sub-phases of the acquire process such as decompresion are operating */
86 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
87 {
88 Status = StatFetching;
89 if (FileSize == 0 && Complete == false)
90 FileSize = Size;
91 }
92 /*}}}*/
93 // Acquire::Item::Done - Item downloaded OK /*{{{*/
94 // ---------------------------------------------------------------------
95 /* */
96 void pkgAcquire::Item::Done(string Message,unsigned long Size,string,
97 pkgAcquire::MethodConfig *Cnf)
98 {
99 // We just downloaded something..
100 string FileName = LookupTag(Message,"Filename");
101 if (Complete == false && FileName == DestFile)
102 {
103 if (Owner->Log != 0)
104 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
105 }
106
107 if (FileSize == 0)
108 FileSize= Size;
109
110 Status = StatDone;
111 ErrorText = string();
112 Owner->Dequeue(this);
113 }
114 /*}}}*/
115 // Acquire::Item::Rename - Rename a file /*{{{*/
116 // ---------------------------------------------------------------------
117 /* This helper function is used by alot of item methods as thier final
118 step */
119 void pkgAcquire::Item::Rename(string From,string To)
120 {
121 if (rename(From.c_str(),To.c_str()) != 0)
122 {
123 char S[300];
124 snprintf(S,sizeof(S),_("rename failed, %s (%s -> %s)."),strerror(errno),
125 From.c_str(),To.c_str());
126 Status = StatError;
127 ErrorText = S;
128 }
129 }
130 /*}}}*/
131
132 // AcqIndex::AcqIndex - Constructor /*{{{*/
133 // ---------------------------------------------------------------------
134 /* The package file is added to the queue and a second class is
135 instantiated to fetch the revision file */
136 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
137 string URI,string URIDesc,string ShortDesc) :
138 Item(Owner), RealURI(URI)
139 {
140 Decompression = false;
141 Erase = false;
142
143 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
144 DestFile += URItoFileName(URI);
145
146 // Create the item
147 Desc.URI = URI + ".bz2";
148 Desc.Description = URIDesc;
149 Desc.Owner = this;
150 Desc.ShortDesc = ShortDesc;
151
152 QueueURI(Desc);
153 }
154 /*}}}*/
155 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
156 // ---------------------------------------------------------------------
157 /* The only header we use is the last-modified header. */
158 string pkgAcqIndex::Custom600Headers()
159 {
160 string Final = _config->FindDir("Dir::State::lists");
161 Final += URItoFileName(RealURI);
162
163 struct stat Buf;
164 if (stat(Final.c_str(),&Buf) != 0)
165 return "\nIndex-File: true";
166
167 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
168 }
169 /*}}}*/
170
171 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
172 {
173 // no .bz2 found, retry with .gz
174 if(Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1) == "bz2") {
175 Desc.URI = Desc.URI.substr(0,Desc.URI.size()-3) + "gz";
176 QueueURI(Desc);
177 return;
178 }
179
180
181 Item::Failed(Message,Cnf);
182 }
183
184
185 // AcqIndex::Done - Finished a fetch /*{{{*/
186 // ---------------------------------------------------------------------
187 /* This goes through a number of states.. On the initial fetch the
188 method could possibly return an alternate filename which points
189 to the uncompressed version of the file. If this is so the file
190 is copied into the partial directory. In all other cases the file
191 is decompressed with a gzip uri. */
192 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5,
193 pkgAcquire::MethodConfig *Cfg)
194 {
195 Item::Done(Message,Size,MD5,Cfg);
196
197 if (Decompression == true)
198 {
199 // Done, move it into position
200 string FinalFile = _config->FindDir("Dir::State::lists");
201 FinalFile += URItoFileName(RealURI);
202 Rename(DestFile,FinalFile);
203 chmod(FinalFile.c_str(),0644);
204
205 /* We restore the original name to DestFile so that the clean operation
206 will work OK */
207 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
208 DestFile += URItoFileName(RealURI);
209
210 // Remove the compressed version.
211 if (Erase == true)
212 unlink(DestFile.c_str());
213 return;
214 }
215
216 Erase = false;
217 Complete = true;
218
219 // Handle the unzipd case
220 string FileName = LookupTag(Message,"Alt-Filename");
221 if (FileName.empty() == false)
222 {
223 // The files timestamp matches
224 if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
225 return;
226
227 Decompression = true;
228 Local = true;
229 DestFile += ".decomp";
230 Desc.URI = "copy:" + FileName;
231 QueueURI(Desc);
232 Mode = "copy";
233 return;
234 }
235
236 FileName = LookupTag(Message,"Filename");
237 if (FileName.empty() == true)
238 {
239 Status = StatError;
240 ErrorText = "Method gave a blank filename";
241 }
242
243 // The files timestamp matches
244 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
245 return;
246
247 if (FileName == DestFile)
248 Erase = true;
249 else
250 Local = true;
251
252 string compExt = Desc.URI.substr(Desc.URI.size()-3,Desc.URI.size()-1);
253 char *decompProg;
254 if(compExt == "bz2")
255 decompProg = "bzip2";
256 else if(compExt == ".gz")
257 decompProg = "gzip";
258 else {
259 _error->Error("Unsupported extension: %s", compExt.c_str());
260 return;
261 }
262
263 Decompression = true;
264 DestFile += ".decomp";
265 Desc.URI = string(decompProg) + ":" + FileName;
266 QueueURI(Desc);
267 Mode = decompProg;
268 }
269 /*}}}*/
270
271 // AcqIndexRel::pkgAcqIndexRel - Constructor /*{{{*/
272 // ---------------------------------------------------------------------
273 /* The Release file is added to the queue */
274 pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
275 string URI,string URIDesc,string ShortDesc) :
276 Item(Owner), RealURI(URI)
277 {
278 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
279 DestFile += URItoFileName(URI);
280
281 // Create the item
282 Desc.URI = URI;
283 Desc.Description = URIDesc;
284 Desc.ShortDesc = ShortDesc;
285 Desc.Owner = this;
286
287 QueueURI(Desc);
288 }
289 /*}}}*/
290 // AcqIndexRel::Custom600Headers - Insert custom request headers /*{{{*/
291 // ---------------------------------------------------------------------
292 /* The only header we use is the last-modified header. */
293 string pkgAcqIndexRel::Custom600Headers()
294 {
295 string Final = _config->FindDir("Dir::State::lists");
296 Final += URItoFileName(RealURI);
297
298 struct stat Buf;
299 if (stat(Final.c_str(),&Buf) != 0)
300 return "\nIndex-File: true";
301
302 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
303 }
304 /*}}}*/
305 // AcqIndexRel::Done - Item downloaded OK /*{{{*/
306 // ---------------------------------------------------------------------
307 /* The release file was not placed into the download directory then
308 a copy URI is generated and it is copied there otherwise the file
309 in the partial directory is moved into .. and the URI is finished. */
310 void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5,
311 pkgAcquire::MethodConfig *Cfg)
312 {
313 Item::Done(Message,Size,MD5,Cfg);
314
315 string FileName = LookupTag(Message,"Filename");
316 if (FileName.empty() == true)
317 {
318 Status = StatError;
319 ErrorText = "Method gave a blank filename";
320 return;
321 }
322
323 Complete = true;
324
325 // The files timestamp matches
326 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
327 return;
328
329 // We have to copy it into place
330 if (FileName != DestFile)
331 {
332 Local = true;
333 Desc.URI = "copy:" + FileName;
334 QueueURI(Desc);
335 return;
336 }
337
338 // Done, move it into position
339 string FinalFile = _config->FindDir("Dir::State::lists");
340 FinalFile += URItoFileName(RealURI);
341 Rename(DestFile,FinalFile);
342
343 chmod(FinalFile.c_str(),0644);
344 }
345 /*}}}*/
346 // AcqIndexRel::Failed - Silence failure messages for missing rel files /*{{{*/
347 // ---------------------------------------------------------------------
348 /* */
349 void pkgAcqIndexRel::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
350 {
351 if (Cnf->LocalOnly == true ||
352 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
353 {
354 // Ignore this
355 Status = StatDone;
356 Complete = false;
357 Dequeue();
358 return;
359 }
360
361 Item::Failed(Message,Cnf);
362 }
363 /*}}}*/
364
365 // AcqArchive::AcqArchive - Constructor /*{{{*/
366 // ---------------------------------------------------------------------
367 /* This just sets up the initial fetch environment and queues the first
368 possibilitiy */
369 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
370 pkgRecords *Recs,pkgCache::VerIterator const &Version,
371 string &StoreFilename) :
372 Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
373 StoreFilename(StoreFilename), Vf(Version.FileList())
374 {
375 Retries = _config->FindI("Acquire::Retries",0);
376
377 if (Version.Arch() == 0)
378 {
379 _error->Error(_("I wasn't able to locate a file for the %s package. "
380 "This might mean you need to manually fix this package. "
381 "(due to missing arch)"),
382 Version.ParentPkg().Name());
383 return;
384 }
385
386 /* We need to find a filename to determine the extension. We make the
387 assumption here that all the available sources for this version share
388 the same extension.. */
389 // Skip not source sources, they do not have file fields.
390 for (; Vf.end() == false; Vf++)
391 {
392 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
393 continue;
394 break;
395 }
396
397 // Does not really matter here.. we are going to fail out below
398 if (Vf.end() != true)
399 {
400 // If this fails to get a file name we will bomb out below.
401 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
402 if (_error->PendingError() == true)
403 return;
404
405 // Generate the final file name as: package_version_arch.foo
406 StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
407 QuoteString(Version.VerStr(),"_:") + '_' +
408 QuoteString(Version.Arch(),"_:.") +
409 "." + flExtension(Parse.FileName());
410 }
411
412 // Select a source
413 if (QueueNext() == false && _error->PendingError() == false)
414 _error->Error(_("I wasn't able to locate file for the %s package. "
415 "This might mean you need to manually fix this package."),
416 Version.ParentPkg().Name());
417 }
418 /*}}}*/
419 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
420 // ---------------------------------------------------------------------
421 /* This queues the next available file version for download. It checks if
422 the archive is already available in the cache and stashs the MD5 for
423 checking later. */
424 bool pkgAcqArchive::QueueNext()
425 {
426 for (; Vf.end() == false; Vf++)
427 {
428 // Ignore not source sources
429 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
430 continue;
431
432 // Try to cross match against the source list
433 pkgIndexFile *Index;
434 if (Sources->FindIndex(Vf.File(),Index) == false)
435 continue;
436
437 // Grab the text package record
438 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
439 if (_error->PendingError() == true)
440 return false;
441
442 string PkgFile = Parse.FileName();
443 MD5 = Parse.MD5Hash();
444 if (PkgFile.empty() == true)
445 return _error->Error(_("The package index files are corrupted. No Filename: "
446 "field for package %s."),
447 Version.ParentPkg().Name());
448
449 // See if we already have the file. (Legacy filenames)
450 FileSize = Version->Size;
451 string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
452 struct stat Buf;
453 if (stat(FinalFile.c_str(),&Buf) == 0)
454 {
455 // Make sure the size matches
456 if ((unsigned)Buf.st_size == Version->Size)
457 {
458 Complete = true;
459 Local = true;
460 Status = StatDone;
461 StoreFilename = DestFile = FinalFile;
462 return true;
463 }
464
465 /* Hmm, we have a file and its size does not match, this means it is
466 an old style mismatched arch */
467 unlink(FinalFile.c_str());
468 }
469
470 // Check it again using the new style output filenames
471 FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
472 if (stat(FinalFile.c_str(),&Buf) == 0)
473 {
474 // Make sure the size matches
475 if ((unsigned)Buf.st_size == Version->Size)
476 {
477 Complete = true;
478 Local = true;
479 Status = StatDone;
480 StoreFilename = DestFile = FinalFile;
481 return true;
482 }
483
484 /* Hmm, we have a file and its size does not match, this shouldnt
485 happen.. */
486 unlink(FinalFile.c_str());
487 }
488
489 DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
490
491 // Check the destination file
492 if (stat(DestFile.c_str(),&Buf) == 0)
493 {
494 // Hmm, the partial file is too big, erase it
495 if ((unsigned)Buf.st_size > Version->Size)
496 unlink(DestFile.c_str());
497 else
498 PartialSize = Buf.st_size;
499 }
500
501 // Create the item
502 Local = false;
503 Desc.URI = Index->ArchiveURI(PkgFile);
504 Desc.Description = Index->ArchiveInfo(Version);
505 Desc.Owner = this;
506 Desc.ShortDesc = Version.ParentPkg().Name();
507 QueueURI(Desc);
508
509 Vf++;
510 return true;
511 }
512 return false;
513 }
514 /*}}}*/
515 // AcqArchive::Done - Finished fetching /*{{{*/
516 // ---------------------------------------------------------------------
517 /* */
518 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash,
519 pkgAcquire::MethodConfig *Cfg)
520 {
521 Item::Done(Message,Size,Md5Hash,Cfg);
522
523 // Check the size
524 if (Size != Version->Size)
525 {
526 Status = StatError;
527 ErrorText = _("Size mismatch");
528 return;
529 }
530
531 // Check the md5
532 if (Md5Hash.empty() == false && MD5.empty() == false)
533 {
534 if (Md5Hash != MD5)
535 {
536 Status = StatError;
537 ErrorText = _("MD5Sum mismatch");
538 Rename(DestFile,DestFile + ".FAILED");
539 return;
540 }
541 }
542
543 // Grab the output filename
544 string FileName = LookupTag(Message,"Filename");
545 if (FileName.empty() == true)
546 {
547 Status = StatError;
548 ErrorText = "Method gave a blank filename";
549 return;
550 }
551
552 Complete = true;
553
554 // Reference filename
555 if (FileName != DestFile)
556 {
557 StoreFilename = DestFile = FileName;
558 Local = true;
559 return;
560 }
561
562 // Done, move it into position
563 string FinalFile = _config->FindDir("Dir::Cache::Archives");
564 FinalFile += flNotDir(StoreFilename);
565 Rename(DestFile,FinalFile);
566
567 StoreFilename = DestFile = FinalFile;
568 Complete = true;
569 }
570 /*}}}*/
571 // AcqArchive::Failed - Failure handler /*{{{*/
572 // ---------------------------------------------------------------------
573 /* Here we try other sources */
574 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
575 {
576 ErrorText = LookupTag(Message,"Message");
577
578 /* We don't really want to retry on failed media swaps, this prevents
579 that. An interesting observation is that permanent failures are not
580 recorded. */
581 if (Cnf->Removable == true &&
582 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
583 {
584 // Vf = Version.FileList();
585 while (Vf.end() == false) Vf++;
586 StoreFilename = string();
587 Item::Failed(Message,Cnf);
588 return;
589 }
590
591 if (QueueNext() == false)
592 {
593 // This is the retry counter
594 if (Retries != 0 &&
595 Cnf->LocalOnly == false &&
596 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
597 {
598 Retries--;
599 Vf = Version.FileList();
600 if (QueueNext() == true)
601 return;
602 }
603
604 StoreFilename = string();
605 Item::Failed(Message,Cnf);
606 }
607 }
608 /*}}}*/
609 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
610 // ---------------------------------------------------------------------
611 /* */
612 void pkgAcqArchive::Finished()
613 {
614 if (Status == pkgAcquire::Item::StatDone &&
615 Complete == true)
616 return;
617 StoreFilename = string();
618 }
619 /*}}}*/
620
621 // AcqFile::pkgAcqFile - Constructor /*{{{*/
622 // ---------------------------------------------------------------------
623 /* The file is added to the queue */
624 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
625 unsigned long Size,string Dsc,string ShortDesc) :
626 Item(Owner), Md5Hash(MD5)
627 {
628 Retries = _config->FindI("Acquire::Retries",0);
629
630 DestFile = flNotDir(URI);
631
632 // Create the item
633 Desc.URI = URI;
634 Desc.Description = Dsc;
635 Desc.Owner = this;
636
637 // Set the short description to the archive component
638 Desc.ShortDesc = ShortDesc;
639
640 // Get the transfer sizes
641 FileSize = Size;
642 struct stat Buf;
643 if (stat(DestFile.c_str(),&Buf) == 0)
644 {
645 // Hmm, the partial file is too big, erase it
646 if ((unsigned)Buf.st_size > Size)
647 unlink(DestFile.c_str());
648 else
649 PartialSize = Buf.st_size;
650 }
651
652 QueueURI(Desc);
653 }
654 /*}}}*/
655 // AcqFile::Done - Item downloaded OK /*{{{*/
656 // ---------------------------------------------------------------------
657 /* */
658 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5,
659 pkgAcquire::MethodConfig *Cnf)
660 {
661 // Check the md5
662 if (Md5Hash.empty() == false && MD5.empty() == false)
663 {
664 if (Md5Hash != MD5)
665 {
666 Status = StatError;
667 ErrorText = "MD5Sum mismatch";
668 Rename(DestFile,DestFile + ".FAILED");
669 return;
670 }
671 }
672
673 Item::Done(Message,Size,MD5,Cnf);
674
675 string FileName = LookupTag(Message,"Filename");
676 if (FileName.empty() == true)
677 {
678 Status = StatError;
679 ErrorText = "Method gave a blank filename";
680 return;
681 }
682
683 Complete = true;
684
685 // The files timestamp matches
686 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
687 return;
688
689 // We have to copy it into place
690 if (FileName != DestFile)
691 {
692 Local = true;
693 if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
694 Cnf->Removable == true)
695 {
696 Desc.URI = "copy:" + FileName;
697 QueueURI(Desc);
698 return;
699 }
700
701 // Erase the file if it is a symlink so we can overwrite it
702 struct stat St;
703 if (lstat(DestFile.c_str(),&St) == 0)
704 {
705 if (S_ISLNK(St.st_mode) != 0)
706 unlink(DestFile.c_str());
707 }
708
709 // Symlink the file
710 if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
711 {
712 ErrorText = "Link to " + DestFile + " failure ";
713 Status = StatError;
714 Complete = false;
715 }
716 }
717 }
718 /*}}}*/
719 // AcqFile::Failed - Failure handler /*{{{*/
720 // ---------------------------------------------------------------------
721 /* Here we try other sources */
722 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
723 {
724 ErrorText = LookupTag(Message,"Message");
725
726 // This is the retry counter
727 if (Retries != 0 &&
728 Cnf->LocalOnly == false &&
729 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
730 {
731 Retries--;
732 QueueURI(Desc);
733 return;
734 }
735
736 Item::Failed(Message,Cnf);
737 }
738 /*}}}*/