Join with aliencode
[ntk/apt.git] / apt-pkg / acquire-item.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire-item.cc,v 1.42 2001/02/20 07:03:17 jgg Exp $
4 /* ######################################################################
5
6 Acquire Item - Item to acquire
7
8 Each item can download to exactly one file at a time. This means you
9 cannot create an item that fetches two uri's to two files at the same
10 time. The pkgAcqIndex class creates a second class upon instantiation
11 to fetch the other index files because of this.
12
13 ##################################################################### */
14 /*}}}*/
15 // Include Files /*{{{*/
16 #ifdef __GNUG__
17 #pragma implementation "apt-pkg/acquire-item.h"
18 #endif
19 #include <apt-pkg/acquire-item.h>
20 #include <apt-pkg/configuration.h>
21 #include <apt-pkg/sourcelist.h>
22 #include <apt-pkg/error.h>
23 #include <apt-pkg/strutl.h>
24 #include <apt-pkg/fileutl.h>
25
26 #include <apti18n.h>
27
28 #include <sys/stat.h>
29 #include <unistd.h>
30 #include <errno.h>
31 #include <string.h>
32 #include <stdio.h>
33 /*}}}*/
34
35 // Acquire::Item::Item - Constructor /*{{{*/
36 // ---------------------------------------------------------------------
37 /* */
38 pkgAcquire::Item::Item(pkgAcquire *Owner) : Owner(Owner), FileSize(0),
39 PartialSize(0), Mode(0), ID(0), Complete(false),
40 Local(false), QueueCounter(0)
41 {
42 Owner->Add(this);
43 Status = StatIdle;
44 }
45 /*}}}*/
46 // Acquire::Item::~Item - Destructor /*{{{*/
47 // ---------------------------------------------------------------------
48 /* */
49 pkgAcquire::Item::~Item()
50 {
51 Owner->Remove(this);
52 }
53 /*}}}*/
54 // Acquire::Item::Failed - Item failed to download /*{{{*/
55 // ---------------------------------------------------------------------
56 /* We return to an idle state if there are still other queues that could
57 fetch this object */
58 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
59 {
60 Status = StatIdle;
61 ErrorText = LookupTag(Message,"Message");
62 if (QueueCounter <= 1)
63 {
64 /* This indicates that the file is not available right now but might
65 be sometime later. If we do a retry cycle then this should be
66 retried [CDROMs] */
67 if (Cnf->LocalOnly == true &&
68 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
69 {
70 Status = StatIdle;
71 Dequeue();
72 return;
73 }
74
75 Status = StatError;
76 Dequeue();
77 }
78 }
79 /*}}}*/
80 // Acquire::Item::Start - Item has begun to download /*{{{*/
81 // ---------------------------------------------------------------------
82 /* Stash status and the file size. Note that setting Complete means
83 sub-phases of the acquire process such as decompresion are operating */
84 void pkgAcquire::Item::Start(string /*Message*/,unsigned long Size)
85 {
86 Status = StatFetching;
87 if (FileSize == 0 && Complete == false)
88 FileSize = Size;
89 }
90 /*}}}*/
91 // Acquire::Item::Done - Item downloaded OK /*{{{*/
92 // ---------------------------------------------------------------------
93 /* */
94 void pkgAcquire::Item::Done(string Message,unsigned long Size,string,
95 pkgAcquire::MethodConfig *Cnf)
96 {
97 // We just downloaded something..
98 string FileName = LookupTag(Message,"Filename");
99 if (Complete == false && FileName == DestFile)
100 {
101 if (Owner->Log != 0)
102 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
103 }
104
105 if (FileSize == 0)
106 FileSize= Size;
107
108 Status = StatDone;
109 ErrorText = string();
110 Owner->Dequeue(this);
111 }
112 /*}}}*/
113 // Acquire::Item::Rename - Rename a file /*{{{*/
114 // ---------------------------------------------------------------------
115 /* This helper function is used by alot of item methods as thier final
116 step */
117 void pkgAcquire::Item::Rename(string From,string To)
118 {
119 if (rename(From.c_str(),To.c_str()) != 0)
120 {
121 char S[300];
122 sprintf(S,_("rename failed, %s (%s -> %s)."),strerror(errno),
123 From.c_str(),To.c_str());
124 Status = StatError;
125 ErrorText = S;
126 }
127 }
128 /*}}}*/
129
130 // AcqIndex::AcqIndex - Constructor /*{{{*/
131 // ---------------------------------------------------------------------
132 /* The package file is added to the queue and a second class is
133 instantiated to fetch the revision file */
134 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
135 string URI,string URIDesc,string ShortDesc) :
136 Item(Owner), RealURI(URI)
137 {
138 Decompression = false;
139 Erase = false;
140
141 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
142 DestFile += URItoFileName(URI);
143
144 // Create the item
145 Desc.URI = URI + ".gz";
146 Desc.Description = URIDesc;
147 Desc.Owner = this;
148 Desc.ShortDesc = ShortDesc;
149
150 QueueURI(Desc);
151 }
152 /*}}}*/
153 // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/
154 // ---------------------------------------------------------------------
155 /* The only header we use is the last-modified header. */
156 string pkgAcqIndex::Custom600Headers()
157 {
158 string Final = _config->FindDir("Dir::State::lists");
159 Final += URItoFileName(RealURI);
160
161 struct stat Buf;
162 if (stat(Final.c_str(),&Buf) != 0)
163 return "\nIndex-File: true";
164
165 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
166 }
167 /*}}}*/
168 // AcqIndex::Done - Finished a fetch /*{{{*/
169 // ---------------------------------------------------------------------
170 /* This goes through a number of states.. On the initial fetch the
171 method could possibly return an alternate filename which points
172 to the uncompressed version of the file. If this is so the file
173 is copied into the partial directory. In all other cases the file
174 is decompressed with a gzip uri. */
175 void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5,
176 pkgAcquire::MethodConfig *Cfg)
177 {
178 Item::Done(Message,Size,MD5,Cfg);
179
180 if (Decompression == true)
181 {
182 // Done, move it into position
183 string FinalFile = _config->FindDir("Dir::State::lists");
184 FinalFile += URItoFileName(RealURI);
185 Rename(DestFile,FinalFile);
186
187 /* We restore the original name to DestFile so that the clean operation
188 will work OK */
189 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
190 DestFile += URItoFileName(RealURI);
191
192 // Remove the compressed version.
193 if (Erase == true)
194 unlink(DestFile.c_str());
195 return;
196 }
197
198 Erase = false;
199 Complete = true;
200
201 // Handle the unzipd case
202 string FileName = LookupTag(Message,"Alt-Filename");
203 if (FileName.empty() == false)
204 {
205 // The files timestamp matches
206 if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
207 return;
208
209 Decompression = true;
210 Local = true;
211 DestFile += ".decomp";
212 Desc.URI = "copy:" + FileName;
213 QueueURI(Desc);
214 Mode = "copy";
215 return;
216 }
217
218 FileName = LookupTag(Message,"Filename");
219 if (FileName.empty() == true)
220 {
221 Status = StatError;
222 ErrorText = "Method gave a blank filename";
223 }
224
225 // The files timestamp matches
226 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
227 return;
228
229 if (FileName == DestFile)
230 Erase = true;
231 else
232 Local = true;
233
234 Decompression = true;
235 DestFile += ".decomp";
236 Desc.URI = "gzip:" + FileName;
237 QueueURI(Desc);
238 Mode = "gzip";
239 }
240 /*}}}*/
241
242 // AcqIndexRel::pkgAcqIndexRel - Constructor /*{{{*/
243 // ---------------------------------------------------------------------
244 /* The Release file is added to the queue */
245 pkgAcqIndexRel::pkgAcqIndexRel(pkgAcquire *Owner,
246 string URI,string URIDesc,string ShortDesc) :
247 Item(Owner), RealURI(URI)
248 {
249 DestFile = _config->FindDir("Dir::State::lists") + "partial/";
250 DestFile += URItoFileName(URI);
251
252 // Create the item
253 Desc.URI = URI;
254 Desc.Description = URIDesc;
255 Desc.ShortDesc = ShortDesc;
256 Desc.Owner = this;
257
258 QueueURI(Desc);
259 }
260 /*}}}*/
261 // AcqIndexRel::Custom600Headers - Insert custom request headers /*{{{*/
262 // ---------------------------------------------------------------------
263 /* The only header we use is the last-modified header. */
264 string pkgAcqIndexRel::Custom600Headers()
265 {
266 string Final = _config->FindDir("Dir::State::lists");
267 Final += URItoFileName(RealURI);
268
269 struct stat Buf;
270 if (stat(Final.c_str(),&Buf) != 0)
271 return "\nIndex-File: true";
272
273 return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
274 }
275 /*}}}*/
276 // AcqIndexRel::Done - Item downloaded OK /*{{{*/
277 // ---------------------------------------------------------------------
278 /* The release file was not placed into the download directory then
279 a copy URI is generated and it is copied there otherwise the file
280 in the partial directory is moved into .. and the URI is finished. */
281 void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5,
282 pkgAcquire::MethodConfig *Cfg)
283 {
284 Item::Done(Message,Size,MD5,Cfg);
285
286 string FileName = LookupTag(Message,"Filename");
287 if (FileName.empty() == true)
288 {
289 Status = StatError;
290 ErrorText = "Method gave a blank filename";
291 return;
292 }
293
294 Complete = true;
295
296 // The files timestamp matches
297 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
298 return;
299
300 // We have to copy it into place
301 if (FileName != DestFile)
302 {
303 Local = true;
304 Desc.URI = "copy:" + FileName;
305 QueueURI(Desc);
306 return;
307 }
308
309 // Done, move it into position
310 string FinalFile = _config->FindDir("Dir::State::lists");
311 FinalFile += URItoFileName(RealURI);
312 Rename(DestFile,FinalFile);
313 }
314 /*}}}*/
315 // AcqIndexRel::Failed - Silence failure messages for missing rel files /*{{{*/
316 // ---------------------------------------------------------------------
317 /* */
318 void pkgAcqIndexRel::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
319 {
320 if (Cnf->LocalOnly == true ||
321 StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
322 {
323 // Ignore this
324 Status = StatDone;
325 Complete = false;
326 Dequeue();
327 return;
328 }
329
330 Item::Failed(Message,Cnf);
331 }
332 /*}}}*/
333
334 // AcqArchive::AcqArchive - Constructor /*{{{*/
335 // ---------------------------------------------------------------------
336 /* This just sets up the initial fetch environment and queues the first
337 possibilitiy */
338 pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
339 pkgRecords *Recs,pkgCache::VerIterator const &Version,
340 string &StoreFilename) :
341 Item(Owner), Version(Version), Sources(Sources), Recs(Recs),
342 StoreFilename(StoreFilename), Vf(Version.FileList())
343 {
344 Retries = _config->FindI("Acquire::Retries",0);
345
346 if (Version.Arch() == 0)
347 {
348 _error->Error(_("I wasn't able to locate file for the %s package. "
349 "This might mean you need to manually fix this package. (due to missing arch)"),
350 Version.ParentPkg().Name());
351 return;
352 }
353
354 /* We need to find a filename to determine the extension. We make the
355 assumption here that all the available sources for this version share
356 the same extension.. */
357 // Skip not source sources, they do not have file fields.
358 for (; Vf.end() == false; Vf++)
359 {
360 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
361 continue;
362 break;
363 }
364
365 // Does not really matter here.. we are going to fail out below
366 if (Vf.end() != true)
367 {
368 // If this fails to get a file name we will bomb out below.
369 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
370 if (_error->PendingError() == true)
371 return;
372
373 // Generate the final file name as: package_version_arch.foo
374 StoreFilename = QuoteString(Version.ParentPkg().Name(),"_:") + '_' +
375 QuoteString(Version.VerStr(),"_:") + '_' +
376 QuoteString(Version.Arch(),"_:.") +
377 "." + flExtension(Parse.FileName());
378 }
379
380 // Select a source
381 if (QueueNext() == false && _error->PendingError() == false)
382 _error->Error(_("I wasn't able to locate file for the %s package. "
383 "This might mean you need to manually fix this package."),
384 Version.ParentPkg().Name());
385 }
386 /*}}}*/
387 // AcqArchive::QueueNext - Queue the next file source /*{{{*/
388 // ---------------------------------------------------------------------
389 /* This queues the next available file version for download. It checks if
390 the archive is already available in the cache and stashs the MD5 for
391 checking later. */
392 bool pkgAcqArchive::QueueNext()
393 {
394 for (; Vf.end() == false; Vf++)
395 {
396 // Ignore not source sources
397 if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
398 continue;
399
400 // Try to cross match against the source list
401 pkgIndexFile *Index;
402 if (Sources->FindIndex(Vf.File(),Index) == false)
403 continue;
404
405 // Grab the text package record
406 pkgRecords::Parser &Parse = Recs->Lookup(Vf);
407 if (_error->PendingError() == true)
408 return false;
409
410 string PkgFile = Parse.FileName();
411 MD5 = Parse.MD5Hash();
412 if (PkgFile.empty() == true)
413 return _error->Error(_("The package index files are corrupted. No Filename: "
414 "field for package %s."),
415 Version.ParentPkg().Name());
416
417 // See if we already have the file. (Legacy filenames)
418 FileSize = Version->Size;
419 string FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(PkgFile);
420 struct stat Buf;
421 if (stat(FinalFile.c_str(),&Buf) == 0)
422 {
423 // Make sure the size matches
424 if ((unsigned)Buf.st_size == Version->Size)
425 {
426 Complete = true;
427 Local = true;
428 Status = StatDone;
429 StoreFilename = DestFile = FinalFile;
430 return true;
431 }
432
433 /* Hmm, we have a file and its size does not match, this means it is
434 an old style mismatched arch */
435 unlink(FinalFile.c_str());
436 }
437
438 // Check it again using the new style output filenames
439 FinalFile = _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
440 if (stat(FinalFile.c_str(),&Buf) == 0)
441 {
442 // Make sure the size matches
443 if ((unsigned)Buf.st_size == Version->Size)
444 {
445 Complete = true;
446 Local = true;
447 Status = StatDone;
448 StoreFilename = DestFile = FinalFile;
449 return true;
450 }
451
452 /* Hmm, we have a file and its size does not match, this shouldnt
453 happen.. */
454 unlink(FinalFile.c_str());
455 }
456
457 DestFile = _config->FindDir("Dir::Cache::Archives") + "partial/" + flNotDir(StoreFilename);
458
459 // Check the destination file
460 if (stat(DestFile.c_str(),&Buf) == 0)
461 {
462 // Hmm, the partial file is too big, erase it
463 if ((unsigned)Buf.st_size > Version->Size)
464 unlink(DestFile.c_str());
465 else
466 PartialSize = Buf.st_size;
467 }
468
469 // Create the item
470 Local = false;
471 Desc.URI = Index->ArchiveURI(PkgFile);
472 Desc.Description = Index->ArchiveInfo(Version);
473 Desc.Owner = this;
474 Desc.ShortDesc = Version.ParentPkg().Name();
475 QueueURI(Desc);
476
477 Vf++;
478 return true;
479 }
480 return false;
481 }
482 /*}}}*/
483 // AcqArchive::Done - Finished fetching /*{{{*/
484 // ---------------------------------------------------------------------
485 /* */
486 void pkgAcqArchive::Done(string Message,unsigned long Size,string Md5Hash,
487 pkgAcquire::MethodConfig *Cfg)
488 {
489 Item::Done(Message,Size,Md5Hash,Cfg);
490
491 // Check the size
492 if (Size != Version->Size)
493 {
494 Status = StatError;
495 ErrorText = _("Size mismatch");
496 return;
497 }
498
499 // Check the md5
500 if (Md5Hash.empty() == false && MD5.empty() == false)
501 {
502 if (Md5Hash != MD5)
503 {
504 Status = StatError;
505 ErrorText = _("MD5Sum mismatch");
506 Rename(DestFile,DestFile + ".FAILED");
507 return;
508 }
509 }
510
511 // Grab the output filename
512 string FileName = LookupTag(Message,"Filename");
513 if (FileName.empty() == true)
514 {
515 Status = StatError;
516 ErrorText = "Method gave a blank filename";
517 return;
518 }
519
520 Complete = true;
521
522 // Reference filename
523 if (FileName != DestFile)
524 {
525 StoreFilename = DestFile = FileName;
526 Local = true;
527 return;
528 }
529
530 // Done, move it into position
531 string FinalFile = _config->FindDir("Dir::Cache::Archives");
532 FinalFile += flNotDir(StoreFilename);
533 Rename(DestFile,FinalFile);
534
535 StoreFilename = DestFile = FinalFile;
536 Complete = true;
537 }
538 /*}}}*/
539 // AcqArchive::Failed - Failure handler /*{{{*/
540 // ---------------------------------------------------------------------
541 /* Here we try other sources */
542 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
543 {
544 ErrorText = LookupTag(Message,"Message");
545
546 /* We don't really want to retry on failed media swaps, this prevents
547 that. An interesting observation is that permanent failures are not
548 recorded. */
549 if (Cnf->Removable == true &&
550 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
551 {
552 // Vf = Version.FileList();
553 while (Vf.end() == false) Vf++;
554 StoreFilename = string();
555 Item::Failed(Message,Cnf);
556 return;
557 }
558
559 if (QueueNext() == false)
560 {
561 // This is the retry counter
562 if (Retries != 0 &&
563 Cnf->LocalOnly == false &&
564 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
565 {
566 Retries--;
567 Vf = Version.FileList();
568 if (QueueNext() == true)
569 return;
570 }
571
572 StoreFilename = string();
573 Item::Failed(Message,Cnf);
574 }
575 }
576 /*}}}*/
577 // AcqArchive::Finished - Fetching has finished, tidy up /*{{{*/
578 // ---------------------------------------------------------------------
579 /* */
580 void pkgAcqArchive::Finished()
581 {
582 if (Status == pkgAcquire::Item::StatDone &&
583 Complete == true)
584 return;
585 StoreFilename = string();
586 }
587 /*}}}*/
588
589 // AcqFile::pkgAcqFile - Constructor /*{{{*/
590 // ---------------------------------------------------------------------
591 /* The file is added to the queue */
592 pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string MD5,
593 unsigned long Size,string Dsc,string ShortDesc) :
594 Item(Owner), Md5Hash(MD5)
595 {
596 Retries = _config->FindI("Acquire::Retries",0);
597
598 DestFile = flNotDir(URI);
599
600 // Create the item
601 Desc.URI = URI;
602 Desc.Description = Dsc;
603 Desc.Owner = this;
604
605 // Set the short description to the archive component
606 Desc.ShortDesc = ShortDesc;
607
608 // Get the transfer sizes
609 FileSize = Size;
610 struct stat Buf;
611 if (stat(DestFile.c_str(),&Buf) == 0)
612 {
613 // Hmm, the partial file is too big, erase it
614 if ((unsigned)Buf.st_size > Size)
615 unlink(DestFile.c_str());
616 else
617 PartialSize = Buf.st_size;
618 }
619
620 QueueURI(Desc);
621 }
622 /*}}}*/
623 // AcqFile::Done - Item downloaded OK /*{{{*/
624 // ---------------------------------------------------------------------
625 /* */
626 void pkgAcqFile::Done(string Message,unsigned long Size,string MD5,
627 pkgAcquire::MethodConfig *Cnf)
628 {
629 // Check the md5
630 if (Md5Hash.empty() == false && MD5.empty() == false)
631 {
632 if (Md5Hash != MD5)
633 {
634 Status = StatError;
635 ErrorText = "MD5Sum mismatch";
636 Rename(DestFile,DestFile + ".FAILED");
637 return;
638 }
639 }
640
641 Item::Done(Message,Size,MD5,Cnf);
642
643 string FileName = LookupTag(Message,"Filename");
644 if (FileName.empty() == true)
645 {
646 Status = StatError;
647 ErrorText = "Method gave a blank filename";
648 return;
649 }
650
651 Complete = true;
652
653 // The files timestamp matches
654 if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
655 return;
656
657 // We have to copy it into place
658 if (FileName != DestFile)
659 {
660 Local = true;
661 if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
662 Cnf->Removable == true)
663 {
664 Desc.URI = "copy:" + FileName;
665 QueueURI(Desc);
666 return;
667 }
668
669 // Erase the file if it is a symlink so we can overwrite it
670 struct stat St;
671 if (lstat(DestFile.c_str(),&St) == 0)
672 {
673 if (S_ISLNK(St.st_mode) != 0)
674 unlink(DestFile.c_str());
675 }
676
677 // Symlink the file
678 if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
679 {
680 ErrorText = "Link to " + DestFile + " failure ";
681 Status = StatError;
682 Complete = false;
683 }
684 }
685 }
686 /*}}}*/
687 // AcqFile::Failed - Failure handler /*{{{*/
688 // ---------------------------------------------------------------------
689 /* Here we try other sources */
690 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
691 {
692 ErrorText = LookupTag(Message,"Message");
693
694 // This is the retry counter
695 if (Retries != 0 &&
696 Cnf->LocalOnly == false &&
697 StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
698 {
699 Retries--;
700 QueueURI(Desc);
701 return;
702 }
703
704 Item::Failed(Message,Cnf);
705 }
706 /*}}}*/