* apt-pkg/policy.cc:
[ntk/apt.git] / apt-pkg / acquire.h
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: acquire.h,v 1.29.2.1 2003/12/24 23:09:17 mdz Exp $
4 /* ######################################################################
5
6 Acquire - File Acquiration
7
8 This module contians the Acquire system. It is responsible for bringing
9 files into the local pathname space. It deals with URIs for files and
10 URI handlers responsible for downloading or finding the URIs.
11
12 Each file to download is represented by an Acquire::Item class subclassed
13 into a specialization. The Item class can add itself to several URI
14 acquire queues each prioritized by the download scheduler. When the
15 system is run the proper URI handlers are spawned and the the acquire
16 queues are fed into the handlers by the schedular until the queues are
17 empty. This allows for an Item to be downloaded from an alternate source
18 if the first try turns out to fail. It also alows concurrent downloading
19 of multiple items from multiple sources as well as dynamic balancing
20 of load between the sources.
21
22 Schedualing of downloads is done on a first ask first get basis. This
23 preserves the order of the download as much as possible. And means the
24 fastest source will tend to process the largest number of files.
25
26 Internal methods and queues for performing gzip decompression,
27 md5sum hashing and file copying are provided to allow items to apply
28 a number of transformations to the data files they are working with.
29
30 ##################################################################### */
31 /*}}}*/
32
33 /** \defgroup acquire Acquire system {{{
34 *
35 * \brief The Acquire system is responsible for retrieving files from
36 * local or remote URIs and postprocessing them (for instance,
37 * verifying their authenticity). The core class in this system is
38 * pkgAcquire, which is responsible for managing the download queues
39 * during the download. There is at least one download queue for
40 * each supported protocol; protocols such as http may provide one
41 * queue per host.
42 *
43 * Each file to download is represented by a subclass of
44 * pkgAcquire::Item. The files add themselves to the download
45 * queue(s) by providing their URI information to
46 * pkgAcquire::Item::QueueURI, which calls pkgAcquire::Enqueue.
47 *
48 * Once the system is set up, the Run method will spawn subprocesses
49 * to handle the enqueued URIs; the scheduler will then take items
50 * from the queues and feed them into the handlers until the queues
51 * are empty.
52 *
53 * \todo Acquire supports inserting an object into several queues at
54 * once, but it is not clear what its behavior in this case is, and
55 * no subclass of pkgAcquire::Item seems to actually use this
56 * capability.
57 */ /*}}}*/
58
59 /** \addtogroup acquire
60 *
61 * @{
62 *
63 * \file acquire.h
64 */
65
66 #ifndef PKGLIB_ACQUIRE_H
67 #define PKGLIB_ACQUIRE_H
68
69 #include <apt-pkg/macros.h>
70 #include <apt-pkg/weakptr.h>
71
72 #include <vector>
73 #include <string>
74
75 using std::vector;
76 using std::string;
77
78
79 #include <sys/time.h>
80 #include <unistd.h>
81
82 class pkgAcquireStatus;
83
84 /** \brief The core download scheduler. {{{
85 *
86 * This class represents an ongoing download. It manages the lists
87 * of active and pending downloads and handles setting up and tearing
88 * down download-related structures.
89 *
90 * \todo Why all the protected data items and methods?
91 */
92 class pkgAcquire
93 {
94 public:
95
96 class Item;
97 class Queue;
98 class Worker;
99 struct MethodConfig;
100 struct ItemDesc;
101 friend class Item;
102 friend class Queue;
103
104 typedef vector<Item *>::iterator ItemIterator;
105 typedef vector<Item *>::const_iterator ItemCIterator;
106
107 protected:
108
109 /** \brief A list of items to download.
110 *
111 * This is built monotonically as items are created and only
112 * emptied when the download shuts down.
113 */
114 vector<Item *> Items;
115
116 /** \brief The head of the list of active queues.
117 *
118 * \todo why a hand-managed list of queues instead of std::list or
119 * std::set?
120 */
121 Queue *Queues;
122
123 /** \brief The head of the list of active workers.
124 *
125 * \todo why a hand-managed list of workers instead of std::list
126 * or std::set?
127 */
128 Worker *Workers;
129
130 /** \brief The head of the list of acquire method configurations.
131 *
132 * Each protocol (http, ftp, gzip, etc) via which files can be
133 * fetched can have a representation in this list. The
134 * configuration data is filled in by parsing the 100 Capabilities
135 * string output by a method on startup (see
136 * pkgAcqMethod::pkgAcqMethod and pkgAcquire::GetConfig).
137 *
138 * \todo why a hand-managed config dictionary instead of std::map?
139 */
140 MethodConfig *Configs;
141
142 /** \brief The progress indicator for this download. */
143 pkgAcquireStatus *Log;
144
145 /** \brief The total size of the files which are to be fetched.
146 *
147 * This is not necessarily the total number of bytes to download
148 * when, e.g., download resumption and list updates via patches
149 * are taken into account.
150 */
151 unsigned long ToFetch;
152
153 // Configurable parameters for the scheduler
154
155 /** \brief Represents the queuing strategy for remote URIs. */
156 enum QueueStrategy {
157 /** \brief Generate one queue for each protocol/host combination; downloads from
158 * multiple hosts can proceed in parallel.
159 */
160 QueueHost,
161 /** \brief Generate a single queue for each protocol; serialize
162 * downloads from multiple hosts.
163 */
164 QueueAccess} QueueMode;
165
166 /** \brief If \b true, debugging information will be dumped to std::clog. */
167 bool const Debug;
168 /** \brief If \b true, a download is currently in progress. */
169 bool Running;
170
171 /** \brief Add the given item to the list of items. */
172 void Add(Item *Item);
173
174 /** \brief Remove the given item from the list of items. */
175 void Remove(Item *Item);
176
177 /** \brief Add the given worker to the list of workers. */
178 void Add(Worker *Work);
179
180 /** \brief Remove the given worker from the list of workers. */
181 void Remove(Worker *Work);
182
183 /** \brief Insert the given fetch request into the appropriate queue.
184 *
185 * \param Item The URI to download and the item to download it
186 * for. Copied by value into the queue; no reference to Item is
187 * retained.
188 */
189 void Enqueue(ItemDesc &Item);
190
191 /** \brief Remove all fetch requests for this item from all queues. */
192 void Dequeue(Item *Item);
193
194 /** \brief Determine the fetch method and queue of a URI.
195 *
196 * \param URI The URI to fetch.
197 *
198 * \param[out] Config A location in which to place the method via
199 * which the URI is to be fetched.
200 *
201 * \return the string-name of the queue in which a fetch request
202 * for the given URI should be placed.
203 */
204 string QueueName(string URI,MethodConfig const *&Config);
205
206 /** \brief Build up the set of file descriptors upon which select() should
207 * block.
208 *
209 * The default implementation inserts the file descriptors
210 * corresponding to active downloads.
211 *
212 * \param[out] Fd The largest file descriptor in the generated sets.
213 *
214 * \param[out] RSet The set of file descriptors that should be
215 * watched for input.
216 *
217 * \param[out] WSet The set of file descriptors that should be
218 * watched for output.
219 */
220 virtual void SetFds(int &Fd,fd_set *RSet,fd_set *WSet);
221
222 /** Handle input from and output to file descriptors which select()
223 * has determined are ready. The default implementation
224 * dispatches to all active downloads.
225 *
226 * \param RSet The set of file descriptors that are ready for
227 * input.
228 *
229 * \param WSet The set of file descriptors that are ready for
230 * output.
231 */
232 virtual void RunFds(fd_set *RSet,fd_set *WSet);
233
234 /** \brief Check for idle queues with ready-to-fetch items.
235 *
236 * Called by pkgAcquire::Queue::Done each time an item is dequeued
237 * but remains on some queues; i.e., another queue should start
238 * fetching it.
239 */
240 void Bump();
241
242 public:
243
244 /** \brief Retrieve information about a fetch method by name.
245 *
246 * \param Access The name of the method to look up.
247 *
248 * \return the method whose name is Access, or \b NULL if no such method exists.
249 */
250 MethodConfig *GetConfig(string Access);
251
252 /** \brief Provides information on how a download terminated. */
253 enum RunResult {
254 /** \brief All files were fetched successfully. */
255 Continue,
256
257 /** \brief Some files failed to download. */
258 Failed,
259
260 /** \brief The download was cancelled by the user (i.e., #Log's
261 * pkgAcquireStatus::Pulse() method returned \b false).
262 */
263 Cancelled};
264
265 /** \brief Download all the items that have been Add()ed to this
266 * download process.
267 *
268 * This method will block until the download completes, invoking
269 * methods on #Log to report on the progress of the download.
270 *
271 * \param PulseInterval The method pkgAcquireStatus::Pulse will be
272 * invoked on #Log at intervals of PulseInterval milliseconds.
273 *
274 * \return the result of the download.
275 */
276 RunResult Run(int PulseInterval=500000);
277
278 /** \brief Remove all items from this download process, terminate
279 * all download workers, and empty all queues.
280 */
281 void Shutdown();
282
283 /** \brief Get the first #Worker object.
284 *
285 * \return the first active worker in this download process.
286 */
287 inline Worker *WorkersBegin() {return Workers;};
288
289 /** \brief Advance to the next #Worker object.
290 *
291 * \return the worker immediately following I, or \b NULL if none
292 * exists.
293 */
294 Worker *WorkerStep(Worker *I);
295
296 /** \brief Get the head of the list of items. */
297 inline ItemIterator ItemsBegin() {return Items.begin();};
298
299 /** \brief Get the end iterator of the list of items. */
300 inline ItemIterator ItemsEnd() {return Items.end();};
301
302 // Iterate over queued Item URIs
303 class UriIterator;
304 /** \brief Get the head of the list of enqueued item URIs.
305 *
306 * This iterator will step over every element of every active
307 * queue.
308 */
309 UriIterator UriBegin();
310 /** \brief Get the end iterator of the list of enqueued item URIs. */
311 UriIterator UriEnd();
312
313 /** Deletes each entry in the given directory that is not being
314 * downloaded by this object. For instance, when downloading new
315 * list files, calling Clean() will delete the old ones.
316 *
317 * \param Dir The directory to be cleaned out.
318 *
319 * \return \b true if the directory exists and is readable.
320 */
321 bool Clean(string Dir);
322
323 /** \return the total size in bytes of all the items included in
324 * this download.
325 */
326 unsigned long long TotalNeeded();
327
328 /** \return the size in bytes of all non-local items included in
329 * this download.
330 */
331 unsigned long long FetchNeeded();
332
333 /** \return the amount of data to be fetched that is already
334 * present on the filesystem.
335 */
336 unsigned long long PartialPresent();
337
338 /** \brief Delayed constructor
339 *
340 * \param Progress indicator associated with this download or
341 * \b NULL for none. This object is not owned by the
342 * download process and will not be deleted when the pkgAcquire
343 * object is destroyed. Naturally, it should live for at least as
344 * long as the pkgAcquire object does.
345 * \param Lock defines a lock file that should be acquired to ensure
346 * only one Acquire class is in action at the time or an empty string
347 * if no lock file should be used.
348 */
349 bool Setup(pkgAcquireStatus *Progress = NULL, string const &Lock = "");
350
351 void SetLog(pkgAcquireStatus *Progress) { Log = Progress; }
352
353 /** \brief Construct a new pkgAcquire. */
354 pkgAcquire(pkgAcquireStatus *Log) __deprecated;
355 pkgAcquire();
356
357 /** \brief Destroy this pkgAcquire object.
358 *
359 * Destroys all queue, method, and item objects associated with
360 * this download.
361 */
362 virtual ~pkgAcquire();
363
364 private:
365 /** \brief FD of the Lock file we acquire in Setup (if any) */
366 int LockFD;
367 };
368
369 /** \brief Represents a single download source from which an item
370 * should be downloaded.
371 *
372 * An item may have several assocated ItemDescs over its lifetime.
373 */
374 struct pkgAcquire::ItemDesc : public WeakPointable
375 {
376 /** \brief The URI from which to download this item. */
377 string URI;
378 /** brief A description of this item. */
379 string Description;
380 /** brief A shorter description of this item. */
381 string ShortDesc;
382 /** brief The underlying item which is to be downloaded. */
383 Item *Owner;
384 };
385 /*}}}*/
386 /** \brief A single download queue in a pkgAcquire object. {{{
387 *
388 * \todo Why so many protected values?
389 */
390 class pkgAcquire::Queue
391 {
392 friend class pkgAcquire;
393 friend class pkgAcquire::UriIterator;
394 friend class pkgAcquire::Worker;
395
396 /** \brief The next queue in the pkgAcquire object's list of queues. */
397 Queue *Next;
398
399 protected:
400
401 /** \brief A single item placed in this queue. */
402 struct QItem : pkgAcquire::ItemDesc
403 {
404 /** \brief The next item in the queue. */
405 QItem *Next;
406 /** \brief The worker associated with this item, if any. */
407 pkgAcquire::Worker *Worker;
408
409 /** \brief Assign the ItemDesc portion of this QItem from
410 * another ItemDesc
411 */
412 void operator =(pkgAcquire::ItemDesc const &I)
413 {
414 URI = I.URI;
415 Description = I.Description;
416 ShortDesc = I.ShortDesc;
417 Owner = I.Owner;
418 };
419 };
420
421 /** \brief The name of this queue. */
422 string Name;
423
424 /** \brief The head of the list of items contained in this queue.
425 *
426 * \todo why a by-hand list instead of an STL structure?
427 */
428 QItem *Items;
429
430 /** \brief The head of the list of workers associated with this queue.
431 *
432 * \todo This is plural because support exists in Queue for
433 * multiple workers. However, it does not appear that there is
434 * any way to actually associate more than one worker with a
435 * queue.
436 *
437 * \todo Why not just use a std::set?
438 */
439 pkgAcquire::Worker *Workers;
440
441 /** \brief the download scheduler with which this queue is associated. */
442 pkgAcquire *Owner;
443
444 /** \brief The number of entries in this queue that are currently
445 * being downloaded.
446 */
447 signed long PipeDepth;
448
449 /** \brief The maximum number of entries that this queue will
450 * attempt to download at once.
451 */
452 unsigned long MaxPipeDepth;
453
454 public:
455
456 /** \brief Insert the given fetch request into this queue.
457 *
458 * \return \b true if the queuing was successful. May return
459 * \b false if the Item is already in the queue
460 */
461 bool Enqueue(ItemDesc &Item);
462
463 /** \brief Remove all fetch requests for the given item from this queue.
464 *
465 * \return \b true if at least one request was removed from the queue.
466 */
467 bool Dequeue(Item *Owner);
468
469 /** \brief Locate an item in this queue.
470 *
471 * \param URI A URI to match against.
472 * \param Owner A pkgAcquire::Worker to match against.
473 *
474 * \return the first item in the queue whose URI is #URI and that
475 * is being downloaded by #Owner.
476 */
477 QItem *FindItem(string URI,pkgAcquire::Worker *Owner);
478
479 /** Presumably this should start downloading an item?
480 *
481 * \todo Unimplemented. Implement it or remove?
482 */
483 bool ItemStart(QItem *Itm,unsigned long Size);
484
485 /** \brief Remove the given item from this queue and set its state
486 * to pkgAcquire::Item::StatDone.
487 *
488 * If this is the only queue containing the item, the item is also
489 * removed from the main queue by calling pkgAcquire::Dequeue.
490 *
491 * \param Itm The item to remove.
492 *
493 * \return \b true if no errors are encountered.
494 */
495 bool ItemDone(QItem *Itm);
496
497 /** \brief Start the worker process associated with this queue.
498 *
499 * If a worker process is already associated with this queue,
500 * this is equivalent to calling Cycle().
501 *
502 * \return \b true if the startup was successful.
503 */
504 bool Startup();
505
506 /** \brief Shut down the worker process associated with this queue.
507 *
508 * \param Final If \b true, then the process is stopped unconditionally.
509 * Otherwise, it is only stopped if it does not need cleanup
510 * as indicated by the pkgAcqMethod::NeedsCleanup member of
511 * its configuration.
512 *
513 * \return \b true.
514 */
515 bool Shutdown(bool Final);
516
517 /** \brief Send idle items to the worker process.
518 *
519 * Fills up the pipeline by inserting idle items into the worker's queue.
520 */
521 bool Cycle();
522
523 /** \brief Check for items that could be enqueued.
524 *
525 * Call this after an item placed in multiple queues has gone from
526 * the pkgAcquire::Item::StatFetching state to the
527 * pkgAcquire::Item::StatIdle state, to possibly refill an empty queue.
528 * This is an alias for Cycle().
529 *
530 * \todo Why both this and Cycle()? Are they expected to be
531 * different someday?
532 */
533 void Bump();
534
535 /** \brief Create a new Queue.
536 *
537 * \param Name The name of the new queue.
538 * \param Owner The download process that owns the new queue.
539 */
540 Queue(string Name,pkgAcquire *Owner);
541
542 /** Shut down all the worker processes associated with this queue
543 * and empty the queue.
544 */
545 ~Queue();
546 };
547 /*}}}*/
548 /** \brief Iterates over all the URIs being fetched by a pkgAcquire object. {{{*/
549 class pkgAcquire::UriIterator
550 {
551 /** The next queue to iterate over. */
552 pkgAcquire::Queue *CurQ;
553 /** The item that we currently point at. */
554 pkgAcquire::Queue::QItem *CurItem;
555
556 public:
557
558 inline void operator ++() {operator ++(0);};
559
560 void operator ++(int)
561 {
562 CurItem = CurItem->Next;
563 while (CurItem == 0 && CurQ != 0)
564 {
565 CurItem = CurQ->Items;
566 CurQ = CurQ->Next;
567 }
568 };
569
570 inline pkgAcquire::ItemDesc const *operator ->() const {return CurItem;};
571 inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;};
572 inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;};
573
574 /** \brief Create a new UriIterator.
575 *
576 * \param Q The queue over which this UriIterator should iterate.
577 */
578 UriIterator(pkgAcquire::Queue *Q) : CurQ(Q), CurItem(0)
579 {
580 while (CurItem == 0 && CurQ != 0)
581 {
582 CurItem = CurQ->Items;
583 CurQ = CurQ->Next;
584 }
585 }
586 };
587 /*}}}*/
588 /** \brief Information about the properties of a single acquire method. {{{*/
589 struct pkgAcquire::MethodConfig
590 {
591 /** \brief The next link on the acquire method list.
592 *
593 * \todo Why not an STL container?
594 */
595 MethodConfig *Next;
596
597 /** \brief The name of this acquire method (e.g., http). */
598 string Access;
599
600 /** \brief The implementation version of this acquire method. */
601 string Version;
602
603 /** \brief If \b true, only one download queue should be created for this
604 * method.
605 */
606 bool SingleInstance;
607
608 /** \brief If \b true, this method supports pipelined downloading. */
609 bool Pipeline;
610
611 /** \brief If \b true, the worker process should send the entire
612 * APT configuration tree to the fetch subprocess when it starts
613 * up.
614 */
615 bool SendConfig;
616
617 /** \brief If \b true, this fetch method does not require network access;
618 * all files are to be acquired from the local disk.
619 */
620 bool LocalOnly;
621
622 /** \brief If \b true, the subprocess has to carry out some cleanup
623 * actions before shutting down.
624 *
625 * For instance, the cdrom method needs to unmount the CD after it
626 * finishes.
627 */
628 bool NeedsCleanup;
629
630 /** \brief If \b true, this fetch method acquires files from removable media. */
631 bool Removable;
632
633 /** \brief Set up the default method parameters.
634 *
635 * All fields are initialized to NULL, "", or \b false as
636 * appropriate.
637 */
638 MethodConfig();
639 };
640 /*}}}*/
641 /** \brief A monitor object for downloads controlled by the pkgAcquire class. {{{
642 *
643 * \todo Why protected members?
644 *
645 * \todo Should the double members be uint64_t?
646 */
647 class pkgAcquireStatus
648 {
649 protected:
650
651 /** \brief The last time at which this monitor object was updated. */
652 struct timeval Time;
653
654 /** \brief The time at which the download started. */
655 struct timeval StartTime;
656
657 /** \brief The number of bytes fetched as of the previous call to
658 * pkgAcquireStatus::Pulse, including local items.
659 */
660 double LastBytes;
661
662 /** \brief The current rate of download as of the most recent call
663 * to pkgAcquireStatus::Pulse, in bytes per second.
664 */
665 double CurrentCPS;
666
667 /** \brief The number of bytes fetched as of the most recent call
668 * to pkgAcquireStatus::Pulse, including local items.
669 */
670 double CurrentBytes;
671
672 /** \brief The total number of bytes that need to be fetched.
673 *
674 * \warning This member is inaccurate, as new items might be
675 * enqueued while the download is in progress!
676 */
677 double TotalBytes;
678
679 /** \brief The total number of bytes accounted for by items that
680 * were successfully fetched.
681 */
682 double FetchedBytes;
683
684 /** \brief The amount of time that has elapsed since the download
685 * started.
686 */
687 unsigned long ElapsedTime;
688
689 /** \brief The total number of items that need to be fetched.
690 *
691 * \warning This member is inaccurate, as new items might be
692 * enqueued while the download is in progress!
693 */
694 unsigned long TotalItems;
695
696 /** \brief The number of items that have been successfully downloaded. */
697 unsigned long CurrentItems;
698
699 public:
700
701 /** \brief If \b true, the download scheduler should call Pulse()
702 * at the next available opportunity.
703 */
704 bool Update;
705
706 /** \brief If \b true, extra Pulse() invocations will be performed.
707 *
708 * With this option set, Pulse() will be called every time that a
709 * download item starts downloading, finishes downloading, or
710 * terminates with an error.
711 */
712 bool MorePulses;
713
714 /** \brief Invoked when a local or remote file has been completely fetched.
715 *
716 * \param Size The size of the file fetched.
717 *
718 * \param ResumePoint How much of the file was already fetched.
719 */
720 virtual void Fetched(unsigned long Size,unsigned long ResumePoint);
721
722 /** \brief Invoked when the user should be prompted to change the
723 * inserted removable media.
724 *
725 * This method should not return until the user has confirmed to
726 * the user interface that the media change is complete.
727 *
728 * \param Media The name of the media type that should be changed.
729 *
730 * \param Drive The identifying name of the drive whose media
731 * should be changed.
732 *
733 * \return \b true if the user confirms the media change, \b
734 * false if it is cancelled.
735 *
736 * \todo This is a horrible blocking monster; it should be CPSed
737 * with prejudice.
738 */
739 virtual bool MediaChange(string Media,string Drive) = 0;
740
741 /** \brief Invoked when an item is confirmed to be up-to-date.
742
743 * For instance, when an HTTP download is informed that the file on
744 * the server was not modified.
745 */
746 virtual void IMSHit(pkgAcquire::ItemDesc &/*Itm*/) {};
747
748 /** \brief Invoked when some of an item's data is fetched. */
749 virtual void Fetch(pkgAcquire::ItemDesc &/*Itm*/) {};
750
751 /** \brief Invoked when an item is successfully and completely fetched. */
752 virtual void Done(pkgAcquire::ItemDesc &/*Itm*/) {};
753
754 /** \brief Invoked when the process of fetching an item encounters
755 * a fatal error.
756 */
757 virtual void Fail(pkgAcquire::ItemDesc &/*Itm*/) {};
758
759 /** \brief Periodically invoked while the Acquire process is underway.
760 *
761 * Subclasses should first call pkgAcquireStatus::Pulse(), then
762 * update their status output. The download process is blocked
763 * while Pulse() is being called.
764 *
765 * \return \b false if the user asked to cancel the whole Acquire process.
766 *
767 * \see pkgAcquire::Run
768 */
769 virtual bool Pulse(pkgAcquire *Owner);
770
771 /** \brief Invoked when the Acquire process starts running. */
772 virtual void Start();
773
774 /** \brief Invoked when the Acquire process stops running. */
775 virtual void Stop();
776
777 /** \brief Initialize all counters to 0 and the time to the current time. */
778 pkgAcquireStatus();
779 virtual ~pkgAcquireStatus() {};
780 };
781 /*}}}*/
782 /** @} */
783
784 #endif