1 // -*- mode: cpp; mode: fold -*-
3 // $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
4 /* ######################################################################
6 Mirror Aquire Method - This is the Mirror aquire method for APT.
8 ##################################################################### */
10 // Include Files /*{{{*/
11 #include <apt-pkg/fileutl.h>
12 #include <apt-pkg/acquire-method.h>
13 #include <apt-pkg/acquire-item.h>
14 #include <apt-pkg/acquire.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/hashes.h>
17 #include <apt-pkg/sourcelist.h>
26 #include <sys/types.h>
27 #include <sys/utsname.h>
40 * - works with http (only!)
41 * - always picks the first mirror from the list
42 * - call out to problem reporting script
43 * - supports "deb mirror://host/path/to/mirror-list/// dist component"
44 * - uses pkgAcqMethod::FailReason() to have a string representation
45 * of the failure that is also send to LP
48 * - deal with runing as non-root because we can't write to the lists
49 dir then -> use the cached mirror file
50 * - better method to download than having a pkgAcquire interface here
51 * and better error handling there!
52 * - support more than http
56 MirrorMethod::MirrorMethod()
57 : HttpMethod(), DownloadedMirrorFile(false)
61 // HttpMethod::Configuration - Handle a configuration message /*{{{*/
62 // ---------------------------------------------------------------------
63 /* We stash the desired pipeline depth */
64 bool MirrorMethod::Configuration(string Message
)
66 if (pkgAcqMethod::Configuration(Message
) == false)
68 Debug
= _config
->FindB("Debug::Acquire::mirror",false);
74 // clean the mirrors dir based on ttl information
75 bool MirrorMethod::Clean(string Dir
)
77 vector
<metaIndex
*>::const_iterator I
;
80 clog
<< "MirrorMethod::Clean(): " << Dir
<< endl
;
83 return _error
->Error("will not clean: '/'");
89 DIR *D
= opendir(Dir
.c_str());
91 return _error
->Errno("opendir",_("Unable to read %s"),Dir
.c_str());
93 string StartDir
= SafeGetCWD();
94 if (chdir(Dir
.c_str()) != 0)
97 return _error
->Errno("chdir",_("Unable to change to %s"),Dir
.c_str());
100 for (struct dirent
*Dir
= readdir(D
); Dir
!= 0; Dir
= readdir(D
))
103 if (strcmp(Dir
->d_name
,"lock") == 0 ||
104 strcmp(Dir
->d_name
,"partial") == 0 ||
105 strcmp(Dir
->d_name
,".") == 0 ||
106 strcmp(Dir
->d_name
,"..") == 0)
109 // see if we have that uri
110 for(I
=list
.begin(); I
!= list
.end(); I
++)
112 string uri
= (*I
)->GetURI();
113 if(uri
.find("mirror://") != 0)
115 string BaseUri
= uri
.substr(0,uri
.size()-1);
116 if (URItoFileName(BaseUri
) == Dir
->d_name
)
119 // nothing found, nuke it
124 chdir(StartDir
.c_str());
130 bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str
)
132 // not that great to use pkgAcquire here, but we do not have
133 // any other way right now
134 string fetch
= BaseUri
;
135 fetch
.replace(0,strlen("mirror://"),"http://");
138 clog
<< "MirrorMethod::DownloadMirrorFile(): '" << fetch
<< "'"
139 << " to " << MirrorFile
<< endl
;
142 new pkgAcqFile(&Fetcher
, fetch
, "", 0, "", "", "", MirrorFile
);
143 bool res
= (Fetcher
.Run() == pkgAcquire::Continue
);
145 DownloadedMirrorFile
= true;
146 chmod(MirrorFile
.c_str(), 0644);
151 clog
<< "MirrorMethod::DownloadMirrorFile() success: " << res
<< endl
;
156 // Randomizes the lines in the mirror file, this is used so that
157 // we spread the load on the mirrors evenly
158 bool MirrorMethod::RandomizeMirrorFile(string mirror_file
)
160 vector
<string
> content
;
163 if (!FileExists(mirror_file
))
167 ifstream
in(mirror_file
.c_str());
168 while ( !in
.eof() ) {
170 content
.push_back(line
);
173 // we want the file to be random for each different machine, but also
174 // "stable" on the same machine. this is to avoid running into out-of-sync
175 // issues (i.e. Release/Release.gpg different on each mirror)
178 if(uname(&buf
) == 0) {
179 for(i
=0,seed
=1; buf
.nodename
[i
] != 0; i
++) {
180 seed
= seed
* 31 + buf
.nodename
[i
];
184 random_shuffle(content
.begin(), content
.end());
187 ofstream
out(mirror_file
.c_str());
188 while ( !content
.empty()) {
189 line
= content
.back();
197 /* convert a the Queue->Uri back to the mirror base uri and look
198 * at all mirrors we have for this, this is needed as queue->uri
199 * may point to different mirrors (if TryNextMirror() was run)
201 void MirrorMethod::CurrentQueueUriToMirror()
203 // already in mirror:// style so nothing to do
204 if(Queue
->Uri
.find("mirror://") == 0)
207 // find current mirror and select next one
208 for (vector
<string
>::const_iterator mirror
= AllMirrors
.begin();
209 mirror
!= AllMirrors
.end(); ++mirror
)
211 if (Queue
->Uri
.find(*mirror
) == 0)
213 Queue
->Uri
.replace(0, mirror
->length(), BaseUri
);
217 _error
->Error("Internal error: Failed to convert %s back to %s",
218 Queue
->Uri
.c_str(), BaseUri
.c_str());
221 bool MirrorMethod::TryNextMirror()
223 // find current mirror and select next one
224 for (vector
<string
>::const_iterator mirror
= AllMirrors
.begin();
225 mirror
!= AllMirrors
.end(); ++mirror
)
227 if (Queue
->Uri
.find(*mirror
) != 0)
230 vector
<string
>::const_iterator nextmirror
= mirror
+ 1;
231 if (nextmirror
== AllMirrors
.end())
233 Queue
->Uri
.replace(0, mirror
->length(), *nextmirror
);
235 clog
<< "TryNextMirror: " << Queue
->Uri
<< endl
;
238 UsedMirror
= *nextmirror
;
239 Log("Switching mirror");
244 clog
<< "TryNextMirror could not find another mirror to try" << endl
;
249 bool MirrorMethod::InitMirrors()
251 // if we do not have a MirrorFile, fallback
252 if(!FileExists(MirrorFile
))
254 // FIXME: fallback to a default mirror here instead
255 // and provide a config option to define that default
256 return _error
->Error(_("No mirror file '%s' found "), MirrorFile
.c_str());
259 if (access(MirrorFile
.c_str(), R_OK
) != 0)
261 // FIXME: fallback to a default mirror here instead
262 // and provide a config option to define that default
263 return _error
->Error(_("Can not read mirror file '%s'"), MirrorFile
.c_str());
266 // FIXME: make the mirror selection more clever, do not
267 // just use the first one!
268 // BUT: we can not make this random, the mirror has to be
269 // stable accross session, because otherwise we can
270 // get into sync issues (got indexfiles from mirror A,
271 // but packages from mirror B - one might be out of date etc)
272 ifstream
in(MirrorFile
.c_str());
278 // ignore lines that start with #
279 if (s
.find("#") == 0)
281 // ignore empty lines
284 // ignore non http lines
285 if (s
.find("http://") != 0)
288 AllMirrors
.push_back(s
);
290 Mirror
= AllMirrors
[0];
295 string
MirrorMethod::GetMirrorFileName(string mirror_uri_str
)
298 - a mirror_uri_str looks like this:
299 mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg
301 - the matching source.list entry
302 deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main
304 - we actually want to go after:
305 http://people.ubuntu.com/~mvo/apt/mirror/mirrors
307 And we need to save the BaseUri for later:
308 - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors
310 FIXME: what if we have two similar prefixes?
311 mirror://people.ubuntu.com/~mvo/mirror
312 mirror://people.ubuntu.com/~mvo/mirror2
313 then mirror_uri_str looks like:
314 mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg
315 mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg
316 we search sources.list and find:
317 mirror://people.ubuntu.com/~mvo/apt/mirror
318 in both cases! So we need to apply some domain knowledge here :( and
319 check for /dists/ or /Release.gpg as suffixes
323 std::cerr
<< "GetMirrorFileName: " << mirror_uri_str
<< std::endl
;
325 // read sources.list and find match
326 vector
<metaIndex
*>::const_iterator I
;
329 for(I
=list
.begin(); I
!= list
.end(); I
++)
331 string uristr
= (*I
)->GetURI();
333 std::cerr
<< "Checking: " << uristr
<< std::endl
;
334 if(uristr
.substr(0,strlen("mirror://")) != string("mirror://"))
336 // find matching uri in sources.list
337 if(mirror_uri_str
.substr(0,uristr
.size()) == uristr
)
340 std::cerr
<< "found BaseURI: " << uristr
<< std::endl
;
341 BaseUri
= uristr
.substr(0,uristr
.size()-1);
345 name
= _config
->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri
);
349 cerr
<< "base-uri: " << BaseUri
<< endl
;
350 cerr
<< "mirror-file: " << name
<< endl
;
355 // MirrorMethod::Fetch - Fetch an item /*{{{*/
356 // ---------------------------------------------------------------------
357 /* This adds an item to the pipeline. We keep the pipeline at a fixed
359 bool MirrorMethod::Fetch(FetchItem
*Itm
)
362 clog
<< "MirrorMethod::Fetch()" << endl
;
364 // the http method uses Fetch(0) as a way to update the pipeline,
365 // just let it do its work in this case - Fetch() with a valid
366 // Itm will always run before the first Fetch(0)
368 return HttpMethod::Fetch(Itm
);
370 // if we don't have the name of the mirror file on disk yet,
371 // calculate it now (can be derived from the uri)
372 if(MirrorFile
.empty())
373 MirrorFile
= GetMirrorFileName(Itm
->Uri
);
375 // download mirror file once (if we are after index files)
376 if(Itm
->IndexFile
&& !DownloadedMirrorFile
)
378 Clean(_config
->FindDir("Dir::State::mirrors"));
379 if (DownloadMirrorFile(Itm
->Uri
))
380 RandomizeMirrorFile(MirrorFile
);
383 if(AllMirrors
.empty()) {
385 // no valid mirror selected, something went wrong downloading
386 // from the master mirror site most likely and there is
387 // no old mirror file availalbe
392 if(Itm
->Uri
.find("mirror://") != string::npos
)
393 Itm
->Uri
.replace(0,BaseUri
.size(), Mirror
);
396 clog
<< "Fetch: " << Itm
->Uri
<< endl
<< endl
;
398 // now run the real fetcher
399 return HttpMethod::Fetch(Itm
);
402 void MirrorMethod::Fail(string Err
,bool Transient
)
404 // FIXME: TryNextMirror is not ideal for indexfile as we may
405 // run into auth issues
408 clog
<< "Failure to get " << Queue
->Uri
<< endl
;
410 // try the next mirror on fail (if its not a expected failure,
411 // e.g. translations are ok to ignore)
412 if (!Queue
->FailIgnore
&& TryNextMirror())
415 // all mirrors failed, so bail out
417 strprintf(s
, _("[Mirror: %s]"), Mirror
.c_str());
420 CurrentQueueUriToMirror();
421 pkgAcqMethod::Fail(Err
, Transient
);
424 void MirrorMethod::URIStart(FetchResult
&Res
)
426 CurrentQueueUriToMirror();
427 pkgAcqMethod::URIStart(Res
);
430 void MirrorMethod::URIDone(FetchResult
&Res
,FetchResult
*Alt
)
432 CurrentQueueUriToMirror();
433 pkgAcqMethod::URIDone(Res
, Alt
);
439 setlocale(LC_ALL
, "");