Better handling of missing services
[ntk/apt.git] / methods / http.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: http.cc,v 1.36 1999/07/18 23:06:56 jgg Exp $
4 /* ######################################################################
5
6 HTTP Aquire Method - This is the HTTP aquire method for APT.
7
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
9 pipelining, range, if-range and so on. It accepts on the command line
10 a list of url destination pairs and writes to stdout the status of the
11 operation as defined in the APT method spec.
12
13 It is based on a doubly buffered select loop. All the requests are
14 fed into a single output buffer that is constantly fed out the
15 socket. This provides ideal pipelining as in many cases all of the
16 requests will fit into a single packet. The input socket is buffered
17 the same way and fed into the fd for the file.
18
19 This double buffering provides fairly substantial transfer rates,
20 compared to wget the http method is about 4% faster. Most importantly,
21 when HTTP is compared with FTP as a protocol the speed difference is
22 huge. In tests over the internet from two sites to llug (via ATM) this
23 program got 230k/s sustained http transfer rates. FTP on the other
24 hand topped out at 170k/s. That combined with the time to setup the
25 FTP connection makes HTTP a vastly superior protocol.
26
27 ##################################################################### */
28 /*}}}*/
29 // Include Files /*{{{*/
30 #include <apt-pkg/fileutl.h>
31 #include <apt-pkg/acquire-method.h>
32 #include <apt-pkg/error.h>
33 #include <apt-pkg/md5.h>
34
35 #include <sys/stat.h>
36 #include <sys/time.h>
37 #include <utime.h>
38 #include <unistd.h>
39 #include <signal.h>
40 #include <stdio.h>
41 #include <errno.h>
42
43 // Internet stuff
44 #include <netdb.h>
45
46 #include "connect.h"
47 #include "rfc2553emu.h"
48 #include "http.h"
49
50 /*}}}*/
51
52 string HttpMethod::FailFile;
53 int HttpMethod::FailFd = -1;
54 time_t HttpMethod::FailTime = 0;
55 unsigned long PipelineDepth = 10;
56 unsigned long TimeOut = 120;
57
58 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
59 // ---------------------------------------------------------------------
60 /* */
61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
62 {
63 Buf = new unsigned char[Size];
64 Reset();
65 }
66 /*}}}*/
67 // CircleBuf::Reset - Reset to the default state /*{{{*/
68 // ---------------------------------------------------------------------
69 /* */
70 void CircleBuf::Reset()
71 {
72 InP = 0;
73 OutP = 0;
74 StrPos = 0;
75 MaxGet = (unsigned int)-1;
76 OutQueue = string();
77 if (MD5 != 0)
78 {
79 delete MD5;
80 MD5 = new MD5Summation;
81 }
82 };
83 /*}}}*/
84 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
85 // ---------------------------------------------------------------------
86 /* This fills up the buffer with as much data as is in the FD, assuming it
87 is non-blocking.. */
88 bool CircleBuf::Read(int Fd)
89 {
90 while (1)
91 {
92 // Woops, buffer is full
93 if (InP - OutP == Size)
94 return true;
95
96 // Write the buffer segment
97 int Res;
98 Res = read(Fd,Buf + (InP%Size),LeftRead());
99
100 if (Res == 0)
101 return false;
102 if (Res < 0)
103 {
104 if (errno == EAGAIN)
105 return true;
106 return false;
107 }
108
109 if (InP == 0)
110 gettimeofday(&Start,0);
111 InP += Res;
112 }
113 }
114 /*}}}*/
115 // CircleBuf::Read - Put the string into the buffer /*{{{*/
116 // ---------------------------------------------------------------------
117 /* This will hold the string in and fill the buffer with it as it empties */
118 bool CircleBuf::Read(string Data)
119 {
120 OutQueue += Data;
121 FillOut();
122 return true;
123 }
124 /*}}}*/
125 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
126 // ---------------------------------------------------------------------
127 /* */
128 void CircleBuf::FillOut()
129 {
130 if (OutQueue.empty() == true)
131 return;
132 while (1)
133 {
134 // Woops, buffer is full
135 if (InP - OutP == Size)
136 return;
137
138 // Write the buffer segment
139 unsigned long Sz = LeftRead();
140 if (OutQueue.length() - StrPos < Sz)
141 Sz = OutQueue.length() - StrPos;
142 memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
143
144 // Advance
145 StrPos += Sz;
146 InP += Sz;
147 if (OutQueue.length() == StrPos)
148 {
149 StrPos = 0;
150 OutQueue = "";
151 return;
152 }
153 }
154 }
155 /*}}}*/
156 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
157 // ---------------------------------------------------------------------
158 /* This empties the buffer into the FD. */
159 bool CircleBuf::Write(int Fd)
160 {
161 while (1)
162 {
163 FillOut();
164
165 // Woops, buffer is empty
166 if (OutP == InP)
167 return true;
168
169 if (OutP == MaxGet)
170 return true;
171
172 // Write the buffer segment
173 int Res;
174 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
175
176 if (Res == 0)
177 return false;
178 if (Res < 0)
179 {
180 if (errno == EAGAIN)
181 return true;
182
183 return false;
184 }
185
186 if (MD5 != 0)
187 MD5->Add(Buf + (OutP%Size),Res);
188
189 OutP += Res;
190 }
191 }
192 /*}}}*/
193 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
194 // ---------------------------------------------------------------------
195 /* This copies till the first empty line */
196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
197 {
198 // We cheat and assume it is unneeded to have more than one buffer load
199 for (unsigned long I = OutP; I < InP; I++)
200 {
201 if (Buf[I%Size] != '\n')
202 continue;
203 for (I++; I < InP && Buf[I%Size] == '\r'; I++);
204
205 if (Single == false)
206 {
207 if (Buf[I%Size] != '\n')
208 continue;
209 for (I++; I < InP && Buf[I%Size] == '\r'; I++);
210 }
211
212 if (I > InP)
213 I = InP;
214
215 Data = "";
216 while (OutP < I)
217 {
218 unsigned long Sz = LeftWrite();
219 if (Sz == 0)
220 return false;
221 if (I - OutP < LeftWrite())
222 Sz = I - OutP;
223 Data += string((char *)(Buf + (OutP%Size)),Sz);
224 OutP += Sz;
225 }
226 return true;
227 }
228 return false;
229 }
230 /*}}}*/
231 // CircleBuf::Stats - Print out stats information /*{{{*/
232 // ---------------------------------------------------------------------
233 /* */
234 void CircleBuf::Stats()
235 {
236 if (InP == 0)
237 return;
238
239 struct timeval Stop;
240 gettimeofday(&Stop,0);
241 /* float Diff = Stop.tv_sec - Start.tv_sec +
242 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
243 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
244 }
245 /*}}}*/
246
247 // ServerState::ServerState - Constructor /*{{{*/
248 // ---------------------------------------------------------------------
249 /* */
250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
251 In(64*1024), Out(4*1024),
252 ServerName(Srv)
253 {
254 Reset();
255 }
256 /*}}}*/
257 // ServerState::Open - Open a connection to the server /*{{{*/
258 // ---------------------------------------------------------------------
259 /* This opens a connection to the server. */
260 string LastHost;
261 int LastPort = 0;
262 struct addrinfo *LastHostAddr = 0;
263 bool ServerState::Open()
264 {
265 // Use the already open connection if possible.
266 if (ServerFd != -1)
267 return true;
268
269 Close();
270 In.Reset();
271 Out.Reset();
272
273 // Determine the proxy setting
274 if (getenv("http_proxy") == 0)
275 {
276 string DefProxy = _config->Find("Acquire::http::Proxy");
277 string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
278 if (SpecificProxy.empty() == false)
279 {
280 if (SpecificProxy == "DIRECT")
281 Proxy = "";
282 else
283 Proxy = SpecificProxy;
284 }
285 else
286 Proxy = DefProxy;
287 }
288 else
289 Proxy = getenv("http_proxy");
290
291 // Determine what host and port to use based on the proxy settings
292 int Port = 0;
293 string Host;
294 if (Proxy.empty() == true)
295 {
296 if (ServerName.Port != 0)
297 Port = ServerName.Port;
298 Host = ServerName.Host;
299 }
300 else
301 {
302 if (Proxy.Port != 0)
303 Port = Proxy.Port;
304 Host = Proxy.Host;
305 }
306
307 // Connect to the remote server
308 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
309 return false;
310
311 return true;
312 }
313 /*}}}*/
314 // ServerState::Close - Close a connection to the server /*{{{*/
315 // ---------------------------------------------------------------------
316 /* */
317 bool ServerState::Close()
318 {
319 close(ServerFd);
320 ServerFd = -1;
321 return true;
322 }
323 /*}}}*/
324 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
325 // ---------------------------------------------------------------------
326 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
327 parse error occured */
328 int ServerState::RunHeaders()
329 {
330 State = Header;
331
332 Owner->Status("Waiting for file");
333
334 Major = 0;
335 Minor = 0;
336 Result = 0;
337 Size = 0;
338 StartPos = 0;
339 Encoding = Closes;
340 HaveContent = false;
341 time(&Date);
342
343 do
344 {
345 string Data;
346 if (In.WriteTillEl(Data) == false)
347 continue;
348
349 for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
350 {
351 string::const_iterator J = I;
352 for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
353 if (HeaderLine(string(I,J-I)) == false)
354 return 2;
355 I = J;
356 }
357 return 0;
358 }
359 while (Owner->Go(false,this) == true);
360
361 return 1;
362 }
363 /*}}}*/
364 // ServerState::RunData - Transfer the data from the socket /*{{{*/
365 // ---------------------------------------------------------------------
366 /* */
367 bool ServerState::RunData()
368 {
369 State = Data;
370
371 // Chunked transfer encoding is fun..
372 if (Encoding == Chunked)
373 {
374 while (1)
375 {
376 // Grab the block size
377 bool Last = true;
378 string Data;
379 In.Limit(-1);
380 do
381 {
382 if (In.WriteTillEl(Data,true) == true)
383 break;
384 }
385 while ((Last = Owner->Go(false,this)) == true);
386
387 if (Last == false)
388 return false;
389
390 // See if we are done
391 unsigned long Len = strtol(Data.c_str(),0,16);
392 if (Len == 0)
393 {
394 In.Limit(-1);
395
396 // We have to remove the entity trailer
397 Last = true;
398 do
399 {
400 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
401 break;
402 }
403 while ((Last = Owner->Go(false,this)) == true);
404 if (Last == false)
405 return false;
406 return !_error->PendingError();
407 }
408
409 // Transfer the block
410 In.Limit(Len);
411 while (Owner->Go(true,this) == true)
412 if (In.IsLimit() == true)
413 break;
414
415 // Error
416 if (In.IsLimit() == false)
417 return false;
418
419 // The server sends an extra new line before the next block specifier..
420 In.Limit(-1);
421 Last = true;
422 do
423 {
424 if (In.WriteTillEl(Data,true) == true)
425 break;
426 }
427 while ((Last = Owner->Go(false,this)) == true);
428 if (Last == false)
429 return false;
430 }
431 }
432 else
433 {
434 /* Closes encoding is used when the server did not specify a size, the
435 loss of the connection means we are done */
436 if (Encoding == Closes)
437 In.Limit(-1);
438 else
439 In.Limit(Size - StartPos);
440
441 // Just transfer the whole block.
442 do
443 {
444 if (In.IsLimit() == false)
445 continue;
446
447 In.Limit(-1);
448 return !_error->PendingError();
449 }
450 while (Owner->Go(true,this) == true);
451 }
452
453 return Owner->Flush(this) && !_error->PendingError();
454 }
455 /*}}}*/
456 // ServerState::HeaderLine - Process a header line /*{{{*/
457 // ---------------------------------------------------------------------
458 /* */
459 bool ServerState::HeaderLine(string Line)
460 {
461 if (Line.empty() == true)
462 return true;
463
464 // The http server might be trying to do something evil.
465 if (Line.length() >= MAXLEN)
466 return _error->Error("Got a single header line over %u chars",MAXLEN);
467
468 string::size_type Pos = Line.find(' ');
469 if (Pos == string::npos || Pos+1 > Line.length())
470 {
471 // Blah, some servers use "connection:closes", evil.
472 Pos = Line.find(':');
473 if (Pos == string::npos || Pos + 2 > Line.length())
474 return _error->Error("Bad header line");
475 Pos++;
476 }
477
478 // Parse off any trailing spaces between the : and the next word.
479 string::size_type Pos2 = Pos;
480 while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
481 Pos2++;
482
483 string Tag = string(Line,0,Pos);
484 string Val = string(Line,Pos2);
485
486 if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
487 {
488 // Evil servers return no version
489 if (Line[4] == '/')
490 {
491 if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
492 &Result,Code) != 4)
493 return _error->Error("The http server sent an invalid reply header");
494 }
495 else
496 {
497 Major = 0;
498 Minor = 9;
499 if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
500 return _error->Error("The http server sent an invalid reply header");
501 }
502
503 return true;
504 }
505
506 if (stringcasecmp(Tag,"Content-Length:") == 0)
507 {
508 if (Encoding == Closes)
509 Encoding = Stream;
510 HaveContent = true;
511
512 // The length is already set from the Content-Range header
513 if (StartPos != 0)
514 return true;
515
516 if (sscanf(Val.c_str(),"%lu",&Size) != 1)
517 return _error->Error("The http server sent an invalid Content-Length header");
518 return true;
519 }
520
521 if (stringcasecmp(Tag,"Content-Type:") == 0)
522 {
523 HaveContent = true;
524 return true;
525 }
526
527 if (stringcasecmp(Tag,"Content-Range:") == 0)
528 {
529 HaveContent = true;
530
531 if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
532 return _error->Error("The http server sent an invalid Content-Range header");
533 if ((unsigned)StartPos > Size)
534 return _error->Error("This http server has broken range support");
535 return true;
536 }
537
538 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
539 {
540 HaveContent = true;
541 if (stringcasecmp(Val,"chunked") == 0)
542 Encoding = Chunked;
543
544 return true;
545 }
546
547 if (stringcasecmp(Tag,"Last-Modified:") == 0)
548 {
549 if (StrToTime(Val,Date) == false)
550 return _error->Error("Unknown date format");
551 return true;
552 }
553
554 return true;
555 }
556 /*}}}*/
557
558 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
559 // ---------------------------------------------------------------------
560 /* This places the http request in the outbound buffer */
561 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
562 {
563 URI Uri = Itm->Uri;
564
565 // The HTTP server expects a hostname with a trailing :port
566 char Buf[1000];
567 string ProperHost = Uri.Host;
568 if (Uri.Port != 0)
569 {
570 sprintf(Buf,":%u",Uri.Port);
571 ProperHost += Buf;
572 }
573
574 // Just in case.
575 if (Itm->Uri.length() >= sizeof(Buf))
576 abort();
577
578 /* Build the request. We include a keep-alive header only for non-proxy
579 requests. This is to tweak old http/1.0 servers that do support keep-alive
580 but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
581 will glitch HTTP/1.0 proxies because they do not filter it out and
582 pass it on, HTTP/1.1 says the connection should default to keep alive
583 and we expect the proxy to do this */
584 if (Proxy.empty() == true)
585 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
586 QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
587 else
588 {
589 /* Generate a cache control header if necessary. We place a max
590 cache age on index files, optionally set a no-cache directive
591 and a no-store directive for archives. */
592 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
593 Itm->Uri.c_str(),ProperHost.c_str());
594 if (_config->FindB("Acquire::http::No-Cache",false) == true)
595 strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
596 else
597 {
598 if (Itm->IndexFile == true)
599 sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
600 _config->FindI("Acquire::http::Max-Age",60*60*24));
601 else
602 {
603 if (_config->FindB("Acquire::http::No-Store",false) == true)
604 strcat(Buf,"Cache-Control: no-store\r\n");
605 }
606 }
607 }
608
609 string Req = Buf;
610
611 // Check for a partial file
612 struct stat SBuf;
613 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
614 {
615 // In this case we send an if-range query with a range header
616 sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
617 TimeRFC1123(SBuf.st_mtime).c_str());
618 Req += Buf;
619 }
620 else
621 {
622 if (Itm->LastModified != 0)
623 {
624 sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
625 Req += Buf;
626 }
627 }
628
629 if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
630 Req += string("Proxy-Authorization: Basic ") +
631 Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
632
633 Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
634 // cerr << Req << endl;
635
636 Out.Read(Req);
637 }
638 /*}}}*/
639 // HttpMethod::Go - Run a single loop /*{{{*/
640 // ---------------------------------------------------------------------
641 /* This runs the select loop over the server FDs, Output file FDs and
642 stdin. */
643 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
644 {
645 // Server has closed the connection
646 if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
647 ToFile == false))
648 return false;
649
650 fd_set rfds,wfds,efds;
651 FD_ZERO(&rfds);
652 FD_ZERO(&wfds);
653 FD_ZERO(&efds);
654
655 // Add the server
656 if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
657 FD_SET(Srv->ServerFd,&wfds);
658 if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
659 FD_SET(Srv->ServerFd,&rfds);
660
661 // Add the file
662 int FileFD = -1;
663 if (File != 0)
664 FileFD = File->Fd();
665
666 if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
667 FD_SET(FileFD,&wfds);
668
669 // Add stdin
670 FD_SET(STDIN_FILENO,&rfds);
671
672 // Error Set
673 if (FileFD != -1)
674 FD_SET(FileFD,&efds);
675 if (Srv->ServerFd != -1)
676 FD_SET(Srv->ServerFd,&efds);
677
678 // Figure out the max fd
679 int MaxFd = FileFD;
680 if (MaxFd < Srv->ServerFd)
681 MaxFd = Srv->ServerFd;
682
683 // Select
684 struct timeval tv;
685 tv.tv_sec = TimeOut;
686 tv.tv_usec = 0;
687 int Res = 0;
688 if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
689 return _error->Errno("select","Select failed");
690
691 if (Res == 0)
692 {
693 _error->Error("Connection timed out");
694 return ServerDie(Srv);
695 }
696
697 // Some kind of exception (error) on the sockets, die
698 if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
699 (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
700 return _error->Error("Socket Exception");
701
702 // Handle server IO
703 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
704 {
705 errno = 0;
706 if (Srv->In.Read(Srv->ServerFd) == false)
707 return ServerDie(Srv);
708 }
709
710 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
711 {
712 errno = 0;
713 if (Srv->Out.Write(Srv->ServerFd) == false)
714 return ServerDie(Srv);
715 }
716
717 // Send data to the file
718 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
719 {
720 if (Srv->In.Write(FileFD) == false)
721 return _error->Errno("write","Error writing to output file");
722 }
723
724 // Handle commands from APT
725 if (FD_ISSET(STDIN_FILENO,&rfds))
726 {
727 if (Run(true) != 0)
728 exit(100);
729 }
730
731 return true;
732 }
733 /*}}}*/
734 // HttpMethod::Flush - Dump the buffer into the file /*{{{*/
735 // ---------------------------------------------------------------------
736 /* This takes the current input buffer from the Server FD and writes it
737 into the file */
738 bool HttpMethod::Flush(ServerState *Srv)
739 {
740 if (File != 0)
741 {
742 SetNonBlock(File->Fd(),false);
743 if (Srv->In.WriteSpace() == false)
744 return true;
745
746 while (Srv->In.WriteSpace() == true)
747 {
748 if (Srv->In.Write(File->Fd()) == false)
749 return _error->Errno("write","Error writing to file");
750 if (Srv->In.IsLimit() == true)
751 return true;
752 }
753
754 if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
755 return true;
756 }
757 return false;
758 }
759 /*}}}*/
760 // HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
761 // ---------------------------------------------------------------------
762 /* */
763 bool HttpMethod::ServerDie(ServerState *Srv)
764 {
765 unsigned int LErrno = errno;
766
767 // Dump the buffer to the file
768 if (Srv->State == ServerState::Data)
769 {
770 SetNonBlock(File->Fd(),false);
771 while (Srv->In.WriteSpace() == true)
772 {
773 if (Srv->In.Write(File->Fd()) == false)
774 return _error->Errno("write","Error writing to the file");
775
776 // Done
777 if (Srv->In.IsLimit() == true)
778 return true;
779 }
780 }
781
782 // See if this is because the server finished the data stream
783 if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
784 Srv->Encoding != ServerState::Closes)
785 {
786 Srv->Close();
787 if (LErrno == 0)
788 return _error->Error("Error reading from server Remote end closed connection");
789 errno = LErrno;
790 return _error->Errno("read","Error reading from server");
791 }
792 else
793 {
794 Srv->In.Limit(-1);
795
796 // Nothing left in the buffer
797 if (Srv->In.WriteSpace() == false)
798 return false;
799
800 // We may have got multiple responses back in one packet..
801 Srv->Close();
802 return true;
803 }
804
805 return false;
806 }
807 /*}}}*/
808 // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
809 // ---------------------------------------------------------------------
810 /* We look at the header data we got back from the server and decide what
811 to do. Returns
812 0 - File is open,
813 1 - IMS hit
814 3 - Unrecoverable error
815 4 - Error with error content page
816 5 - Unrecoverable non-server error (close the connection) */
817 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
818 {
819 // Not Modified
820 if (Srv->Result == 304)
821 {
822 unlink(Queue->DestFile.c_str());
823 Res.IMSHit = true;
824 Res.LastModified = Queue->LastModified;
825 return 1;
826 }
827
828 /* We have a reply we dont handle. This should indicate a perm server
829 failure */
830 if (Srv->Result < 200 || Srv->Result >= 300)
831 {
832 _error->Error("%u %s",Srv->Result,Srv->Code);
833 if (Srv->HaveContent == true)
834 return 4;
835 return 3;
836 }
837
838 // This is some sort of 2xx 'data follows' reply
839 Res.LastModified = Srv->Date;
840 Res.Size = Srv->Size;
841
842 // Open the file
843 delete File;
844 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
845 if (_error->PendingError() == true)
846 return 5;
847
848 FailFile = Queue->DestFile;
849 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
850 FailFd = File->Fd();
851 FailTime = Srv->Date;
852
853 // Set the expected size
854 if (Srv->StartPos >= 0)
855 {
856 Res.ResumePoint = Srv->StartPos;
857 ftruncate(File->Fd(),Srv->StartPos);
858 }
859
860 // Set the start point
861 lseek(File->Fd(),0,SEEK_END);
862
863 delete Srv->In.MD5;
864 Srv->In.MD5 = new MD5Summation;
865
866 // Fill the MD5 Hash if the file is non-empty (resume)
867 if (Srv->StartPos > 0)
868 {
869 lseek(File->Fd(),0,SEEK_SET);
870 if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
871 {
872 _error->Errno("read","Problem hashing file");
873 return 5;
874 }
875 lseek(File->Fd(),0,SEEK_END);
876 }
877
878 SetNonBlock(File->Fd(),true);
879 return 0;
880 }
881 /*}}}*/
882 // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
883 // ---------------------------------------------------------------------
884 /* This closes and timestamps the open file. This is neccessary to get
885 resume behavoir on user abort */
886 void HttpMethod::SigTerm(int)
887 {
888 if (FailFd == -1)
889 exit(100);
890 close(FailFd);
891
892 // Timestamp
893 struct utimbuf UBuf;
894 UBuf.actime = FailTime;
895 UBuf.modtime = FailTime;
896 utime(FailFile.c_str(),&UBuf);
897
898 exit(100);
899 }
900 /*}}}*/
901 // HttpMethod::Fetch - Fetch an item /*{{{*/
902 // ---------------------------------------------------------------------
903 /* This adds an item to the pipeline. We keep the pipeline at a fixed
904 depth. */
905 bool HttpMethod::Fetch(FetchItem *)
906 {
907 if (Server == 0)
908 return true;
909
910 // Queue the requests
911 int Depth = -1;
912 bool Tail = false;
913 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
914 {
915 // Make sure we stick with the same server
916 if (Server->Comp(I->Uri) == false)
917 break;
918 if (QueueBack == I)
919 Tail = true;
920 if (Tail == true)
921 {
922 QueueBack = I->Next;
923 SendReq(I,Server->Out);
924 continue;
925 }
926 }
927
928 return true;
929 };
930 /*}}}*/
931 // HttpMethod::Configuration - Handle a configuration message /*{{{*/
932 // ---------------------------------------------------------------------
933 /* We stash the desired pipeline depth */
934 bool HttpMethod::Configuration(string Message)
935 {
936 if (pkgAcqMethod::Configuration(Message) == false)
937 return false;
938
939 TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
940 PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
941 PipelineDepth);
942
943 return true;
944 }
945 /*}}}*/
946 // HttpMethod::Loop - Main loop /*{{{*/
947 // ---------------------------------------------------------------------
948 /* */
949 int HttpMethod::Loop()
950 {
951 signal(SIGTERM,SigTerm);
952 signal(SIGINT,SigTerm);
953
954 Server = 0;
955
956 int FailCounter = 0;
957 while (1)
958 {
959 // We have no commands, wait for some to arrive
960 if (Queue == 0)
961 {
962 if (WaitFd(STDIN_FILENO) == false)
963 return 0;
964 }
965
966 // Run messages
967 if (Run(true) != 0)
968 return 100;
969
970 if (Queue == 0)
971 continue;
972
973 // Connect to the server
974 if (Server == 0 || Server->Comp(Queue->Uri) == false)
975 {
976 delete Server;
977 Server = new ServerState(Queue->Uri,this);
978 }
979
980 // Reset the pipeline
981 if (Server->ServerFd == -1)
982 QueueBack = Queue;
983
984 // Connnect to the host
985 if (Server->Open() == false)
986 {
987 Fail(true);
988 delete Server;
989 Server = 0;
990 continue;
991 }
992
993 // Fill the pipeline.
994 Fetch(0);
995
996 // Fetch the next URL header data from the server.
997 switch (Server->RunHeaders())
998 {
999 case 0:
1000 break;
1001
1002 // The header data is bad
1003 case 2:
1004 {
1005 _error->Error("Bad header Data");
1006 Fail(true);
1007 continue;
1008 }
1009
1010 // The server closed a connection during the header get..
1011 default:
1012 case 1:
1013 {
1014 FailCounter++;
1015 _error->Discard();
1016 Server->Close();
1017
1018 if (FailCounter >= 2)
1019 {
1020 Fail("Connection failed",true);
1021 FailCounter = 0;
1022 }
1023
1024 continue;
1025 }
1026 };
1027
1028 // Decide what to do.
1029 FetchResult Res;
1030 Res.Filename = Queue->DestFile;
1031 switch (DealWithHeaders(Res,Server))
1032 {
1033 // Ok, the file is Open
1034 case 0:
1035 {
1036 URIStart(Res);
1037
1038 // Run the data
1039 bool Result = Server->RunData();
1040
1041 // Close the file, destroy the FD object and timestamp it
1042 FailFd = -1;
1043 delete File;
1044 File = 0;
1045
1046 // Timestamp
1047 struct utimbuf UBuf;
1048 time(&UBuf.actime);
1049 UBuf.actime = Server->Date;
1050 UBuf.modtime = Server->Date;
1051 utime(Queue->DestFile.c_str(),&UBuf);
1052
1053 // Send status to APT
1054 if (Result == true)
1055 {
1056 Res.MD5Sum = Server->In.MD5->Result();
1057 URIDone(Res);
1058 }
1059 else
1060 Fail(true);
1061
1062 break;
1063 }
1064
1065 // IMS hit
1066 case 1:
1067 {
1068 URIDone(Res);
1069 break;
1070 }
1071
1072 // Hard server error, not found or something
1073 case 3:
1074 {
1075 Fail();
1076 break;
1077 }
1078
1079 // Hard internal error, kill the connection and fail
1080 case 5:
1081 {
1082 Fail();
1083 Server->Close();
1084 break;
1085 }
1086
1087 // We need to flush the data, the header is like a 404 w/ error text
1088 case 4:
1089 {
1090 Fail();
1091
1092 // Send to content to dev/null
1093 File = new FileFd("/dev/null",FileFd::WriteExists);
1094 Server->RunData();
1095 delete File;
1096 File = 0;
1097 break;
1098 }
1099
1100 default:
1101 Fail("Internal error");
1102 break;
1103 }
1104
1105 FailCounter = 0;
1106 }
1107
1108 return 0;
1109 }
1110 /*}}}*/
1111
1112 int main()
1113 {
1114 HttpMethod Mth;
1115
1116 return Mth.Loop();
1117 }