Lots and lots of i18n updates.
[ntk/apt.git] / methods / http.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: http.cc,v 1.55 2003/02/10 07:34:41 doogie Exp $
4 /* ######################################################################
5
6 HTTP Aquire Method - This is the HTTP aquire method for APT.
7
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
9 pipelining, range, if-range and so on.
10
11 It is based on a doubly buffered select loop. A groupe of requests are
12 fed into a single output buffer that is constantly fed out the
13 socket. This provides ideal pipelining as in many cases all of the
14 requests will fit into a single packet. The input socket is buffered
15 the same way and fed into the fd for the file (may be a pipe in future).
16
17 This double buffering provides fairly substantial transfer rates,
18 compared to wget the http method is about 4% faster. Most importantly,
19 when HTTP is compared with FTP as a protocol the speed difference is
20 huge. In tests over the internet from two sites to llug (via ATM) this
21 program got 230k/s sustained http transfer rates. FTP on the other
22 hand topped out at 170k/s. That combined with the time to setup the
23 FTP connection makes HTTP a vastly superior protocol.
24
25 ##################################################################### */
26 /*}}}*/
27 // Include Files /*{{{*/
28 #include <apti18n.h>
29 #include <apt-pkg/fileutl.h>
30 #include <apt-pkg/acquire-method.h>
31 #include <apt-pkg/error.h>
32 #include <apt-pkg/hashes.h>
33
34 #include <sys/stat.h>
35 #include <sys/time.h>
36 #include <utime.h>
37 #include <unistd.h>
38 #include <signal.h>
39 #include <stdio.h>
40 #include <errno.h>
41 #include <string.h>
42 #include <iostream>
43
44 // Internet stuff
45 #include <netdb.h>
46
47 #include "connect.h"
48 #include "rfc2553emu.h"
49 #include "http.h"
50
51 /*}}}*/
52 using namespace std;
53
54 string HttpMethod::FailFile;
55 int HttpMethod::FailFd = -1;
56 time_t HttpMethod::FailTime = 0;
57 unsigned long PipelineDepth = 10;
58 unsigned long TimeOut = 120;
59 bool Debug = false;
60
61 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
62 // ---------------------------------------------------------------------
63 /* */
64 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
65 {
66 Buf = new unsigned char[Size];
67 Reset();
68 }
69 /*}}}*/
70 // CircleBuf::Reset - Reset to the default state /*{{{*/
71 // ---------------------------------------------------------------------
72 /* */
73 void CircleBuf::Reset()
74 {
75 InP = 0;
76 OutP = 0;
77 StrPos = 0;
78 MaxGet = (unsigned int)-1;
79 OutQueue = string();
80 if (Hash != 0)
81 {
82 delete Hash;
83 Hash = new Hashes;
84 }
85 };
86 /*}}}*/
87 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
88 // ---------------------------------------------------------------------
89 /* This fills up the buffer with as much data as is in the FD, assuming it
90 is non-blocking.. */
91 bool CircleBuf::Read(int Fd)
92 {
93 while (1)
94 {
95 // Woops, buffer is full
96 if (InP - OutP == Size)
97 return true;
98
99 // Write the buffer segment
100 int Res;
101 Res = read(Fd,Buf + (InP%Size),LeftRead());
102
103 if (Res == 0)
104 return false;
105 if (Res < 0)
106 {
107 if (errno == EAGAIN)
108 return true;
109 return false;
110 }
111
112 if (InP == 0)
113 gettimeofday(&Start,0);
114 InP += Res;
115 }
116 }
117 /*}}}*/
118 // CircleBuf::Read - Put the string into the buffer /*{{{*/
119 // ---------------------------------------------------------------------
120 /* This will hold the string in and fill the buffer with it as it empties */
121 bool CircleBuf::Read(string Data)
122 {
123 OutQueue += Data;
124 FillOut();
125 return true;
126 }
127 /*}}}*/
128 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
129 // ---------------------------------------------------------------------
130 /* */
131 void CircleBuf::FillOut()
132 {
133 if (OutQueue.empty() == true)
134 return;
135 while (1)
136 {
137 // Woops, buffer is full
138 if (InP - OutP == Size)
139 return;
140
141 // Write the buffer segment
142 unsigned long Sz = LeftRead();
143 if (OutQueue.length() - StrPos < Sz)
144 Sz = OutQueue.length() - StrPos;
145 memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
146
147 // Advance
148 StrPos += Sz;
149 InP += Sz;
150 if (OutQueue.length() == StrPos)
151 {
152 StrPos = 0;
153 OutQueue = "";
154 return;
155 }
156 }
157 }
158 /*}}}*/
159 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
160 // ---------------------------------------------------------------------
161 /* This empties the buffer into the FD. */
162 bool CircleBuf::Write(int Fd)
163 {
164 while (1)
165 {
166 FillOut();
167
168 // Woops, buffer is empty
169 if (OutP == InP)
170 return true;
171
172 if (OutP == MaxGet)
173 return true;
174
175 // Write the buffer segment
176 int Res;
177 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
178
179 if (Res == 0)
180 return false;
181 if (Res < 0)
182 {
183 if (errno == EAGAIN)
184 return true;
185
186 return false;
187 }
188
189 if (Hash != 0)
190 Hash->Add(Buf + (OutP%Size),Res);
191
192 OutP += Res;
193 }
194 }
195 /*}}}*/
196 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
197 // ---------------------------------------------------------------------
198 /* This copies till the first empty line */
199 bool CircleBuf::WriteTillEl(string &Data,bool Single)
200 {
201 // We cheat and assume it is unneeded to have more than one buffer load
202 for (unsigned long I = OutP; I < InP; I++)
203 {
204 if (Buf[I%Size] != '\n')
205 continue;
206 for (I++; I < InP && Buf[I%Size] == '\r'; I++);
207
208 if (Single == false)
209 {
210 if (Buf[I%Size] != '\n')
211 continue;
212 for (I++; I < InP && Buf[I%Size] == '\r'; I++);
213 }
214
215 if (I > InP)
216 I = InP;
217
218 Data = "";
219 while (OutP < I)
220 {
221 unsigned long Sz = LeftWrite();
222 if (Sz == 0)
223 return false;
224 if (I - OutP < LeftWrite())
225 Sz = I - OutP;
226 Data += string((char *)(Buf + (OutP%Size)),Sz);
227 OutP += Sz;
228 }
229 return true;
230 }
231 return false;
232 }
233 /*}}}*/
234 // CircleBuf::Stats - Print out stats information /*{{{*/
235 // ---------------------------------------------------------------------
236 /* */
237 void CircleBuf::Stats()
238 {
239 if (InP == 0)
240 return;
241
242 struct timeval Stop;
243 gettimeofday(&Stop,0);
244 /* float Diff = Stop.tv_sec - Start.tv_sec +
245 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
246 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
247 }
248 /*}}}*/
249
250 // ServerState::ServerState - Constructor /*{{{*/
251 // ---------------------------------------------------------------------
252 /* */
253 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
254 In(64*1024), Out(4*1024),
255 ServerName(Srv)
256 {
257 Reset();
258 }
259 /*}}}*/
260 // ServerState::Open - Open a connection to the server /*{{{*/
261 // ---------------------------------------------------------------------
262 /* This opens a connection to the server. */
263 bool ServerState::Open()
264 {
265 // Use the already open connection if possible.
266 if (ServerFd != -1)
267 return true;
268
269 Close();
270 In.Reset();
271 Out.Reset();
272 Persistent = true;
273
274 // Determine the proxy setting
275 if (getenv("http_proxy") == 0)
276 {
277 string DefProxy = _config->Find("Acquire::http::Proxy");
278 string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
279 if (SpecificProxy.empty() == false)
280 {
281 if (SpecificProxy == "DIRECT")
282 Proxy = "";
283 else
284 Proxy = SpecificProxy;
285 }
286 else
287 Proxy = DefProxy;
288 }
289 else
290 Proxy = getenv("http_proxy");
291
292 // Parse no_proxy, a , separated list of domains
293 if (getenv("no_proxy") != 0)
294 {
295 if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
296 Proxy = "";
297 }
298
299 // Determine what host and port to use based on the proxy settings
300 int Port = 0;
301 string Host;
302 if (Proxy.empty() == true || Proxy.Host.empty() == true)
303 {
304 if (ServerName.Port != 0)
305 Port = ServerName.Port;
306 Host = ServerName.Host;
307 }
308 else
309 {
310 if (Proxy.Port != 0)
311 Port = Proxy.Port;
312 Host = Proxy.Host;
313 }
314
315 // Connect to the remote server
316 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
317 return false;
318
319 return true;
320 }
321 /*}}}*/
322 // ServerState::Close - Close a connection to the server /*{{{*/
323 // ---------------------------------------------------------------------
324 /* */
325 bool ServerState::Close()
326 {
327 close(ServerFd);
328 ServerFd = -1;
329 return true;
330 }
331 /*}}}*/
332 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
333 // ---------------------------------------------------------------------
334 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
335 parse error occured */
336 int ServerState::RunHeaders()
337 {
338 State = Header;
339
340 Owner->Status(_("Waiting for file"));
341
342 Major = 0;
343 Minor = 0;
344 Result = 0;
345 Size = 0;
346 StartPos = 0;
347 Encoding = Closes;
348 HaveContent = false;
349 time(&Date);
350
351 do
352 {
353 string Data;
354 if (In.WriteTillEl(Data) == false)
355 continue;
356
357 if (Debug == true)
358 clog << Data;
359
360 for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
361 {
362 string::const_iterator J = I;
363 for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
364 if (HeaderLine(string(I,J)) == false)
365 return 2;
366 I = J;
367 }
368
369 // 100 Continue is a Nop...
370 if (Result == 100)
371 continue;
372
373 // Tidy up the connection persistance state.
374 if (Encoding == Closes && HaveContent == true)
375 Persistent = false;
376
377 return 0;
378 }
379 while (Owner->Go(false,this) == true);
380
381 return 1;
382 }
383 /*}}}*/
384 // ServerState::RunData - Transfer the data from the socket /*{{{*/
385 // ---------------------------------------------------------------------
386 /* */
387 bool ServerState::RunData()
388 {
389 State = Data;
390
391 // Chunked transfer encoding is fun..
392 if (Encoding == Chunked)
393 {
394 while (1)
395 {
396 // Grab the block size
397 bool Last = true;
398 string Data;
399 In.Limit(-1);
400 do
401 {
402 if (In.WriteTillEl(Data,true) == true)
403 break;
404 }
405 while ((Last = Owner->Go(false,this)) == true);
406
407 if (Last == false)
408 return false;
409
410 // See if we are done
411 unsigned long Len = strtol(Data.c_str(),0,16);
412 if (Len == 0)
413 {
414 In.Limit(-1);
415
416 // We have to remove the entity trailer
417 Last = true;
418 do
419 {
420 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
421 break;
422 }
423 while ((Last = Owner->Go(false,this)) == true);
424 if (Last == false)
425 return false;
426 return !_error->PendingError();
427 }
428
429 // Transfer the block
430 In.Limit(Len);
431 while (Owner->Go(true,this) == true)
432 if (In.IsLimit() == true)
433 break;
434
435 // Error
436 if (In.IsLimit() == false)
437 return false;
438
439 // The server sends an extra new line before the next block specifier..
440 In.Limit(-1);
441 Last = true;
442 do
443 {
444 if (In.WriteTillEl(Data,true) == true)
445 break;
446 }
447 while ((Last = Owner->Go(false,this)) == true);
448 if (Last == false)
449 return false;
450 }
451 }
452 else
453 {
454 /* Closes encoding is used when the server did not specify a size, the
455 loss of the connection means we are done */
456 if (Encoding == Closes)
457 In.Limit(-1);
458 else
459 In.Limit(Size - StartPos);
460
461 // Just transfer the whole block.
462 do
463 {
464 if (In.IsLimit() == false)
465 continue;
466
467 In.Limit(-1);
468 return !_error->PendingError();
469 }
470 while (Owner->Go(true,this) == true);
471 }
472
473 return Owner->Flush(this) && !_error->PendingError();
474 }
475 /*}}}*/
476 // ServerState::HeaderLine - Process a header line /*{{{*/
477 // ---------------------------------------------------------------------
478 /* */
479 bool ServerState::HeaderLine(string Line)
480 {
481 if (Line.empty() == true)
482 return true;
483
484 // The http server might be trying to do something evil.
485 if (Line.length() >= MAXLEN)
486 return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
487
488 string::size_type Pos = Line.find(' ');
489 if (Pos == string::npos || Pos+1 > Line.length())
490 {
491 // Blah, some servers use "connection:closes", evil.
492 Pos = Line.find(':');
493 if (Pos == string::npos || Pos + 2 > Line.length())
494 return _error->Error(_("Bad header line"));
495 Pos++;
496 }
497
498 // Parse off any trailing spaces between the : and the next word.
499 string::size_type Pos2 = Pos;
500 while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
501 Pos2++;
502
503 string Tag = string(Line,0,Pos);
504 string Val = string(Line,Pos2);
505
506 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
507 {
508 // Evil servers return no version
509 if (Line[4] == '/')
510 {
511 if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
512 &Result,Code) != 4)
513 return _error->Error(_("The http server sent an invalid reply header"));
514 }
515 else
516 {
517 Major = 0;
518 Minor = 9;
519 if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
520 return _error->Error(_("The http server sent an invalid reply header"));
521 }
522
523 /* Check the HTTP response header to get the default persistance
524 state. */
525 if (Major < 1)
526 Persistent = false;
527 else
528 {
529 if (Major == 1 && Minor <= 0)
530 Persistent = false;
531 else
532 Persistent = true;
533 }
534
535 return true;
536 }
537
538 if (stringcasecmp(Tag,"Content-Length:") == 0)
539 {
540 if (Encoding == Closes)
541 Encoding = Stream;
542 HaveContent = true;
543
544 // The length is already set from the Content-Range header
545 if (StartPos != 0)
546 return true;
547
548 if (sscanf(Val.c_str(),"%lu",&Size) != 1)
549 return _error->Error(_("The http server sent an invalid Content-Length header"));
550 return true;
551 }
552
553 if (stringcasecmp(Tag,"Content-Type:") == 0)
554 {
555 HaveContent = true;
556 return true;
557 }
558
559 if (stringcasecmp(Tag,"Content-Range:") == 0)
560 {
561 HaveContent = true;
562
563 if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
564 return _error->Error(_("The http server sent an invalid Content-Range header"));
565 if ((unsigned)StartPos > Size)
566 return _error->Error(_("This http server has broken range support"));
567 return true;
568 }
569
570 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
571 {
572 HaveContent = true;
573 if (stringcasecmp(Val,"chunked") == 0)
574 Encoding = Chunked;
575 return true;
576 }
577
578 if (stringcasecmp(Tag,"Connection:") == 0)
579 {
580 if (stringcasecmp(Val,"close") == 0)
581 Persistent = false;
582 if (stringcasecmp(Val,"keep-alive") == 0)
583 Persistent = true;
584 return true;
585 }
586
587 if (stringcasecmp(Tag,"Last-Modified:") == 0)
588 {
589 if (StrToTime(Val,Date) == false)
590 return _error->Error(_("Unknown date format"));
591 return true;
592 }
593
594 return true;
595 }
596 /*}}}*/
597
598 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
599 // ---------------------------------------------------------------------
600 /* This places the http request in the outbound buffer */
601 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
602 {
603 URI Uri = Itm->Uri;
604
605 // The HTTP server expects a hostname with a trailing :port
606 char Buf[1000];
607 string ProperHost = Uri.Host;
608 if (Uri.Port != 0)
609 {
610 sprintf(Buf,":%u",Uri.Port);
611 ProperHost += Buf;
612 }
613
614 // Just in case.
615 if (Itm->Uri.length() >= sizeof(Buf))
616 abort();
617
618 /* Build the request. We include a keep-alive header only for non-proxy
619 requests. This is to tweak old http/1.0 servers that do support keep-alive
620 but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
621 will glitch HTTP/1.0 proxies because they do not filter it out and
622 pass it on, HTTP/1.1 says the connection should default to keep alive
623 and we expect the proxy to do this */
624 if (Proxy.empty() == true)
625 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
626 QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
627 else
628 {
629 /* Generate a cache control header if necessary. We place a max
630 cache age on index files, optionally set a no-cache directive
631 and a no-store directive for archives. */
632 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
633 Itm->Uri.c_str(),ProperHost.c_str());
634 if (_config->FindB("Acquire::http::No-Cache",false) == true)
635 strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
636 else
637 {
638 if (Itm->IndexFile == true)
639 sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
640 _config->FindI("Acquire::http::Max-Age",60*60*24));
641 else
642 {
643 if (_config->FindB("Acquire::http::No-Store",false) == true)
644 strcat(Buf,"Cache-Control: no-store\r\n");
645 }
646 }
647 }
648
649 string Req = Buf;
650
651 // Check for a partial file
652 struct stat SBuf;
653 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
654 {
655 // In this case we send an if-range query with a range header
656 sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
657 TimeRFC1123(SBuf.st_mtime).c_str());
658 Req += Buf;
659 }
660 else
661 {
662 if (Itm->LastModified != 0)
663 {
664 sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
665 Req += Buf;
666 }
667 }
668
669 if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
670 Req += string("Proxy-Authorization: Basic ") +
671 Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
672
673 if (Uri.User.empty() == false || Uri.Password.empty() == false)
674 Req += string("Authorization: Basic ") +
675 Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
676
677 Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
678
679 if (Debug == true)
680 cerr << Req << endl;
681
682 Out.Read(Req);
683 }
684 /*}}}*/
685 // HttpMethod::Go - Run a single loop /*{{{*/
686 // ---------------------------------------------------------------------
687 /* This runs the select loop over the server FDs, Output file FDs and
688 stdin. */
689 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
690 {
691 // Server has closed the connection
692 if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
693 ToFile == false))
694 return false;
695
696 fd_set rfds,wfds;
697 FD_ZERO(&rfds);
698 FD_ZERO(&wfds);
699
700 /* Add the server. We only send more requests if the connection will
701 be persisting */
702 if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
703 && Srv->Persistent == true)
704 FD_SET(Srv->ServerFd,&wfds);
705 if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
706 FD_SET(Srv->ServerFd,&rfds);
707
708 // Add the file
709 int FileFD = -1;
710 if (File != 0)
711 FileFD = File->Fd();
712
713 if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
714 FD_SET(FileFD,&wfds);
715
716 // Add stdin
717 FD_SET(STDIN_FILENO,&rfds);
718
719 // Figure out the max fd
720 int MaxFd = FileFD;
721 if (MaxFd < Srv->ServerFd)
722 MaxFd = Srv->ServerFd;
723
724 // Select
725 struct timeval tv;
726 tv.tv_sec = TimeOut;
727 tv.tv_usec = 0;
728 int Res = 0;
729 if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
730 {
731 if (errno == EINTR)
732 return true;
733 return _error->Errno("select",_("Select failed"));
734 }
735
736 if (Res == 0)
737 {
738 _error->Error(_("Connection timed out"));
739 return ServerDie(Srv);
740 }
741
742 // Handle server IO
743 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
744 {
745 errno = 0;
746 if (Srv->In.Read(Srv->ServerFd) == false)
747 return ServerDie(Srv);
748 }
749
750 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
751 {
752 errno = 0;
753 if (Srv->Out.Write(Srv->ServerFd) == false)
754 return ServerDie(Srv);
755 }
756
757 // Send data to the file
758 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
759 {
760 if (Srv->In.Write(FileFD) == false)
761 return _error->Errno("write",_("Error writing to output file"));
762 }
763
764 // Handle commands from APT
765 if (FD_ISSET(STDIN_FILENO,&rfds))
766 {
767 if (Run(true) != -1)
768 exit(100);
769 }
770
771 return true;
772 }
773 /*}}}*/
774 // HttpMethod::Flush - Dump the buffer into the file /*{{{*/
775 // ---------------------------------------------------------------------
776 /* This takes the current input buffer from the Server FD and writes it
777 into the file */
778 bool HttpMethod::Flush(ServerState *Srv)
779 {
780 if (File != 0)
781 {
782 SetNonBlock(File->Fd(),false);
783 if (Srv->In.WriteSpace() == false)
784 return true;
785
786 while (Srv->In.WriteSpace() == true)
787 {
788 if (Srv->In.Write(File->Fd()) == false)
789 return _error->Errno("write",_("Error writing to file"));
790 if (Srv->In.IsLimit() == true)
791 return true;
792 }
793
794 if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
795 return true;
796 }
797 return false;
798 }
799 /*}}}*/
800 // HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
801 // ---------------------------------------------------------------------
802 /* */
803 bool HttpMethod::ServerDie(ServerState *Srv)
804 {
805 unsigned int LErrno = errno;
806
807 // Dump the buffer to the file
808 if (Srv->State == ServerState::Data)
809 {
810 SetNonBlock(File->Fd(),false);
811 while (Srv->In.WriteSpace() == true)
812 {
813 if (Srv->In.Write(File->Fd()) == false)
814 return _error->Errno("write",_("Error writing to the file"));
815
816 // Done
817 if (Srv->In.IsLimit() == true)
818 return true;
819 }
820 }
821
822 // See if this is because the server finished the data stream
823 if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
824 Srv->Encoding != ServerState::Closes)
825 {
826 Srv->Close();
827 if (LErrno == 0)
828 return _error->Error(_("Error reading from server Remote end closed connection"));
829 errno = LErrno;
830 return _error->Errno("read",_("Error reading from server"));
831 }
832 else
833 {
834 Srv->In.Limit(-1);
835
836 // Nothing left in the buffer
837 if (Srv->In.WriteSpace() == false)
838 return false;
839
840 // We may have got multiple responses back in one packet..
841 Srv->Close();
842 return true;
843 }
844
845 return false;
846 }
847 /*}}}*/
848 // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
849 // ---------------------------------------------------------------------
850 /* We look at the header data we got back from the server and decide what
851 to do. Returns
852 0 - File is open,
853 1 - IMS hit
854 3 - Unrecoverable error
855 4 - Error with error content page
856 5 - Unrecoverable non-server error (close the connection) */
857 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
858 {
859 // Not Modified
860 if (Srv->Result == 304)
861 {
862 unlink(Queue->DestFile.c_str());
863 Res.IMSHit = true;
864 Res.LastModified = Queue->LastModified;
865 return 1;
866 }
867
868 /* We have a reply we dont handle. This should indicate a perm server
869 failure */
870 if (Srv->Result < 200 || Srv->Result >= 300)
871 {
872 _error->Error("%u %s",Srv->Result,Srv->Code);
873 if (Srv->HaveContent == true)
874 return 4;
875 return 3;
876 }
877
878 // This is some sort of 2xx 'data follows' reply
879 Res.LastModified = Srv->Date;
880 Res.Size = Srv->Size;
881
882 // Open the file
883 delete File;
884 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
885 if (_error->PendingError() == true)
886 return 5;
887
888 FailFile = Queue->DestFile;
889 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
890 FailFd = File->Fd();
891 FailTime = Srv->Date;
892
893 // Set the expected size
894 if (Srv->StartPos >= 0)
895 {
896 Res.ResumePoint = Srv->StartPos;
897 ftruncate(File->Fd(),Srv->StartPos);
898 }
899
900 // Set the start point
901 lseek(File->Fd(),0,SEEK_END);
902
903 delete Srv->In.Hash;
904 Srv->In.Hash = new Hashes;
905
906 // Fill the Hash if the file is non-empty (resume)
907 if (Srv->StartPos > 0)
908 {
909 lseek(File->Fd(),0,SEEK_SET);
910 if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
911 {
912 _error->Errno("read",_("Problem hashing file"));
913 return 5;
914 }
915 lseek(File->Fd(),0,SEEK_END);
916 }
917
918 SetNonBlock(File->Fd(),true);
919 return 0;
920 }
921 /*}}}*/
922 // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
923 // ---------------------------------------------------------------------
924 /* This closes and timestamps the open file. This is neccessary to get
925 resume behavoir on user abort */
926 void HttpMethod::SigTerm(int)
927 {
928 if (FailFd == -1)
929 _exit(100);
930 close(FailFd);
931
932 // Timestamp
933 struct utimbuf UBuf;
934 UBuf.actime = FailTime;
935 UBuf.modtime = FailTime;
936 utime(FailFile.c_str(),&UBuf);
937
938 _exit(100);
939 }
940 /*}}}*/
941 // HttpMethod::Fetch - Fetch an item /*{{{*/
942 // ---------------------------------------------------------------------
943 /* This adds an item to the pipeline. We keep the pipeline at a fixed
944 depth. */
945 bool HttpMethod::Fetch(FetchItem *)
946 {
947 if (Server == 0)
948 return true;
949
950 // Queue the requests
951 int Depth = -1;
952 bool Tail = false;
953 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
954 I = I->Next, Depth++)
955 {
956 // If pipelining is disabled, we only queue 1 request
957 if (Server->Pipeline == false && Depth >= 0)
958 break;
959
960 // Make sure we stick with the same server
961 if (Server->Comp(I->Uri) == false)
962 break;
963 if (QueueBack == I)
964 Tail = true;
965 if (Tail == true)
966 {
967 QueueBack = I->Next;
968 SendReq(I,Server->Out);
969 continue;
970 }
971 }
972
973 return true;
974 };
975 /*}}}*/
976 // HttpMethod::Configuration - Handle a configuration message /*{{{*/
977 // ---------------------------------------------------------------------
978 /* We stash the desired pipeline depth */
979 bool HttpMethod::Configuration(string Message)
980 {
981 if (pkgAcqMethod::Configuration(Message) == false)
982 return false;
983
984 TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
985 PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
986 PipelineDepth);
987 Debug = _config->FindB("Debug::Acquire::http",false);
988
989 return true;
990 }
991 /*}}}*/
992 // HttpMethod::Loop - Main loop /*{{{*/
993 // ---------------------------------------------------------------------
994 /* */
995 int HttpMethod::Loop()
996 {
997 signal(SIGTERM,SigTerm);
998 signal(SIGINT,SigTerm);
999
1000 Server = 0;
1001
1002 int FailCounter = 0;
1003 while (1)
1004 {
1005 // We have no commands, wait for some to arrive
1006 if (Queue == 0)
1007 {
1008 if (WaitFd(STDIN_FILENO) == false)
1009 return 0;
1010 }
1011
1012 /* Run messages, we can accept 0 (no message) if we didn't
1013 do a WaitFd above.. Otherwise the FD is closed. */
1014 int Result = Run(true);
1015 if (Result != -1 && (Result != 0 || Queue == 0))
1016 return 100;
1017
1018 if (Queue == 0)
1019 continue;
1020
1021 // Connect to the server
1022 if (Server == 0 || Server->Comp(Queue->Uri) == false)
1023 {
1024 delete Server;
1025 Server = new ServerState(Queue->Uri,this);
1026 }
1027
1028 /* If the server has explicitly said this is the last connection
1029 then we pre-emptively shut down the pipeline and tear down
1030 the connection. This will speed up HTTP/1.0 servers a tad
1031 since we don't have to wait for the close sequence to
1032 complete */
1033 if (Server->Persistent == false)
1034 Server->Close();
1035
1036 // Reset the pipeline
1037 if (Server->ServerFd == -1)
1038 QueueBack = Queue;
1039
1040 // Connnect to the host
1041 if (Server->Open() == false)
1042 {
1043 Fail(true);
1044 delete Server;
1045 Server = 0;
1046 continue;
1047 }
1048
1049 // Fill the pipeline.
1050 Fetch(0);
1051
1052 // Fetch the next URL header data from the server.
1053 switch (Server->RunHeaders())
1054 {
1055 case 0:
1056 break;
1057
1058 // The header data is bad
1059 case 2:
1060 {
1061 _error->Error(_("Bad header Data"));
1062 Fail(true);
1063 RotateDNS();
1064 continue;
1065 }
1066
1067 // The server closed a connection during the header get..
1068 default:
1069 case 1:
1070 {
1071 FailCounter++;
1072 _error->Discard();
1073 Server->Close();
1074 Server->Pipeline = false;
1075
1076 if (FailCounter >= 2)
1077 {
1078 Fail(_("Connection failed"),true);
1079 FailCounter = 0;
1080 }
1081
1082 RotateDNS();
1083 continue;
1084 }
1085 };
1086
1087 // Decide what to do.
1088 FetchResult Res;
1089 Res.Filename = Queue->DestFile;
1090 switch (DealWithHeaders(Res,Server))
1091 {
1092 // Ok, the file is Open
1093 case 0:
1094 {
1095 URIStart(Res);
1096
1097 // Run the data
1098 bool Result = Server->RunData();
1099
1100 /* If the server is sending back sizeless responses then fill in
1101 the size now */
1102 if (Res.Size == 0)
1103 Res.Size = File->Size();
1104
1105 // Close the file, destroy the FD object and timestamp it
1106 FailFd = -1;
1107 delete File;
1108 File = 0;
1109
1110 // Timestamp
1111 struct utimbuf UBuf;
1112 time(&UBuf.actime);
1113 UBuf.actime = Server->Date;
1114 UBuf.modtime = Server->Date;
1115 utime(Queue->DestFile.c_str(),&UBuf);
1116
1117 // Send status to APT
1118 if (Result == true)
1119 {
1120 Res.TakeHashes(*Server->In.Hash);
1121 URIDone(Res);
1122 }
1123 else
1124 Fail(true);
1125
1126 break;
1127 }
1128
1129 // IMS hit
1130 case 1:
1131 {
1132 URIDone(Res);
1133 break;
1134 }
1135
1136 // Hard server error, not found or something
1137 case 3:
1138 {
1139 Fail();
1140 break;
1141 }
1142
1143 // Hard internal error, kill the connection and fail
1144 case 5:
1145 {
1146 delete File;
1147 File = 0;
1148
1149 Fail();
1150 RotateDNS();
1151 Server->Close();
1152 break;
1153 }
1154
1155 // We need to flush the data, the header is like a 404 w/ error text
1156 case 4:
1157 {
1158 Fail();
1159
1160 // Send to content to dev/null
1161 File = new FileFd("/dev/null",FileFd::WriteExists);
1162 Server->RunData();
1163 delete File;
1164 File = 0;
1165 break;
1166 }
1167
1168 default:
1169 Fail(_("Internal error"));
1170 break;
1171 }
1172
1173 FailCounter = 0;
1174 }
1175
1176 return 0;
1177 }
1178 /*}}}*/
1179
1180 int main()
1181 {
1182 HttpMethod Mth;
1183
1184 return Mth.Loop();
1185 }
1186
1187