correct some style/performance/warnings from cppcheck
[ntk/apt.git] / methods / http.cc
CommitLineData
be4401bf
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
2cbcabd8 3// $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
be4401bf
AL
4/* ######################################################################
5
ae58a985 6 HTTP Acquire Method - This is the HTTP aquire method for APT.
be4401bf
AL
7
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
e836f356
AL
9 pipelining, range, if-range and so on.
10
11 It is based on a doubly buffered select loop. A groupe of requests are
be4401bf
AL
12 fed into a single output buffer that is constantly fed out the
13 socket. This provides ideal pipelining as in many cases all of the
14 requests will fit into a single packet. The input socket is buffered
e836f356 15 the same way and fed into the fd for the file (may be a pipe in future).
be4401bf
AL
16
17 This double buffering provides fairly substantial transfer rates,
18 compared to wget the http method is about 4% faster. Most importantly,
19 when HTTP is compared with FTP as a protocol the speed difference is
20 huge. In tests over the internet from two sites to llug (via ATM) this
21 program got 230k/s sustained http transfer rates. FTP on the other
22 hand topped out at 170k/s. That combined with the time to setup the
23 FTP connection makes HTTP a vastly superior protocol.
24
25 ##################################################################### */
26 /*}}}*/
27// Include Files /*{{{*/
ea542140
DK
28#include <config.h>
29
be4401bf
AL
30#include <apt-pkg/fileutl.h>
31#include <apt-pkg/acquire-method.h>
472ff00e 32#include <apt-pkg/configuration.h>
be4401bf 33#include <apt-pkg/error.h>
63b1700f 34#include <apt-pkg/hashes.h>
592b7800 35#include <apt-pkg/netrc.h>
be4401bf
AL
36
37#include <sys/stat.h>
38#include <sys/time.h>
39#include <utime.h>
40#include <unistd.h>
492f957a 41#include <signal.h>
be4401bf 42#include <stdio.h>
65a1e968 43#include <errno.h>
42195eb2 44#include <string.h>
74865d5d 45#include <climits>
42195eb2 46#include <iostream>
15d7e515 47#include <map>
592b7800 48
be4401bf 49// Internet stuff
0837bd25 50#include <netdb.h>
be4401bf 51
59b46c41 52#include "config.h"
0837bd25 53#include "connect.h"
934b6582 54#include "rfc2553emu.h"
be4401bf 55#include "http.h"
ea542140
DK
56
57#include <apti18n.h>
be4401bf 58 /*}}}*/
42195eb2 59using namespace std;
be4401bf 60
650faab0
DK
61unsigned long long CircleBuf::BwReadLimit=0;
62unsigned long long CircleBuf::BwTickReadData=0;
7c6e2dc7
MV
63struct timeval CircleBuf::BwReadTick={0,0};
64const unsigned int CircleBuf::BW_HZ=10;
7273e494 65
be4401bf
AL
66// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
67// ---------------------------------------------------------------------
68/* */
650faab0 69CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
be4401bf
AL
70{
71 Buf = new unsigned char[Size];
72 Reset();
7c6e2dc7
MV
73
74 CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
be4401bf
AL
75}
76 /*}}}*/
77// CircleBuf::Reset - Reset to the default state /*{{{*/
78// ---------------------------------------------------------------------
79/* */
80void CircleBuf::Reset()
81{
82 InP = 0;
83 OutP = 0;
84 StrPos = 0;
650faab0 85 MaxGet = (unsigned long long)-1;
be4401bf 86 OutQueue = string();
63b1700f 87 if (Hash != 0)
be4401bf 88 {
63b1700f
AL
89 delete Hash;
90 Hash = new Hashes;
be4401bf
AL
91 }
92};
93 /*}}}*/
94// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
95// ---------------------------------------------------------------------
96/* This fills up the buffer with as much data as is in the FD, assuming it
97 is non-blocking.. */
98bool CircleBuf::Read(int Fd)
99{
100 while (1)
101 {
102 // Woops, buffer is full
103 if (InP - OutP == Size)
104 return true;
7c6e2dc7
MV
105
106 // what's left to read in this tick
9ce3cfc9 107 unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
7c6e2dc7
MV
108
109 if(CircleBuf::BwReadLimit) {
110 struct timeval now;
111 gettimeofday(&now,0);
112
650faab0 113 unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
7c6e2dc7
MV
114 now.tv_usec-CircleBuf::BwReadTick.tv_usec;
115 if(d > 1000000/BW_HZ) {
116 CircleBuf::BwReadTick = now;
117 CircleBuf::BwTickReadData = 0;
118 }
119
120 if(CircleBuf::BwTickReadData >= BwReadMax) {
121 usleep(1000000/BW_HZ);
122 return true;
123 }
124 }
125
be4401bf 126 // Write the buffer segment
650faab0 127 ssize_t Res;
7c6e2dc7
MV
128 if(CircleBuf::BwReadLimit) {
129 Res = read(Fd,Buf + (InP%Size),
130 BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
131 } else
132 Res = read(Fd,Buf + (InP%Size),LeftRead());
be4401bf 133
7c6e2dc7
MV
134 if(Res > 0 && BwReadLimit > 0)
135 CircleBuf::BwTickReadData += Res;
136
be4401bf
AL
137 if (Res == 0)
138 return false;
139 if (Res < 0)
140 {
141 if (errno == EAGAIN)
142 return true;
143 return false;
144 }
145
146 if (InP == 0)
147 gettimeofday(&Start,0);
148 InP += Res;
149 }
150}
151 /*}}}*/
152// CircleBuf::Read - Put the string into the buffer /*{{{*/
153// ---------------------------------------------------------------------
154/* This will hold the string in and fill the buffer with it as it empties */
155bool CircleBuf::Read(string Data)
156{
157 OutQueue += Data;
158 FillOut();
159 return true;
160}
161 /*}}}*/
162// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
163// ---------------------------------------------------------------------
164/* */
165void CircleBuf::FillOut()
166{
167 if (OutQueue.empty() == true)
168 return;
169 while (1)
170 {
171 // Woops, buffer is full
172 if (InP - OutP == Size)
173 return;
174
175 // Write the buffer segment
650faab0 176 unsigned long long Sz = LeftRead();
be4401bf
AL
177 if (OutQueue.length() - StrPos < Sz)
178 Sz = OutQueue.length() - StrPos;
42195eb2 179 memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
be4401bf
AL
180
181 // Advance
182 StrPos += Sz;
183 InP += Sz;
184 if (OutQueue.length() == StrPos)
185 {
186 StrPos = 0;
187 OutQueue = "";
188 return;
189 }
190 }
191}
192 /*}}}*/
193// CircleBuf::Write - Write from the buffer into a FD /*{{{*/
194// ---------------------------------------------------------------------
195/* This empties the buffer into the FD. */
196bool CircleBuf::Write(int Fd)
197{
198 while (1)
199 {
200 FillOut();
201
202 // Woops, buffer is empty
203 if (OutP == InP)
204 return true;
205
206 if (OutP == MaxGet)
207 return true;
208
209 // Write the buffer segment
650faab0 210 ssize_t Res;
be4401bf
AL
211 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
212
213 if (Res == 0)
214 return false;
215 if (Res < 0)
216 {
217 if (errno == EAGAIN)
218 return true;
219
220 return false;
221 }
222
63b1700f
AL
223 if (Hash != 0)
224 Hash->Add(Buf + (OutP%Size),Res);
be4401bf
AL
225
226 OutP += Res;
227 }
228}
229 /*}}}*/
230// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
231// ---------------------------------------------------------------------
232/* This copies till the first empty line */
233bool CircleBuf::WriteTillEl(string &Data,bool Single)
234{
235 // We cheat and assume it is unneeded to have more than one buffer load
650faab0 236 for (unsigned long long I = OutP; I < InP; I++)
be4401bf
AL
237 {
238 if (Buf[I%Size] != '\n')
239 continue;
2cbcabd8 240 ++I;
be4401bf
AL
241
242 if (Single == false)
243 {
2cbcabd8
AL
244 if (I < InP && Buf[I%Size] == '\r')
245 ++I;
927c393f
MV
246 if (I >= InP || Buf[I%Size] != '\n')
247 continue;
248 ++I;
be4401bf
AL
249 }
250
be4401bf
AL
251 Data = "";
252 while (OutP < I)
253 {
650faab0 254 unsigned long long Sz = LeftWrite();
be4401bf
AL
255 if (Sz == 0)
256 return false;
927c393f 257 if (I - OutP < Sz)
be4401bf
AL
258 Sz = I - OutP;
259 Data += string((char *)(Buf + (OutP%Size)),Sz);
260 OutP += Sz;
261 }
262 return true;
263 }
264 return false;
265}
266 /*}}}*/
267// CircleBuf::Stats - Print out stats information /*{{{*/
268// ---------------------------------------------------------------------
269/* */
270void CircleBuf::Stats()
271{
272 if (InP == 0)
273 return;
274
275 struct timeval Stop;
276 gettimeofday(&Stop,0);
277/* float Diff = Stop.tv_sec - Start.tv_sec +
278 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
279 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
280}
281 /*}}}*/
472ff00e
DK
282CircleBuf::~CircleBuf()
283{
284 delete [] Buf;
285 delete Hash;
286}
be4401bf 287
7330f4df
DK
288// HttpServerState::HttpServerState - Constructor /*{{{*/
289HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
be4401bf 290{
7330f4df 291 TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
be4401bf
AL
292 Reset();
293}
294 /*}}}*/
7330f4df 295// HttpServerState::Open - Open a connection to the server /*{{{*/
be4401bf
AL
296// ---------------------------------------------------------------------
297/* This opens a connection to the server. */
7330f4df 298bool HttpServerState::Open()
be4401bf 299{
92e889c8
AL
300 // Use the already open connection if possible.
301 if (ServerFd != -1)
302 return true;
303
be4401bf 304 Close();
492f957a
AL
305 In.Reset();
306 Out.Reset();
e836f356
AL
307 Persistent = true;
308
492f957a 309 // Determine the proxy setting
788a8f42
EL
310 string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
311 if (!SpecificProxy.empty())
492f957a 312 {
788a8f42
EL
313 if (SpecificProxy == "DIRECT")
314 Proxy = "";
315 else
316 Proxy = SpecificProxy;
352c2768 317 }
492f957a 318 else
788a8f42
EL
319 {
320 string DefProxy = _config->Find("Acquire::http::Proxy");
321 if (!DefProxy.empty())
322 {
323 Proxy = DefProxy;
324 }
325 else
326 {
327 char* result = getenv("http_proxy");
328 Proxy = result ? result : "";
329 }
330 }
352c2768 331
f8081133 332 // Parse no_proxy, a , separated list of domains
9e2a06ff
AL
333 if (getenv("no_proxy") != 0)
334 {
f8081133
AL
335 if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
336 Proxy = "";
337 }
338
492f957a 339 // Determine what host and port to use based on the proxy settings
934b6582 340 int Port = 0;
492f957a 341 string Host;
dd1fd92b 342 if (Proxy.empty() == true || Proxy.Host.empty() == true)
be4401bf 343 {
92e889c8
AL
344 if (ServerName.Port != 0)
345 Port = ServerName.Port;
be4401bf
AL
346 Host = ServerName.Host;
347 }
348 else
349 {
92e889c8
AL
350 if (Proxy.Port != 0)
351 Port = Proxy.Port;
be4401bf
AL
352 Host = Proxy.Host;
353 }
354
0837bd25 355 // Connect to the remote server
9505213b 356 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
0837bd25 357 return false;
3000ccea 358
be4401bf
AL
359 return true;
360}
361 /*}}}*/
7330f4df 362// HttpServerState::Close - Close a connection to the server /*{{{*/
be4401bf
AL
363// ---------------------------------------------------------------------
364/* */
7330f4df 365bool HttpServerState::Close()
be4401bf
AL
366{
367 close(ServerFd);
368 ServerFd = -1;
be4401bf
AL
369 return true;
370}
371 /*}}}*/
7330f4df
DK
372// HttpServerState::RunData - Transfer the data from the socket /*{{{*/
373bool HttpServerState::RunData(FileFd * const File)
be4401bf
AL
374{
375 State = Data;
376
377 // Chunked transfer encoding is fun..
378 if (Encoding == Chunked)
379 {
380 while (1)
381 {
382 // Grab the block size
383 bool Last = true;
384 string Data;
385 In.Limit(-1);
386 do
387 {
388 if (In.WriteTillEl(Data,true) == true)
389 break;
390 }
7330f4df 391 while ((Last = Go(false, File)) == true);
be4401bf
AL
392
393 if (Last == false)
394 return false;
395
396 // See if we are done
650faab0 397 unsigned long long Len = strtoull(Data.c_str(),0,16);
be4401bf
AL
398 if (Len == 0)
399 {
400 In.Limit(-1);
401
402 // We have to remove the entity trailer
403 Last = true;
404 do
405 {
406 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
407 break;
408 }
7330f4df 409 while ((Last = Go(false, File)) == true);
be4401bf
AL
410 if (Last == false)
411 return false;
e1b96638 412 return !_error->PendingError();
be4401bf
AL
413 }
414
415 // Transfer the block
416 In.Limit(Len);
7330f4df 417 while (Go(true, File) == true)
be4401bf
AL
418 if (In.IsLimit() == true)
419 break;
420
421 // Error
422 if (In.IsLimit() == false)
423 return false;
424
425 // The server sends an extra new line before the next block specifier..
426 In.Limit(-1);
427 Last = true;
428 do
429 {
430 if (In.WriteTillEl(Data,true) == true)
431 break;
432 }
7330f4df 433 while ((Last = Go(false, File)) == true);
be4401bf
AL
434 if (Last == false)
435 return false;
92e889c8 436 }
be4401bf
AL
437 }
438 else
439 {
440 /* Closes encoding is used when the server did not specify a size, the
441 loss of the connection means we are done */
442 if (Encoding == Closes)
443 In.Limit(-1);
444 else
445 In.Limit(Size - StartPos);
446
447 // Just transfer the whole block.
448 do
449 {
450 if (In.IsLimit() == false)
451 continue;
452
453 In.Limit(-1);
e1b96638 454 return !_error->PendingError();
be4401bf 455 }
7330f4df 456 while (Go(true, File) == true);
be4401bf
AL
457 }
458
7330f4df 459 return Owner->Flush() && !_error->PendingError();
be4401bf
AL
460}
461 /*}}}*/
7330f4df 462bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
be4401bf 463{
7330f4df
DK
464 return In.WriteTillEl(Data);
465}
466 /*}}}*/
467bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
468{
469 return Go(ToFile, File);
470}
471 /*}}}*/
472bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
473{
474 return Out.Read(Data);
475}
476 /*}}}*/
477bool HttpServerState::IsOpen() /*{{{*/
478{
479 return (ServerFd != -1);
480}
481 /*}}}*/
482bool HttpServerState::InitHashes(FileFd &File) /*{{{*/
483{
484 delete In.Hash;
485 In.Hash = new Hashes;
30456e14 486
7330f4df
DK
487 // Set the expected size and read file for the hashes
488 if (StartPos >= 0)
c901051d 489 {
7330f4df
DK
490 File.Truncate(StartPos);
491
492 return In.Hash->AddFD(File, StartPos);
c901051d 493 }
7330f4df
DK
494 return true;
495}
496 /*}}}*/
497Hashes * HttpServerState::GetHashes() /*{{{*/
498{
499 return In.Hash;
500}
501 /*}}}*/
502// HttpServerState::Die - The server has closed the connection. /*{{{*/
503bool HttpServerState::Die(FileFd &File)
504{
505 unsigned int LErrno = errno;
be4401bf 506
7330f4df
DK
507 // Dump the buffer to the file
508 if (State == ServerState::Data)
be4401bf 509 {
7330f4df
DK
510 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
511 // can't be set
512 if (File.Name() != "/dev/null")
513 SetNonBlock(File.Fd(),false);
514 while (In.WriteSpace() == true)
be4401bf 515 {
7330f4df
DK
516 if (In.Write(File.Fd()) == false)
517 return _error->Errno("write",_("Error writing to the file"));
e836f356 518
7330f4df
DK
519 // Done
520 if (In.IsLimit() == true)
521 return true;
e836f356 522 }
7330f4df 523 }
b2e465d6 524
7330f4df
DK
525 // See if this is because the server finished the data stream
526 if (In.IsLimit() == false && State != HttpServerState::Header &&
527 Encoding != HttpServerState::Closes)
be4401bf 528 {
7330f4df
DK
529 Close();
530 if (LErrno == 0)
531 return _error->Error(_("Error reading from server. Remote end closed connection"));
532 errno = LErrno;
533 return _error->Errno("read",_("Error reading from server"));
be4401bf 534 }
7330f4df 535 else
92e889c8 536 {
7330f4df
DK
537 In.Limit(-1);
538
539 // Nothing left in the buffer
540 if (In.WriteSpace() == false)
541 return false;
542
543 // We may have got multiple responses back in one packet..
544 Close();
92e889c8
AL
545 return true;
546 }
331e8396 547
7330f4df
DK
548 return false;
549}
550 /*}}}*/
551// HttpServerState::Flush - Dump the buffer into the file /*{{{*/
552// ---------------------------------------------------------------------
553/* This takes the current input buffer from the Server FD and writes it
554 into the file */
555bool HttpServerState::Flush(FileFd * const File)
556{
557 if (File != NULL)
558 {
559 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
560 // can't be set
561 if (File->Name() != "/dev/null")
562 SetNonBlock(File->Fd(),false);
563 if (In.WriteSpace() == false)
564 return true;
565
566 while (In.WriteSpace() == true)
331e8396 567 {
7330f4df
DK
568 if (In.Write(File->Fd()) == false)
569 return _error->Errno("write",_("Error writing to file"));
570 if (In.IsLimit() == true)
571 return true;
331e8396 572 }
7330f4df
DK
573
574 if (In.IsLimit() == true || Encoding == ServerState::Closes)
575 return true;
be4401bf 576 }
7330f4df
DK
577 return false;
578}
579 /*}}}*/
580// HttpServerState::Go - Run a single loop /*{{{*/
581// ---------------------------------------------------------------------
582/* This runs the select loop over the server FDs, Output file FDs and
583 stdin. */
584bool HttpServerState::Go(bool ToFile, FileFd * const File)
585{
586 // Server has closed the connection
587 if (ServerFd == -1 && (In.WriteSpace() == false ||
588 ToFile == false))
589 return false;
590
591 fd_set rfds,wfds;
592 FD_ZERO(&rfds);
593 FD_ZERO(&wfds);
594
595 /* Add the server. We only send more requests if the connection will
596 be persisting */
597 if (Out.WriteSpace() == true && ServerFd != -1
598 && Persistent == true)
599 FD_SET(ServerFd,&wfds);
600 if (In.ReadSpace() == true && ServerFd != -1)
601 FD_SET(ServerFd,&rfds);
be4401bf 602
7330f4df
DK
603 // Add the file
604 int FileFD = -1;
605 if (File != NULL)
606 FileFD = File->Fd();
607
608 if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
609 FD_SET(FileFD,&wfds);
610
611 // Add stdin
612 if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
613 FD_SET(STDIN_FILENO,&rfds);
614
615 // Figure out the max fd
616 int MaxFd = FileFD;
617 if (MaxFd < ServerFd)
618 MaxFd = ServerFd;
619
620 // Select
621 struct timeval tv;
622 tv.tv_sec = TimeOut;
623 tv.tv_usec = 0;
624 int Res = 0;
625 if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
be4401bf 626 {
7330f4df
DK
627 if (errno == EINTR)
628 return true;
629 return _error->Errno("select",_("Select failed"));
be4401bf 630 }
7330f4df
DK
631
632 if (Res == 0)
e836f356 633 {
7330f4df
DK
634 _error->Error(_("Connection timed out"));
635 return Die(*File);
e836f356
AL
636 }
637
7330f4df
DK
638 // Handle server IO
639 if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
be4401bf 640 {
7330f4df
DK
641 errno = 0;
642 if (In.Read(ServerFd) == false)
643 return Die(*File);
644 }
645
646 if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
647 {
648 errno = 0;
649 if (Out.Write(ServerFd) == false)
650 return Die(*File);
be4401bf
AL
651 }
652
7330f4df
DK
653 // Send data to the file
654 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
15d7e515 655 {
7330f4df
DK
656 if (In.Write(FileFD) == false)
657 return _error->Errno("write",_("Error writing to output file"));
15d7e515
MV
658 }
659
7330f4df
DK
660 // Handle commands from APT
661 if (FD_ISSET(STDIN_FILENO,&rfds))
662 {
663 if (Owner->Run(true) != -1)
664 exit(100);
665 }
666
be4401bf
AL
667 return true;
668}
669 /*}}}*/
670
671// HttpMethod::SendReq - Send the HTTP request /*{{{*/
672// ---------------------------------------------------------------------
673/* This places the http request in the outbound buffer */
7330f4df 674void HttpMethod::SendReq(FetchItem *Itm)
be4401bf
AL
675{
676 URI Uri = Itm->Uri;
c1a22377 677
be4401bf 678 // The HTTP server expects a hostname with a trailing :port
c1a22377 679 char Buf[1000];
5b63d2a9
MV
680 string ProperHost;
681
682 if (Uri.Host.find(':') != string::npos)
683 ProperHost = '[' + Uri.Host + ']';
684 else
685 ProperHost = Uri.Host;
be4401bf
AL
686 if (Uri.Port != 0)
687 {
688 sprintf(Buf,":%u",Uri.Port);
689 ProperHost += Buf;
690 }
691
c1a22377
AL
692 // Just in case.
693 if (Itm->Uri.length() >= sizeof(Buf))
694 abort();
f2380a78
DK
695
696 /* RFC 2616 ยง5.1.2 requires absolute URIs for requests to proxies,
697 but while its a must for all servers to accept absolute URIs,
698 it is assumed clients will sent an absolute path for non-proxies */
699 std::string requesturi;
7330f4df 700 if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
f2380a78
DK
701 requesturi = Uri.Path;
702 else
703 requesturi = Itm->Uri;
704
705 // The "+" is encoded as a workaround for a amazon S3 bug
706 // see LP bugs #1003633 and #1086997.
707 requesturi = QuoteString(requesturi, "+~ ");
708
2b9c9b7f
RG
709 /* Build the request. No keep-alive is included as it is the default
710 in 1.1, can cause problems with proxies, and we are an HTTP/1.1
711 client anyway.
712 C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
2b9c9b7f 713 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
f2380a78
DK
714 requesturi.c_str(),ProperHost.c_str());
715
c9cd3b70
MV
716 // generate a cache control header (if needed)
717 if (_config->FindB("Acquire::http::No-Cache",false) == true)
718 {
719 strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
720 }
721 else
722 {
723 if (Itm->IndexFile == true)
c1a22377 724 {
c9cd3b70
MV
725 sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
726 _config->FindI("Acquire::http::Max-Age",0));
727 }
728 else
729 {
730 if (_config->FindB("Acquire::http::No-Store",false) == true)
731 strcat(Buf,"Cache-Control: no-store\r\n");
c1a22377
AL
732 }
733 }
106e6740 734
6f4501f9
DK
735 // If we ask for uncompressed files servers might respond with content-
736 // negotation which lets us end up with compressed files we do not support,
737 // see 657029, 657560 and co, so if we have no extension on the request
738 // ask for text only. As a sidenote: If there is nothing to negotate servers
739 // seem to be nice and ignore it.
740 if (_config->FindB("Acquire::http::SendAccept", true) == true)
741 {
742 size_t const filepos = Itm->Uri.find_last_of('/');
743 string const file = Itm->Uri.substr(filepos + 1);
744 if (flExtension(file) == file)
745 strcat(Buf,"Accept: text/*\r\n");
746 }
747
be4401bf 748 string Req = Buf;
492f957a 749
be4401bf
AL
750 // Check for a partial file
751 struct stat SBuf;
752 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
753 {
754 // In this case we send an if-range query with a range header
78c72d0c 755 sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size,
be4401bf
AL
756 TimeRFC1123(SBuf.st_mtime).c_str());
757 Req += Buf;
758 }
759 else
760 {
761 if (Itm->LastModified != 0)
762 {
763 sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
764 Req += Buf;
765 }
766 }
767
7330f4df 768 if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
8d64c395 769 Req += string("Proxy-Authorization: Basic ") +
7330f4df 770 Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) + "\r\n";
be4401bf 771
1de1f703 772 maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
b2e465d6 773 if (Uri.User.empty() == false || Uri.Password.empty() == false)
592b7800 774 {
b2e465d6
AL
775 Req += string("Authorization: Basic ") +
776 Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
592b7800 777 }
9f542bae 778 Req += "User-Agent: " + _config->Find("Acquire::http::User-Agent",
335e2c82 779 "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") + "\r\n\r\n";
c98b1307
AL
780
781 if (Debug == true)
782 cerr << Req << endl;
c1a22377 783
7330f4df 784 Server->WriteResponse(Req);
be4401bf
AL
785}
786 /*}}}*/
85f72a56
AL
787// HttpMethod::Configuration - Handle a configuration message /*{{{*/
788// ---------------------------------------------------------------------
789/* We stash the desired pipeline depth */
790bool HttpMethod::Configuration(string Message)
791{
7330f4df 792 if (ServerMethod::Configuration(Message) == false)
85f72a56 793 return false;
7330f4df 794
15d7e515 795 AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
30456e14
AL
796 PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
797 PipelineDepth);
c98b1307 798 Debug = _config->FindB("Debug::Acquire::http",false);
bd49b02c
MV
799
800 // Get the proxy to use
801 AutoDetectProxy();
802
85f72a56
AL
803 return true;
804}
805 /*}}}*/
bd49b02c
MV
806// HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/
807// ---------------------------------------------------------------------
808/* */
809bool HttpMethod::AutoDetectProxy()
810{
c08cf1dc
MV
811 // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
812 // name without the dash ("-")
813 AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect",
814 _config->Find("Acquire::http::ProxyAutoDetect"));
815
bd49b02c
MV
816 if (AutoDetectProxyCmd.empty())
817 return true;
818
819 if (Debug)
820 clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl;
821
822 int Pipes[2] = {-1,-1};
823 if (pipe(Pipes) != 0)
824 return _error->Errno("pipe", "Failed to create Pipe");
825
826 pid_t Process = ExecFork();
827 if (Process == 0)
828 {
08ded4d6 829 close(Pipes[0]);
bd49b02c
MV
830 dup2(Pipes[1],STDOUT_FILENO);
831 SetCloseExec(STDOUT_FILENO,false);
08ded4d6 832
bd49b02c
MV
833 const char *Args[2];
834 Args[0] = AutoDetectProxyCmd.c_str();
835 Args[1] = 0;
836 execv(Args[0],(char **)Args);
837 cerr << "Failed to exec method " << Args[0] << endl;
838 _exit(100);
839 }
840 char buf[512];
841 int InFd = Pipes[0];
08ded4d6 842 close(Pipes[1]);
c1042000 843 int res = read(InFd, buf, sizeof(buf)-1);
08ded4d6
MV
844 ExecWait(Process, "ProxyAutoDetect", true);
845
846 if (res < 0)
bd49b02c 847 return _error->Errno("read", "Failed to read");
08ded4d6
MV
848 if (res == 0)
849 return _error->Warning("ProxyAutoDetect returned no data");
850
851 // add trailing \0
852 buf[res] = 0;
853
bd49b02c
MV
854 if (Debug)
855 clog << "auto detect command returned: '" << buf << "'" << endl;
856
857 if (strstr(buf, "http://") == buf)
858 _config->Set("Acquire::http::proxy", _strstrip(buf));
859
860 return true;
861}
862 /*}}}*/
7330f4df
DK
863ServerState * HttpMethod::CreateServerState(URI uri) /*{{{*/
864{
865 return new HttpServerState(uri, this);
866}
867 /*}}}*/
fd46d305
DK
868void HttpMethod::RotateDNS() /*{{{*/
869{
870 ::RotateDNS();
871}
872 /*}}}*/