methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.33 1999/05/28 07:04:45 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 /*#include <netinet/in.h>
  45 #include <sys/socket.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>*/
  48
  49 #include "rfc2553emu.h"
  50 #include "http.h"
  51
  52                                                                         /*}}}*/
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59
  60 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* */
  63 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  64 {
  65    Buf = new unsigned char[Size];
  66    Reset();
  67 }
  68                                                                         /*}}}*/
  69 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* */
  72 void CircleBuf::Reset()
  73 {
  74    InP = 0;
  75    OutP = 0;
  76    StrPos = 0;
  77    MaxGet = (unsigned int)-1;
  78    OutQueue = string();
  79    if (MD5 != 0)
  80    {
  81       delete MD5;
  82       MD5 = new MD5Summation;
  83    }
  84 };
  85                                                                         /*}}}*/
  86 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* This fills up the buffer with as much data as is in the FD, assuming it
  89    is non-blocking.. */
  90 bool CircleBuf::Read(int Fd)
  91 {
  92    while (1)
  93    {
  94       // Woops, buffer is full
  95       if (InP - OutP == Size)
  96          return true;
  97
  98       // Write the buffer segment
  99       int Res;
 100       Res = read(Fd,Buf + (InP%Size),LeftRead());
 101
 102       if (Res == 0)
 103          return false;
 104       if (Res < 0)
 105       {
 106          if (errno == EAGAIN)
 107             return true;
 108          return false;
 109       }
 110
 111       if (InP == 0)
 112          gettimeofday(&Start,0);
 113       InP += Res;
 114    }
 115 }
 116                                                                         /*}}}*/
 117 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* This will hold the string in and fill the buffer with it as it empties */
 120 bool CircleBuf::Read(string Data)
 121 {
 122    OutQueue += Data;
 123    FillOut();
 124    return true;
 125 }
 126                                                                         /*}}}*/
 127 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* */
 130 void CircleBuf::FillOut()
 131 {
 132    if (OutQueue.empty() == true)
 133       return;
 134    while (1)
 135    {
 136       // Woops, buffer is full
 137       if (InP - OutP == Size)
 138          return;
 139
 140       // Write the buffer segment
 141       unsigned long Sz = LeftRead();
 142       if (OutQueue.length() - StrPos < Sz)
 143          Sz = OutQueue.length() - StrPos;
 144       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 145
 146       // Advance
 147       StrPos += Sz;
 148       InP += Sz;
 149       if (OutQueue.length() == StrPos)
 150       {
 151          StrPos = 0;
 152          OutQueue = "";
 153          return;
 154       }
 155    }
 156 }
 157                                                                         /*}}}*/
 158 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 159 // ---------------------------------------------------------------------
 160 /* This empties the buffer into the FD. */
 161 bool CircleBuf::Write(int Fd)
 162 {
 163    while (1)
 164    {
 165       FillOut();
 166
 167       // Woops, buffer is empty
 168       if (OutP == InP)
 169          return true;
 170
 171       if (OutP == MaxGet)
 172          return true;
 173
 174       // Write the buffer segment
 175       int Res;
 176       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 177
 178       if (Res == 0)
 179          return false;
 180       if (Res < 0)
 181       {
 182          if (errno == EAGAIN)
 183             return true;
 184
 185          return false;
 186       }
 187
 188       if (MD5 != 0)
 189          MD5->Add(Buf + (OutP%Size),Res);
 190
 191       OutP += Res;
 192    }
 193 }
 194                                                                         /*}}}*/
 195 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 196 // ---------------------------------------------------------------------
 197 /* This copies till the first empty line */
 198 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 199 {
 200    // We cheat and assume it is unneeded to have more than one buffer load
 201    for (unsigned long I = OutP; I < InP; I++)
 202    {
 203       if (Buf[I%Size] != '\n')
 204          continue;
 205       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 206
 207       if (Single == false)
 208       {
 209          if (Buf[I%Size] != '\n')
 210             continue;
 211          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 212       }
 213
 214       if (I > InP)
 215          I = InP;
 216
 217       Data = "";
 218       while (OutP < I)
 219       {
 220          unsigned long Sz = LeftWrite();
 221          if (Sz == 0)
 222             return false;
 223          if (I - OutP < LeftWrite())
 224             Sz = I - OutP;
 225          Data += string((char *)(Buf + (OutP%Size)),Sz);
 226          OutP += Sz;
 227       }
 228       return true;
 229    }
 230    return false;
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* */
 236 void CircleBuf::Stats()
 237 {
 238    if (InP == 0)
 239       return;
 240
 241    struct timeval Stop;
 242    gettimeofday(&Stop,0);
 243 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 244              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 245    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 246 }
 247                                                                         /*}}}*/
 248
 249 // ServerState::ServerState - Constructor                               /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 253                         In(64*1024), Out(4*1024),
 254                         ServerName(Srv)
 255 {
 256    Reset();
 257 }
 258                                                                         /*}}}*/
 259 // ServerState::Open - Open a connection to the server                  /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* This opens a connection to the server. */
 262 string LastHost;
 263 int LastPort = 0;
 264 struct addrinfo *LastHostAddr = 0;
 265 bool ServerState::Open()
 266 {
 267    // Use the already open connection if possible.
 268    if (ServerFd != -1)
 269       return true;
 270
 271    Close();
 272    In.Reset();
 273    Out.Reset();
 274
 275    // Determine the proxy setting
 276    if (getenv("http_proxy") == 0)
 277    {
 278       string DefProxy = _config->Find("Acquire::http::Proxy");
 279       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 280       if (SpecificProxy.empty() == false)
 281       {
 282          if (SpecificProxy == "DIRECT")
 283             Proxy = "";
 284          else
 285             Proxy = SpecificProxy;
 286       }
 287       else
 288          Proxy = DefProxy;
 289    }
 290    else
 291       Proxy = getenv("http_proxy");
 292
 293    // Determine what host and port to use based on the proxy settings
 294    int Port = 0;
 295    string Host;
 296    if (Proxy.empty() == true)
 297    {
 298       if (ServerName.Port != 0)
 299          Port = ServerName.Port;
 300       Host = ServerName.Host;
 301    }
 302    else
 303    {
 304       if (Proxy.Port != 0)
 305          Port = Proxy.Port;
 306       Host = Proxy.Host;
 307    }
 308
 309    /* We used a cached address record.. Yes this is against the spec but
 310       the way we have setup our rotating dns suggests that this is more
 311       sensible */
 312    if (LastHost != Host || LastPort != Port)
 313    {
 314       Owner->Status("Connecting to %s",Host.c_str());
 315
 316       // Lookup the host
 317       char S[30] = "http";
 318       if (Port != 0)
 319          snprintf(S,sizeof(S),"%u",Port);
 320
 321       // Free the old address structure
 322       if (LastHostAddr != 0)
 323       {
 324          freeaddrinfo(LastHostAddr);
 325          LastHostAddr = 0;
 326       }
 327
 328       // We only understand SOCK_STREAM sockets.
 329       struct addrinfo Hints;
 330       memset(&Hints,0,sizeof(Hints));
 331       Hints.ai_socktype = SOCK_STREAM;
 332
 333       // Resolve both the host and service simultaneously
 334       if (getaddrinfo(Host.c_str(),S,&Hints,&LastHostAddr) != 0 ||
 335           LastHostAddr == 0)
 336          return _error->Error("Could not resolve '%s'",Host.c_str());
 337
 338       LastHost = Host;
 339       LastPort = Port;
 340    }
 341
 342    // Get the printable IP address
 343    char Name[NI_MAXHOST];
 344    Name[0] = 0;
 345    getnameinfo(LastHostAddr->ai_addr,LastHostAddr->ai_addrlen,
 346                Name,sizeof(Name),0,0,NI_NUMERICHOST);
 347    Owner->Status("Connecting to %s (%s)",Host.c_str(),Name);
 348
 349    // Get a socket
 350    if ((ServerFd = socket(LastHostAddr->ai_family,LastHostAddr->ai_socktype,
 351                           LastHostAddr->ai_protocol)) < 0)
 352       return _error->Errno("socket","Could not create a socket");
 353    SetNonBlock(ServerFd,true);
 354    if (connect(ServerFd,LastHostAddr->ai_addr,LastHostAddr->ai_addrlen) < 0 &&
 355        errno != EINPROGRESS)
 356       return _error->Errno("connect","Cannot initiate the connection "
 357                            "to %s (%s).",Host.c_str(),Name);
 358
 359    /* This implements a timeout for connect by opening the connection
 360       nonblocking */
 361    if (WaitFd(ServerFd,true,TimeOut) == false)
 362       return _error->Error("Could not connect to %s (%s), "
 363                            "connection timed out",Host.c_str(),Name);
 364    unsigned int Err;
 365    unsigned int Len = sizeof(Err);
 366    if (getsockopt(ServerFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 367       return _error->Errno("getsockopt","Failed");
 368    if (Err != 0)
 369       return _error->Error("Could not connect to %s (%s).",Host.c_str(),Name);
 370
 371    return true;
 372 }
 373                                                                         /*}}}*/
 374 // ServerState::Close - Close a connection to the server                /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* */
 377 bool ServerState::Close()
 378 {
 379    close(ServerFd);
 380    ServerFd = -1;
 381    return true;
 382 }
 383                                                                         /*}}}*/
 384 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 385 // ---------------------------------------------------------------------
 386 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 387    parse error occured */
 388 int ServerState::RunHeaders()
 389 {
 390    State = Header;
 391
 392    Owner->Status("Waiting for file");
 393
 394    Major = 0;
 395    Minor = 0;
 396    Result = 0;
 397    Size = 0;
 398    StartPos = 0;
 399    Encoding = Closes;
 400    HaveContent = false;
 401    time(&Date);
 402
 403    do
 404    {
 405       string Data;
 406       if (In.WriteTillEl(Data) == false)
 407          continue;
 408
 409       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 410       {
 411          string::const_iterator J = I;
 412          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 413          if (HeaderLine(string(I,J-I)) == false)
 414             return 2;
 415          I = J;
 416       }
 417       return 0;
 418    }
 419    while (Owner->Go(false,this) == true);
 420
 421    return 1;
 422 }
 423                                                                         /*}}}*/
 424 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 425 // ---------------------------------------------------------------------
 426 /* */
 427 bool ServerState::RunData()
 428 {
 429    State = Data;
 430
 431    // Chunked transfer encoding is fun..
 432    if (Encoding == Chunked)
 433    {
 434       while (1)
 435       {
 436          // Grab the block size
 437          bool Last = true;
 438          string Data;
 439          In.Limit(-1);
 440          do
 441          {
 442             if (In.WriteTillEl(Data,true) == true)
 443                break;
 444          }
 445          while ((Last = Owner->Go(false,this)) == true);
 446
 447          if (Last == false)
 448             return false;
 449
 450          // See if we are done
 451          unsigned long Len = strtol(Data.c_str(),0,16);
 452          if (Len == 0)
 453          {
 454             In.Limit(-1);
 455
 456             // We have to remove the entity trailer
 457             Last = true;
 458             do
 459             {
 460                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 461                   break;
 462             }
 463             while ((Last = Owner->Go(false,this)) == true);
 464             if (Last == false)
 465                return false;
 466             return !_error->PendingError();
 467          }
 468
 469          // Transfer the block
 470          In.Limit(Len);
 471          while (Owner->Go(true,this) == true)
 472             if (In.IsLimit() == true)
 473                break;
 474
 475          // Error
 476          if (In.IsLimit() == false)
 477             return false;
 478
 479          // The server sends an extra new line before the next block specifier..
 480          In.Limit(-1);
 481          Last = true;
 482          do
 483          {
 484             if (In.WriteTillEl(Data,true) == true)
 485                break;
 486          }
 487          while ((Last = Owner->Go(false,this)) == true);
 488          if (Last == false)
 489             return false;
 490       }
 491    }
 492    else
 493    {
 494       /* Closes encoding is used when the server did not specify a size, the
 495          loss of the connection means we are done */
 496       if (Encoding == Closes)
 497          In.Limit(-1);
 498       else
 499          In.Limit(Size - StartPos);
 500
 501       // Just transfer the whole block.
 502       do
 503       {
 504          if (In.IsLimit() == false)
 505             continue;
 506
 507          In.Limit(-1);
 508          return !_error->PendingError();
 509       }
 510       while (Owner->Go(true,this) == true);
 511    }
 512
 513    return Owner->Flush(this) && !_error->PendingError();
 514 }
 515                                                                         /*}}}*/
 516 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 517 // ---------------------------------------------------------------------
 518 /* */
 519 bool ServerState::HeaderLine(string Line)
 520 {
 521    if (Line.empty() == true)
 522       return true;
 523
 524    // The http server might be trying to do something evil.
 525    if (Line.length() >= MAXLEN)
 526       return _error->Error("Got a single header line over %u chars",MAXLEN);
 527
 528    string::size_type Pos = Line.find(' ');
 529    if (Pos == string::npos || Pos+1 > Line.length())
 530    {
 531       // Blah, some servers use "connection:closes", evil.
 532       Pos = Line.find(':');
 533       if (Pos == string::npos || Pos + 2 > Line.length())
 534          return _error->Error("Bad header line");
 535       Pos++;
 536    }
 537
 538    // Parse off any trailing spaces between the : and the next word.
 539    string::size_type Pos2 = Pos;
 540    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 541       Pos2++;
 542
 543    string Tag = string(Line,0,Pos);
 544    string Val = string(Line,Pos2);
 545
 546    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 547    {
 548       // Evil servers return no version
 549       if (Line[4] == '/')
 550       {
 551          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 552                     &Result,Code) != 4)
 553             return _error->Error("The http server sent an invalid reply header");
 554       }
 555       else
 556       {
 557          Major = 0;
 558          Minor = 9;
 559          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 560             return _error->Error("The http server sent an invalid reply header");
 561       }
 562
 563       return true;
 564    }
 565
 566    if (stringcasecmp(Tag,"Content-Length:") == 0)
 567    {
 568       if (Encoding == Closes)
 569          Encoding = Stream;
 570       HaveContent = true;
 571
 572       // The length is already set from the Content-Range header
 573       if (StartPos != 0)
 574          return true;
 575
 576       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 577          return _error->Error("The http server sent an invalid Content-Length header");
 578       return true;
 579    }
 580
 581    if (stringcasecmp(Tag,"Content-Type:") == 0)
 582    {
 583       HaveContent = true;
 584       return true;
 585    }
 586
 587    if (stringcasecmp(Tag,"Content-Range:") == 0)
 588    {
 589       HaveContent = true;
 590
 591       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 592          return _error->Error("The http server sent an invalid Content-Range header");
 593       if ((unsigned)StartPos > Size)
 594          return _error->Error("This http server has broken range support");
 595       return true;
 596    }
 597
 598    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 599    {
 600       HaveContent = true;
 601       if (stringcasecmp(Val,"chunked") == 0)
 602          Encoding = Chunked;
 603
 604       return true;
 605    }
 606
 607    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 608    {
 609       if (StrToTime(Val,Date) == false)
 610          return _error->Error("Unknown date format");
 611       return true;
 612    }
 613
 614    return true;
 615 }
 616                                                                         /*}}}*/
 617
 618 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 619 // ---------------------------------------------------------------------
 620 /* This places the http request in the outbound buffer */
 621 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 622 {
 623    URI Uri = Itm->Uri;
 624
 625    // The HTTP server expects a hostname with a trailing :port
 626    char Buf[1000];
 627    string ProperHost = Uri.Host;
 628    if (Uri.Port != 0)
 629    {
 630       sprintf(Buf,":%u",Uri.Port);
 631       ProperHost += Buf;
 632    }
 633
 634    // Just in case.
 635    if (Itm->Uri.length() >= sizeof(Buf))
 636        abort();
 637
 638    /* Build the request. We include a keep-alive header only for non-proxy
 639       requests. This is to tweak old http/1.0 servers that do support keep-alive
 640       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 641       will glitch HTTP/1.0 proxies because they do not filter it out and
 642       pass it on, HTTP/1.1 says the connection should default to keep alive
 643       and we expect the proxy to do this */
 644    if (Proxy.empty() == true)
 645       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 646               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 647    else
 648    {
 649       /* Generate a cache control header if necessary. We place a max
 650          cache age on index files, optionally set a no-cache directive
 651          and a no-store directive for archives. */
 652       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 653               Itm->Uri.c_str(),ProperHost.c_str());
 654       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 655          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 656       else
 657       {
 658          if (Itm->IndexFile == true)
 659             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 660                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 661          else
 662          {
 663             if (_config->FindB("Acquire::http::No-Store",false) == true)
 664                strcat(Buf,"Cache-Control: no-store\r\n");
 665          }
 666       }
 667    }
 668
 669    string Req = Buf;
 670
 671    // Check for a partial file
 672    struct stat SBuf;
 673    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 674    {
 675       // In this case we send an if-range query with a range header
 676       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 677               TimeRFC1123(SBuf.st_mtime).c_str());
 678       Req += Buf;
 679    }
 680    else
 681    {
 682       if (Itm->LastModified != 0)
 683       {
 684          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 685          Req += Buf;
 686       }
 687    }
 688
 689    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 690       Req += string("Proxy-Authorization: Basic ") +
 691           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 692
 693    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 694 //   cerr << Req << endl;
 695
 696    Out.Read(Req);
 697 }
 698                                                                         /*}}}*/
 699 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 700 // ---------------------------------------------------------------------
 701 /* This runs the select loop over the server FDs, Output file FDs and
 702    stdin. */
 703 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 704 {
 705    // Server has closed the connection
 706    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 707       return false;
 708
 709    fd_set rfds,wfds,efds;
 710    FD_ZERO(&rfds);
 711    FD_ZERO(&wfds);
 712    FD_ZERO(&efds);
 713
 714    // Add the server
 715    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 716       FD_SET(Srv->ServerFd,&wfds);
 717    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 718       FD_SET(Srv->ServerFd,&rfds);
 719
 720    // Add the file
 721    int FileFD = -1;
 722    if (File != 0)
 723       FileFD = File->Fd();
 724
 725    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 726       FD_SET(FileFD,&wfds);
 727
 728    // Add stdin
 729    FD_SET(STDIN_FILENO,&rfds);
 730
 731    // Error Set
 732    if (FileFD != -1)
 733       FD_SET(FileFD,&efds);
 734    if (Srv->ServerFd != -1)
 735       FD_SET(Srv->ServerFd,&efds);
 736
 737    // Figure out the max fd
 738    int MaxFd = FileFD;
 739    if (MaxFd < Srv->ServerFd)
 740       MaxFd = Srv->ServerFd;
 741
 742    // Select
 743    struct timeval tv;
 744    tv.tv_sec = TimeOut;
 745    tv.tv_usec = 0;
 746    int Res = 0;
 747    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 748       return _error->Errno("select","Select failed");
 749
 750    if (Res == 0)
 751    {
 752       _error->Error("Connection timed out");
 753       return ServerDie(Srv);
 754    }
 755
 756    // Some kind of exception (error) on the sockets, die
 757    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 758        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 759       return _error->Error("Socket Exception");
 760
 761    // Handle server IO
 762    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 763    {
 764       errno = 0;
 765       if (Srv->In.Read(Srv->ServerFd) == false)
 766          return ServerDie(Srv);
 767    }
 768
 769    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 770    {
 771       errno = 0;
 772       if (Srv->Out.Write(Srv->ServerFd) == false)
 773          return ServerDie(Srv);
 774    }
 775
 776    // Send data to the file
 777    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 778    {
 779       if (Srv->In.Write(FileFD) == false)
 780          return _error->Errno("write","Error writing to output file");
 781    }
 782
 783    // Handle commands from APT
 784    if (FD_ISSET(STDIN_FILENO,&rfds))
 785    {
 786       if (Run(true) != 0)
 787          exit(100);
 788    }
 789
 790    return true;
 791 }
 792                                                                         /*}}}*/
 793 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 794 // ---------------------------------------------------------------------
 795 /* This takes the current input buffer from the Server FD and writes it
 796    into the file */
 797 bool HttpMethod::Flush(ServerState *Srv)
 798 {
 799    if (File != 0)
 800    {
 801       SetNonBlock(File->Fd(),false);
 802       if (Srv->In.WriteSpace() == false)
 803          return true;
 804
 805       while (Srv->In.WriteSpace() == true)
 806       {
 807          if (Srv->In.Write(File->Fd()) == false)
 808             return _error->Errno("write","Error writing to file");
 809          if (Srv->In.IsLimit() == true)
 810             return true;
 811       }
 812
 813       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 814          return true;
 815    }
 816    return false;
 817 }
 818                                                                         /*}}}*/
 819 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 820 // ---------------------------------------------------------------------
 821 /* */
 822 bool HttpMethod::ServerDie(ServerState *Srv)
 823 {
 824    unsigned int LErrno = errno;
 825
 826    // Dump the buffer to the file
 827    if (Srv->State == ServerState::Data)
 828    {
 829       SetNonBlock(File->Fd(),false);
 830       while (Srv->In.WriteSpace() == true)
 831       {
 832          if (Srv->In.Write(File->Fd()) == false)
 833             return _error->Errno("write","Error writing to the file");
 834
 835          // Done
 836          if (Srv->In.IsLimit() == true)
 837             return true;
 838       }
 839    }
 840
 841    // See if this is because the server finished the data stream
 842    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 843        Srv->Encoding != ServerState::Closes)
 844    {
 845       Srv->Close();
 846       if (LErrno == 0)
 847          return _error->Error("Error reading from server Remote end closed connection");
 848       errno = LErrno;
 849       return _error->Errno("read","Error reading from server");
 850    }
 851    else
 852    {
 853       Srv->In.Limit(-1);
 854
 855       // Nothing left in the buffer
 856       if (Srv->In.WriteSpace() == false)
 857          return false;
 858
 859       // We may have got multiple responses back in one packet..
 860       Srv->Close();
 861       return true;
 862    }
 863
 864    return false;
 865 }
 866                                                                         /*}}}*/
 867 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 868 // ---------------------------------------------------------------------
 869 /* We look at the header data we got back from the server and decide what
 870    to do. Returns
 871      0 - File is open,
 872      1 - IMS hit
 873      3 - Unrecoverable error
 874      4 - Error with error content page
 875      5 - Unrecoverable non-server error (close the connection) */
 876 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 877 {
 878    // Not Modified
 879    if (Srv->Result == 304)
 880    {
 881       unlink(Queue->DestFile.c_str());
 882       Res.IMSHit = true;
 883       Res.LastModified = Queue->LastModified;
 884       return 1;
 885    }
 886
 887    /* We have a reply we dont handle. This should indicate a perm server
 888       failure */
 889    if (Srv->Result < 200 || Srv->Result >= 300)
 890    {
 891       _error->Error("%u %s",Srv->Result,Srv->Code);
 892       if (Srv->HaveContent == true)
 893          return 4;
 894       return 3;
 895    }
 896
 897    // This is some sort of 2xx 'data follows' reply
 898    Res.LastModified = Srv->Date;
 899    Res.Size = Srv->Size;
 900
 901    // Open the file
 902    delete File;
 903    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 904    if (_error->PendingError() == true)
 905       return 5;
 906
 907    FailFile = Queue->DestFile;
 908    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 909    FailFd = File->Fd();
 910    FailTime = Srv->Date;
 911
 912    // Set the expected size
 913    if (Srv->StartPos >= 0)
 914    {
 915       Res.ResumePoint = Srv->StartPos;
 916       ftruncate(File->Fd(),Srv->StartPos);
 917    }
 918
 919    // Set the start point
 920    lseek(File->Fd(),0,SEEK_END);
 921
 922    delete Srv->In.MD5;
 923    Srv->In.MD5 = new MD5Summation;
 924
 925    // Fill the MD5 Hash if the file is non-empty (resume)
 926    if (Srv->StartPos > 0)
 927    {
 928       lseek(File->Fd(),0,SEEK_SET);
 929       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 930       {
 931          _error->Errno("read","Problem hashing file");
 932          return 5;
 933       }
 934       lseek(File->Fd(),0,SEEK_END);
 935    }
 936
 937    SetNonBlock(File->Fd(),true);
 938    return 0;
 939 }
 940                                                                         /*}}}*/
 941 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 942 // ---------------------------------------------------------------------
 943 /* This closes and timestamps the open file. This is neccessary to get
 944    resume behavoir on user abort */
 945 void HttpMethod::SigTerm(int)
 946 {
 947    if (FailFd == -1)
 948       exit(100);
 949    close(FailFd);
 950
 951    // Timestamp
 952    struct utimbuf UBuf;
 953    UBuf.actime = FailTime;
 954    UBuf.modtime = FailTime;
 955    utime(FailFile.c_str(),&UBuf);
 956
 957    exit(100);
 958 }
 959                                                                         /*}}}*/
 960 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 961 // ---------------------------------------------------------------------
 962 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 963    depth. */
 964 bool HttpMethod::Fetch(FetchItem *)
 965 {
 966    if (Server == 0)
 967       return true;
 968
 969    // Queue the requests
 970    int Depth = -1;
 971    bool Tail = false;
 972    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 973    {
 974       // Make sure we stick with the same server
 975       if (Server->Comp(I->Uri) == false)
 976          break;
 977       if (QueueBack == I)
 978          Tail = true;
 979       if (Tail == true)
 980       {
 981          QueueBack = I->Next;
 982          SendReq(I,Server->Out);
 983          continue;
 984       }
 985    }
 986
 987    return true;
 988 };
 989                                                                         /*}}}*/
 990 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 991 // ---------------------------------------------------------------------
 992 /* We stash the desired pipeline depth */
 993 bool HttpMethod::Configuration(string Message)
 994 {
 995    if (pkgAcqMethod::Configuration(Message) == false)
 996       return false;
 997
 998    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 999    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1000                                   PipelineDepth);
1001
1002    return true;
1003 }
1004                                                                         /*}}}*/
1005 // HttpMethod::Loop - Main loop                                         /*{{{*/
1006 // ---------------------------------------------------------------------
1007 /* */
1008 int HttpMethod::Loop()
1009 {
1010    signal(SIGTERM,SigTerm);
1011    signal(SIGINT,SigTerm);
1012
1013    Server = 0;
1014
1015    int FailCounter = 0;
1016    while (1)
1017    {
1018       // We have no commands, wait for some to arrive
1019       if (Queue == 0)
1020       {
1021          if (WaitFd(STDIN_FILENO) == false)
1022             return 0;
1023       }
1024
1025       // Run messages
1026       if (Run(true) != 0)
1027          return 100;
1028
1029       if (Queue == 0)
1030          continue;
1031
1032       // Connect to the server
1033       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1034       {
1035          delete Server;
1036          Server = new ServerState(Queue->Uri,this);
1037       }
1038
1039       // Reset the pipeline
1040       if (Server->ServerFd == -1)
1041          QueueBack = Queue;
1042
1043       // Connnect to the host
1044       if (Server->Open() == false)
1045       {
1046          Fail(true);
1047          delete Server;
1048          Server = 0;
1049          continue;
1050       }
1051
1052       // Fill the pipeline.
1053       Fetch(0);
1054
1055       // Fetch the next URL header data from the server.
1056       switch (Server->RunHeaders())
1057       {
1058          case 0:
1059          break;
1060
1061          // The header data is bad
1062          case 2:
1063          {
1064             _error->Error("Bad header Data");
1065             Fail(true);
1066             continue;
1067          }
1068
1069          // The server closed a connection during the header get..
1070          default:
1071          case 1:
1072          {
1073             FailCounter++;
1074             _error->Discard();
1075             Server->Close();
1076
1077             if (FailCounter >= 2)
1078             {
1079                Fail("Connection timed out",true);
1080                FailCounter = 0;
1081             }
1082
1083             continue;
1084          }
1085       };
1086
1087       // Decide what to do.
1088       FetchResult Res;
1089       Res.Filename = Queue->DestFile;
1090       switch (DealWithHeaders(Res,Server))
1091       {
1092          // Ok, the file is Open
1093          case 0:
1094          {
1095             URIStart(Res);
1096
1097             // Run the data
1098             bool Result =  Server->RunData();
1099
1100             // Close the file, destroy the FD object and timestamp it
1101             FailFd = -1;
1102             delete File;
1103             File = 0;
1104
1105             // Timestamp
1106             struct utimbuf UBuf;
1107             time(&UBuf.actime);
1108             UBuf.actime = Server->Date;
1109             UBuf.modtime = Server->Date;
1110             utime(Queue->DestFile.c_str(),&UBuf);
1111
1112             // Send status to APT
1113             if (Result == true)
1114             {
1115                Res.MD5Sum = Server->In.MD5->Result();
1116                URIDone(Res);
1117             }
1118             else
1119                Fail(true);
1120
1121             break;
1122          }
1123
1124          // IMS hit
1125          case 1:
1126          {
1127             URIDone(Res);
1128             break;
1129          }
1130
1131          // Hard server error, not found or something
1132          case 3:
1133          {
1134             Fail();
1135             break;
1136          }
1137
1138          // Hard internal error, kill the connection and fail
1139          case 5:
1140          {
1141             Fail();
1142             Server->Close();
1143             break;
1144          }
1145
1146          // We need to flush the data, the header is like a 404 w/ error text
1147          case 4:
1148          {
1149             Fail();
1150
1151             // Send to content to dev/null
1152             File = new FileFd("/dev/null",FileFd::WriteExists);
1153             Server->RunData();
1154             delete File;
1155             File = 0;
1156             break;
1157          }
1158
1159          default:
1160          Fail("Internal error");
1161          break;
1162       }
1163
1164       FailCounter = 0;
1165    }
1166
1167    return 0;
1168 }
1169                                                                         /*}}}*/
1170
1171 int main()
1172 {
1173    HttpMethod Mth;
1174
1175    return Mth.Loop();
1176 }