methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.52 2001/05/27 23:53:55 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51 using namespace std;
  52
  53 string HttpMethod::FailFile;
  54 int HttpMethod::FailFd = -1;
  55 time_t HttpMethod::FailTime = 0;
  56 unsigned long PipelineDepth = 10;
  57 unsigned long TimeOut = 120;
  58 bool Debug = false;
  59
  60 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* */
  63 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  64 {
  65    Buf = new unsigned char[Size];
  66    Reset();
  67 }
  68                                                                         /*}}}*/
  69 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* */
  72 void CircleBuf::Reset()
  73 {
  74    InP = 0;
  75    OutP = 0;
  76    StrPos = 0;
  77    MaxGet = (unsigned int)-1;
  78    OutQueue = string();
  79    if (Hash != 0)
  80    {
  81       delete Hash;
  82       Hash = new Hashes;
  83    }
  84 };
  85                                                                         /*}}}*/
  86 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* This fills up the buffer with as much data as is in the FD, assuming it
  89    is non-blocking.. */
  90 bool CircleBuf::Read(int Fd)
  91 {
  92    while (1)
  93    {
  94       // Woops, buffer is full
  95       if (InP - OutP == Size)
  96          return true;
  97
  98       // Write the buffer segment
  99       int Res;
 100       Res = read(Fd,Buf + (InP%Size),LeftRead());
 101
 102       if (Res == 0)
 103          return false;
 104       if (Res < 0)
 105       {
 106          if (errno == EAGAIN)
 107             return true;
 108          return false;
 109       }
 110
 111       if (InP == 0)
 112          gettimeofday(&Start,0);
 113       InP += Res;
 114    }
 115 }
 116                                                                         /*}}}*/
 117 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* This will hold the string in and fill the buffer with it as it empties */
 120 bool CircleBuf::Read(string Data)
 121 {
 122    OutQueue += Data;
 123    FillOut();
 124    return true;
 125 }
 126                                                                         /*}}}*/
 127 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* */
 130 void CircleBuf::FillOut()
 131 {
 132    if (OutQueue.empty() == true)
 133       return;
 134    while (1)
 135    {
 136       // Woops, buffer is full
 137       if (InP - OutP == Size)
 138          return;
 139
 140       // Write the buffer segment
 141       unsigned long Sz = LeftRead();
 142       if (OutQueue.length() - StrPos < Sz)
 143          Sz = OutQueue.length() - StrPos;
 144       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 145
 146       // Advance
 147       StrPos += Sz;
 148       InP += Sz;
 149       if (OutQueue.length() == StrPos)
 150       {
 151          StrPos = 0;
 152          OutQueue = "";
 153          return;
 154       }
 155    }
 156 }
 157                                                                         /*}}}*/
 158 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 159 // ---------------------------------------------------------------------
 160 /* This empties the buffer into the FD. */
 161 bool CircleBuf::Write(int Fd)
 162 {
 163    while (1)
 164    {
 165       FillOut();
 166
 167       // Woops, buffer is empty
 168       if (OutP == InP)
 169          return true;
 170
 171       if (OutP == MaxGet)
 172          return true;
 173
 174       // Write the buffer segment
 175       int Res;
 176       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 177
 178       if (Res == 0)
 179          return false;
 180       if (Res < 0)
 181       {
 182          if (errno == EAGAIN)
 183             return true;
 184
 185          return false;
 186       }
 187
 188       if (Hash != 0)
 189          Hash->Add(Buf + (OutP%Size),Res);
 190
 191       OutP += Res;
 192    }
 193 }
 194                                                                         /*}}}*/
 195 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 196 // ---------------------------------------------------------------------
 197 /* This copies till the first empty line */
 198 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 199 {
 200    // We cheat and assume it is unneeded to have more than one buffer load
 201    for (unsigned long I = OutP; I < InP; I++)
 202    {
 203       if (Buf[I%Size] != '\n')
 204          continue;
 205       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 206
 207       if (Single == false)
 208       {
 209          if (Buf[I%Size] != '\n')
 210             continue;
 211          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 212       }
 213
 214       if (I > InP)
 215          I = InP;
 216
 217       Data = "";
 218       while (OutP < I)
 219       {
 220          unsigned long Sz = LeftWrite();
 221          if (Sz == 0)
 222             return false;
 223          if (I - OutP < LeftWrite())
 224             Sz = I - OutP;
 225          Data += string((char *)(Buf + (OutP%Size)),Sz);
 226          OutP += Sz;
 227       }
 228       return true;
 229    }
 230    return false;
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* */
 236 void CircleBuf::Stats()
 237 {
 238    if (InP == 0)
 239       return;
 240
 241    struct timeval Stop;
 242    gettimeofday(&Stop,0);
 243 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 244              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 245    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 246 }
 247                                                                         /*}}}*/
 248
 249 // ServerState::ServerState - Constructor                               /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 253                         In(64*1024), Out(4*1024),
 254                         ServerName(Srv)
 255 {
 256    Reset();
 257 }
 258                                                                         /*}}}*/
 259 // ServerState::Open - Open a connection to the server                  /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* This opens a connection to the server. */
 262 bool ServerState::Open()
 263 {
 264    // Use the already open connection if possible.
 265    if (ServerFd != -1)
 266       return true;
 267
 268    Close();
 269    In.Reset();
 270    Out.Reset();
 271    Persistent = true;
 272
 273    // Determine the proxy setting
 274    if (getenv("http_proxy") == 0)
 275    {
 276       string DefProxy = _config->Find("Acquire::http::Proxy");
 277       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 278       if (SpecificProxy.empty() == false)
 279       {
 280          if (SpecificProxy == "DIRECT")
 281             Proxy = "";
 282          else
 283             Proxy = SpecificProxy;
 284       }
 285       else
 286          Proxy = DefProxy;
 287    }
 288    else
 289       Proxy = getenv("http_proxy");
 290
 291    // Parse no_proxy, a , separated list of domains
 292    if (getenv("no_proxy") != 0)
 293    {
 294       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 295          Proxy = "";
 296    }
 297
 298    // Determine what host and port to use based on the proxy settings
 299    int Port = 0;
 300    string Host;
 301    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 302    {
 303       if (ServerName.Port != 0)
 304          Port = ServerName.Port;
 305       Host = ServerName.Host;
 306    }
 307    else
 308    {
 309       if (Proxy.Port != 0)
 310          Port = Proxy.Port;
 311       Host = Proxy.Host;
 312    }
 313
 314    // Connect to the remote server
 315    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 316       return false;
 317
 318    return true;
 319 }
 320                                                                         /*}}}*/
 321 // ServerState::Close - Close a connection to the server                /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 bool ServerState::Close()
 325 {
 326    close(ServerFd);
 327    ServerFd = -1;
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 334    parse error occured */
 335 int ServerState::RunHeaders()
 336 {
 337    State = Header;
 338
 339    Owner->Status("Waiting for file");
 340
 341    Major = 0;
 342    Minor = 0;
 343    Result = 0;
 344    Size = 0;
 345    StartPos = 0;
 346    Encoding = Closes;
 347    HaveContent = false;
 348    time(&Date);
 349
 350    do
 351    {
 352       string Data;
 353       if (In.WriteTillEl(Data) == false)
 354          continue;
 355
 356       if (Debug == true)
 357          clog << Data;
 358
 359       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 360       {
 361          string::const_iterator J = I;
 362          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 363          if (HeaderLine(string(I,J)) == false)
 364             return 2;
 365          I = J;
 366       }
 367
 368       // 100 Continue is a Nop...
 369       if (Result == 100)
 370          continue;
 371
 372       // Tidy up the connection persistance state.
 373       if (Encoding == Closes && HaveContent == true)
 374          Persistent = false;
 375
 376       return 0;
 377    }
 378    while (Owner->Go(false,this) == true);
 379
 380    return 1;
 381 }
 382                                                                         /*}}}*/
 383 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 384 // ---------------------------------------------------------------------
 385 /* */
 386 bool ServerState::RunData()
 387 {
 388    State = Data;
 389
 390    // Chunked transfer encoding is fun..
 391    if (Encoding == Chunked)
 392    {
 393       while (1)
 394       {
 395          // Grab the block size
 396          bool Last = true;
 397          string Data;
 398          In.Limit(-1);
 399          do
 400          {
 401             if (In.WriteTillEl(Data,true) == true)
 402                break;
 403          }
 404          while ((Last = Owner->Go(false,this)) == true);
 405
 406          if (Last == false)
 407             return false;
 408
 409          // See if we are done
 410          unsigned long Len = strtol(Data.c_str(),0,16);
 411          if (Len == 0)
 412          {
 413             In.Limit(-1);
 414
 415             // We have to remove the entity trailer
 416             Last = true;
 417             do
 418             {
 419                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 420                   break;
 421             }
 422             while ((Last = Owner->Go(false,this)) == true);
 423             if (Last == false)
 424                return false;
 425             return !_error->PendingError();
 426          }
 427
 428          // Transfer the block
 429          In.Limit(Len);
 430          while (Owner->Go(true,this) == true)
 431             if (In.IsLimit() == true)
 432                break;
 433
 434          // Error
 435          if (In.IsLimit() == false)
 436             return false;
 437
 438          // The server sends an extra new line before the next block specifier..
 439          In.Limit(-1);
 440          Last = true;
 441          do
 442          {
 443             if (In.WriteTillEl(Data,true) == true)
 444                break;
 445          }
 446          while ((Last = Owner->Go(false,this)) == true);
 447          if (Last == false)
 448             return false;
 449       }
 450    }
 451    else
 452    {
 453       /* Closes encoding is used when the server did not specify a size, the
 454          loss of the connection means we are done */
 455       if (Encoding == Closes)
 456          In.Limit(-1);
 457       else
 458          In.Limit(Size - StartPos);
 459
 460       // Just transfer the whole block.
 461       do
 462       {
 463          if (In.IsLimit() == false)
 464             continue;
 465
 466          In.Limit(-1);
 467          return !_error->PendingError();
 468       }
 469       while (Owner->Go(true,this) == true);
 470    }
 471
 472    return Owner->Flush(this) && !_error->PendingError();
 473 }
 474                                                                         /*}}}*/
 475 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* */
 478 bool ServerState::HeaderLine(string Line)
 479 {
 480    if (Line.empty() == true)
 481       return true;
 482
 483    // The http server might be trying to do something evil.
 484    if (Line.length() >= MAXLEN)
 485       return _error->Error("Got a single header line over %u chars",MAXLEN);
 486
 487    string::size_type Pos = Line.find(' ');
 488    if (Pos == string::npos || Pos+1 > Line.length())
 489    {
 490       // Blah, some servers use "connection:closes", evil.
 491       Pos = Line.find(':');
 492       if (Pos == string::npos || Pos + 2 > Line.length())
 493          return _error->Error("Bad header line");
 494       Pos++;
 495    }
 496
 497    // Parse off any trailing spaces between the : and the next word.
 498    string::size_type Pos2 = Pos;
 499    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 500       Pos2++;
 501
 502    string Tag = string(Line,0,Pos);
 503    string Val = string(Line,Pos2);
 504
 505    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 506    {
 507       // Evil servers return no version
 508       if (Line[4] == '/')
 509       {
 510          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 511                     &Result,Code) != 4)
 512             return _error->Error("The http server sent an invalid reply header");
 513       }
 514       else
 515       {
 516          Major = 0;
 517          Minor = 9;
 518          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 519             return _error->Error("The http server sent an invalid reply header");
 520       }
 521
 522       /* Check the HTTP response header to get the default persistance
 523          state. */
 524       if (Major < 1)
 525          Persistent = false;
 526       else
 527       {
 528          if (Major == 1 && Minor <= 0)
 529             Persistent = false;
 530          else
 531             Persistent = true;
 532       }
 533
 534       return true;
 535    }
 536
 537    if (stringcasecmp(Tag,"Content-Length:") == 0)
 538    {
 539       if (Encoding == Closes)
 540          Encoding = Stream;
 541       HaveContent = true;
 542
 543       // The length is already set from the Content-Range header
 544       if (StartPos != 0)
 545          return true;
 546
 547       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 548          return _error->Error("The http server sent an invalid Content-Length header");
 549       return true;
 550    }
 551
 552    if (stringcasecmp(Tag,"Content-Type:") == 0)
 553    {
 554       HaveContent = true;
 555       return true;
 556    }
 557
 558    if (stringcasecmp(Tag,"Content-Range:") == 0)
 559    {
 560       HaveContent = true;
 561
 562       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 563          return _error->Error("The http server sent an invalid Content-Range header");
 564       if ((unsigned)StartPos > Size)
 565          return _error->Error("This http server has broken range support");
 566       return true;
 567    }
 568
 569    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 570    {
 571       HaveContent = true;
 572       if (stringcasecmp(Val,"chunked") == 0)
 573          Encoding = Chunked;
 574       return true;
 575    }
 576
 577    if (stringcasecmp(Tag,"Connection:") == 0)
 578    {
 579       if (stringcasecmp(Val,"close") == 0)
 580          Persistent = false;
 581       if (stringcasecmp(Val,"keep-alive") == 0)
 582          Persistent = true;
 583       return true;
 584    }
 585
 586    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 587    {
 588       if (StrToTime(Val,Date) == false)
 589          return _error->Error("Unknown date format");
 590       return true;
 591    }
 592
 593    return true;
 594 }
 595                                                                         /*}}}*/
 596
 597 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This places the http request in the outbound buffer */
 600 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 601 {
 602    URI Uri = Itm->Uri;
 603
 604    // The HTTP server expects a hostname with a trailing :port
 605    char Buf[1000];
 606    string ProperHost = Uri.Host;
 607    if (Uri.Port != 0)
 608    {
 609       sprintf(Buf,":%u",Uri.Port);
 610       ProperHost += Buf;
 611    }
 612
 613    // Just in case.
 614    if (Itm->Uri.length() >= sizeof(Buf))
 615        abort();
 616
 617    /* Build the request. We include a keep-alive header only for non-proxy
 618       requests. This is to tweak old http/1.0 servers that do support keep-alive
 619       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 620       will glitch HTTP/1.0 proxies because they do not filter it out and
 621       pass it on, HTTP/1.1 says the connection should default to keep alive
 622       and we expect the proxy to do this */
 623    if (Proxy.empty() == true)
 624       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 625               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 626    else
 627    {
 628       /* Generate a cache control header if necessary. We place a max
 629          cache age on index files, optionally set a no-cache directive
 630          and a no-store directive for archives. */
 631       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 632               Itm->Uri.c_str(),ProperHost.c_str());
 633       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 634          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 635       else
 636       {
 637          if (Itm->IndexFile == true)
 638             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 639                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 640          else
 641          {
 642             if (_config->FindB("Acquire::http::No-Store",false) == true)
 643                strcat(Buf,"Cache-Control: no-store\r\n");
 644          }
 645       }
 646    }
 647
 648    string Req = Buf;
 649
 650    // Check for a partial file
 651    struct stat SBuf;
 652    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 653    {
 654       // In this case we send an if-range query with a range header
 655       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 656               TimeRFC1123(SBuf.st_mtime).c_str());
 657       Req += Buf;
 658    }
 659    else
 660    {
 661       if (Itm->LastModified != 0)
 662       {
 663          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 664          Req += Buf;
 665       }
 666    }
 667
 668    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 669       Req += string("Proxy-Authorization: Basic ") +
 670           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 671
 672    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 673       Req += string("Authorization: Basic ") +
 674           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 675
 676    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 677
 678    if (Debug == true)
 679       cerr << Req << endl;
 680
 681    Out.Read(Req);
 682 }
 683                                                                         /*}}}*/
 684 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 685 // ---------------------------------------------------------------------
 686 /* This runs the select loop over the server FDs, Output file FDs and
 687    stdin. */
 688 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 689 {
 690    // Server has closed the connection
 691    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 692                                ToFile == false))
 693       return false;
 694
 695    fd_set rfds,wfds;
 696    FD_ZERO(&rfds);
 697    FD_ZERO(&wfds);
 698
 699    /* Add the server. We only send more requests if the connection will
 700       be persisting */
 701    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 702        && Srv->Persistent == true)
 703       FD_SET(Srv->ServerFd,&wfds);
 704    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 705       FD_SET(Srv->ServerFd,&rfds);
 706
 707    // Add the file
 708    int FileFD = -1;
 709    if (File != 0)
 710       FileFD = File->Fd();
 711
 712    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 713       FD_SET(FileFD,&wfds);
 714
 715    // Add stdin
 716    FD_SET(STDIN_FILENO,&rfds);
 717
 718    // Figure out the max fd
 719    int MaxFd = FileFD;
 720    if (MaxFd < Srv->ServerFd)
 721       MaxFd = Srv->ServerFd;
 722
 723    // Select
 724    struct timeval tv;
 725    tv.tv_sec = TimeOut;
 726    tv.tv_usec = 0;
 727    int Res = 0;
 728    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 729       return _error->Errno("select","Select failed");
 730
 731    if (Res == 0)
 732    {
 733       _error->Error("Connection timed out");
 734       return ServerDie(Srv);
 735    }
 736
 737    // Handle server IO
 738    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 739    {
 740       errno = 0;
 741       if (Srv->In.Read(Srv->ServerFd) == false)
 742          return ServerDie(Srv);
 743    }
 744
 745    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 746    {
 747       errno = 0;
 748       if (Srv->Out.Write(Srv->ServerFd) == false)
 749          return ServerDie(Srv);
 750    }
 751
 752    // Send data to the file
 753    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 754    {
 755       if (Srv->In.Write(FileFD) == false)
 756          return _error->Errno("write","Error writing to output file");
 757    }
 758
 759    // Handle commands from APT
 760    if (FD_ISSET(STDIN_FILENO,&rfds))
 761    {
 762       if (Run(true) != -1)
 763          exit(100);
 764    }
 765
 766    return true;
 767 }
 768                                                                         /*}}}*/
 769 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 770 // ---------------------------------------------------------------------
 771 /* This takes the current input buffer from the Server FD and writes it
 772    into the file */
 773 bool HttpMethod::Flush(ServerState *Srv)
 774 {
 775    if (File != 0)
 776    {
 777       SetNonBlock(File->Fd(),false);
 778       if (Srv->In.WriteSpace() == false)
 779          return true;
 780
 781       while (Srv->In.WriteSpace() == true)
 782       {
 783          if (Srv->In.Write(File->Fd()) == false)
 784             return _error->Errno("write","Error writing to file");
 785          if (Srv->In.IsLimit() == true)
 786             return true;
 787       }
 788
 789       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 790          return true;
 791    }
 792    return false;
 793 }
 794                                                                         /*}}}*/
 795 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 796 // ---------------------------------------------------------------------
 797 /* */
 798 bool HttpMethod::ServerDie(ServerState *Srv)
 799 {
 800    unsigned int LErrno = errno;
 801
 802    // Dump the buffer to the file
 803    if (Srv->State == ServerState::Data)
 804    {
 805       SetNonBlock(File->Fd(),false);
 806       while (Srv->In.WriteSpace() == true)
 807       {
 808          if (Srv->In.Write(File->Fd()) == false)
 809             return _error->Errno("write","Error writing to the file");
 810
 811          // Done
 812          if (Srv->In.IsLimit() == true)
 813             return true;
 814       }
 815    }
 816
 817    // See if this is because the server finished the data stream
 818    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 819        Srv->Encoding != ServerState::Closes)
 820    {
 821       Srv->Close();
 822       if (LErrno == 0)
 823          return _error->Error("Error reading from server Remote end closed connection");
 824       errno = LErrno;
 825       return _error->Errno("read","Error reading from server");
 826    }
 827    else
 828    {
 829       Srv->In.Limit(-1);
 830
 831       // Nothing left in the buffer
 832       if (Srv->In.WriteSpace() == false)
 833          return false;
 834
 835       // We may have got multiple responses back in one packet..
 836       Srv->Close();
 837       return true;
 838    }
 839
 840    return false;
 841 }
 842                                                                         /*}}}*/
 843 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 844 // ---------------------------------------------------------------------
 845 /* We look at the header data we got back from the server and decide what
 846    to do. Returns
 847      0 - File is open,
 848      1 - IMS hit
 849      3 - Unrecoverable error
 850      4 - Error with error content page
 851      5 - Unrecoverable non-server error (close the connection) */
 852 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 853 {
 854    // Not Modified
 855    if (Srv->Result == 304)
 856    {
 857       unlink(Queue->DestFile.c_str());
 858       Res.IMSHit = true;
 859       Res.LastModified = Queue->LastModified;
 860       return 1;
 861    }
 862
 863    /* We have a reply we dont handle. This should indicate a perm server
 864       failure */
 865    if (Srv->Result < 200 || Srv->Result >= 300)
 866    {
 867       _error->Error("%u %s",Srv->Result,Srv->Code);
 868       if (Srv->HaveContent == true)
 869          return 4;
 870       return 3;
 871    }
 872
 873    // This is some sort of 2xx 'data follows' reply
 874    Res.LastModified = Srv->Date;
 875    Res.Size = Srv->Size;
 876
 877    // Open the file
 878    delete File;
 879    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 880    if (_error->PendingError() == true)
 881       return 5;
 882
 883    FailFile = Queue->DestFile;
 884    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 885    FailFd = File->Fd();
 886    FailTime = Srv->Date;
 887
 888    // Set the expected size
 889    if (Srv->StartPos >= 0)
 890    {
 891       Res.ResumePoint = Srv->StartPos;
 892       ftruncate(File->Fd(),Srv->StartPos);
 893    }
 894
 895    // Set the start point
 896    lseek(File->Fd(),0,SEEK_END);
 897
 898    delete Srv->In.Hash;
 899    Srv->In.Hash = new Hashes;
 900
 901    // Fill the Hash if the file is non-empty (resume)
 902    if (Srv->StartPos > 0)
 903    {
 904       lseek(File->Fd(),0,SEEK_SET);
 905       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 906       {
 907          _error->Errno("read","Problem hashing file");
 908          return 5;
 909       }
 910       lseek(File->Fd(),0,SEEK_END);
 911    }
 912
 913    SetNonBlock(File->Fd(),true);
 914    return 0;
 915 }
 916                                                                         /*}}}*/
 917 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 918 // ---------------------------------------------------------------------
 919 /* This closes and timestamps the open file. This is neccessary to get
 920    resume behavoir on user abort */
 921 void HttpMethod::SigTerm(int)
 922 {
 923    if (FailFd == -1)
 924       _exit(100);
 925    close(FailFd);
 926
 927    // Timestamp
 928    struct utimbuf UBuf;
 929    UBuf.actime = FailTime;
 930    UBuf.modtime = FailTime;
 931    utime(FailFile.c_str(),&UBuf);
 932
 933    _exit(100);
 934 }
 935                                                                         /*}}}*/
 936 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 937 // ---------------------------------------------------------------------
 938 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 939    depth. */
 940 bool HttpMethod::Fetch(FetchItem *)
 941 {
 942    if (Server == 0)
 943       return true;
 944
 945    // Queue the requests
 946    int Depth = -1;
 947    bool Tail = false;
 948    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 949         I = I->Next, Depth++)
 950    {
 951       // If pipelining is disabled, we only queue 1 request
 952       if (Server->Pipeline == false && Depth >= 0)
 953          break;
 954
 955       // Make sure we stick with the same server
 956       if (Server->Comp(I->Uri) == false)
 957          break;
 958       if (QueueBack == I)
 959          Tail = true;
 960       if (Tail == true)
 961       {
 962          QueueBack = I->Next;
 963          SendReq(I,Server->Out);
 964          continue;
 965       }
 966    }
 967
 968    return true;
 969 };
 970                                                                         /*}}}*/
 971 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 972 // ---------------------------------------------------------------------
 973 /* We stash the desired pipeline depth */
 974 bool HttpMethod::Configuration(string Message)
 975 {
 976    if (pkgAcqMethod::Configuration(Message) == false)
 977       return false;
 978
 979    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 980    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 981                                   PipelineDepth);
 982    Debug = _config->FindB("Debug::Acquire::http",false);
 983
 984    return true;
 985 }
 986                                                                         /*}}}*/
 987 // HttpMethod::Loop - Main loop                                         /*{{{*/
 988 // ---------------------------------------------------------------------
 989 /* */
 990 int HttpMethod::Loop()
 991 {
 992    signal(SIGTERM,SigTerm);
 993    signal(SIGINT,SigTerm);
 994
 995    Server = 0;
 996
 997    int FailCounter = 0;
 998    while (1)
 999    {
1000       // We have no commands, wait for some to arrive
1001       if (Queue == 0)
1002       {
1003          if (WaitFd(STDIN_FILENO) == false)
1004             return 0;
1005       }
1006
1007       /* Run messages, we can accept 0 (no message) if we didn't
1008          do a WaitFd above.. Otherwise the FD is closed. */
1009       int Result = Run(true);
1010       if (Result != -1 && (Result != 0 || Queue == 0))
1011          return 100;
1012
1013       if (Queue == 0)
1014          continue;
1015
1016       // Connect to the server
1017       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1018       {
1019          delete Server;
1020          Server = new ServerState(Queue->Uri,this);
1021       }
1022
1023       /* If the server has explicitly said this is the last connection
1024          then we pre-emptively shut down the pipeline and tear down
1025          the connection. This will speed up HTTP/1.0 servers a tad
1026          since we don't have to wait for the close sequence to
1027          complete */
1028       if (Server->Persistent == false)
1029          Server->Close();
1030
1031       // Reset the pipeline
1032       if (Server->ServerFd == -1)
1033          QueueBack = Queue;
1034
1035       // Connnect to the host
1036       if (Server->Open() == false)
1037       {
1038          Fail(true);
1039          delete Server;
1040          Server = 0;
1041          continue;
1042       }
1043
1044       // Fill the pipeline.
1045       Fetch(0);
1046
1047       // Fetch the next URL header data from the server.
1048       switch (Server->RunHeaders())
1049       {
1050          case 0:
1051          break;
1052
1053          // The header data is bad
1054          case 2:
1055          {
1056             _error->Error("Bad header Data");
1057             Fail(true);
1058             RotateDNS();
1059             continue;
1060          }
1061
1062          // The server closed a connection during the header get..
1063          default:
1064          case 1:
1065          {
1066             FailCounter++;
1067             _error->Discard();
1068             Server->Close();
1069             Server->Pipeline = false;
1070
1071             if (FailCounter >= 2)
1072             {
1073                Fail("Connection failed",true);
1074                FailCounter = 0;
1075             }
1076
1077             RotateDNS();
1078             continue;
1079          }
1080       };
1081
1082       // Decide what to do.
1083       FetchResult Res;
1084       Res.Filename = Queue->DestFile;
1085       switch (DealWithHeaders(Res,Server))
1086       {
1087          // Ok, the file is Open
1088          case 0:
1089          {
1090             URIStart(Res);
1091
1092             // Run the data
1093             bool Result =  Server->RunData();
1094
1095             /* If the server is sending back sizeless responses then fill in
1096                the size now */
1097             if (Res.Size == 0)
1098                Res.Size = File->Size();
1099
1100             // Close the file, destroy the FD object and timestamp it
1101             FailFd = -1;
1102             delete File;
1103             File = 0;
1104
1105             // Timestamp
1106             struct utimbuf UBuf;
1107             time(&UBuf.actime);
1108             UBuf.actime = Server->Date;
1109             UBuf.modtime = Server->Date;
1110             utime(Queue->DestFile.c_str(),&UBuf);
1111
1112             // Send status to APT
1113             if (Result == true)
1114             {
1115                Res.TakeHashes(*Server->In.Hash);
1116                URIDone(Res);
1117             }
1118             else
1119                Fail(true);
1120
1121             break;
1122          }
1123
1124          // IMS hit
1125          case 1:
1126          {
1127             URIDone(Res);
1128             break;
1129          }
1130
1131          // Hard server error, not found or something
1132          case 3:
1133          {
1134             Fail();
1135             break;
1136          }
1137
1138          // Hard internal error, kill the connection and fail
1139          case 5:
1140          {
1141             Fail();
1142             RotateDNS();
1143             Server->Close();
1144             break;
1145          }
1146
1147          // We need to flush the data, the header is like a 404 w/ error text
1148          case 4:
1149          {
1150             Fail();
1151
1152             // Send to content to dev/null
1153             File = new FileFd("/dev/null",FileFd::WriteExists);
1154             Server->RunData();
1155             delete File;
1156             File = 0;
1157             break;
1158          }
1159
1160          default:
1161          Fail("Internal error");
1162          break;
1163       }
1164
1165       FailCounter = 0;
1166    }
1167
1168    return 0;
1169 }
1170                                                                         /*}}}*/
1171
1172 int main()
1173 {
1174    HttpMethod Mth;
1175
1176    return Mth.Loop();
1177 }