methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.53 2001/07/01 20:49:09 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51 using namespace std;
  52
  53 string HttpMethod::FailFile;
  54 int HttpMethod::FailFd = -1;
  55 time_t HttpMethod::FailTime = 0;
  56 unsigned long PipelineDepth = 10;
  57 unsigned long TimeOut = 120;
  58 bool Debug = false;
  59
  60 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  61 // ---------------------------------------------------------------------
  62 /* */
  63 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  64 {
  65    Buf = new unsigned char[Size];
  66    Reset();
  67 }
  68                                                                         /*}}}*/
  69 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  70 // ---------------------------------------------------------------------
  71 /* */
  72 void CircleBuf::Reset()
  73 {
  74    InP = 0;
  75    OutP = 0;
  76    StrPos = 0;
  77    MaxGet = (unsigned int)-1;
  78    OutQueue = string();
  79    if (Hash != 0)
  80    {
  81       delete Hash;
  82       Hash = new Hashes;
  83    }
  84 };
  85                                                                         /*}}}*/
  86 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  87 // ---------------------------------------------------------------------
  88 /* This fills up the buffer with as much data as is in the FD, assuming it
  89    is non-blocking.. */
  90 bool CircleBuf::Read(int Fd)
  91 {
  92    while (1)
  93    {
  94       // Woops, buffer is full
  95       if (InP - OutP == Size)
  96          return true;
  97
  98       // Write the buffer segment
  99       int Res;
 100       Res = read(Fd,Buf + (InP%Size),LeftRead());
 101
 102       if (Res == 0)
 103          return false;
 104       if (Res < 0)
 105       {
 106          if (errno == EAGAIN)
 107             return true;
 108          return false;
 109       }
 110
 111       if (InP == 0)
 112          gettimeofday(&Start,0);
 113       InP += Res;
 114    }
 115 }
 116                                                                         /*}}}*/
 117 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 118 // ---------------------------------------------------------------------
 119 /* This will hold the string in and fill the buffer with it as it empties */
 120 bool CircleBuf::Read(string Data)
 121 {
 122    OutQueue += Data;
 123    FillOut();
 124    return true;
 125 }
 126                                                                         /*}}}*/
 127 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 128 // ---------------------------------------------------------------------
 129 /* */
 130 void CircleBuf::FillOut()
 131 {
 132    if (OutQueue.empty() == true)
 133       return;
 134    while (1)
 135    {
 136       // Woops, buffer is full
 137       if (InP - OutP == Size)
 138          return;
 139
 140       // Write the buffer segment
 141       unsigned long Sz = LeftRead();
 142       if (OutQueue.length() - StrPos < Sz)
 143          Sz = OutQueue.length() - StrPos;
 144       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 145
 146       // Advance
 147       StrPos += Sz;
 148       InP += Sz;
 149       if (OutQueue.length() == StrPos)
 150       {
 151          StrPos = 0;
 152          OutQueue = "";
 153          return;
 154       }
 155    }
 156 }
 157                                                                         /*}}}*/
 158 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 159 // ---------------------------------------------------------------------
 160 /* This empties the buffer into the FD. */
 161 bool CircleBuf::Write(int Fd)
 162 {
 163    while (1)
 164    {
 165       FillOut();
 166
 167       // Woops, buffer is empty
 168       if (OutP == InP)
 169          return true;
 170
 171       if (OutP == MaxGet)
 172          return true;
 173
 174       // Write the buffer segment
 175       int Res;
 176       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 177
 178       if (Res == 0)
 179          return false;
 180       if (Res < 0)
 181       {
 182          if (errno == EAGAIN)
 183             return true;
 184
 185          return false;
 186       }
 187
 188       if (Hash != 0)
 189          Hash->Add(Buf + (OutP%Size),Res);
 190
 191       OutP += Res;
 192    }
 193 }
 194                                                                         /*}}}*/
 195 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 196 // ---------------------------------------------------------------------
 197 /* This copies till the first empty line */
 198 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 199 {
 200    // We cheat and assume it is unneeded to have more than one buffer load
 201    for (unsigned long I = OutP; I < InP; I++)
 202    {
 203       if (Buf[I%Size] != '\n')
 204          continue;
 205       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 206
 207       if (Single == false)
 208       {
 209          if (Buf[I%Size] != '\n')
 210             continue;
 211          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 212       }
 213
 214       if (I > InP)
 215          I = InP;
 216
 217       Data = "";
 218       while (OutP < I)
 219       {
 220          unsigned long Sz = LeftWrite();
 221          if (Sz == 0)
 222             return false;
 223          if (I - OutP < LeftWrite())
 224             Sz = I - OutP;
 225          Data += string((char *)(Buf + (OutP%Size)),Sz);
 226          OutP += Sz;
 227       }
 228       return true;
 229    }
 230    return false;
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* */
 236 void CircleBuf::Stats()
 237 {
 238    if (InP == 0)
 239       return;
 240
 241    struct timeval Stop;
 242    gettimeofday(&Stop,0);
 243 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 244              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 245    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 246 }
 247                                                                         /*}}}*/
 248
 249 // ServerState::ServerState - Constructor                               /*{{{*/
 250 // ---------------------------------------------------------------------
 251 /* */
 252 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 253                         In(64*1024), Out(4*1024),
 254                         ServerName(Srv)
 255 {
 256    Reset();
 257 }
 258                                                                         /*}}}*/
 259 // ServerState::Open - Open a connection to the server                  /*{{{*/
 260 // ---------------------------------------------------------------------
 261 /* This opens a connection to the server. */
 262 bool ServerState::Open()
 263 {
 264    // Use the already open connection if possible.
 265    if (ServerFd != -1)
 266       return true;
 267
 268    Close();
 269    In.Reset();
 270    Out.Reset();
 271    Persistent = true;
 272
 273    // Determine the proxy setting
 274    if (getenv("http_proxy") == 0)
 275    {
 276       string DefProxy = _config->Find("Acquire::http::Proxy");
 277       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 278       if (SpecificProxy.empty() == false)
 279       {
 280          if (SpecificProxy == "DIRECT")
 281             Proxy = "";
 282          else
 283             Proxy = SpecificProxy;
 284       }
 285       else
 286          Proxy = DefProxy;
 287    }
 288    else
 289       Proxy = getenv("http_proxy");
 290
 291    // Parse no_proxy, a , separated list of domains
 292    if (getenv("no_proxy") != 0)
 293    {
 294       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 295          Proxy = "";
 296    }
 297
 298    // Determine what host and port to use based on the proxy settings
 299    int Port = 0;
 300    string Host;
 301    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 302    {
 303       if (ServerName.Port != 0)
 304          Port = ServerName.Port;
 305       Host = ServerName.Host;
 306    }
 307    else
 308    {
 309       if (Proxy.Port != 0)
 310          Port = Proxy.Port;
 311       Host = Proxy.Host;
 312    }
 313
 314    // Connect to the remote server
 315    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 316       return false;
 317
 318    return true;
 319 }
 320                                                                         /*}}}*/
 321 // ServerState::Close - Close a connection to the server                /*{{{*/
 322 // ---------------------------------------------------------------------
 323 /* */
 324 bool ServerState::Close()
 325 {
 326    close(ServerFd);
 327    ServerFd = -1;
 328    return true;
 329 }
 330                                                                         /*}}}*/
 331 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 334    parse error occured */
 335 int ServerState::RunHeaders()
 336 {
 337    State = Header;
 338
 339    Owner->Status("Waiting for file");
 340
 341    Major = 0;
 342    Minor = 0;
 343    Result = 0;
 344    Size = 0;
 345    StartPos = 0;
 346    Encoding = Closes;
 347    HaveContent = false;
 348    time(&Date);
 349
 350    do
 351    {
 352       string Data;
 353       if (In.WriteTillEl(Data) == false)
 354          continue;
 355
 356       if (Debug == true)
 357          clog << Data;
 358
 359       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 360       {
 361          string::const_iterator J = I;
 362          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 363          if (HeaderLine(string(I,J)) == false)
 364             return 2;
 365          I = J;
 366       }
 367
 368       // 100 Continue is a Nop...
 369       if (Result == 100)
 370          continue;
 371
 372       // Tidy up the connection persistance state.
 373       if (Encoding == Closes && HaveContent == true)
 374          Persistent = false;
 375
 376       return 0;
 377    }
 378    while (Owner->Go(false,this) == true);
 379
 380    return 1;
 381 }
 382                                                                         /*}}}*/
 383 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 384 // ---------------------------------------------------------------------
 385 /* */
 386 bool ServerState::RunData()
 387 {
 388    State = Data;
 389
 390    // Chunked transfer encoding is fun..
 391    if (Encoding == Chunked)
 392    {
 393       while (1)
 394       {
 395          // Grab the block size
 396          bool Last = true;
 397          string Data;
 398          In.Limit(-1);
 399          do
 400          {
 401             if (In.WriteTillEl(Data,true) == true)
 402                break;
 403          }
 404          while ((Last = Owner->Go(false,this)) == true);
 405
 406          if (Last == false)
 407             return false;
 408
 409          // See if we are done
 410          unsigned long Len = strtol(Data.c_str(),0,16);
 411          if (Len == 0)
 412          {
 413             In.Limit(-1);
 414
 415             // We have to remove the entity trailer
 416             Last = true;
 417             do
 418             {
 419                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 420                   break;
 421             }
 422             while ((Last = Owner->Go(false,this)) == true);
 423             if (Last == false)
 424                return false;
 425             return !_error->PendingError();
 426          }
 427
 428          // Transfer the block
 429          In.Limit(Len);
 430          while (Owner->Go(true,this) == true)
 431             if (In.IsLimit() == true)
 432                break;
 433
 434          // Error
 435          if (In.IsLimit() == false)
 436             return false;
 437
 438          // The server sends an extra new line before the next block specifier..
 439          In.Limit(-1);
 440          Last = true;
 441          do
 442          {
 443             if (In.WriteTillEl(Data,true) == true)
 444                break;
 445          }
 446          while ((Last = Owner->Go(false,this)) == true);
 447          if (Last == false)
 448             return false;
 449       }
 450    }
 451    else
 452    {
 453       /* Closes encoding is used when the server did not specify a size, the
 454          loss of the connection means we are done */
 455       if (Encoding == Closes)
 456          In.Limit(-1);
 457       else
 458          In.Limit(Size - StartPos);
 459
 460       // Just transfer the whole block.
 461       do
 462       {
 463          if (In.IsLimit() == false)
 464             continue;
 465
 466          In.Limit(-1);
 467          return !_error->PendingError();
 468       }
 469       while (Owner->Go(true,this) == true);
 470    }
 471
 472    return Owner->Flush(this) && !_error->PendingError();
 473 }
 474                                                                         /*}}}*/
 475 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* */
 478 bool ServerState::HeaderLine(string Line)
 479 {
 480    if (Line.empty() == true)
 481       return true;
 482
 483    // The http server might be trying to do something evil.
 484    if (Line.length() >= MAXLEN)
 485       return _error->Error("Got a single header line over %u chars",MAXLEN);
 486
 487    string::size_type Pos = Line.find(' ');
 488    if (Pos == string::npos || Pos+1 > Line.length())
 489    {
 490       // Blah, some servers use "connection:closes", evil.
 491       Pos = Line.find(':');
 492       if (Pos == string::npos || Pos + 2 > Line.length())
 493          return _error->Error("Bad header line");
 494       Pos++;
 495    }
 496
 497    // Parse off any trailing spaces between the : and the next word.
 498    string::size_type Pos2 = Pos;
 499    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 500       Pos2++;
 501
 502    string Tag = string(Line,0,Pos);
 503    string Val = string(Line,Pos2);
 504
 505    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 506    {
 507       // Evil servers return no version
 508       if (Line[4] == '/')
 509       {
 510          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 511                     &Result,Code) != 4)
 512             return _error->Error("The http server sent an invalid reply header");
 513       }
 514       else
 515       {
 516          Major = 0;
 517          Minor = 9;
 518          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 519             return _error->Error("The http server sent an invalid reply header");
 520       }
 521
 522       /* Check the HTTP response header to get the default persistance
 523          state. */
 524       if (Major < 1)
 525          Persistent = false;
 526       else
 527       {
 528          if (Major == 1 && Minor <= 0)
 529             Persistent = false;
 530          else
 531             Persistent = true;
 532       }
 533
 534       return true;
 535    }
 536
 537    if (stringcasecmp(Tag,"Content-Length:") == 0)
 538    {
 539       if (Encoding == Closes)
 540          Encoding = Stream;
 541       HaveContent = true;
 542
 543       // The length is already set from the Content-Range header
 544       if (StartPos != 0)
 545          return true;
 546
 547       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 548          return _error->Error("The http server sent an invalid Content-Length header");
 549       return true;
 550    }
 551
 552    if (stringcasecmp(Tag,"Content-Type:") == 0)
 553    {
 554       HaveContent = true;
 555       return true;
 556    }
 557
 558    if (stringcasecmp(Tag,"Content-Range:") == 0)
 559    {
 560       HaveContent = true;
 561
 562       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 563          return _error->Error("The http server sent an invalid Content-Range header");
 564       if ((unsigned)StartPos > Size)
 565          return _error->Error("This http server has broken range support");
 566       return true;
 567    }
 568
 569    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 570    {
 571       HaveContent = true;
 572       if (stringcasecmp(Val,"chunked") == 0)
 573          Encoding = Chunked;
 574       return true;
 575    }
 576
 577    if (stringcasecmp(Tag,"Connection:") == 0)
 578    {
 579       if (stringcasecmp(Val,"close") == 0)
 580          Persistent = false;
 581       if (stringcasecmp(Val,"keep-alive") == 0)
 582          Persistent = true;
 583       return true;
 584    }
 585
 586    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 587    {
 588       if (StrToTime(Val,Date) == false)
 589          return _error->Error("Unknown date format");
 590       return true;
 591    }
 592
 593    return true;
 594 }
 595                                                                         /*}}}*/
 596
 597 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 598 // ---------------------------------------------------------------------
 599 /* This places the http request in the outbound buffer */
 600 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 601 {
 602    URI Uri = Itm->Uri;
 603
 604    // The HTTP server expects a hostname with a trailing :port
 605    char Buf[1000];
 606    string ProperHost = Uri.Host;
 607    if (Uri.Port != 0)
 608    {
 609       sprintf(Buf,":%u",Uri.Port);
 610       ProperHost += Buf;
 611    }
 612
 613    // Just in case.
 614    if (Itm->Uri.length() >= sizeof(Buf))
 615        abort();
 616
 617    /* Build the request. We include a keep-alive header only for non-proxy
 618       requests. This is to tweak old http/1.0 servers that do support keep-alive
 619       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 620       will glitch HTTP/1.0 proxies because they do not filter it out and
 621       pass it on, HTTP/1.1 says the connection should default to keep alive
 622       and we expect the proxy to do this */
 623    if (Proxy.empty() == true)
 624       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 625               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 626    else
 627    {
 628       /* Generate a cache control header if necessary. We place a max
 629          cache age on index files, optionally set a no-cache directive
 630          and a no-store directive for archives. */
 631       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 632               Itm->Uri.c_str(),ProperHost.c_str());
 633       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 634          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 635       else
 636       {
 637          if (Itm->IndexFile == true)
 638             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 639                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 640          else
 641          {
 642             if (_config->FindB("Acquire::http::No-Store",false) == true)
 643                strcat(Buf,"Cache-Control: no-store\r\n");
 644          }
 645       }
 646    }
 647
 648    string Req = Buf;
 649
 650    // Check for a partial file
 651    struct stat SBuf;
 652    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 653    {
 654       // In this case we send an if-range query with a range header
 655       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 656               TimeRFC1123(SBuf.st_mtime).c_str());
 657       Req += Buf;
 658    }
 659    else
 660    {
 661       if (Itm->LastModified != 0)
 662       {
 663          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 664          Req += Buf;
 665       }
 666    }
 667
 668    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 669       Req += string("Proxy-Authorization: Basic ") +
 670           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 671
 672    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 673       Req += string("Authorization: Basic ") +
 674           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 675
 676    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 677
 678    if (Debug == true)
 679       cerr << Req << endl;
 680
 681    Out.Read(Req);
 682 }
 683                                                                         /*}}}*/
 684 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 685 // ---------------------------------------------------------------------
 686 /* This runs the select loop over the server FDs, Output file FDs and
 687    stdin. */
 688 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 689 {
 690    // Server has closed the connection
 691    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 692                                ToFile == false))
 693       return false;
 694
 695    fd_set rfds,wfds;
 696    FD_ZERO(&rfds);
 697    FD_ZERO(&wfds);
 698
 699    /* Add the server. We only send more requests if the connection will
 700       be persisting */
 701    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 702        && Srv->Persistent == true)
 703       FD_SET(Srv->ServerFd,&wfds);
 704    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 705       FD_SET(Srv->ServerFd,&rfds);
 706
 707    // Add the file
 708    int FileFD = -1;
 709    if (File != 0)
 710       FileFD = File->Fd();
 711
 712    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 713       FD_SET(FileFD,&wfds);
 714
 715    // Add stdin
 716    FD_SET(STDIN_FILENO,&rfds);
 717
 718    // Figure out the max fd
 719    int MaxFd = FileFD;
 720    if (MaxFd < Srv->ServerFd)
 721       MaxFd = Srv->ServerFd;
 722
 723    // Select
 724    struct timeval tv;
 725    tv.tv_sec = TimeOut;
 726    tv.tv_usec = 0;
 727    int Res = 0;
 728    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 729    {
 730       if (errno == EINTR)
 731          return true;
 732       return _error->Errno("select","Select failed");
 733    }
 734
 735    if (Res == 0)
 736    {
 737       _error->Error("Connection timed out");
 738       return ServerDie(Srv);
 739    }
 740
 741    // Handle server IO
 742    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 743    {
 744       errno = 0;
 745       if (Srv->In.Read(Srv->ServerFd) == false)
 746          return ServerDie(Srv);
 747    }
 748
 749    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 750    {
 751       errno = 0;
 752       if (Srv->Out.Write(Srv->ServerFd) == false)
 753          return ServerDie(Srv);
 754    }
 755
 756    // Send data to the file
 757    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 758    {
 759       if (Srv->In.Write(FileFD) == false)
 760          return _error->Errno("write","Error writing to output file");
 761    }
 762
 763    // Handle commands from APT
 764    if (FD_ISSET(STDIN_FILENO,&rfds))
 765    {
 766       if (Run(true) != -1)
 767          exit(100);
 768    }
 769
 770    return true;
 771 }
 772                                                                         /*}}}*/
 773 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 774 // ---------------------------------------------------------------------
 775 /* This takes the current input buffer from the Server FD and writes it
 776    into the file */
 777 bool HttpMethod::Flush(ServerState *Srv)
 778 {
 779    if (File != 0)
 780    {
 781       SetNonBlock(File->Fd(),false);
 782       if (Srv->In.WriteSpace() == false)
 783          return true;
 784
 785       while (Srv->In.WriteSpace() == true)
 786       {
 787          if (Srv->In.Write(File->Fd()) == false)
 788             return _error->Errno("write","Error writing to file");
 789          if (Srv->In.IsLimit() == true)
 790             return true;
 791       }
 792
 793       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 794          return true;
 795    }
 796    return false;
 797 }
 798                                                                         /*}}}*/
 799 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 800 // ---------------------------------------------------------------------
 801 /* */
 802 bool HttpMethod::ServerDie(ServerState *Srv)
 803 {
 804    unsigned int LErrno = errno;
 805
 806    // Dump the buffer to the file
 807    if (Srv->State == ServerState::Data)
 808    {
 809       SetNonBlock(File->Fd(),false);
 810       while (Srv->In.WriteSpace() == true)
 811       {
 812          if (Srv->In.Write(File->Fd()) == false)
 813             return _error->Errno("write","Error writing to the file");
 814
 815          // Done
 816          if (Srv->In.IsLimit() == true)
 817             return true;
 818       }
 819    }
 820
 821    // See if this is because the server finished the data stream
 822    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 823        Srv->Encoding != ServerState::Closes)
 824    {
 825       Srv->Close();
 826       if (LErrno == 0)
 827          return _error->Error("Error reading from server Remote end closed connection");
 828       errno = LErrno;
 829       return _error->Errno("read","Error reading from server");
 830    }
 831    else
 832    {
 833       Srv->In.Limit(-1);
 834
 835       // Nothing left in the buffer
 836       if (Srv->In.WriteSpace() == false)
 837          return false;
 838
 839       // We may have got multiple responses back in one packet..
 840       Srv->Close();
 841       return true;
 842    }
 843
 844    return false;
 845 }
 846                                                                         /*}}}*/
 847 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* We look at the header data we got back from the server and decide what
 850    to do. Returns
 851      0 - File is open,
 852      1 - IMS hit
 853      3 - Unrecoverable error
 854      4 - Error with error content page
 855      5 - Unrecoverable non-server error (close the connection) */
 856 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 857 {
 858    // Not Modified
 859    if (Srv->Result == 304)
 860    {
 861       unlink(Queue->DestFile.c_str());
 862       Res.IMSHit = true;
 863       Res.LastModified = Queue->LastModified;
 864       return 1;
 865    }
 866
 867    /* We have a reply we dont handle. This should indicate a perm server
 868       failure */
 869    if (Srv->Result < 200 || Srv->Result >= 300)
 870    {
 871       _error->Error("%u %s",Srv->Result,Srv->Code);
 872       if (Srv->HaveContent == true)
 873          return 4;
 874       return 3;
 875    }
 876
 877    // This is some sort of 2xx 'data follows' reply
 878    Res.LastModified = Srv->Date;
 879    Res.Size = Srv->Size;
 880
 881    // Open the file
 882    delete File;
 883    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 884    if (_error->PendingError() == true)
 885       return 5;
 886
 887    FailFile = Queue->DestFile;
 888    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 889    FailFd = File->Fd();
 890    FailTime = Srv->Date;
 891
 892    // Set the expected size
 893    if (Srv->StartPos >= 0)
 894    {
 895       Res.ResumePoint = Srv->StartPos;
 896       ftruncate(File->Fd(),Srv->StartPos);
 897    }
 898
 899    // Set the start point
 900    lseek(File->Fd(),0,SEEK_END);
 901
 902    delete Srv->In.Hash;
 903    Srv->In.Hash = new Hashes;
 904
 905    // Fill the Hash if the file is non-empty (resume)
 906    if (Srv->StartPos > 0)
 907    {
 908       lseek(File->Fd(),0,SEEK_SET);
 909       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 910       {
 911          _error->Errno("read","Problem hashing file");
 912          return 5;
 913       }
 914       lseek(File->Fd(),0,SEEK_END);
 915    }
 916
 917    SetNonBlock(File->Fd(),true);
 918    return 0;
 919 }
 920                                                                         /*}}}*/
 921 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 922 // ---------------------------------------------------------------------
 923 /* This closes and timestamps the open file. This is neccessary to get
 924    resume behavoir on user abort */
 925 void HttpMethod::SigTerm(int)
 926 {
 927    if (FailFd == -1)
 928       _exit(100);
 929    close(FailFd);
 930
 931    // Timestamp
 932    struct utimbuf UBuf;
 933    UBuf.actime = FailTime;
 934    UBuf.modtime = FailTime;
 935    utime(FailFile.c_str(),&UBuf);
 936
 937    _exit(100);
 938 }
 939                                                                         /*}}}*/
 940 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 941 // ---------------------------------------------------------------------
 942 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 943    depth. */
 944 bool HttpMethod::Fetch(FetchItem *)
 945 {
 946    if (Server == 0)
 947       return true;
 948
 949    // Queue the requests
 950    int Depth = -1;
 951    bool Tail = false;
 952    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 953         I = I->Next, Depth++)
 954    {
 955       // If pipelining is disabled, we only queue 1 request
 956       if (Server->Pipeline == false && Depth >= 0)
 957          break;
 958
 959       // Make sure we stick with the same server
 960       if (Server->Comp(I->Uri) == false)
 961          break;
 962       if (QueueBack == I)
 963          Tail = true;
 964       if (Tail == true)
 965       {
 966          QueueBack = I->Next;
 967          SendReq(I,Server->Out);
 968          continue;
 969       }
 970    }
 971
 972    return true;
 973 };
 974                                                                         /*}}}*/
 975 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 976 // ---------------------------------------------------------------------
 977 /* We stash the desired pipeline depth */
 978 bool HttpMethod::Configuration(string Message)
 979 {
 980    if (pkgAcqMethod::Configuration(Message) == false)
 981       return false;
 982
 983    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 984    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 985                                   PipelineDepth);
 986    Debug = _config->FindB("Debug::Acquire::http",false);
 987
 988    return true;
 989 }
 990                                                                         /*}}}*/
 991 // HttpMethod::Loop - Main loop                                         /*{{{*/
 992 // ---------------------------------------------------------------------
 993 /* */
 994 int HttpMethod::Loop()
 995 {
 996    signal(SIGTERM,SigTerm);
 997    signal(SIGINT,SigTerm);
 998
 999    Server = 0;
1000
1001    int FailCounter = 0;
1002    while (1)
1003    {
1004       // We have no commands, wait for some to arrive
1005       if (Queue == 0)
1006       {
1007          if (WaitFd(STDIN_FILENO) == false)
1008             return 0;
1009       }
1010
1011       /* Run messages, we can accept 0 (no message) if we didn't
1012          do a WaitFd above.. Otherwise the FD is closed. */
1013       int Result = Run(true);
1014       if (Result != -1 && (Result != 0 || Queue == 0))
1015          return 100;
1016
1017       if (Queue == 0)
1018          continue;
1019
1020       // Connect to the server
1021       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1022       {
1023          delete Server;
1024          Server = new ServerState(Queue->Uri,this);
1025       }
1026
1027       /* If the server has explicitly said this is the last connection
1028          then we pre-emptively shut down the pipeline and tear down
1029          the connection. This will speed up HTTP/1.0 servers a tad
1030          since we don't have to wait for the close sequence to
1031          complete */
1032       if (Server->Persistent == false)
1033          Server->Close();
1034
1035       // Reset the pipeline
1036       if (Server->ServerFd == -1)
1037          QueueBack = Queue;
1038
1039       // Connnect to the host
1040       if (Server->Open() == false)
1041       {
1042          Fail(true);
1043          delete Server;
1044          Server = 0;
1045          continue;
1046       }
1047
1048       // Fill the pipeline.
1049       Fetch(0);
1050
1051       // Fetch the next URL header data from the server.
1052       switch (Server->RunHeaders())
1053       {
1054          case 0:
1055          break;
1056
1057          // The header data is bad
1058          case 2:
1059          {
1060             _error->Error("Bad header Data");
1061             Fail(true);
1062             RotateDNS();
1063             continue;
1064          }
1065
1066          // The server closed a connection during the header get..
1067          default:
1068          case 1:
1069          {
1070             FailCounter++;
1071             _error->Discard();
1072             Server->Close();
1073             Server->Pipeline = false;
1074
1075             if (FailCounter >= 2)
1076             {
1077                Fail("Connection failed",true);
1078                FailCounter = 0;
1079             }
1080
1081             RotateDNS();
1082             continue;
1083          }
1084       };
1085
1086       // Decide what to do.
1087       FetchResult Res;
1088       Res.Filename = Queue->DestFile;
1089       switch (DealWithHeaders(Res,Server))
1090       {
1091          // Ok, the file is Open
1092          case 0:
1093          {
1094             URIStart(Res);
1095
1096             // Run the data
1097             bool Result =  Server->RunData();
1098
1099             /* If the server is sending back sizeless responses then fill in
1100                the size now */
1101             if (Res.Size == 0)
1102                Res.Size = File->Size();
1103
1104             // Close the file, destroy the FD object and timestamp it
1105             FailFd = -1;
1106             delete File;
1107             File = 0;
1108
1109             // Timestamp
1110             struct utimbuf UBuf;
1111             time(&UBuf.actime);
1112             UBuf.actime = Server->Date;
1113             UBuf.modtime = Server->Date;
1114             utime(Queue->DestFile.c_str(),&UBuf);
1115
1116             // Send status to APT
1117             if (Result == true)
1118             {
1119                Res.TakeHashes(*Server->In.Hash);
1120                URIDone(Res);
1121             }
1122             else
1123                Fail(true);
1124
1125             break;
1126          }
1127
1128          // IMS hit
1129          case 1:
1130          {
1131             URIDone(Res);
1132             break;
1133          }
1134
1135          // Hard server error, not found or something
1136          case 3:
1137          {
1138             Fail();
1139             break;
1140          }
1141
1142          // Hard internal error, kill the connection and fail
1143          case 5:
1144          {
1145             Fail();
1146             RotateDNS();
1147             Server->Close();
1148             break;
1149          }
1150
1151          // We need to flush the data, the header is like a 404 w/ error text
1152          case 4:
1153          {
1154             Fail();
1155
1156             // Send to content to dev/null
1157             File = new FileFd("/dev/null",FileFd::WriteExists);
1158             Server->RunData();
1159             delete File;
1160             File = 0;
1161             break;
1162          }
1163
1164          default:
1165          Fail("Internal error");
1166          break;
1167       }
1168
1169       FailCounter = 0;
1170    }
1171
1172    return 0;
1173 }
1174                                                                         /*}}}*/
1175
1176 int main()
1177 {
1178    HttpMethod Mth;
1179
1180    return Mth.Loop();
1181 }