methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.10 1998/12/07 07:26:27 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41
  42 // Internet stuff
  43 #include <netinet/in.h>
  44 #include <sys/socket.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47
  48 #include "http.h"
  49                                                                         /*}}}*/
  50
  51 string HttpMethod::FailFile;
  52 int HttpMethod::FailFd = -1;
  53 time_t HttpMethod::FailTime = 0;
  54 unsigned long PipelineDepth = 5;
  55
  56 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  57 // ---------------------------------------------------------------------
  58 /* */
  59 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  60 {
  61    Buf = new unsigned char[Size];
  62    Reset();
  63 }
  64                                                                         /*}}}*/
  65 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  66 // ---------------------------------------------------------------------
  67 /* */
  68 void CircleBuf::Reset()
  69 {
  70    InP = 0;
  71    OutP = 0;
  72    StrPos = 0;
  73    MaxGet = (unsigned int)-1;
  74    OutQueue = string();
  75    if (MD5 != 0)
  76    {
  77       delete MD5;
  78       MD5 = new MD5Summation;
  79    }
  80 };
  81                                                                         /*}}}*/
  82 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  83 // ---------------------------------------------------------------------
  84 /* This fills up the buffer with as much data as is in the FD, assuming it
  85    is non-blocking.. */
  86 bool CircleBuf::Read(int Fd)
  87 {
  88    while (1)
  89    {
  90       // Woops, buffer is full
  91       if (InP - OutP == Size)
  92          return true;
  93
  94       // Write the buffer segment
  95       int Res;
  96       Res = read(Fd,Buf + (InP%Size),LeftRead());
  97
  98       if (Res == 0)
  99          return false;
 100       if (Res < 0)
 101       {
 102          if (errno == EAGAIN)
 103             return true;
 104          return false;
 105       }
 106
 107       if (InP == 0)
 108          gettimeofday(&Start,0);
 109       InP += Res;
 110    }
 111 }
 112                                                                         /*}}}*/
 113 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 114 // ---------------------------------------------------------------------
 115 /* This will hold the string in and fill the buffer with it as it empties */
 116 bool CircleBuf::Read(string Data)
 117 {
 118    OutQueue += Data;
 119    FillOut();
 120    return true;
 121 }
 122                                                                         /*}}}*/
 123 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 124 // ---------------------------------------------------------------------
 125 /* */
 126 void CircleBuf::FillOut()
 127 {
 128    if (OutQueue.empty() == true)
 129       return;
 130    while (1)
 131    {
 132       // Woops, buffer is full
 133       if (InP - OutP == Size)
 134          return;
 135
 136       // Write the buffer segment
 137       unsigned long Sz = LeftRead();
 138       if (OutQueue.length() - StrPos < Sz)
 139          Sz = OutQueue.length() - StrPos;
 140       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 141
 142       // Advance
 143       StrPos += Sz;
 144       InP += Sz;
 145       if (OutQueue.length() == StrPos)
 146       {
 147          StrPos = 0;
 148          OutQueue = "";
 149          return;
 150       }
 151    }
 152 }
 153                                                                         /*}}}*/
 154 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 155 // ---------------------------------------------------------------------
 156 /* This empties the buffer into the FD. */
 157 bool CircleBuf::Write(int Fd)
 158 {
 159    while (1)
 160    {
 161       FillOut();
 162
 163       // Woops, buffer is empty
 164       if (OutP == InP)
 165          return true;
 166
 167       if (OutP == MaxGet)
 168          return true;
 169
 170       // Write the buffer segment
 171       int Res;
 172       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 173
 174       if (Res == 0)
 175          return false;
 176       if (Res < 0)
 177       {
 178          if (errno == EAGAIN)
 179             return true;
 180
 181          return false;
 182       }
 183
 184       if (MD5 != 0)
 185          MD5->Add(Buf + (OutP%Size),Res);
 186
 187       OutP += Res;
 188    }
 189 }
 190                                                                         /*}}}*/
 191 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 192 // ---------------------------------------------------------------------
 193 /* This copies till the first empty line */
 194 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 195 {
 196    // We cheat and assume it is unneeded to have more than one buffer load
 197    for (unsigned long I = OutP; I < InP; I++)
 198    {
 199       if (Buf[I%Size] != '\n')
 200          continue;
 201       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 202
 203       if (Single == false)
 204       {
 205          if (Buf[I%Size] != '\n')
 206             continue;
 207          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 208       }
 209
 210       if (I > InP)
 211          I = InP;
 212
 213       Data = "";
 214       while (OutP < I)
 215       {
 216          unsigned long Sz = LeftWrite();
 217          if (Sz == 0)
 218             return false;
 219          if (I - OutP < LeftWrite())
 220             Sz = I - OutP;
 221          Data += string((char *)(Buf + (OutP%Size)),Sz);
 222          OutP += Sz;
 223       }
 224       return true;
 225    }
 226    return false;
 227 }
 228                                                                         /*}}}*/
 229 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 230 // ---------------------------------------------------------------------
 231 /* */
 232 void CircleBuf::Stats()
 233 {
 234    if (InP == 0)
 235       return;
 236
 237    struct timeval Stop;
 238    gettimeofday(&Stop,0);
 239 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 240              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 241    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 242 }
 243                                                                         /*}}}*/
 244
 245 // ServerState::ServerState - Constructor                               /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* */
 248 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 249                         In(64*1024), Out(1*1024),
 250                         ServerName(Srv)
 251 {
 252    Reset();
 253 }
 254                                                                         /*}}}*/
 255 // ServerState::Open - Open a connection to the server                  /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* This opens a connection to the server. */
 258 string LastHost;
 259 in_addr LastHostA;
 260 bool ServerState::Open()
 261 {
 262    // Use the already open connection if possible.
 263    if (ServerFd != -1)
 264       return true;
 265
 266    Close();
 267    In.Reset();
 268    Out.Reset();
 269
 270    // Determine the proxy setting
 271    if (getenv("http_proxy") == 0)
 272    {
 273       string DefProxy = _config->Find("Acquire::http::Proxy");
 274       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 275       if (SpecificProxy.empty() == false)
 276       {
 277          if (SpecificProxy == "DIRECT")
 278             Proxy = "";
 279          else
 280             Proxy = SpecificProxy;
 281       }
 282       else
 283          Proxy = DefProxy;
 284    }
 285    else
 286       Proxy = getenv("http_proxy");
 287
 288    // Determine what host and port to use based on the proxy settings
 289    int Port = 80;
 290    string Host;
 291    if (Proxy.empty() == true)
 292    {
 293       if (ServerName.Port != 0)
 294          Port = ServerName.Port;
 295       Host = ServerName.Host;
 296    }
 297    else
 298    {
 299       if (Proxy.Port != 0)
 300          Port = Proxy.Port;
 301       Host = Proxy.Host;
 302    }
 303
 304    /* We used a cached address record.. Yes this is against the spec but
 305       the way we have setup our rotating dns suggests that this is more
 306       sensible */
 307    if (LastHost != Host)
 308    {
 309       Owner->Status("Connecting to %s",Host.c_str());
 310
 311       // Lookup the host
 312       hostent *Addr = gethostbyname(Host.c_str());
 313       if (Addr == 0)
 314          return _error->Error("Could not resolve '%s'",Host.c_str());
 315       LastHost = Host;
 316       LastHostA = *(in_addr *)(Addr->h_addr_list[0]);
 317    }
 318
 319    Owner->Status("Connecting to %s (%s)",Host.c_str(),inet_ntoa(LastHostA));
 320
 321    // Get a socket
 322    if ((ServerFd = socket(AF_INET,SOCK_STREAM,0)) < 0)
 323       return _error->Errno("socket","Could not create a socket");
 324
 325    // Connect to the server
 326    struct sockaddr_in server;
 327    server.sin_family = AF_INET;
 328    server.sin_port = htons(Port);
 329    server.sin_addr = LastHostA;
 330    if (connect(ServerFd,(sockaddr *)&server,sizeof(server)) < 0)
 331       return _error->Errno("socket","Could not create a socket");
 332
 333    SetNonBlock(ServerFd,true);
 334    return true;
 335 }
 336                                                                         /*}}}*/
 337 // ServerState::Close - Close a connection to the server                /*{{{*/
 338 // ---------------------------------------------------------------------
 339 /* */
 340 bool ServerState::Close()
 341 {
 342    close(ServerFd);
 343    ServerFd = -1;
 344    return true;
 345 }
 346                                                                         /*}}}*/
 347 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 348 // ---------------------------------------------------------------------
 349 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 350    parse error occured */
 351 int ServerState::RunHeaders()
 352 {
 353    State = Header;
 354
 355    Owner->Status("Waiting for file");
 356
 357    Major = 0;
 358    Minor = 0;
 359    Result = 0;
 360    Size = 0;
 361    StartPos = 0;
 362    Encoding = Closes;
 363    HaveContent = false;
 364    time(&Date);
 365
 366    do
 367    {
 368       string Data;
 369       if (In.WriteTillEl(Data) == false)
 370          continue;
 371
 372       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 373       {
 374          string::const_iterator J = I;
 375          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 376          if (HeaderLine(string(I,J-I)) == false)
 377             return 2;
 378          I = J;
 379       }
 380       return 0;
 381    }
 382    while (Owner->Go(false,this) == true);
 383
 384    return 1;
 385 }
 386                                                                         /*}}}*/
 387 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 388 // ---------------------------------------------------------------------
 389 /* */
 390 bool ServerState::RunData()
 391 {
 392    State = Data;
 393
 394    // Chunked transfer encoding is fun..
 395    if (Encoding == Chunked)
 396    {
 397       while (1)
 398       {
 399          // Grab the block size
 400          bool Last = true;
 401          string Data;
 402          In.Limit(-1);
 403          do
 404          {
 405             if (In.WriteTillEl(Data,true) == true)
 406                break;
 407          }
 408          while ((Last = Owner->Go(false,this)) == true);
 409
 410          if (Last == false)
 411             return false;
 412
 413          // See if we are done
 414          unsigned long Len = strtol(Data.c_str(),0,16);
 415          if (Len == 0)
 416          {
 417             In.Limit(-1);
 418
 419             // We have to remove the entity trailer
 420             Last = true;
 421             do
 422             {
 423                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 424                   break;
 425             }
 426             while ((Last = Owner->Go(false,this)) == true);
 427             if (Last == false)
 428                return false;
 429             return true;
 430          }
 431
 432          // Transfer the block
 433          In.Limit(Len);
 434          while (Owner->Go(true,this) == true)
 435             if (In.IsLimit() == true)
 436                break;
 437
 438          // Error
 439          if (In.IsLimit() == false)
 440             return false;
 441
 442          // The server sends an extra new line before the next block specifier..
 443          In.Limit(-1);
 444          Last = true;
 445          do
 446          {
 447             if (In.WriteTillEl(Data,true) == true)
 448                break;
 449          }
 450          while ((Last = Owner->Go(false,this)) == true);
 451          if (Last == false)
 452             return false;
 453       }
 454    }
 455    else
 456    {
 457       /* Closes encoding is used when the server did not specify a size, the
 458          loss of the connection means we are done */
 459       if (Encoding == Closes)
 460          In.Limit(-1);
 461       else
 462          In.Limit(Size - StartPos);
 463
 464       // Just transfer the whole block.
 465       do
 466       {
 467          if (In.IsLimit() == false)
 468             continue;
 469
 470          In.Limit(-1);
 471          return true;
 472       }
 473       while (Owner->Go(true,this) == true);
 474    }
 475
 476    return Owner->Flush(this);
 477 }
 478                                                                         /*}}}*/
 479 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 480 // ---------------------------------------------------------------------
 481 /* */
 482 bool ServerState::HeaderLine(string Line)
 483 {
 484    if (Line.empty() == true)
 485       return true;
 486
 487    // The http server might be trying to do something evil.
 488    if (Line.length() >= MAXLEN)
 489       return _error->Error("Got a single header line over %u chars",MAXLEN);
 490
 491    string::size_type Pos = Line.find(' ');
 492    if (Pos == string::npos || Pos+1 > Line.length())
 493       return _error->Error("Bad header line");
 494
 495    string Tag = string(Line,0,Pos);
 496    string Val = string(Line,Pos+1);
 497
 498    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 499    {
 500       // Evil servers return no version
 501       if (Line[4] == '/')
 502       {
 503          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 504                     &Result,Code) != 4)
 505             return _error->Error("The http server sent an invalid reply header");
 506       }
 507       else
 508       {
 509          Major = 0;
 510          Minor = 9;
 511          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 512             return _error->Error("The http server sent an invalid reply header");
 513       }
 514
 515       return true;
 516    }
 517
 518    if (stringcasecmp(Tag,"Content-Length:") == 0)
 519    {
 520       if (Encoding == Closes)
 521          Encoding = Stream;
 522       HaveContent = true;
 523
 524       // The length is already set from the Content-Range header
 525       if (StartPos != 0)
 526          return true;
 527
 528       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 529          return _error->Error("The http server sent an invalid Content-Length header");
 530       return true;
 531    }
 532
 533    if (stringcasecmp(Tag,"Content-Type:") == 0)
 534    {
 535       HaveContent = true;
 536       return true;
 537    }
 538
 539    if (stringcasecmp(Tag,"Content-Range:") == 0)
 540    {
 541       HaveContent = true;
 542
 543       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 544          return _error->Error("The http server sent an invalid Content-Range header");
 545       if ((unsigned)StartPos > Size)
 546          return _error->Error("This http server has broken range support");
 547       return true;
 548    }
 549
 550    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 551    {
 552       HaveContent = true;
 553       if (stringcasecmp(Val,"chunked") == 0)
 554          Encoding = Chunked;
 555
 556       return true;
 557    }
 558
 559    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 560    {
 561       if (StrToTime(Val,Date) == false)
 562          return _error->Error("Unknown date format");
 563       return true;
 564    }
 565
 566    return true;
 567 }
 568                                                                         /*}}}*/
 569
 570 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 571 // ---------------------------------------------------------------------
 572 /* This places the http request in the outbound buffer */
 573 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 574 {
 575    URI Uri = Itm->Uri;
 576
 577    // The HTTP server expects a hostname with a trailing :port
 578    char Buf[1000];
 579    string ProperHost = Uri.Host;
 580    if (Uri.Port != 0)
 581    {
 582       sprintf(Buf,":%u",Uri.Port);
 583       ProperHost += Buf;
 584    }
 585
 586    // Just in case.
 587    if (Itm->Uri.length() >= sizeof(Buf))
 588        abort();
 589
 590    /* Build the request. We include a keep-alive header only for non-proxy
 591       requests. This is to tweak old http/1.0 servers that do support keep-alive
 592       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 593       will glitch HTTP/1.0 proxies because they do not filter it out and
 594       pass it on, HTTP/1.1 says the connection should default to keep alive
 595       and we expect the proxy to do this */
 596    if (Proxy.empty() == true)
 597       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 598               Uri.Path.c_str(),ProperHost.c_str());
 599    else
 600    {
 601       /* Generate a cache control header if necessary. We place a max
 602          cache age on index files, optionally set a no-cache directive
 603          and a no-store directive for archives. */
 604       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 605               Itm->Uri.c_str(),ProperHost.c_str());
 606       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 607          strcat(Buf,"Cache-Control: no-cache\r\n");
 608       else
 609       {
 610          if (Itm->IndexFile == true)
 611             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 612                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 613          else
 614          {
 615             if (_config->FindB("Acquire::http::No-Store",false) == true)
 616                strcat(Buf,"Cache-Control: no-store\r\n");
 617          }
 618       }
 619    }
 620
 621    string Req = Buf;
 622
 623    // Check for a partial file
 624    struct stat SBuf;
 625    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 626    {
 627       // In this case we send an if-range query with a range header
 628       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 629               TimeRFC1123(SBuf.st_mtime).c_str());
 630       Req += Buf;
 631    }
 632    else
 633    {
 634       if (Itm->LastModified != 0)
 635       {
 636          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 637          Req += Buf;
 638       }
 639    }
 640
 641 /*   if (ProxyAuth.empty() == false)
 642       Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
 643
 644    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 645 //   cerr << Req << endl;
 646
 647    cerr << Req << endl;
 648    Out.Read(Req);
 649 }
 650                                                                         /*}}}*/
 651 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 652 // ---------------------------------------------------------------------
 653 /* This runs the select loop over the server FDs, Output file FDs and
 654    stdin. */
 655 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 656 {
 657    // Server has closed the connection
 658    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 659       return false;
 660
 661    fd_set rfds,wfds,efds;
 662    FD_ZERO(&rfds);
 663    FD_ZERO(&wfds);
 664    FD_ZERO(&efds);
 665
 666    // Add the server
 667    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 668       FD_SET(Srv->ServerFd,&wfds);
 669    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 670       FD_SET(Srv->ServerFd,&rfds);
 671
 672    // Add the file
 673    int FileFD = -1;
 674    if (File != 0)
 675       FileFD = File->Fd();
 676
 677    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 678       FD_SET(FileFD,&wfds);
 679
 680    // Add stdin
 681    FD_SET(STDIN_FILENO,&rfds);
 682
 683    // Error Set
 684    if (FileFD != -1)
 685       FD_SET(FileFD,&efds);
 686    if (Srv->ServerFd != -1)
 687       FD_SET(Srv->ServerFd,&efds);
 688
 689    // Figure out the max fd
 690    int MaxFd = FileFD;
 691    if (MaxFd < Srv->ServerFd)
 692       MaxFd = Srv->ServerFd;
 693
 694    // Select
 695    struct timeval tv;
 696    tv.tv_sec = 120;
 697    tv.tv_usec = 0;
 698    int Res = 0;
 699    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 700       return _error->Errno("select","Select failed");
 701
 702    if (Res == 0)
 703    {
 704       _error->Error("Connection timed out");
 705       return ServerDie(Srv);
 706    }
 707
 708    // Some kind of exception (error) on the sockets, die
 709    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 710        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 711       return _error->Error("Socket Exception");
 712
 713    // Handle server IO
 714    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 715    {
 716       errno = 0;
 717       if (Srv->In.Read(Srv->ServerFd) == false)
 718          return ServerDie(Srv);
 719    }
 720
 721    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 722    {
 723       errno = 0;
 724       if (Srv->Out.Write(Srv->ServerFd) == false)
 725          return ServerDie(Srv);
 726    }
 727
 728    // Send data to the file
 729    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 730    {
 731       if (Srv->In.Write(FileFD) == false)
 732          return _error->Errno("write","Error writing to output file");
 733    }
 734
 735    // Handle commands from APT
 736    if (FD_ISSET(STDIN_FILENO,&rfds))
 737    {
 738       if (Run(true) != 0)
 739          exit(100);
 740    }
 741
 742    return true;
 743 }
 744                                                                         /*}}}*/
 745 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 746 // ---------------------------------------------------------------------
 747 /* This takes the current input buffer from the Server FD and writes it
 748    into the file */
 749 bool HttpMethod::Flush(ServerState *Srv)
 750 {
 751    if (File != 0)
 752    {
 753       SetNonBlock(File->Fd(),false);
 754       if (Srv->In.WriteSpace() == false)
 755          return true;
 756
 757       while (Srv->In.WriteSpace() == true)
 758       {
 759          if (Srv->In.Write(File->Fd()) == false)
 760             return _error->Errno("write","Error writing to file");
 761          if (Srv->In.IsLimit() == true)
 762             return true;
 763       }
 764
 765       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 766          return true;
 767    }
 768    return false;
 769 }
 770                                                                         /*}}}*/
 771 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 772 // ---------------------------------------------------------------------
 773 /* */
 774 bool HttpMethod::ServerDie(ServerState *Srv)
 775 {
 776    // Dump the buffer to the file
 777    if (Srv->State == ServerState::Data)
 778    {
 779       SetNonBlock(File->Fd(),false);
 780       while (Srv->In.WriteSpace() == true)
 781       {
 782          if (Srv->In.Write(File->Fd()) == false)
 783             return _error->Errno("write","Error writing to the file");
 784
 785          // Done
 786          if (Srv->In.IsLimit() == true)
 787             return true;
 788       }
 789    }
 790
 791    // See if this is because the server finished the data stream
 792    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 793        Srv->Encoding != ServerState::Closes)
 794    {
 795       if (errno == 0)
 796          return _error->Error("Error reading from server Remote end closed connection");
 797       return _error->Errno("read","Error reading from server");
 798    }
 799    else
 800    {
 801       Srv->In.Limit(-1);
 802
 803       // Nothing left in the buffer
 804       if (Srv->In.WriteSpace() == false)
 805          return false;
 806
 807       // We may have got multiple responses back in one packet..
 808       Srv->Close();
 809       return true;
 810    }
 811
 812    return false;
 813 }
 814                                                                         /*}}}*/
 815 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 816 // ---------------------------------------------------------------------
 817 /* We look at the header data we got back from the server and decide what
 818    to do. Returns
 819      0 - File is open,
 820      1 - IMS hit
 821      3 - Unrecoverable error
 822      4 - Error with error content page
 823      5 - Unrecoverable non-server error (close the connection) */
 824 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 825 {
 826    // Not Modified
 827    if (Srv->Result == 304)
 828    {
 829       unlink(Queue->DestFile.c_str());
 830       Res.IMSHit = true;
 831       Res.LastModified = Queue->LastModified;
 832       return 1;
 833    }
 834
 835    /* We have a reply we dont handle. This should indicate a perm server
 836       failure */
 837    if (Srv->Result < 200 || Srv->Result >= 300)
 838    {
 839       _error->Error("%u %s",Srv->Result,Srv->Code);
 840       if (Srv->HaveContent == true)
 841          return 4;
 842       return 3;
 843    }
 844
 845    // This is some sort of 2xx 'data follows' reply
 846    Res.LastModified = Srv->Date;
 847    Res.Size = Srv->Size;
 848
 849    // Open the file
 850    delete File;
 851    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 852    if (_error->PendingError() == true)
 853       return 5;
 854
 855    FailFile = Queue->DestFile;
 856    FailFd = File->Fd();
 857    FailTime = Srv->Date;
 858
 859    // Set the expected size
 860    if (Srv->StartPos >= 0)
 861    {
 862       Res.ResumePoint = Srv->StartPos;
 863       ftruncate(File->Fd(),Srv->StartPos);
 864    }
 865
 866    // Set the start point
 867    lseek(File->Fd(),0,SEEK_END);
 868
 869    delete Srv->In.MD5;
 870    Srv->In.MD5 = new MD5Summation;
 871
 872    // Fill the MD5 Hash if the file is non-empty (resume)
 873    if (Srv->StartPos > 0)
 874    {
 875       lseek(File->Fd(),0,SEEK_SET);
 876       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 877       {
 878          _error->Errno("read","Problem hashing file");
 879          return 5;
 880       }
 881       lseek(File->Fd(),0,SEEK_END);
 882    }
 883
 884    SetNonBlock(File->Fd(),true);
 885    return 0;
 886 }
 887                                                                         /*}}}*/
 888 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 889 // ---------------------------------------------------------------------
 890 /* This closes and timestamps the open file. This is neccessary to get
 891    resume behavoir on user abort */
 892 void HttpMethod::SigTerm(int)
 893 {
 894    if (FailFd == -1)
 895       exit(100);
 896    close(FailFd);
 897
 898    // Timestamp
 899    struct utimbuf UBuf;
 900    time(&UBuf.actime);
 901    UBuf.actime = FailTime;
 902    UBuf.modtime = FailTime;
 903    utime(FailFile.c_str(),&UBuf);
 904
 905    exit(100);
 906 }
 907                                                                         /*}}}*/
 908 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 909 // ---------------------------------------------------------------------
 910 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 911    depth. */
 912 bool HttpMethod::Fetch(FetchItem *)
 913 {
 914    if (Server == 0)
 915       return true;
 916
 917    // Queue the requests
 918    int Depth = -1;
 919    bool Tail = false;
 920    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 921    {
 922       // Make sure we stick with the same server
 923       if (Server->Comp(I->Uri) == false)
 924          break;
 925
 926       if (QueueBack == I)
 927          Tail = true;
 928       if (Tail == true)
 929       {
 930          Depth++;
 931          QueueBack = I->Next;
 932          SendReq(I,Server->Out);
 933          continue;
 934       }
 935    }
 936
 937    return true;
 938 };
 939                                                                         /*}}}*/
 940 // HttpMethod::Loop - Main loop                                         /*{{{*/
 941 // ---------------------------------------------------------------------
 942 /* */
 943 int HttpMethod::Loop()
 944 {
 945    signal(SIGTERM,SigTerm);
 946    signal(SIGINT,SigTerm);
 947
 948    Server = 0;
 949
 950    int FailCounter = 0;
 951    while (1)
 952    {
 953       if (FailCounter >= 2)
 954       {
 955          Fail("Massive Server Brain Damage");
 956          FailCounter = 0;
 957       }
 958
 959       // We have no commands, wait for some to arrive
 960       if (Queue == 0)
 961       {
 962          if (WaitFd(STDIN_FILENO) == false)
 963             return 0;
 964       }
 965
 966       // Run messages
 967       if (Run(true) != 0)
 968          return 100;
 969
 970       if (Queue == 0)
 971          continue;
 972
 973       // Connect to the server
 974       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 975       {
 976          delete Server;
 977          Server = new ServerState(Queue->Uri,this);
 978       }
 979
 980       // Connnect to the host
 981       if (Server->Open() == false)
 982       {
 983          Fail();
 984          continue;
 985       }
 986
 987       // Fill the pipeline.
 988       Fetch(0);
 989
 990       // Fetch the next URL header data from the server.
 991       switch (Server->RunHeaders())
 992       {
 993          case 0:
 994          break;
 995
 996          // The header data is bad
 997          case 2:
 998          {
 999             _error->Error("Bad header Data");
1000             Fail();
1001             continue;
1002          }
1003
1004          // The server closed a connection during the header get..
1005          default:
1006          case 1:
1007          {
1008             FailCounter++;
1009             _error->DumpErrors();
1010             Server->Close();
1011             continue;
1012          }
1013       };
1014
1015       // Decide what to do.
1016       FetchResult Res;
1017       Res.Filename = Queue->DestFile;
1018       switch (DealWithHeaders(Res,Server))
1019       {
1020          // Ok, the file is Open
1021          case 0:
1022          {
1023             URIStart(Res);
1024
1025             // Run the data
1026             bool Result =  Server->RunData();
1027
1028             // Close the file, destroy the FD object and timestamp it
1029             FailFd = -1;
1030             delete File;
1031             File = 0;
1032
1033             // Timestamp
1034             struct utimbuf UBuf;
1035             time(&UBuf.actime);
1036             UBuf.actime = Server->Date;
1037             UBuf.modtime = Server->Date;
1038             utime(Queue->DestFile.c_str(),&UBuf);
1039
1040             // Send status to APT
1041             if (Result == true)
1042             {
1043                Res.MD5Sum = Server->In.MD5->Result();
1044                URIDone(Res);
1045             }
1046             else
1047                Fail();
1048
1049             break;
1050          }
1051
1052          // IMS hit
1053          case 1:
1054          {
1055             URIDone(Res);
1056             break;
1057          }
1058
1059          // Hard server error, not found or something
1060          case 3:
1061          {
1062             Fail();
1063             break;
1064          }
1065
1066          // Hard internal error, kill the connection and fail
1067          case 5:
1068          {
1069             Fail();
1070             Server->Close();
1071             break;
1072          }
1073
1074          // We need to flush the data, the header is like a 404 w/ error text
1075          case 4:
1076          {
1077             Fail();
1078
1079             // Send to content to dev/null
1080             File = new FileFd("/dev/null",FileFd::WriteExists);
1081             Server->RunData();
1082             delete File;
1083             File = 0;
1084             break;
1085          }
1086
1087          default:
1088          Fail("Internal error");
1089          break;
1090       }
1091
1092       FailCounter = 0;
1093    }
1094
1095    return 0;
1096 }
1097                                                                         /*}}}*/
1098
1099 int main()
1100 {
1101    HttpMethod Mth;
1102
1103    return Mth.Loop();
1104 }