methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.41 1999/12/09 03:45:56 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57 bool Debug = false;
  58
  59 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  63 {
  64    Buf = new unsigned char[Size];
  65    Reset();
  66 }
  67                                                                         /*}}}*/
  68 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 void CircleBuf::Reset()
  72 {
  73    InP = 0;
  74    OutP = 0;
  75    StrPos = 0;
  76    MaxGet = (unsigned int)-1;
  77    OutQueue = string();
  78    if (MD5 != 0)
  79    {
  80       delete MD5;
  81       MD5 = new MD5Summation;
  82    }
  83 };
  84                                                                         /*}}}*/
  85 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* This fills up the buffer with as much data as is in the FD, assuming it
  88    is non-blocking.. */
  89 bool CircleBuf::Read(int Fd)
  90 {
  91    while (1)
  92    {
  93       // Woops, buffer is full
  94       if (InP - OutP == Size)
  95          return true;
  96
  97       // Write the buffer segment
  98       int Res;
  99       Res = read(Fd,Buf + (InP%Size),LeftRead());
 100
 101       if (Res == 0)
 102          return false;
 103       if (Res < 0)
 104       {
 105          if (errno == EAGAIN)
 106             return true;
 107          return false;
 108       }
 109
 110       if (InP == 0)
 111          gettimeofday(&Start,0);
 112       InP += Res;
 113    }
 114 }
 115                                                                         /*}}}*/
 116 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 117 // ---------------------------------------------------------------------
 118 /* This will hold the string in and fill the buffer with it as it empties */
 119 bool CircleBuf::Read(string Data)
 120 {
 121    OutQueue += Data;
 122    FillOut();
 123    return true;
 124 }
 125                                                                         /*}}}*/
 126 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* */
 129 void CircleBuf::FillOut()
 130 {
 131    if (OutQueue.empty() == true)
 132       return;
 133    while (1)
 134    {
 135       // Woops, buffer is full
 136       if (InP - OutP == Size)
 137          return;
 138
 139       // Write the buffer segment
 140       unsigned long Sz = LeftRead();
 141       if (OutQueue.length() - StrPos < Sz)
 142          Sz = OutQueue.length() - StrPos;
 143       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 144
 145       // Advance
 146       StrPos += Sz;
 147       InP += Sz;
 148       if (OutQueue.length() == StrPos)
 149       {
 150          StrPos = 0;
 151          OutQueue = "";
 152          return;
 153       }
 154    }
 155 }
 156                                                                         /*}}}*/
 157 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* This empties the buffer into the FD. */
 160 bool CircleBuf::Write(int Fd)
 161 {
 162    while (1)
 163    {
 164       FillOut();
 165
 166       // Woops, buffer is empty
 167       if (OutP == InP)
 168          return true;
 169
 170       if (OutP == MaxGet)
 171          return true;
 172
 173       // Write the buffer segment
 174       int Res;
 175       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 176
 177       if (Res == 0)
 178          return false;
 179       if (Res < 0)
 180       {
 181          if (errno == EAGAIN)
 182             return true;
 183
 184          return false;
 185       }
 186
 187       if (MD5 != 0)
 188          MD5->Add(Buf + (OutP%Size),Res);
 189
 190       OutP += Res;
 191    }
 192 }
 193                                                                         /*}}}*/
 194 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This copies till the first empty line */
 197 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 198 {
 199    // We cheat and assume it is unneeded to have more than one buffer load
 200    for (unsigned long I = OutP; I < InP; I++)
 201    {
 202       if (Buf[I%Size] != '\n')
 203          continue;
 204       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 205
 206       if (Single == false)
 207       {
 208          if (Buf[I%Size] != '\n')
 209             continue;
 210          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 211       }
 212
 213       if (I > InP)
 214          I = InP;
 215
 216       Data = "";
 217       while (OutP < I)
 218       {
 219          unsigned long Sz = LeftWrite();
 220          if (Sz == 0)
 221             return false;
 222          if (I - OutP < LeftWrite())
 223             Sz = I - OutP;
 224          Data += string((char *)(Buf + (OutP%Size)),Sz);
 225          OutP += Sz;
 226       }
 227       return true;
 228    }
 229    return false;
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* */
 235 void CircleBuf::Stats()
 236 {
 237    if (InP == 0)
 238       return;
 239
 240    struct timeval Stop;
 241    gettimeofday(&Stop,0);
 242 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 243              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 244    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 245 }
 246                                                                         /*}}}*/
 247
 248 // ServerState::ServerState - Constructor                               /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* */
 251 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 252                         In(64*1024), Out(4*1024),
 253                         ServerName(Srv)
 254 {
 255    Reset();
 256 }
 257                                                                         /*}}}*/
 258 // ServerState::Open - Open a connection to the server                  /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This opens a connection to the server. */
 261 string LastHost;
 262 int LastPort = 0;
 263 struct addrinfo *LastHostAddr = 0;
 264 bool ServerState::Open()
 265 {
 266    // Use the already open connection if possible.
 267    if (ServerFd != -1)
 268       return true;
 269
 270    Close();
 271    In.Reset();
 272    Out.Reset();
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Parse no_proxy, a , seperated list of hosts
 293    if (getenv("no_proxy") != 0)
 294    {
 295       const char *Start = getenv("no_proxy");
 296       for (const char *Cur = Start; true ; Cur++)
 297       {
 298          if (*Cur != ',' && *Cur != 0)
 299             continue;
 300          if (stringcasecmp(ServerName.Host.begin(),ServerName.Host.end(),
 301                            Start,Cur) == 0)
 302          {
 303             Proxy = "";
 304             break;
 305          }
 306
 307          Start = Cur + 1;
 308          if (*Cur == 0)
 309             break;
 310       }
 311    }
 312
 313    // Determine what host and port to use based on the proxy settings
 314    int Port = 0;
 315    string Host;
 316    if (Proxy.empty() == true)
 317    {
 318       if (ServerName.Port != 0)
 319          Port = ServerName.Port;
 320       Host = ServerName.Host;
 321    }
 322    else
 323    {
 324       if (Proxy.Port != 0)
 325          Port = Proxy.Port;
 326       Host = Proxy.Host;
 327    }
 328
 329    // Connect to the remote server
 330    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 331       return false;
 332
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::Close - Close a connection to the server                /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* */
 339 bool ServerState::Close()
 340 {
 341    close(ServerFd);
 342    ServerFd = -1;
 343    return true;
 344 }
 345                                                                         /*}}}*/
 346 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 347 // ---------------------------------------------------------------------
 348 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 349    parse error occured */
 350 int ServerState::RunHeaders()
 351 {
 352    State = Header;
 353
 354    Owner->Status("Waiting for file");
 355
 356    Major = 0;
 357    Minor = 0;
 358    Result = 0;
 359    Size = 0;
 360    StartPos = 0;
 361    Encoding = Closes;
 362    HaveContent = false;
 363    time(&Date);
 364
 365    do
 366    {
 367       string Data;
 368       if (In.WriteTillEl(Data) == false)
 369          continue;
 370
 371       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 372       {
 373          string::const_iterator J = I;
 374          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 375          if (HeaderLine(string(I,J-I)) == false)
 376             return 2;
 377          I = J;
 378       }
 379       return 0;
 380    }
 381    while (Owner->Go(false,this) == true);
 382
 383    return 1;
 384 }
 385                                                                         /*}}}*/
 386 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 387 // ---------------------------------------------------------------------
 388 /* */
 389 bool ServerState::RunData()
 390 {
 391    State = Data;
 392
 393    // Chunked transfer encoding is fun..
 394    if (Encoding == Chunked)
 395    {
 396       while (1)
 397       {
 398          // Grab the block size
 399          bool Last = true;
 400          string Data;
 401          In.Limit(-1);
 402          do
 403          {
 404             if (In.WriteTillEl(Data,true) == true)
 405                break;
 406          }
 407          while ((Last = Owner->Go(false,this)) == true);
 408
 409          if (Last == false)
 410             return false;
 411
 412          // See if we are done
 413          unsigned long Len = strtol(Data.c_str(),0,16);
 414          if (Len == 0)
 415          {
 416             In.Limit(-1);
 417
 418             // We have to remove the entity trailer
 419             Last = true;
 420             do
 421             {
 422                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 423                   break;
 424             }
 425             while ((Last = Owner->Go(false,this)) == true);
 426             if (Last == false)
 427                return false;
 428             return !_error->PendingError();
 429          }
 430
 431          // Transfer the block
 432          In.Limit(Len);
 433          while (Owner->Go(true,this) == true)
 434             if (In.IsLimit() == true)
 435                break;
 436
 437          // Error
 438          if (In.IsLimit() == false)
 439             return false;
 440
 441          // The server sends an extra new line before the next block specifier..
 442          In.Limit(-1);
 443          Last = true;
 444          do
 445          {
 446             if (In.WriteTillEl(Data,true) == true)
 447                break;
 448          }
 449          while ((Last = Owner->Go(false,this)) == true);
 450          if (Last == false)
 451             return false;
 452       }
 453    }
 454    else
 455    {
 456       /* Closes encoding is used when the server did not specify a size, the
 457          loss of the connection means we are done */
 458       if (Encoding == Closes)
 459          In.Limit(-1);
 460       else
 461          In.Limit(Size - StartPos);
 462
 463       // Just transfer the whole block.
 464       do
 465       {
 466          if (In.IsLimit() == false)
 467             continue;
 468
 469          In.Limit(-1);
 470          return !_error->PendingError();
 471       }
 472       while (Owner->Go(true,this) == true);
 473    }
 474
 475    return Owner->Flush(this) && !_error->PendingError();
 476 }
 477                                                                         /*}}}*/
 478 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 479 // ---------------------------------------------------------------------
 480 /* */
 481 bool ServerState::HeaderLine(string Line)
 482 {
 483    if (Line.empty() == true)
 484       return true;
 485
 486    // The http server might be trying to do something evil.
 487    if (Line.length() >= MAXLEN)
 488       return _error->Error("Got a single header line over %u chars",MAXLEN);
 489
 490    string::size_type Pos = Line.find(' ');
 491    if (Pos == string::npos || Pos+1 > Line.length())
 492    {
 493       // Blah, some servers use "connection:closes", evil.
 494       Pos = Line.find(':');
 495       if (Pos == string::npos || Pos + 2 > Line.length())
 496          return _error->Error("Bad header line");
 497       Pos++;
 498    }
 499
 500    // Parse off any trailing spaces between the : and the next word.
 501    string::size_type Pos2 = Pos;
 502    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 503       Pos2++;
 504
 505    string Tag = string(Line,0,Pos);
 506    string Val = string(Line,Pos2);
 507
 508    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 509    {
 510       // Evil servers return no version
 511       if (Line[4] == '/')
 512       {
 513          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 514                     &Result,Code) != 4)
 515             return _error->Error("The http server sent an invalid reply header");
 516       }
 517       else
 518       {
 519          Major = 0;
 520          Minor = 9;
 521          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 522             return _error->Error("The http server sent an invalid reply header");
 523       }
 524
 525       return true;
 526    }
 527
 528    if (stringcasecmp(Tag,"Content-Length:") == 0)
 529    {
 530       if (Encoding == Closes)
 531          Encoding = Stream;
 532       HaveContent = true;
 533
 534       // The length is already set from the Content-Range header
 535       if (StartPos != 0)
 536          return true;
 537
 538       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 539          return _error->Error("The http server sent an invalid Content-Length header");
 540       return true;
 541    }
 542
 543    if (stringcasecmp(Tag,"Content-Type:") == 0)
 544    {
 545       HaveContent = true;
 546       return true;
 547    }
 548
 549    if (stringcasecmp(Tag,"Content-Range:") == 0)
 550    {
 551       HaveContent = true;
 552
 553       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 554          return _error->Error("The http server sent an invalid Content-Range header");
 555       if ((unsigned)StartPos > Size)
 556          return _error->Error("This http server has broken range support");
 557       return true;
 558    }
 559
 560    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 561    {
 562       HaveContent = true;
 563       if (stringcasecmp(Val,"chunked") == 0)
 564          Encoding = Chunked;
 565
 566       return true;
 567    }
 568
 569    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 570    {
 571       if (StrToTime(Val,Date) == false)
 572          return _error->Error("Unknown date format");
 573       return true;
 574    }
 575
 576    return true;
 577 }
 578                                                                         /*}}}*/
 579
 580 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* This places the http request in the outbound buffer */
 583 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 584 {
 585    URI Uri = Itm->Uri;
 586
 587    // The HTTP server expects a hostname with a trailing :port
 588    char Buf[1000];
 589    string ProperHost = Uri.Host;
 590    if (Uri.Port != 0)
 591    {
 592       sprintf(Buf,":%u",Uri.Port);
 593       ProperHost += Buf;
 594    }
 595
 596    // Just in case.
 597    if (Itm->Uri.length() >= sizeof(Buf))
 598        abort();
 599
 600    /* Build the request. We include a keep-alive header only for non-proxy
 601       requests. This is to tweak old http/1.0 servers that do support keep-alive
 602       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 603       will glitch HTTP/1.0 proxies because they do not filter it out and
 604       pass it on, HTTP/1.1 says the connection should default to keep alive
 605       and we expect the proxy to do this */
 606    if (Proxy.empty() == true)
 607       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 608               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 609    else
 610    {
 611       /* Generate a cache control header if necessary. We place a max
 612          cache age on index files, optionally set a no-cache directive
 613          and a no-store directive for archives. */
 614       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 615               Itm->Uri.c_str(),ProperHost.c_str());
 616       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 617          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 618       else
 619       {
 620          if (Itm->IndexFile == true)
 621             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 622                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 623          else
 624          {
 625             if (_config->FindB("Acquire::http::No-Store",false) == true)
 626                strcat(Buf,"Cache-Control: no-store\r\n");
 627          }
 628       }
 629    }
 630
 631    string Req = Buf;
 632
 633    // Check for a partial file
 634    struct stat SBuf;
 635    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 636    {
 637       // In this case we send an if-range query with a range header
 638       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 639               TimeRFC1123(SBuf.st_mtime).c_str());
 640       Req += Buf;
 641    }
 642    else
 643    {
 644       if (Itm->LastModified != 0)
 645       {
 646          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 647          Req += Buf;
 648       }
 649    }
 650
 651    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 652       Req += string("Proxy-Authorization: Basic ") +
 653           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 654
 655    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 656
 657    if (Debug == true)
 658       cerr << Req << endl;
 659
 660    Out.Read(Req);
 661 }
 662                                                                         /*}}}*/
 663 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 664 // ---------------------------------------------------------------------
 665 /* This runs the select loop over the server FDs, Output file FDs and
 666    stdin. */
 667 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 668 {
 669    // Server has closed the connection
 670    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 671                                ToFile == false))
 672       return false;
 673
 674    fd_set rfds,wfds,efds;
 675    FD_ZERO(&rfds);
 676    FD_ZERO(&wfds);
 677    FD_ZERO(&efds);
 678
 679    // Add the server
 680    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 681       FD_SET(Srv->ServerFd,&wfds);
 682    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 683       FD_SET(Srv->ServerFd,&rfds);
 684
 685    // Add the file
 686    int FileFD = -1;
 687    if (File != 0)
 688       FileFD = File->Fd();
 689
 690    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 691       FD_SET(FileFD,&wfds);
 692
 693    // Add stdin
 694    FD_SET(STDIN_FILENO,&rfds);
 695
 696    // Error Set
 697    if (FileFD != -1)
 698       FD_SET(FileFD,&efds);
 699    if (Srv->ServerFd != -1)
 700       FD_SET(Srv->ServerFd,&efds);
 701
 702    // Figure out the max fd
 703    int MaxFd = FileFD;
 704    if (MaxFd < Srv->ServerFd)
 705       MaxFd = Srv->ServerFd;
 706
 707    // Select
 708    struct timeval tv;
 709    tv.tv_sec = TimeOut;
 710    tv.tv_usec = 0;
 711    int Res = 0;
 712    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 713       return _error->Errno("select","Select failed");
 714
 715    if (Res == 0)
 716    {
 717       _error->Error("Connection timed out");
 718       return ServerDie(Srv);
 719    }
 720
 721    // Some kind of exception (error) on the sockets, die
 722    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 723        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 724       return _error->Error("Socket Exception");
 725
 726    // Handle server IO
 727    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 728    {
 729       errno = 0;
 730       if (Srv->In.Read(Srv->ServerFd) == false)
 731          return ServerDie(Srv);
 732    }
 733
 734    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 735    {
 736       errno = 0;
 737       if (Srv->Out.Write(Srv->ServerFd) == false)
 738          return ServerDie(Srv);
 739    }
 740
 741    // Send data to the file
 742    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 743    {
 744       if (Srv->In.Write(FileFD) == false)
 745          return _error->Errno("write","Error writing to output file");
 746    }
 747
 748    // Handle commands from APT
 749    if (FD_ISSET(STDIN_FILENO,&rfds))
 750    {
 751       if (Run(true) != -1)
 752          exit(100);
 753    }
 754
 755    return true;
 756 }
 757                                                                         /*}}}*/
 758 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 759 // ---------------------------------------------------------------------
 760 /* This takes the current input buffer from the Server FD and writes it
 761    into the file */
 762 bool HttpMethod::Flush(ServerState *Srv)
 763 {
 764    if (File != 0)
 765    {
 766       SetNonBlock(File->Fd(),false);
 767       if (Srv->In.WriteSpace() == false)
 768          return true;
 769
 770       while (Srv->In.WriteSpace() == true)
 771       {
 772          if (Srv->In.Write(File->Fd()) == false)
 773             return _error->Errno("write","Error writing to file");
 774          if (Srv->In.IsLimit() == true)
 775             return true;
 776       }
 777
 778       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 779          return true;
 780    }
 781    return false;
 782 }
 783                                                                         /*}}}*/
 784 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 785 // ---------------------------------------------------------------------
 786 /* */
 787 bool HttpMethod::ServerDie(ServerState *Srv)
 788 {
 789    unsigned int LErrno = errno;
 790
 791    // Dump the buffer to the file
 792    if (Srv->State == ServerState::Data)
 793    {
 794       SetNonBlock(File->Fd(),false);
 795       while (Srv->In.WriteSpace() == true)
 796       {
 797          if (Srv->In.Write(File->Fd()) == false)
 798             return _error->Errno("write","Error writing to the file");
 799
 800          // Done
 801          if (Srv->In.IsLimit() == true)
 802             return true;
 803       }
 804    }
 805
 806    // See if this is because the server finished the data stream
 807    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 808        Srv->Encoding != ServerState::Closes)
 809    {
 810       Srv->Close();
 811       if (LErrno == 0)
 812          return _error->Error("Error reading from server Remote end closed connection");
 813       errno = LErrno;
 814       return _error->Errno("read","Error reading from server");
 815    }
 816    else
 817    {
 818       Srv->In.Limit(-1);
 819
 820       // Nothing left in the buffer
 821       if (Srv->In.WriteSpace() == false)
 822          return false;
 823
 824       // We may have got multiple responses back in one packet..
 825       Srv->Close();
 826       return true;
 827    }
 828
 829    return false;
 830 }
 831                                                                         /*}}}*/
 832 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 833 // ---------------------------------------------------------------------
 834 /* We look at the header data we got back from the server and decide what
 835    to do. Returns
 836      0 - File is open,
 837      1 - IMS hit
 838      3 - Unrecoverable error
 839      4 - Error with error content page
 840      5 - Unrecoverable non-server error (close the connection) */
 841 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 842 {
 843    // Not Modified
 844    if (Srv->Result == 304)
 845    {
 846       unlink(Queue->DestFile.c_str());
 847       Res.IMSHit = true;
 848       Res.LastModified = Queue->LastModified;
 849       return 1;
 850    }
 851
 852    /* We have a reply we dont handle. This should indicate a perm server
 853       failure */
 854    if (Srv->Result < 200 || Srv->Result >= 300)
 855    {
 856       _error->Error("%u %s",Srv->Result,Srv->Code);
 857       if (Srv->HaveContent == true)
 858          return 4;
 859       return 3;
 860    }
 861
 862    // This is some sort of 2xx 'data follows' reply
 863    Res.LastModified = Srv->Date;
 864    Res.Size = Srv->Size;
 865
 866    // Open the file
 867    delete File;
 868    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 869    if (_error->PendingError() == true)
 870       return 5;
 871
 872    FailFile = Queue->DestFile;
 873    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 874    FailFd = File->Fd();
 875    FailTime = Srv->Date;
 876
 877    // Set the expected size
 878    if (Srv->StartPos >= 0)
 879    {
 880       Res.ResumePoint = Srv->StartPos;
 881       ftruncate(File->Fd(),Srv->StartPos);
 882    }
 883
 884    // Set the start point
 885    lseek(File->Fd(),0,SEEK_END);
 886
 887    delete Srv->In.MD5;
 888    Srv->In.MD5 = new MD5Summation;
 889
 890    // Fill the MD5 Hash if the file is non-empty (resume)
 891    if (Srv->StartPos > 0)
 892    {
 893       lseek(File->Fd(),0,SEEK_SET);
 894       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 895       {
 896          _error->Errno("read","Problem hashing file");
 897          return 5;
 898       }
 899       lseek(File->Fd(),0,SEEK_END);
 900    }
 901
 902    SetNonBlock(File->Fd(),true);
 903    return 0;
 904 }
 905                                                                         /*}}}*/
 906 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 907 // ---------------------------------------------------------------------
 908 /* This closes and timestamps the open file. This is neccessary to get
 909    resume behavoir on user abort */
 910 void HttpMethod::SigTerm(int)
 911 {
 912    if (FailFd == -1)
 913       _exit(100);
 914    close(FailFd);
 915
 916    // Timestamp
 917    struct utimbuf UBuf;
 918    UBuf.actime = FailTime;
 919    UBuf.modtime = FailTime;
 920    utime(FailFile.c_str(),&UBuf);
 921
 922    _exit(100);
 923 }
 924                                                                         /*}}}*/
 925 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 926 // ---------------------------------------------------------------------
 927 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 928    depth. */
 929 bool HttpMethod::Fetch(FetchItem *)
 930 {
 931    if (Server == 0)
 932       return true;
 933
 934    // Queue the requests
 935    int Depth = -1;
 936    bool Tail = false;
 937    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 938         I = I->Next, Depth++)
 939    {
 940       // If pipelining is disabled, we only queue 1 request
 941       if (Server->Pipeline == false && Depth >= 0)
 942          break;
 943
 944       // Make sure we stick with the same server
 945       if (Server->Comp(I->Uri) == false)
 946          break;
 947       if (QueueBack == I)
 948          Tail = true;
 949       if (Tail == true)
 950       {
 951          QueueBack = I->Next;
 952          SendReq(I,Server->Out);
 953          continue;
 954       }
 955    }
 956
 957    return true;
 958 };
 959                                                                         /*}}}*/
 960 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 961 // ---------------------------------------------------------------------
 962 /* We stash the desired pipeline depth */
 963 bool HttpMethod::Configuration(string Message)
 964 {
 965    if (pkgAcqMethod::Configuration(Message) == false)
 966       return false;
 967
 968    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 969    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 970                                   PipelineDepth);
 971    Debug = _config->FindB("Debug::Acquire::http",false);
 972
 973    return true;
 974 }
 975                                                                         /*}}}*/
 976 // HttpMethod::Loop - Main loop                                         /*{{{*/
 977 // ---------------------------------------------------------------------
 978 /* */
 979 int HttpMethod::Loop()
 980 {
 981    signal(SIGTERM,SigTerm);
 982    signal(SIGINT,SigTerm);
 983
 984    Server = 0;
 985
 986    int FailCounter = 0;
 987    while (1)
 988    {
 989       // We have no commands, wait for some to arrive
 990       if (Queue == 0)
 991       {
 992          if (WaitFd(STDIN_FILENO) == false)
 993             return 0;
 994       }
 995
 996       /* Run messages, we can accept 0 (no message) if we didn't
 997          do a WaitFd above.. Otherwise the FD is closed. */
 998       int Result = Run(true);
 999       if (Result != -1 && (Result != 0 || Queue == 0))
1000          return 100;
1001
1002       if (Queue == 0)
1003          continue;
1004
1005       // Connect to the server
1006       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1007       {
1008          delete Server;
1009          Server = new ServerState(Queue->Uri,this);
1010       }
1011
1012       // Reset the pipeline
1013       if (Server->ServerFd == -1)
1014          QueueBack = Queue;
1015
1016       // Connnect to the host
1017       if (Server->Open() == false)
1018       {
1019          Fail(true);
1020          delete Server;
1021          Server = 0;
1022          continue;
1023       }
1024
1025       // Fill the pipeline.
1026       Fetch(0);
1027
1028       // Fetch the next URL header data from the server.
1029       switch (Server->RunHeaders())
1030       {
1031          case 0:
1032          break;
1033
1034          // The header data is bad
1035          case 2:
1036          {
1037             _error->Error("Bad header Data");
1038             Fail(true);
1039             continue;
1040          }
1041
1042          // The server closed a connection during the header get..
1043          default:
1044          case 1:
1045          {
1046             FailCounter++;
1047             _error->Discard();
1048             Server->Close();
1049             Server->Pipeline = false;
1050
1051             if (FailCounter >= 2)
1052             {
1053                Fail("Connection failed",true);
1054                FailCounter = 0;
1055             }
1056
1057             continue;
1058          }
1059       };
1060
1061       // Decide what to do.
1062       FetchResult Res;
1063       Res.Filename = Queue->DestFile;
1064       switch (DealWithHeaders(Res,Server))
1065       {
1066          // Ok, the file is Open
1067          case 0:
1068          {
1069             URIStart(Res);
1070
1071             // Run the data
1072             bool Result =  Server->RunData();
1073
1074             // Close the file, destroy the FD object and timestamp it
1075             FailFd = -1;
1076             delete File;
1077             File = 0;
1078
1079             // Timestamp
1080             struct utimbuf UBuf;
1081             time(&UBuf.actime);
1082             UBuf.actime = Server->Date;
1083             UBuf.modtime = Server->Date;
1084             utime(Queue->DestFile.c_str(),&UBuf);
1085
1086             // Send status to APT
1087             if (Result == true)
1088             {
1089                Res.MD5Sum = Server->In.MD5->Result();
1090                URIDone(Res);
1091             }
1092             else
1093                Fail(true);
1094
1095             break;
1096          }
1097
1098          // IMS hit
1099          case 1:
1100          {
1101             URIDone(Res);
1102             break;
1103          }
1104
1105          // Hard server error, not found or something
1106          case 3:
1107          {
1108             Fail();
1109             break;
1110          }
1111
1112          // Hard internal error, kill the connection and fail
1113          case 5:
1114          {
1115             Fail();
1116             Server->Close();
1117             break;
1118          }
1119
1120          // We need to flush the data, the header is like a 404 w/ error text
1121          case 4:
1122          {
1123             Fail();
1124
1125             // Send to content to dev/null
1126             File = new FileFd("/dev/null",FileFd::WriteExists);
1127             Server->RunData();
1128             delete File;
1129             File = 0;
1130             break;
1131          }
1132
1133          default:
1134          Fail("Internal error");
1135          break;
1136       }
1137
1138       FailCounter = 0;
1139    }
1140
1141    return 0;
1142 }
1143                                                                         /*}}}*/
1144
1145 int main()
1146 {
1147    HttpMethod Mth;
1148
1149    return Mth.Loop();
1150 }