methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include "connect.h"
  48 #include "rfc2553emu.h"
  49 #include "http.h"
  50
  51                                                                         /*}}}*/
  52 using namespace std;
  53
  54 string HttpMethod::FailFile;
  55 int HttpMethod::FailFd = -1;
  56 time_t HttpMethod::FailTime = 0;
  57 unsigned long PipelineDepth = 10;
  58 unsigned long TimeOut = 120;
  59 bool Debug = false;
  60
  61
  62 unsigned long CircleBuf::BwReadLimit=0;
  63 unsigned long CircleBuf::BwTickReadData=0;
  64 struct timeval CircleBuf::BwReadTick={0,0};
  65 const unsigned int CircleBuf::BW_HZ=10;
  66
  67 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  71 {
  72    Buf = new unsigned char[Size];
  73    Reset();
  74
  75    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  76 }
  77                                                                         /*}}}*/
  78 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  79 // ---------------------------------------------------------------------
  80 /* */
  81 void CircleBuf::Reset()
  82 {
  83    InP = 0;
  84    OutP = 0;
  85    StrPos = 0;
  86    MaxGet = (unsigned int)-1;
  87    OutQueue = string();
  88    if (Hash != 0)
  89    {
  90       delete Hash;
  91       Hash = new Hashes;
  92    }
  93 };
  94                                                                         /*}}}*/
  95 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* This fills up the buffer with as much data as is in the FD, assuming it
  98    is non-blocking.. */
  99 bool CircleBuf::Read(int Fd)
 100 {
 101    unsigned long BwReadMax;
 102
 103    while (1)
 104    {
 105       // Woops, buffer is full
 106       if (InP - OutP == Size)
 107          return true;
 108
 109       // what's left to read in this tick
 110       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 111
 112       if(CircleBuf::BwReadLimit) {
 113          struct timeval now;
 114          gettimeofday(&now,0);
 115
 116          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 117             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 118          if(d > 1000000/BW_HZ) {
 119             CircleBuf::BwReadTick = now;
 120             CircleBuf::BwTickReadData = 0;
 121          }
 122
 123          if(CircleBuf::BwTickReadData >= BwReadMax) {
 124             usleep(1000000/BW_HZ);
 125             return true;
 126          }
 127       }
 128
 129       // Write the buffer segment
 130       int Res;
 131       if(CircleBuf::BwReadLimit) {
 132          Res = read(Fd,Buf + (InP%Size),
 133                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 134       } else
 135          Res = read(Fd,Buf + (InP%Size),LeftRead());
 136
 137       if(Res > 0 && BwReadLimit > 0)
 138          CircleBuf::BwTickReadData += Res;
 139
 140       if (Res == 0)
 141          return false;
 142       if (Res < 0)
 143       {
 144          if (errno == EAGAIN)
 145             return true;
 146          return false;
 147       }
 148
 149       if (InP == 0)
 150          gettimeofday(&Start,0);
 151       InP += Res;
 152    }
 153 }
 154                                                                         /*}}}*/
 155 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 156 // ---------------------------------------------------------------------
 157 /* This will hold the string in and fill the buffer with it as it empties */
 158 bool CircleBuf::Read(string Data)
 159 {
 160    OutQueue += Data;
 161    FillOut();
 162    return true;
 163 }
 164                                                                         /*}}}*/
 165 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 166 // ---------------------------------------------------------------------
 167 /* */
 168 void CircleBuf::FillOut()
 169 {
 170    if (OutQueue.empty() == true)
 171       return;
 172    while (1)
 173    {
 174       // Woops, buffer is full
 175       if (InP - OutP == Size)
 176          return;
 177
 178       // Write the buffer segment
 179       unsigned long Sz = LeftRead();
 180       if (OutQueue.length() - StrPos < Sz)
 181          Sz = OutQueue.length() - StrPos;
 182       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 183
 184       // Advance
 185       StrPos += Sz;
 186       InP += Sz;
 187       if (OutQueue.length() == StrPos)
 188       {
 189          StrPos = 0;
 190          OutQueue = "";
 191          return;
 192       }
 193    }
 194 }
 195                                                                         /*}}}*/
 196 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 197 // ---------------------------------------------------------------------
 198 /* This empties the buffer into the FD. */
 199 bool CircleBuf::Write(int Fd)
 200 {
 201    while (1)
 202    {
 203       FillOut();
 204
 205       // Woops, buffer is empty
 206       if (OutP == InP)
 207          return true;
 208
 209       if (OutP == MaxGet)
 210          return true;
 211
 212       // Write the buffer segment
 213       int Res;
 214       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 215
 216       if (Res == 0)
 217          return false;
 218       if (Res < 0)
 219       {
 220          if (errno == EAGAIN)
 221             return true;
 222
 223          return false;
 224       }
 225
 226       if (Hash != 0)
 227          Hash->Add(Buf + (OutP%Size),Res);
 228
 229       OutP += Res;
 230    }
 231 }
 232                                                                         /*}}}*/
 233 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 234 // ---------------------------------------------------------------------
 235 /* This copies till the first empty line */
 236 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 237 {
 238    // We cheat and assume it is unneeded to have more than one buffer load
 239    for (unsigned long I = OutP; I < InP; I++)
 240    {
 241       if (Buf[I%Size] != '\n')
 242          continue;
 243       ++I;
 244       if (I < InP  && Buf[I%Size] == '\r')
 245          ++I;
 246
 247       if (Single == false)
 248       {
 249          if (Buf[I%Size] != '\n')
 250             continue;
 251          ++I;
 252          if (I < InP  && Buf[I%Size] == '\r')
 253             ++I;
 254       }
 255
 256       if (I > InP)
 257          I = InP;
 258
 259       Data = "";
 260       while (OutP < I)
 261       {
 262          unsigned long Sz = LeftWrite();
 263          if (Sz == 0)
 264             return false;
 265          if (I - OutP < LeftWrite())
 266             Sz = I - OutP;
 267          Data += string((char *)(Buf + (OutP%Size)),Sz);
 268          OutP += Sz;
 269       }
 270       return true;
 271    }
 272    return false;
 273 }
 274                                                                         /*}}}*/
 275 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 276 // ---------------------------------------------------------------------
 277 /* */
 278 void CircleBuf::Stats()
 279 {
 280    if (InP == 0)
 281       return;
 282
 283    struct timeval Stop;
 284    gettimeofday(&Stop,0);
 285 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 286              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 287    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 288 }
 289                                                                         /*}}}*/
 290
 291 // ServerState::ServerState - Constructor                               /*{{{*/
 292 // ---------------------------------------------------------------------
 293 /* */
 294 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 295                         In(64*1024), Out(4*1024),
 296                         ServerName(Srv)
 297 {
 298    Reset();
 299 }
 300                                                                         /*}}}*/
 301 // ServerState::Open - Open a connection to the server                  /*{{{*/
 302 // ---------------------------------------------------------------------
 303 /* This opens a connection to the server. */
 304 bool ServerState::Open()
 305 {
 306    // Use the already open connection if possible.
 307    if (ServerFd != -1)
 308       return true;
 309
 310    Close();
 311    In.Reset();
 312    Out.Reset();
 313    Persistent = true;
 314
 315    // Determine the proxy setting
 316    if (getenv("http_proxy") == 0)
 317    {
 318       string DefProxy = _config->Find("Acquire::http::Proxy");
 319       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 320       if (SpecificProxy.empty() == false)
 321       {
 322          if (SpecificProxy == "DIRECT")
 323             Proxy = "";
 324          else
 325             Proxy = SpecificProxy;
 326       }
 327       else
 328          Proxy = DefProxy;
 329    }
 330    else
 331       Proxy = getenv("http_proxy");
 332
 333    // Parse no_proxy, a , separated list of domains
 334    if (getenv("no_proxy") != 0)
 335    {
 336       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 337          Proxy = "";
 338    }
 339
 340    // Determine what host and port to use based on the proxy settings
 341    int Port = 0;
 342    string Host;
 343    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 344    {
 345       if (ServerName.Port != 0)
 346          Port = ServerName.Port;
 347       Host = ServerName.Host;
 348    }
 349    else
 350    {
 351       if (Proxy.Port != 0)
 352          Port = Proxy.Port;
 353       Host = Proxy.Host;
 354    }
 355
 356    // Connect to the remote server
 357    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 358       return false;
 359
 360    return true;
 361 }
 362                                                                         /*}}}*/
 363 // ServerState::Close - Close a connection to the server                /*{{{*/
 364 // ---------------------------------------------------------------------
 365 /* */
 366 bool ServerState::Close()
 367 {
 368    close(ServerFd);
 369    ServerFd = -1;
 370    return true;
 371 }
 372                                                                         /*}}}*/
 373 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 374 // ---------------------------------------------------------------------
 375 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 376    parse error occured */
 377 int ServerState::RunHeaders()
 378 {
 379    State = Header;
 380
 381    Owner->Status(_("Waiting for headers"));
 382
 383    Major = 0;
 384    Minor = 0;
 385    Result = 0;
 386    Size = 0;
 387    StartPos = 0;
 388    Encoding = Closes;
 389    HaveContent = false;
 390    time(&Date);
 391
 392    do
 393    {
 394       string Data;
 395       if (In.WriteTillEl(Data) == false)
 396          continue;
 397
 398       if (Debug == true)
 399          clog << Data;
 400
 401       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 402       {
 403          string::const_iterator J = I;
 404          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 405          if (HeaderLine(string(I,J)) == false)
 406             return 2;
 407          I = J;
 408       }
 409
 410       // 100 Continue is a Nop...
 411       if (Result == 100)
 412          continue;
 413
 414       // Tidy up the connection persistance state.
 415       if (Encoding == Closes && HaveContent == true)
 416          Persistent = false;
 417
 418       return 0;
 419    }
 420    while (Owner->Go(false,this) == true);
 421
 422    return 1;
 423 }
 424                                                                         /*}}}*/
 425 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 426 // ---------------------------------------------------------------------
 427 /* */
 428 bool ServerState::RunData()
 429 {
 430    State = Data;
 431
 432    // Chunked transfer encoding is fun..
 433    if (Encoding == Chunked)
 434    {
 435       while (1)
 436       {
 437          // Grab the block size
 438          bool Last = true;
 439          string Data;
 440          In.Limit(-1);
 441          do
 442          {
 443             if (In.WriteTillEl(Data,true) == true)
 444                break;
 445          }
 446          while ((Last = Owner->Go(false,this)) == true);
 447
 448          if (Last == false)
 449             return false;
 450
 451          // See if we are done
 452          unsigned long Len = strtol(Data.c_str(),0,16);
 453          if (Len == 0)
 454          {
 455             In.Limit(-1);
 456
 457             // We have to remove the entity trailer
 458             Last = true;
 459             do
 460             {
 461                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 462                   break;
 463             }
 464             while ((Last = Owner->Go(false,this)) == true);
 465             if (Last == false)
 466                return false;
 467             return !_error->PendingError();
 468          }
 469
 470          // Transfer the block
 471          In.Limit(Len);
 472          while (Owner->Go(true,this) == true)
 473             if (In.IsLimit() == true)
 474                break;
 475
 476          // Error
 477          if (In.IsLimit() == false)
 478             return false;
 479
 480          // The server sends an extra new line before the next block specifier..
 481          In.Limit(-1);
 482          Last = true;
 483          do
 484          {
 485             if (In.WriteTillEl(Data,true) == true)
 486                break;
 487          }
 488          while ((Last = Owner->Go(false,this)) == true);
 489          if (Last == false)
 490             return false;
 491       }
 492    }
 493    else
 494    {
 495       /* Closes encoding is used when the server did not specify a size, the
 496          loss of the connection means we are done */
 497       if (Encoding == Closes)
 498          In.Limit(-1);
 499       else
 500          In.Limit(Size - StartPos);
 501
 502       // Just transfer the whole block.
 503       do
 504       {
 505          if (In.IsLimit() == false)
 506             continue;
 507
 508          In.Limit(-1);
 509          return !_error->PendingError();
 510       }
 511       while (Owner->Go(true,this) == true);
 512    }
 513
 514    return Owner->Flush(this) && !_error->PendingError();
 515 }
 516                                                                         /*}}}*/
 517 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 518 // ---------------------------------------------------------------------
 519 /* */
 520 bool ServerState::HeaderLine(string Line)
 521 {
 522    if (Line.empty() == true)
 523       return true;
 524
 525    // The http server might be trying to do something evil.
 526    if (Line.length() >= MAXLEN)
 527       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 528
 529    string::size_type Pos = Line.find(' ');
 530    if (Pos == string::npos || Pos+1 > Line.length())
 531    {
 532       // Blah, some servers use "connection:closes", evil.
 533       Pos = Line.find(':');
 534       if (Pos == string::npos || Pos + 2 > Line.length())
 535          return _error->Error(_("Bad header line"));
 536       Pos++;
 537    }
 538
 539    // Parse off any trailing spaces between the : and the next word.
 540    string::size_type Pos2 = Pos;
 541    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 542       Pos2++;
 543
 544    string Tag = string(Line,0,Pos);
 545    string Val = string(Line,Pos2);
 546
 547    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 548    {
 549       // Evil servers return no version
 550       if (Line[4] == '/')
 551       {
 552          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 553                     &Result,Code) != 4)
 554             return _error->Error(_("The HTTP server sent an invalid reply header"));
 555       }
 556       else
 557       {
 558          Major = 0;
 559          Minor = 9;
 560          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 561             return _error->Error(_("The HTTP server sent an invalid reply header"));
 562       }
 563
 564       /* Check the HTTP response header to get the default persistance
 565          state. */
 566       if (Major < 1)
 567          Persistent = false;
 568       else
 569       {
 570          if (Major == 1 && Minor <= 0)
 571             Persistent = false;
 572          else
 573             Persistent = true;
 574       }
 575
 576       return true;
 577    }
 578
 579    if (stringcasecmp(Tag,"Content-Length:") == 0)
 580    {
 581       if (Encoding == Closes)
 582          Encoding = Stream;
 583       HaveContent = true;
 584
 585       // The length is already set from the Content-Range header
 586       if (StartPos != 0)
 587          return true;
 588
 589       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 590          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 591       return true;
 592    }
 593
 594    if (stringcasecmp(Tag,"Content-Type:") == 0)
 595    {
 596       HaveContent = true;
 597       return true;
 598    }
 599
 600    if (stringcasecmp(Tag,"Content-Range:") == 0)
 601    {
 602       HaveContent = true;
 603
 604       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 605          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 606       if ((unsigned)StartPos > Size)
 607          return _error->Error(_("This HTTP server has broken range support"));
 608       return true;
 609    }
 610
 611    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 612    {
 613       HaveContent = true;
 614       if (stringcasecmp(Val,"chunked") == 0)
 615          Encoding = Chunked;
 616       return true;
 617    }
 618
 619    if (stringcasecmp(Tag,"Connection:") == 0)
 620    {
 621       if (stringcasecmp(Val,"close") == 0)
 622          Persistent = false;
 623       if (stringcasecmp(Val,"keep-alive") == 0)
 624          Persistent = true;
 625       return true;
 626    }
 627
 628    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 629    {
 630       if (StrToTime(Val,Date) == false)
 631          return _error->Error(_("Unknown date format"));
 632       return true;
 633    }
 634
 635    return true;
 636 }
 637                                                                         /*}}}*/
 638
 639 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This places the http request in the outbound buffer */
 642 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 643 {
 644    URI Uri = Itm->Uri;
 645
 646    // The HTTP server expects a hostname with a trailing :port
 647    char Buf[1000];
 648    string ProperHost = Uri.Host;
 649    if (Uri.Port != 0)
 650    {
 651       sprintf(Buf,":%u",Uri.Port);
 652       ProperHost += Buf;
 653    }
 654
 655    // Just in case.
 656    if (Itm->Uri.length() >= sizeof(Buf))
 657        abort();
 658
 659    /* Build the request. We include a keep-alive header only for non-proxy
 660       requests. This is to tweak old http/1.0 servers that do support keep-alive
 661       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 662       will glitch HTTP/1.0 proxies because they do not filter it out and
 663       pass it on, HTTP/1.1 says the connection should default to keep alive
 664       and we expect the proxy to do this */
 665    if (Proxy.empty() == true)
 666       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 667               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 668    else
 669    {
 670       /* Generate a cache control header if necessary. We place a max
 671          cache age on index files, optionally set a no-cache directive
 672          and a no-store directive for archives. */
 673       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 674               Itm->Uri.c_str(),ProperHost.c_str());
 675       // only generate a cache control header if we actually want to
 676       // use a cache
 677       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 678       {
 679          if (Itm->IndexFile == true)
 680             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 681                     _config->FindI("Acquire::http::Max-Age",0));
 682          else
 683          {
 684             if (_config->FindB("Acquire::http::No-Store",false) == true)
 685                strcat(Buf,"Cache-Control: no-store\r\n");
 686          }
 687       }
 688    }
 689    // generate a no-cache header if needed
 690    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 691       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 692
 693
 694    string Req = Buf;
 695
 696    // Check for a partial file
 697    struct stat SBuf;
 698    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 699    {
 700       // In this case we send an if-range query with a range header
 701       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 702               TimeRFC1123(SBuf.st_mtime).c_str());
 703       Req += Buf;
 704    }
 705    else
 706    {
 707       if (Itm->LastModified != 0)
 708       {
 709          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 710          Req += Buf;
 711       }
 712    }
 713
 714    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 715       Req += string("Proxy-Authorization: Basic ") +
 716           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 717
 718    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 719       Req += string("Authorization: Basic ") +
 720           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 721
 722    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 723
 724    if (Debug == true)
 725       cerr << Req << endl;
 726
 727    Out.Read(Req);
 728 }
 729                                                                         /*}}}*/
 730 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 731 // ---------------------------------------------------------------------
 732 /* This runs the select loop over the server FDs, Output file FDs and
 733    stdin. */
 734 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 735 {
 736    // Server has closed the connection
 737    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 738                                ToFile == false))
 739       return false;
 740
 741    fd_set rfds,wfds;
 742    FD_ZERO(&rfds);
 743    FD_ZERO(&wfds);
 744
 745    /* Add the server. We only send more requests if the connection will
 746       be persisting */
 747    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 748        && Srv->Persistent == true)
 749       FD_SET(Srv->ServerFd,&wfds);
 750    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 751       FD_SET(Srv->ServerFd,&rfds);
 752
 753    // Add the file
 754    int FileFD = -1;
 755    if (File != 0)
 756       FileFD = File->Fd();
 757
 758    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 759       FD_SET(FileFD,&wfds);
 760
 761    // Add stdin
 762    FD_SET(STDIN_FILENO,&rfds);
 763
 764    // Figure out the max fd
 765    int MaxFd = FileFD;
 766    if (MaxFd < Srv->ServerFd)
 767       MaxFd = Srv->ServerFd;
 768
 769    // Select
 770    struct timeval tv;
 771    tv.tv_sec = TimeOut;
 772    tv.tv_usec = 0;
 773    int Res = 0;
 774    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 775    {
 776       if (errno == EINTR)
 777          return true;
 778       return _error->Errno("select",_("Select failed"));
 779    }
 780
 781    if (Res == 0)
 782    {
 783       _error->Error(_("Connection timed out"));
 784       return ServerDie(Srv);
 785    }
 786
 787    // Handle server IO
 788    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 789    {
 790       errno = 0;
 791       if (Srv->In.Read(Srv->ServerFd) == false)
 792          return ServerDie(Srv);
 793    }
 794
 795    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 796    {
 797       errno = 0;
 798       if (Srv->Out.Write(Srv->ServerFd) == false)
 799          return ServerDie(Srv);
 800    }
 801
 802    // Send data to the file
 803    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 804    {
 805       if (Srv->In.Write(FileFD) == false)
 806          return _error->Errno("write",_("Error writing to output file"));
 807    }
 808
 809    // Handle commands from APT
 810    if (FD_ISSET(STDIN_FILENO,&rfds))
 811    {
 812       if (Run(true) != -1)
 813          exit(100);
 814    }
 815
 816    return true;
 817 }
 818                                                                         /*}}}*/
 819 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 820 // ---------------------------------------------------------------------
 821 /* This takes the current input buffer from the Server FD and writes it
 822    into the file */
 823 bool HttpMethod::Flush(ServerState *Srv)
 824 {
 825    if (File != 0)
 826    {
 827       SetNonBlock(File->Fd(),false);
 828       if (Srv->In.WriteSpace() == false)
 829          return true;
 830
 831       while (Srv->In.WriteSpace() == true)
 832       {
 833          if (Srv->In.Write(File->Fd()) == false)
 834             return _error->Errno("write",_("Error writing to file"));
 835          if (Srv->In.IsLimit() == true)
 836             return true;
 837       }
 838
 839       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 840          return true;
 841    }
 842    return false;
 843 }
 844                                                                         /*}}}*/
 845 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 846 // ---------------------------------------------------------------------
 847 /* */
 848 bool HttpMethod::ServerDie(ServerState *Srv)
 849 {
 850    unsigned int LErrno = errno;
 851
 852    // Dump the buffer to the file
 853    if (Srv->State == ServerState::Data)
 854    {
 855       SetNonBlock(File->Fd(),false);
 856       while (Srv->In.WriteSpace() == true)
 857       {
 858          if (Srv->In.Write(File->Fd()) == false)
 859             return _error->Errno("write",_("Error writing to the file"));
 860
 861          // Done
 862          if (Srv->In.IsLimit() == true)
 863             return true;
 864       }
 865    }
 866
 867    // See if this is because the server finished the data stream
 868    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 869        Srv->Encoding != ServerState::Closes)
 870    {
 871       Srv->Close();
 872       if (LErrno == 0)
 873          return _error->Error(_("Error reading from server. Remote end closed connection"));
 874       errno = LErrno;
 875       return _error->Errno("read",_("Error reading from server"));
 876    }
 877    else
 878    {
 879       Srv->In.Limit(-1);
 880
 881       // Nothing left in the buffer
 882       if (Srv->In.WriteSpace() == false)
 883          return false;
 884
 885       // We may have got multiple responses back in one packet..
 886       Srv->Close();
 887       return true;
 888    }
 889
 890    return false;
 891 }
 892                                                                         /*}}}*/
 893 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 894 // ---------------------------------------------------------------------
 895 /* We look at the header data we got back from the server and decide what
 896    to do. Returns
 897      0 - File is open,
 898      1 - IMS hit
 899      3 - Unrecoverable error
 900      4 - Error with error content page
 901      5 - Unrecoverable non-server error (close the connection) */
 902 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 903 {
 904    // Not Modified
 905    if (Srv->Result == 304)
 906    {
 907       unlink(Queue->DestFile.c_str());
 908       Res.IMSHit = true;
 909       Res.LastModified = Queue->LastModified;
 910       return 1;
 911    }
 912
 913    /* We have a reply we dont handle. This should indicate a perm server
 914       failure */
 915    if (Srv->Result < 200 || Srv->Result >= 300)
 916    {
 917       _error->Error("%u %s",Srv->Result,Srv->Code);
 918       if (Srv->HaveContent == true)
 919          return 4;
 920       return 3;
 921    }
 922
 923    // This is some sort of 2xx 'data follows' reply
 924    Res.LastModified = Srv->Date;
 925    Res.Size = Srv->Size;
 926
 927    // Open the file
 928    delete File;
 929    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 930    if (_error->PendingError() == true)
 931       return 5;
 932
 933    FailFile = Queue->DestFile;
 934    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 935    FailFd = File->Fd();
 936    FailTime = Srv->Date;
 937
 938    // Set the expected size
 939    if (Srv->StartPos >= 0)
 940    {
 941       Res.ResumePoint = Srv->StartPos;
 942       ftruncate(File->Fd(),Srv->StartPos);
 943    }
 944
 945    // Set the start point
 946    lseek(File->Fd(),0,SEEK_END);
 947
 948    delete Srv->In.Hash;
 949    Srv->In.Hash = new Hashes;
 950
 951    // Fill the Hash if the file is non-empty (resume)
 952    if (Srv->StartPos > 0)
 953    {
 954       lseek(File->Fd(),0,SEEK_SET);
 955       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 956       {
 957          _error->Errno("read",_("Problem hashing file"));
 958          return 5;
 959       }
 960       lseek(File->Fd(),0,SEEK_END);
 961    }
 962
 963    SetNonBlock(File->Fd(),true);
 964    return 0;
 965 }
 966                                                                         /*}}}*/
 967 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 968 // ---------------------------------------------------------------------
 969 /* This closes and timestamps the open file. This is neccessary to get
 970    resume behavoir on user abort */
 971 void HttpMethod::SigTerm(int)
 972 {
 973    if (FailFd == -1)
 974       _exit(100);
 975    close(FailFd);
 976
 977    // Timestamp
 978    struct utimbuf UBuf;
 979    UBuf.actime = FailTime;
 980    UBuf.modtime = FailTime;
 981    utime(FailFile.c_str(),&UBuf);
 982
 983    _exit(100);
 984 }
 985                                                                         /*}}}*/
 986 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 987 // ---------------------------------------------------------------------
 988 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 989    depth. */
 990 bool HttpMethod::Fetch(FetchItem *)
 991 {
 992    if (Server == 0)
 993       return true;
 994
 995    // Queue the requests
 996    int Depth = -1;
 997    bool Tail = false;
 998    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 999         I = I->Next, Depth++)
1000    {
1001       // If pipelining is disabled, we only queue 1 request
1002       if (Server->Pipeline == false && Depth >= 0)
1003          break;
1004
1005       // Make sure we stick with the same server
1006       if (Server->Comp(I->Uri) == false)
1007          break;
1008       if (QueueBack == I)
1009          Tail = true;
1010       if (Tail == true)
1011       {
1012          QueueBack = I->Next;
1013          SendReq(I,Server->Out);
1014          continue;
1015       }
1016    }
1017
1018    return true;
1019 };
1020                                                                         /*}}}*/
1021 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1022 // ---------------------------------------------------------------------
1023 /* We stash the desired pipeline depth */
1024 bool HttpMethod::Configuration(string Message)
1025 {
1026    if (pkgAcqMethod::Configuration(Message) == false)
1027       return false;
1028
1029    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1030    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1031                                   PipelineDepth);
1032    Debug = _config->FindB("Debug::Acquire::http",false);
1033
1034    return true;
1035 }
1036                                                                         /*}}}*/
1037 // HttpMethod::Loop - Main loop                                         /*{{{*/
1038 // ---------------------------------------------------------------------
1039 /* */
1040 int HttpMethod::Loop()
1041 {
1042    signal(SIGTERM,SigTerm);
1043    signal(SIGINT,SigTerm);
1044
1045    Server = 0;
1046
1047    int FailCounter = 0;
1048    while (1)
1049    {
1050       // We have no commands, wait for some to arrive
1051       if (Queue == 0)
1052       {
1053          if (WaitFd(STDIN_FILENO) == false)
1054             return 0;
1055       }
1056
1057       /* Run messages, we can accept 0 (no message) if we didn't
1058          do a WaitFd above.. Otherwise the FD is closed. */
1059       int Result = Run(true);
1060       if (Result != -1 && (Result != 0 || Queue == 0))
1061          return 100;
1062
1063       if (Queue == 0)
1064          continue;
1065
1066       // Connect to the server
1067       if (Server == 0 || Server->Comp(Queue->Uri) == false)
1068       {
1069          delete Server;
1070          Server = new ServerState(Queue->Uri,this);
1071       }
1072
1073       /* If the server has explicitly said this is the last connection
1074          then we pre-emptively shut down the pipeline and tear down
1075          the connection. This will speed up HTTP/1.0 servers a tad
1076          since we don't have to wait for the close sequence to
1077          complete */
1078       if (Server->Persistent == false)
1079          Server->Close();
1080
1081       // Reset the pipeline
1082       if (Server->ServerFd == -1)
1083          QueueBack = Queue;
1084
1085       // Connnect to the host
1086       if (Server->Open() == false)
1087       {
1088          Fail(true);
1089          delete Server;
1090          Server = 0;
1091          continue;
1092       }
1093
1094       // Fill the pipeline.
1095       Fetch(0);
1096
1097       // Fetch the next URL header data from the server.
1098       switch (Server->RunHeaders())
1099       {
1100          case 0:
1101          break;
1102
1103          // The header data is bad
1104          case 2:
1105          {
1106             _error->Error(_("Bad header data"));
1107             Fail(true);
1108             RotateDNS();
1109             continue;
1110          }
1111
1112          // The server closed a connection during the header get..
1113          default:
1114          case 1:
1115          {
1116             FailCounter++;
1117             _error->Discard();
1118             Server->Close();
1119             Server->Pipeline = false;
1120
1121             if (FailCounter >= 2)
1122             {
1123                Fail(_("Connection failed"),true);
1124                FailCounter = 0;
1125             }
1126
1127             RotateDNS();
1128             continue;
1129          }
1130       };
1131
1132       // Decide what to do.
1133       FetchResult Res;
1134       Res.Filename = Queue->DestFile;
1135       switch (DealWithHeaders(Res,Server))
1136       {
1137          // Ok, the file is Open
1138          case 0:
1139          {
1140             URIStart(Res);
1141
1142             // Run the data
1143             bool Result =  Server->RunData();
1144
1145             /* If the server is sending back sizeless responses then fill in
1146                the size now */
1147             if (Res.Size == 0)
1148                Res.Size = File->Size();
1149
1150             // Close the file, destroy the FD object and timestamp it
1151             FailFd = -1;
1152             delete File;
1153             File = 0;
1154
1155             // Timestamp
1156             struct utimbuf UBuf;
1157             time(&UBuf.actime);
1158             UBuf.actime = Server->Date;
1159             UBuf.modtime = Server->Date;
1160             utime(Queue->DestFile.c_str(),&UBuf);
1161
1162             // Send status to APT
1163             if (Result == true)
1164             {
1165                Res.TakeHashes(*Server->In.Hash);
1166                URIDone(Res);
1167             }
1168             else
1169                Fail(true);
1170
1171             break;
1172          }
1173
1174          // IMS hit
1175          case 1:
1176          {
1177             URIDone(Res);
1178             break;
1179          }
1180
1181          // Hard server error, not found or something
1182          case 3:
1183          {
1184             Fail();
1185             break;
1186          }
1187
1188          // Hard internal error, kill the connection and fail
1189          case 5:
1190          {
1191             delete File;
1192             File = 0;
1193
1194             Fail();
1195             RotateDNS();
1196             Server->Close();
1197             break;
1198          }
1199
1200          // We need to flush the data, the header is like a 404 w/ error text
1201          case 4:
1202          {
1203             Fail();
1204
1205             // Send to content to dev/null
1206             File = new FileFd("/dev/null",FileFd::WriteExists);
1207             Server->RunData();
1208             delete File;
1209             File = 0;
1210             break;
1211          }
1212
1213          default:
1214          Fail(_("Internal error"));
1215          break;
1216       }
1217
1218       FailCounter = 0;
1219    }
1220
1221    return 0;
1222 }
1223                                                                         /*}}}*/
1224
1225 int main()
1226 {
1227    setlocale(LC_ALL, "");
1228
1229    HttpMethod Mth;
1230
1231    return Mth.Loop();
1232 }
1233
1234