methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.42 1999/12/10 08:53:43 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57 bool Debug = false;
  58
  59 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  60 // ---------------------------------------------------------------------
  61 /* */
  62 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  63 {
  64    Buf = new unsigned char[Size];
  65    Reset();
  66 }
  67                                                                         /*}}}*/
  68 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 void CircleBuf::Reset()
  72 {
  73    InP = 0;
  74    OutP = 0;
  75    StrPos = 0;
  76    MaxGet = (unsigned int)-1;
  77    OutQueue = string();
  78    if (MD5 != 0)
  79    {
  80       delete MD5;
  81       MD5 = new MD5Summation;
  82    }
  83 };
  84                                                                         /*}}}*/
  85 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* This fills up the buffer with as much data as is in the FD, assuming it
  88    is non-blocking.. */
  89 bool CircleBuf::Read(int Fd)
  90 {
  91    while (1)
  92    {
  93       // Woops, buffer is full
  94       if (InP - OutP == Size)
  95          return true;
  96
  97       // Write the buffer segment
  98       int Res;
  99       Res = read(Fd,Buf + (InP%Size),LeftRead());
 100
 101       if (Res == 0)
 102          return false;
 103       if (Res < 0)
 104       {
 105          if (errno == EAGAIN)
 106             return true;
 107          return false;
 108       }
 109
 110       if (InP == 0)
 111          gettimeofday(&Start,0);
 112       InP += Res;
 113    }
 114 }
 115                                                                         /*}}}*/
 116 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 117 // ---------------------------------------------------------------------
 118 /* This will hold the string in and fill the buffer with it as it empties */
 119 bool CircleBuf::Read(string Data)
 120 {
 121    OutQueue += Data;
 122    FillOut();
 123    return true;
 124 }
 125                                                                         /*}}}*/
 126 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 127 // ---------------------------------------------------------------------
 128 /* */
 129 void CircleBuf::FillOut()
 130 {
 131    if (OutQueue.empty() == true)
 132       return;
 133    while (1)
 134    {
 135       // Woops, buffer is full
 136       if (InP - OutP == Size)
 137          return;
 138
 139       // Write the buffer segment
 140       unsigned long Sz = LeftRead();
 141       if (OutQueue.length() - StrPos < Sz)
 142          Sz = OutQueue.length() - StrPos;
 143       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 144
 145       // Advance
 146       StrPos += Sz;
 147       InP += Sz;
 148       if (OutQueue.length() == StrPos)
 149       {
 150          StrPos = 0;
 151          OutQueue = "";
 152          return;
 153       }
 154    }
 155 }
 156                                                                         /*}}}*/
 157 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 158 // ---------------------------------------------------------------------
 159 /* This empties the buffer into the FD. */
 160 bool CircleBuf::Write(int Fd)
 161 {
 162    while (1)
 163    {
 164       FillOut();
 165
 166       // Woops, buffer is empty
 167       if (OutP == InP)
 168          return true;
 169
 170       if (OutP == MaxGet)
 171          return true;
 172
 173       // Write the buffer segment
 174       int Res;
 175       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 176
 177       if (Res == 0)
 178          return false;
 179       if (Res < 0)
 180       {
 181          if (errno == EAGAIN)
 182             return true;
 183
 184          return false;
 185       }
 186
 187       if (MD5 != 0)
 188          MD5->Add(Buf + (OutP%Size),Res);
 189
 190       OutP += Res;
 191    }
 192 }
 193                                                                         /*}}}*/
 194 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This copies till the first empty line */
 197 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 198 {
 199    // We cheat and assume it is unneeded to have more than one buffer load
 200    for (unsigned long I = OutP; I < InP; I++)
 201    {
 202       if (Buf[I%Size] != '\n')
 203          continue;
 204       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 205
 206       if (Single == false)
 207       {
 208          if (Buf[I%Size] != '\n')
 209             continue;
 210          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 211       }
 212
 213       if (I > InP)
 214          I = InP;
 215
 216       Data = "";
 217       while (OutP < I)
 218       {
 219          unsigned long Sz = LeftWrite();
 220          if (Sz == 0)
 221             return false;
 222          if (I - OutP < LeftWrite())
 223             Sz = I - OutP;
 224          Data += string((char *)(Buf + (OutP%Size)),Sz);
 225          OutP += Sz;
 226       }
 227       return true;
 228    }
 229    return false;
 230 }
 231                                                                         /*}}}*/
 232 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 233 // ---------------------------------------------------------------------
 234 /* */
 235 void CircleBuf::Stats()
 236 {
 237    if (InP == 0)
 238       return;
 239
 240    struct timeval Stop;
 241    gettimeofday(&Stop,0);
 242 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 243              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 244    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 245 }
 246                                                                         /*}}}*/
 247
 248 // ServerState::ServerState - Constructor                               /*{{{*/
 249 // ---------------------------------------------------------------------
 250 /* */
 251 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 252                         In(64*1024), Out(4*1024),
 253                         ServerName(Srv)
 254 {
 255    Reset();
 256 }
 257                                                                         /*}}}*/
 258 // ServerState::Open - Open a connection to the server                  /*{{{*/
 259 // ---------------------------------------------------------------------
 260 /* This opens a connection to the server. */
 261 string LastHost;
 262 int LastPort = 0;
 263 struct addrinfo *LastHostAddr = 0;
 264 bool ServerState::Open()
 265 {
 266    // Use the already open connection if possible.
 267    if (ServerFd != -1)
 268       return true;
 269
 270    Close();
 271    In.Reset();
 272    Out.Reset();
 273
 274    // Determine the proxy setting
 275    if (getenv("http_proxy") == 0)
 276    {
 277       string DefProxy = _config->Find("Acquire::http::Proxy");
 278       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 279       if (SpecificProxy.empty() == false)
 280       {
 281          if (SpecificProxy == "DIRECT")
 282             Proxy = "";
 283          else
 284             Proxy = SpecificProxy;
 285       }
 286       else
 287          Proxy = DefProxy;
 288    }
 289    else
 290       Proxy = getenv("http_proxy");
 291
 292    // Parse no_proxy, a , seperated list of hosts
 293    if (getenv("no_proxy") != 0)
 294    {
 295       const char *Start = getenv("no_proxy");
 296       for (const char *Cur = Start; true ; Cur++)
 297       {
 298          if (*Cur != ',' && *Cur != 0)
 299             continue;
 300          if (stringcasecmp(ServerName.Host.begin(),ServerName.Host.end(),
 301                            Start,Cur) == 0)
 302          {
 303             Proxy = "";
 304             break;
 305          }
 306
 307          Start = Cur + 1;
 308          if (*Cur == 0)
 309             break;
 310       }
 311    }
 312
 313    // Determine what host and port to use based on the proxy settings
 314    int Port = 0;
 315    string Host;
 316    if (Proxy.empty() == true)
 317    {
 318       if (ServerName.Port != 0)
 319          Port = ServerName.Port;
 320       Host = ServerName.Host;
 321    }
 322    else
 323    {
 324       if (Proxy.Port != 0)
 325          Port = Proxy.Port;
 326       Host = Proxy.Host;
 327    }
 328
 329    // Connect to the remote server
 330    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 331       return false;
 332
 333    return true;
 334 }
 335                                                                         /*}}}*/
 336 // ServerState::Close - Close a connection to the server                /*{{{*/
 337 // ---------------------------------------------------------------------
 338 /* */
 339 bool ServerState::Close()
 340 {
 341    close(ServerFd);
 342    ServerFd = -1;
 343    return true;
 344 }
 345                                                                         /*}}}*/
 346 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 347 // ---------------------------------------------------------------------
 348 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 349    parse error occured */
 350 int ServerState::RunHeaders()
 351 {
 352    State = Header;
 353
 354    Owner->Status("Waiting for file");
 355
 356    Major = 0;
 357    Minor = 0;
 358    Result = 0;
 359    Size = 0;
 360    StartPos = 0;
 361    Encoding = Closes;
 362    HaveContent = false;
 363    time(&Date);
 364
 365    do
 366    {
 367       string Data;
 368       if (In.WriteTillEl(Data) == false)
 369          continue;
 370
 371       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 372       {
 373          string::const_iterator J = I;
 374          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 375          if (HeaderLine(string(I,J-I)) == false)
 376             return 2;
 377          I = J;
 378       }
 379       return 0;
 380    }
 381    while (Owner->Go(false,this) == true);
 382
 383    return 1;
 384 }
 385                                                                         /*}}}*/
 386 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 387 // ---------------------------------------------------------------------
 388 /* */
 389 bool ServerState::RunData()
 390 {
 391    State = Data;
 392
 393    // Chunked transfer encoding is fun..
 394    if (Encoding == Chunked)
 395    {
 396       while (1)
 397       {
 398          // Grab the block size
 399          bool Last = true;
 400          string Data;
 401          In.Limit(-1);
 402          do
 403          {
 404             if (In.WriteTillEl(Data,true) == true)
 405                break;
 406          }
 407          while ((Last = Owner->Go(false,this)) == true);
 408
 409          if (Last == false)
 410             return false;
 411
 412          // See if we are done
 413          unsigned long Len = strtol(Data.c_str(),0,16);
 414          if (Len == 0)
 415          {
 416             In.Limit(-1);
 417
 418             // We have to remove the entity trailer
 419             Last = true;
 420             do
 421             {
 422                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 423                   break;
 424             }
 425             while ((Last = Owner->Go(false,this)) == true);
 426             if (Last == false)
 427                return false;
 428             return !_error->PendingError();
 429          }
 430
 431          // Transfer the block
 432          In.Limit(Len);
 433          while (Owner->Go(true,this) == true)
 434             if (In.IsLimit() == true)
 435                break;
 436
 437          // Error
 438          if (In.IsLimit() == false)
 439             return false;
 440
 441          // The server sends an extra new line before the next block specifier..
 442          In.Limit(-1);
 443          Last = true;
 444          do
 445          {
 446             if (In.WriteTillEl(Data,true) == true)
 447                break;
 448          }
 449          while ((Last = Owner->Go(false,this)) == true);
 450          if (Last == false)
 451             return false;
 452       }
 453    }
 454    else
 455    {
 456       /* Closes encoding is used when the server did not specify a size, the
 457          loss of the connection means we are done */
 458       if (Encoding == Closes)
 459          In.Limit(-1);
 460       else
 461          In.Limit(Size - StartPos);
 462
 463       // Just transfer the whole block.
 464       do
 465       {
 466          if (In.IsLimit() == false)
 467             continue;
 468
 469          In.Limit(-1);
 470          return !_error->PendingError();
 471       }
 472       while (Owner->Go(true,this) == true);
 473    }
 474
 475    return Owner->Flush(this) && !_error->PendingError();
 476 }
 477                                                                         /*}}}*/
 478 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 479 // ---------------------------------------------------------------------
 480 /* */
 481 bool ServerState::HeaderLine(string Line)
 482 {
 483    if (Line.empty() == true)
 484       return true;
 485
 486    // The http server might be trying to do something evil.
 487    if (Line.length() >= MAXLEN)
 488       return _error->Error("Got a single header line over %u chars",MAXLEN);
 489
 490    string::size_type Pos = Line.find(' ');
 491    if (Pos == string::npos || Pos+1 > Line.length())
 492    {
 493       // Blah, some servers use "connection:closes", evil.
 494       Pos = Line.find(':');
 495       if (Pos == string::npos || Pos + 2 > Line.length())
 496          return _error->Error("Bad header line");
 497       Pos++;
 498    }
 499
 500    // Parse off any trailing spaces between the : and the next word.
 501    string::size_type Pos2 = Pos;
 502    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 503       Pos2++;
 504
 505    string Tag = string(Line,0,Pos);
 506    string Val = string(Line,Pos2);
 507
 508    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 509    {
 510       // Evil servers return no version
 511       if (Line[4] == '/')
 512       {
 513          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 514                     &Result,Code) != 4)
 515             return _error->Error("The http server sent an invalid reply header");
 516       }
 517       else
 518       {
 519          Major = 0;
 520          Minor = 9;
 521          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 522             return _error->Error("The http server sent an invalid reply header");
 523       }
 524
 525       return true;
 526    }
 527
 528    if (stringcasecmp(Tag,"Content-Length:") == 0)
 529    {
 530       if (Encoding == Closes)
 531          Encoding = Stream;
 532       HaveContent = true;
 533
 534       // The length is already set from the Content-Range header
 535       if (StartPos != 0)
 536          return true;
 537
 538       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 539          return _error->Error("The http server sent an invalid Content-Length header");
 540       return true;
 541    }
 542
 543    if (stringcasecmp(Tag,"Content-Type:") == 0)
 544    {
 545       HaveContent = true;
 546       return true;
 547    }
 548
 549    if (stringcasecmp(Tag,"Content-Range:") == 0)
 550    {
 551       HaveContent = true;
 552
 553       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 554          return _error->Error("The http server sent an invalid Content-Range header");
 555       if ((unsigned)StartPos > Size)
 556          return _error->Error("This http server has broken range support");
 557       return true;
 558    }
 559
 560    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 561    {
 562       HaveContent = true;
 563       if (stringcasecmp(Val,"chunked") == 0)
 564          Encoding = Chunked;
 565
 566       return true;
 567    }
 568
 569    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 570    {
 571       if (StrToTime(Val,Date) == false)
 572          return _error->Error("Unknown date format");
 573       return true;
 574    }
 575
 576    return true;
 577 }
 578                                                                         /*}}}*/
 579
 580 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 581 // ---------------------------------------------------------------------
 582 /* This places the http request in the outbound buffer */
 583 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 584 {
 585    URI Uri = Itm->Uri;
 586
 587    // The HTTP server expects a hostname with a trailing :port
 588    char Buf[1000];
 589    string ProperHost = Uri.Host;
 590    if (Uri.Port != 0)
 591    {
 592       sprintf(Buf,":%u",Uri.Port);
 593       ProperHost += Buf;
 594    }
 595
 596    // Just in case.
 597    if (Itm->Uri.length() >= sizeof(Buf))
 598        abort();
 599
 600    /* Build the request. We include a keep-alive header only for non-proxy
 601       requests. This is to tweak old http/1.0 servers that do support keep-alive
 602       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 603       will glitch HTTP/1.0 proxies because they do not filter it out and
 604       pass it on, HTTP/1.1 says the connection should default to keep alive
 605       and we expect the proxy to do this */
 606    if (Proxy.empty() == true)
 607       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 608               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 609    else
 610    {
 611       /* Generate a cache control header if necessary. We place a max
 612          cache age on index files, optionally set a no-cache directive
 613          and a no-store directive for archives. */
 614       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 615               Itm->Uri.c_str(),ProperHost.c_str());
 616       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 617          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 618       else
 619       {
 620          if (Itm->IndexFile == true)
 621             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 622                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 623          else
 624          {
 625             if (_config->FindB("Acquire::http::No-Store",false) == true)
 626                strcat(Buf,"Cache-Control: no-store\r\n");
 627          }
 628       }
 629    }
 630
 631    string Req = Buf;
 632
 633    // Check for a partial file
 634    struct stat SBuf;
 635    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 636    {
 637       // In this case we send an if-range query with a range header
 638       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 639               TimeRFC1123(SBuf.st_mtime).c_str());
 640       Req += Buf;
 641    }
 642    else
 643    {
 644       if (Itm->LastModified != 0)
 645       {
 646          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 647          Req += Buf;
 648       }
 649    }
 650
 651    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 652       Req += string("Proxy-Authorization: Basic ") +
 653           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 654
 655    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 656
 657    if (Debug == true)
 658       cerr << Req << endl;
 659
 660    Out.Read(Req);
 661 }
 662                                                                         /*}}}*/
 663 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 664 // ---------------------------------------------------------------------
 665 /* This runs the select loop over the server FDs, Output file FDs and
 666    stdin. */
 667 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 668 {
 669    // Server has closed the connection
 670    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 671                                ToFile == false))
 672       return false;
 673
 674    fd_set rfds,wfds;
 675    FD_ZERO(&rfds);
 676    FD_ZERO(&wfds);
 677
 678    // Add the server
 679    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 680       FD_SET(Srv->ServerFd,&wfds);
 681    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 682       FD_SET(Srv->ServerFd,&rfds);
 683
 684    // Add the file
 685    int FileFD = -1;
 686    if (File != 0)
 687       FileFD = File->Fd();
 688
 689    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 690       FD_SET(FileFD,&wfds);
 691
 692    // Add stdin
 693    FD_SET(STDIN_FILENO,&rfds);
 694
 695    // Figure out the max fd
 696    int MaxFd = FileFD;
 697    if (MaxFd < Srv->ServerFd)
 698       MaxFd = Srv->ServerFd;
 699
 700    // Select
 701    struct timeval tv;
 702    tv.tv_sec = TimeOut;
 703    tv.tv_usec = 0;
 704    int Res = 0;
 705    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 706       return _error->Errno("select","Select failed");
 707
 708    if (Res == 0)
 709    {
 710       _error->Error("Connection timed out");
 711       return ServerDie(Srv);
 712    }
 713
 714    // Handle server IO
 715    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 716    {
 717       errno = 0;
 718       if (Srv->In.Read(Srv->ServerFd) == false)
 719          return ServerDie(Srv);
 720    }
 721
 722    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 723    {
 724       errno = 0;
 725       if (Srv->Out.Write(Srv->ServerFd) == false)
 726          return ServerDie(Srv);
 727    }
 728
 729    // Send data to the file
 730    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 731    {
 732       if (Srv->In.Write(FileFD) == false)
 733          return _error->Errno("write","Error writing to output file");
 734    }
 735
 736    // Handle commands from APT
 737    if (FD_ISSET(STDIN_FILENO,&rfds))
 738    {
 739       if (Run(true) != -1)
 740          exit(100);
 741    }
 742
 743    return true;
 744 }
 745                                                                         /*}}}*/
 746 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 747 // ---------------------------------------------------------------------
 748 /* This takes the current input buffer from the Server FD and writes it
 749    into the file */
 750 bool HttpMethod::Flush(ServerState *Srv)
 751 {
 752    if (File != 0)
 753    {
 754       SetNonBlock(File->Fd(),false);
 755       if (Srv->In.WriteSpace() == false)
 756          return true;
 757
 758       while (Srv->In.WriteSpace() == true)
 759       {
 760          if (Srv->In.Write(File->Fd()) == false)
 761             return _error->Errno("write","Error writing to file");
 762          if (Srv->In.IsLimit() == true)
 763             return true;
 764       }
 765
 766       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 767          return true;
 768    }
 769    return false;
 770 }
 771                                                                         /*}}}*/
 772 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 773 // ---------------------------------------------------------------------
 774 /* */
 775 bool HttpMethod::ServerDie(ServerState *Srv)
 776 {
 777    unsigned int LErrno = errno;
 778
 779    // Dump the buffer to the file
 780    if (Srv->State == ServerState::Data)
 781    {
 782       SetNonBlock(File->Fd(),false);
 783       while (Srv->In.WriteSpace() == true)
 784       {
 785          if (Srv->In.Write(File->Fd()) == false)
 786             return _error->Errno("write","Error writing to the file");
 787
 788          // Done
 789          if (Srv->In.IsLimit() == true)
 790             return true;
 791       }
 792    }
 793
 794    // See if this is because the server finished the data stream
 795    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 796        Srv->Encoding != ServerState::Closes)
 797    {
 798       Srv->Close();
 799       if (LErrno == 0)
 800          return _error->Error("Error reading from server Remote end closed connection");
 801       errno = LErrno;
 802       return _error->Errno("read","Error reading from server");
 803    }
 804    else
 805    {
 806       Srv->In.Limit(-1);
 807
 808       // Nothing left in the buffer
 809       if (Srv->In.WriteSpace() == false)
 810          return false;
 811
 812       // We may have got multiple responses back in one packet..
 813       Srv->Close();
 814       return true;
 815    }
 816
 817    return false;
 818 }
 819                                                                         /*}}}*/
 820 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 821 // ---------------------------------------------------------------------
 822 /* We look at the header data we got back from the server and decide what
 823    to do. Returns
 824      0 - File is open,
 825      1 - IMS hit
 826      3 - Unrecoverable error
 827      4 - Error with error content page
 828      5 - Unrecoverable non-server error (close the connection) */
 829 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 830 {
 831    // Not Modified
 832    if (Srv->Result == 304)
 833    {
 834       unlink(Queue->DestFile.c_str());
 835       Res.IMSHit = true;
 836       Res.LastModified = Queue->LastModified;
 837       return 1;
 838    }
 839
 840    /* We have a reply we dont handle. This should indicate a perm server
 841       failure */
 842    if (Srv->Result < 200 || Srv->Result >= 300)
 843    {
 844       _error->Error("%u %s",Srv->Result,Srv->Code);
 845       if (Srv->HaveContent == true)
 846          return 4;
 847       return 3;
 848    }
 849
 850    // This is some sort of 2xx 'data follows' reply
 851    Res.LastModified = Srv->Date;
 852    Res.Size = Srv->Size;
 853
 854    // Open the file
 855    delete File;
 856    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 857    if (_error->PendingError() == true)
 858       return 5;
 859
 860    FailFile = Queue->DestFile;
 861    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 862    FailFd = File->Fd();
 863    FailTime = Srv->Date;
 864
 865    // Set the expected size
 866    if (Srv->StartPos >= 0)
 867    {
 868       Res.ResumePoint = Srv->StartPos;
 869       ftruncate(File->Fd(),Srv->StartPos);
 870    }
 871
 872    // Set the start point
 873    lseek(File->Fd(),0,SEEK_END);
 874
 875    delete Srv->In.MD5;
 876    Srv->In.MD5 = new MD5Summation;
 877
 878    // Fill the MD5 Hash if the file is non-empty (resume)
 879    if (Srv->StartPos > 0)
 880    {
 881       lseek(File->Fd(),0,SEEK_SET);
 882       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 883       {
 884          _error->Errno("read","Problem hashing file");
 885          return 5;
 886       }
 887       lseek(File->Fd(),0,SEEK_END);
 888    }
 889
 890    SetNonBlock(File->Fd(),true);
 891    return 0;
 892 }
 893                                                                         /*}}}*/
 894 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 895 // ---------------------------------------------------------------------
 896 /* This closes and timestamps the open file. This is neccessary to get
 897    resume behavoir on user abort */
 898 void HttpMethod::SigTerm(int)
 899 {
 900    if (FailFd == -1)
 901       _exit(100);
 902    close(FailFd);
 903
 904    // Timestamp
 905    struct utimbuf UBuf;
 906    UBuf.actime = FailTime;
 907    UBuf.modtime = FailTime;
 908    utime(FailFile.c_str(),&UBuf);
 909
 910    _exit(100);
 911 }
 912                                                                         /*}}}*/
 913 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 914 // ---------------------------------------------------------------------
 915 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 916    depth. */
 917 bool HttpMethod::Fetch(FetchItem *)
 918 {
 919    if (Server == 0)
 920       return true;
 921
 922    // Queue the requests
 923    int Depth = -1;
 924    bool Tail = false;
 925    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 926         I = I->Next, Depth++)
 927    {
 928       // If pipelining is disabled, we only queue 1 request
 929       if (Server->Pipeline == false && Depth >= 0)
 930          break;
 931
 932       // Make sure we stick with the same server
 933       if (Server->Comp(I->Uri) == false)
 934          break;
 935       if (QueueBack == I)
 936          Tail = true;
 937       if (Tail == true)
 938       {
 939          QueueBack = I->Next;
 940          SendReq(I,Server->Out);
 941          continue;
 942       }
 943    }
 944
 945    return true;
 946 };
 947                                                                         /*}}}*/
 948 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 949 // ---------------------------------------------------------------------
 950 /* We stash the desired pipeline depth */
 951 bool HttpMethod::Configuration(string Message)
 952 {
 953    if (pkgAcqMethod::Configuration(Message) == false)
 954       return false;
 955
 956    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 957    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 958                                   PipelineDepth);
 959    Debug = _config->FindB("Debug::Acquire::http",false);
 960
 961    return true;
 962 }
 963                                                                         /*}}}*/
 964 // HttpMethod::Loop - Main loop                                         /*{{{*/
 965 // ---------------------------------------------------------------------
 966 /* */
 967 int HttpMethod::Loop()
 968 {
 969    signal(SIGTERM,SigTerm);
 970    signal(SIGINT,SigTerm);
 971
 972    Server = 0;
 973
 974    int FailCounter = 0;
 975    while (1)
 976    {
 977       // We have no commands, wait for some to arrive
 978       if (Queue == 0)
 979       {
 980          if (WaitFd(STDIN_FILENO) == false)
 981             return 0;
 982       }
 983
 984       /* Run messages, we can accept 0 (no message) if we didn't
 985          do a WaitFd above.. Otherwise the FD is closed. */
 986       int Result = Run(true);
 987       if (Result != -1 && (Result != 0 || Queue == 0))
 988          return 100;
 989
 990       if (Queue == 0)
 991          continue;
 992
 993       // Connect to the server
 994       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 995       {
 996          delete Server;
 997          Server = new ServerState(Queue->Uri,this);
 998       }
 999
1000       // Reset the pipeline
1001       if (Server->ServerFd == -1)
1002          QueueBack = Queue;
1003
1004       // Connnect to the host
1005       if (Server->Open() == false)
1006       {
1007          Fail(true);
1008          delete Server;
1009          Server = 0;
1010          continue;
1011       }
1012
1013       // Fill the pipeline.
1014       Fetch(0);
1015
1016       // Fetch the next URL header data from the server.
1017       switch (Server->RunHeaders())
1018       {
1019          case 0:
1020          break;
1021
1022          // The header data is bad
1023          case 2:
1024          {
1025             _error->Error("Bad header Data");
1026             Fail(true);
1027             continue;
1028          }
1029
1030          // The server closed a connection during the header get..
1031          default:
1032          case 1:
1033          {
1034             FailCounter++;
1035             _error->Discard();
1036             Server->Close();
1037             Server->Pipeline = false;
1038
1039             if (FailCounter >= 2)
1040             {
1041                Fail("Connection failed",true);
1042                FailCounter = 0;
1043             }
1044
1045             continue;
1046          }
1047       };
1048
1049       // Decide what to do.
1050       FetchResult Res;
1051       Res.Filename = Queue->DestFile;
1052       switch (DealWithHeaders(Res,Server))
1053       {
1054          // Ok, the file is Open
1055          case 0:
1056          {
1057             URIStart(Res);
1058
1059             // Run the data
1060             bool Result =  Server->RunData();
1061
1062             // Close the file, destroy the FD object and timestamp it
1063             FailFd = -1;
1064             delete File;
1065             File = 0;
1066
1067             // Timestamp
1068             struct utimbuf UBuf;
1069             time(&UBuf.actime);
1070             UBuf.actime = Server->Date;
1071             UBuf.modtime = Server->Date;
1072             utime(Queue->DestFile.c_str(),&UBuf);
1073
1074             // Send status to APT
1075             if (Result == true)
1076             {
1077                Res.MD5Sum = Server->In.MD5->Result();
1078                URIDone(Res);
1079             }
1080             else
1081                Fail(true);
1082
1083             break;
1084          }
1085
1086          // IMS hit
1087          case 1:
1088          {
1089             URIDone(Res);
1090             break;
1091          }
1092
1093          // Hard server error, not found or something
1094          case 3:
1095          {
1096             Fail();
1097             break;
1098          }
1099
1100          // Hard internal error, kill the connection and fail
1101          case 5:
1102          {
1103             Fail();
1104             Server->Close();
1105             break;
1106          }
1107
1108          // We need to flush the data, the header is like a 404 w/ error text
1109          case 4:
1110          {
1111             Fail();
1112
1113             // Send to content to dev/null
1114             File = new FileFd("/dev/null",FileFd::WriteExists);
1115             Server->RunData();
1116             delete File;
1117             File = 0;
1118             break;
1119          }
1120
1121          default:
1122          Fail("Internal error");
1123          break;
1124       }
1125
1126       FailCounter = 0;
1127    }
1128
1129    return 0;
1130 }
1131                                                                         /*}}}*/
1132
1133 int main()
1134 {
1135    HttpMethod Mth;
1136
1137    return Mth.Loop();
1138 }