methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.34 1999/05/29 03:25:03 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57
  58 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* */
  61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  62 {
  63    Buf = new unsigned char[Size];
  64    Reset();
  65 }
  66                                                                         /*}}}*/
  67 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 void CircleBuf::Reset()
  71 {
  72    InP = 0;
  73    OutP = 0;
  74    StrPos = 0;
  75    MaxGet = (unsigned int)-1;
  76    OutQueue = string();
  77    if (MD5 != 0)
  78    {
  79       delete MD5;
  80       MD5 = new MD5Summation;
  81    }
  82 };
  83                                                                         /*}}}*/
  84 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* This fills up the buffer with as much data as is in the FD, assuming it
  87    is non-blocking.. */
  88 bool CircleBuf::Read(int Fd)
  89 {
  90    while (1)
  91    {
  92       // Woops, buffer is full
  93       if (InP - OutP == Size)
  94          return true;
  95
  96       // Write the buffer segment
  97       int Res;
  98       Res = read(Fd,Buf + (InP%Size),LeftRead());
  99
 100       if (Res == 0)
 101          return false;
 102       if (Res < 0)
 103       {
 104          if (errno == EAGAIN)
 105             return true;
 106          return false;
 107       }
 108
 109       if (InP == 0)
 110          gettimeofday(&Start,0);
 111       InP += Res;
 112    }
 113 }
 114                                                                         /*}}}*/
 115 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* This will hold the string in and fill the buffer with it as it empties */
 118 bool CircleBuf::Read(string Data)
 119 {
 120    OutQueue += Data;
 121    FillOut();
 122    return true;
 123 }
 124                                                                         /*}}}*/
 125 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* */
 128 void CircleBuf::FillOut()
 129 {
 130    if (OutQueue.empty() == true)
 131       return;
 132    while (1)
 133    {
 134       // Woops, buffer is full
 135       if (InP - OutP == Size)
 136          return;
 137
 138       // Write the buffer segment
 139       unsigned long Sz = LeftRead();
 140       if (OutQueue.length() - StrPos < Sz)
 141          Sz = OutQueue.length() - StrPos;
 142       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 143
 144       // Advance
 145       StrPos += Sz;
 146       InP += Sz;
 147       if (OutQueue.length() == StrPos)
 148       {
 149          StrPos = 0;
 150          OutQueue = "";
 151          return;
 152       }
 153    }
 154 }
 155                                                                         /*}}}*/
 156 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* This empties the buffer into the FD. */
 159 bool CircleBuf::Write(int Fd)
 160 {
 161    while (1)
 162    {
 163       FillOut();
 164
 165       // Woops, buffer is empty
 166       if (OutP == InP)
 167          return true;
 168
 169       if (OutP == MaxGet)
 170          return true;
 171
 172       // Write the buffer segment
 173       int Res;
 174       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 175
 176       if (Res == 0)
 177          return false;
 178       if (Res < 0)
 179       {
 180          if (errno == EAGAIN)
 181             return true;
 182
 183          return false;
 184       }
 185
 186       if (MD5 != 0)
 187          MD5->Add(Buf + (OutP%Size),Res);
 188
 189       OutP += Res;
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This copies till the first empty line */
 196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 197 {
 198    // We cheat and assume it is unneeded to have more than one buffer load
 199    for (unsigned long I = OutP; I < InP; I++)
 200    {
 201       if (Buf[I%Size] != '\n')
 202          continue;
 203       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 204
 205       if (Single == false)
 206       {
 207          if (Buf[I%Size] != '\n')
 208             continue;
 209          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 210       }
 211
 212       if (I > InP)
 213          I = InP;
 214
 215       Data = "";
 216       while (OutP < I)
 217       {
 218          unsigned long Sz = LeftWrite();
 219          if (Sz == 0)
 220             return false;
 221          if (I - OutP < LeftWrite())
 222             Sz = I - OutP;
 223          Data += string((char *)(Buf + (OutP%Size)),Sz);
 224          OutP += Sz;
 225       }
 226       return true;
 227    }
 228    return false;
 229 }
 230                                                                         /*}}}*/
 231 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 232 // ---------------------------------------------------------------------
 233 /* */
 234 void CircleBuf::Stats()
 235 {
 236    if (InP == 0)
 237       return;
 238
 239    struct timeval Stop;
 240    gettimeofday(&Stop,0);
 241 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 242              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 243    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerState::ServerState - Constructor                               /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 251                         In(64*1024), Out(4*1024),
 252                         ServerName(Srv)
 253 {
 254    Reset();
 255 }
 256                                                                         /*}}}*/
 257 // ServerState::Open - Open a connection to the server                  /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This opens a connection to the server. */
 260 string LastHost;
 261 int LastPort = 0;
 262 struct addrinfo *LastHostAddr = 0;
 263 bool ServerState::Open()
 264 {
 265    // Use the already open connection if possible.
 266    if (ServerFd != -1)
 267       return true;
 268
 269    Close();
 270    In.Reset();
 271    Out.Reset();
 272
 273    // Determine the proxy setting
 274    if (getenv("http_proxy") == 0)
 275    {
 276       string DefProxy = _config->Find("Acquire::http::Proxy");
 277       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 278       if (SpecificProxy.empty() == false)
 279       {
 280          if (SpecificProxy == "DIRECT")
 281             Proxy = "";
 282          else
 283             Proxy = SpecificProxy;
 284       }
 285       else
 286          Proxy = DefProxy;
 287    }
 288    else
 289       Proxy = getenv("http_proxy");
 290
 291    // Determine what host and port to use based on the proxy settings
 292    int Port = 0;
 293    string Host;
 294    if (Proxy.empty() == true)
 295    {
 296       if (ServerName.Port != 0)
 297          Port = ServerName.Port;
 298       Host = ServerName.Host;
 299    }
 300    else
 301    {
 302       if (Proxy.Port != 0)
 303          Port = Proxy.Port;
 304       Host = Proxy.Host;
 305    }
 306
 307    // Connect to the remote server
 308    if (Connect(Host,Port,"http",ServerFd,TimeOut,Owner) == false)
 309       return false;
 310
 311    return true;
 312 }
 313                                                                         /*}}}*/
 314 // ServerState::Close - Close a connection to the server                /*{{{*/
 315 // ---------------------------------------------------------------------
 316 /* */
 317 bool ServerState::Close()
 318 {
 319    close(ServerFd);
 320    ServerFd = -1;
 321    return true;
 322 }
 323                                                                         /*}}}*/
 324 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 327    parse error occured */
 328 int ServerState::RunHeaders()
 329 {
 330    State = Header;
 331
 332    Owner->Status("Waiting for file");
 333
 334    Major = 0;
 335    Minor = 0;
 336    Result = 0;
 337    Size = 0;
 338    StartPos = 0;
 339    Encoding = Closes;
 340    HaveContent = false;
 341    time(&Date);
 342
 343    do
 344    {
 345       string Data;
 346       if (In.WriteTillEl(Data) == false)
 347          continue;
 348
 349       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 350       {
 351          string::const_iterator J = I;
 352          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 353          if (HeaderLine(string(I,J-I)) == false)
 354             return 2;
 355          I = J;
 356       }
 357       return 0;
 358    }
 359    while (Owner->Go(false,this) == true);
 360
 361    return 1;
 362 }
 363                                                                         /*}}}*/
 364 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* */
 367 bool ServerState::RunData()
 368 {
 369    State = Data;
 370
 371    // Chunked transfer encoding is fun..
 372    if (Encoding == Chunked)
 373    {
 374       while (1)
 375       {
 376          // Grab the block size
 377          bool Last = true;
 378          string Data;
 379          In.Limit(-1);
 380          do
 381          {
 382             if (In.WriteTillEl(Data,true) == true)
 383                break;
 384          }
 385          while ((Last = Owner->Go(false,this)) == true);
 386
 387          if (Last == false)
 388             return false;
 389
 390          // See if we are done
 391          unsigned long Len = strtol(Data.c_str(),0,16);
 392          if (Len == 0)
 393          {
 394             In.Limit(-1);
 395
 396             // We have to remove the entity trailer
 397             Last = true;
 398             do
 399             {
 400                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 401                   break;
 402             }
 403             while ((Last = Owner->Go(false,this)) == true);
 404             if (Last == false)
 405                return false;
 406             return !_error->PendingError();
 407          }
 408
 409          // Transfer the block
 410          In.Limit(Len);
 411          while (Owner->Go(true,this) == true)
 412             if (In.IsLimit() == true)
 413                break;
 414
 415          // Error
 416          if (In.IsLimit() == false)
 417             return false;
 418
 419          // The server sends an extra new line before the next block specifier..
 420          In.Limit(-1);
 421          Last = true;
 422          do
 423          {
 424             if (In.WriteTillEl(Data,true) == true)
 425                break;
 426          }
 427          while ((Last = Owner->Go(false,this)) == true);
 428          if (Last == false)
 429             return false;
 430       }
 431    }
 432    else
 433    {
 434       /* Closes encoding is used when the server did not specify a size, the
 435          loss of the connection means we are done */
 436       if (Encoding == Closes)
 437          In.Limit(-1);
 438       else
 439          In.Limit(Size - StartPos);
 440
 441       // Just transfer the whole block.
 442       do
 443       {
 444          if (In.IsLimit() == false)
 445             continue;
 446
 447          In.Limit(-1);
 448          return !_error->PendingError();
 449       }
 450       while (Owner->Go(true,this) == true);
 451    }
 452
 453    return Owner->Flush(this) && !_error->PendingError();
 454 }
 455                                                                         /*}}}*/
 456 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* */
 459 bool ServerState::HeaderLine(string Line)
 460 {
 461    if (Line.empty() == true)
 462       return true;
 463
 464    // The http server might be trying to do something evil.
 465    if (Line.length() >= MAXLEN)
 466       return _error->Error("Got a single header line over %u chars",MAXLEN);
 467
 468    string::size_type Pos = Line.find(' ');
 469    if (Pos == string::npos || Pos+1 > Line.length())
 470    {
 471       // Blah, some servers use "connection:closes", evil.
 472       Pos = Line.find(':');
 473       if (Pos == string::npos || Pos + 2 > Line.length())
 474          return _error->Error("Bad header line");
 475       Pos++;
 476    }
 477
 478    // Parse off any trailing spaces between the : and the next word.
 479    string::size_type Pos2 = Pos;
 480    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 481       Pos2++;
 482
 483    string Tag = string(Line,0,Pos);
 484    string Val = string(Line,Pos2);
 485
 486    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 487    {
 488       // Evil servers return no version
 489       if (Line[4] == '/')
 490       {
 491          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 492                     &Result,Code) != 4)
 493             return _error->Error("The http server sent an invalid reply header");
 494       }
 495       else
 496       {
 497          Major = 0;
 498          Minor = 9;
 499          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 500             return _error->Error("The http server sent an invalid reply header");
 501       }
 502
 503       return true;
 504    }
 505
 506    if (stringcasecmp(Tag,"Content-Length:") == 0)
 507    {
 508       if (Encoding == Closes)
 509          Encoding = Stream;
 510       HaveContent = true;
 511
 512       // The length is already set from the Content-Range header
 513       if (StartPos != 0)
 514          return true;
 515
 516       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 517          return _error->Error("The http server sent an invalid Content-Length header");
 518       return true;
 519    }
 520
 521    if (stringcasecmp(Tag,"Content-Type:") == 0)
 522    {
 523       HaveContent = true;
 524       return true;
 525    }
 526
 527    if (stringcasecmp(Tag,"Content-Range:") == 0)
 528    {
 529       HaveContent = true;
 530
 531       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 532          return _error->Error("The http server sent an invalid Content-Range header");
 533       if ((unsigned)StartPos > Size)
 534          return _error->Error("This http server has broken range support");
 535       return true;
 536    }
 537
 538    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 539    {
 540       HaveContent = true;
 541       if (stringcasecmp(Val,"chunked") == 0)
 542          Encoding = Chunked;
 543
 544       return true;
 545    }
 546
 547    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 548    {
 549       if (StrToTime(Val,Date) == false)
 550          return _error->Error("Unknown date format");
 551       return true;
 552    }
 553
 554    return true;
 555 }
 556                                                                         /*}}}*/
 557
 558 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 559 // ---------------------------------------------------------------------
 560 /* This places the http request in the outbound buffer */
 561 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 562 {
 563    URI Uri = Itm->Uri;
 564
 565    // The HTTP server expects a hostname with a trailing :port
 566    char Buf[1000];
 567    string ProperHost = Uri.Host;
 568    if (Uri.Port != 0)
 569    {
 570       sprintf(Buf,":%u",Uri.Port);
 571       ProperHost += Buf;
 572    }
 573
 574    // Just in case.
 575    if (Itm->Uri.length() >= sizeof(Buf))
 576        abort();
 577
 578    /* Build the request. We include a keep-alive header only for non-proxy
 579       requests. This is to tweak old http/1.0 servers that do support keep-alive
 580       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 581       will glitch HTTP/1.0 proxies because they do not filter it out and
 582       pass it on, HTTP/1.1 says the connection should default to keep alive
 583       and we expect the proxy to do this */
 584    if (Proxy.empty() == true)
 585       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 586               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 587    else
 588    {
 589       /* Generate a cache control header if necessary. We place a max
 590          cache age on index files, optionally set a no-cache directive
 591          and a no-store directive for archives. */
 592       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 593               Itm->Uri.c_str(),ProperHost.c_str());
 594       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 595          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 596       else
 597       {
 598          if (Itm->IndexFile == true)
 599             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 600                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 601          else
 602          {
 603             if (_config->FindB("Acquire::http::No-Store",false) == true)
 604                strcat(Buf,"Cache-Control: no-store\r\n");
 605          }
 606       }
 607    }
 608
 609    string Req = Buf;
 610
 611    // Check for a partial file
 612    struct stat SBuf;
 613    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 614    {
 615       // In this case we send an if-range query with a range header
 616       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 617               TimeRFC1123(SBuf.st_mtime).c_str());
 618       Req += Buf;
 619    }
 620    else
 621    {
 622       if (Itm->LastModified != 0)
 623       {
 624          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 625          Req += Buf;
 626       }
 627    }
 628
 629    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 630       Req += string("Proxy-Authorization: Basic ") +
 631           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 632
 633    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 634 //   cerr << Req << endl;
 635
 636    Out.Read(Req);
 637 }
 638                                                                         /*}}}*/
 639 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This runs the select loop over the server FDs, Output file FDs and
 642    stdin. */
 643 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 644 {
 645    // Server has closed the connection
 646    if (Srv->ServerFd == -1 && Srv->In.WriteSpace() == false)
 647       return false;
 648
 649    fd_set rfds,wfds,efds;
 650    FD_ZERO(&rfds);
 651    FD_ZERO(&wfds);
 652    FD_ZERO(&efds);
 653
 654    // Add the server
 655    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 656       FD_SET(Srv->ServerFd,&wfds);
 657    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 658       FD_SET(Srv->ServerFd,&rfds);
 659
 660    // Add the file
 661    int FileFD = -1;
 662    if (File != 0)
 663       FileFD = File->Fd();
 664
 665    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 666       FD_SET(FileFD,&wfds);
 667
 668    // Add stdin
 669    FD_SET(STDIN_FILENO,&rfds);
 670
 671    // Error Set
 672    if (FileFD != -1)
 673       FD_SET(FileFD,&efds);
 674    if (Srv->ServerFd != -1)
 675       FD_SET(Srv->ServerFd,&efds);
 676
 677    // Figure out the max fd
 678    int MaxFd = FileFD;
 679    if (MaxFd < Srv->ServerFd)
 680       MaxFd = Srv->ServerFd;
 681
 682    // Select
 683    struct timeval tv;
 684    tv.tv_sec = TimeOut;
 685    tv.tv_usec = 0;
 686    int Res = 0;
 687    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 688       return _error->Errno("select","Select failed");
 689
 690    if (Res == 0)
 691    {
 692       _error->Error("Connection timed out");
 693       return ServerDie(Srv);
 694    }
 695
 696    // Some kind of exception (error) on the sockets, die
 697    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 698        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 699       return _error->Error("Socket Exception");
 700
 701    // Handle server IO
 702    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 703    {
 704       errno = 0;
 705       if (Srv->In.Read(Srv->ServerFd) == false)
 706          return ServerDie(Srv);
 707    }
 708
 709    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 710    {
 711       errno = 0;
 712       if (Srv->Out.Write(Srv->ServerFd) == false)
 713          return ServerDie(Srv);
 714    }
 715
 716    // Send data to the file
 717    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 718    {
 719       if (Srv->In.Write(FileFD) == false)
 720          return _error->Errno("write","Error writing to output file");
 721    }
 722
 723    // Handle commands from APT
 724    if (FD_ISSET(STDIN_FILENO,&rfds))
 725    {
 726       if (Run(true) != 0)
 727          exit(100);
 728    }
 729
 730    return true;
 731 }
 732                                                                         /*}}}*/
 733 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 734 // ---------------------------------------------------------------------
 735 /* This takes the current input buffer from the Server FD and writes it
 736    into the file */
 737 bool HttpMethod::Flush(ServerState *Srv)
 738 {
 739    if (File != 0)
 740    {
 741       SetNonBlock(File->Fd(),false);
 742       if (Srv->In.WriteSpace() == false)
 743          return true;
 744
 745       while (Srv->In.WriteSpace() == true)
 746       {
 747          if (Srv->In.Write(File->Fd()) == false)
 748             return _error->Errno("write","Error writing to file");
 749          if (Srv->In.IsLimit() == true)
 750             return true;
 751       }
 752
 753       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 754          return true;
 755    }
 756    return false;
 757 }
 758                                                                         /*}}}*/
 759 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 760 // ---------------------------------------------------------------------
 761 /* */
 762 bool HttpMethod::ServerDie(ServerState *Srv)
 763 {
 764    unsigned int LErrno = errno;
 765
 766    // Dump the buffer to the file
 767    if (Srv->State == ServerState::Data)
 768    {
 769       SetNonBlock(File->Fd(),false);
 770       while (Srv->In.WriteSpace() == true)
 771       {
 772          if (Srv->In.Write(File->Fd()) == false)
 773             return _error->Errno("write","Error writing to the file");
 774
 775          // Done
 776          if (Srv->In.IsLimit() == true)
 777             return true;
 778       }
 779    }
 780
 781    // See if this is because the server finished the data stream
 782    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 783        Srv->Encoding != ServerState::Closes)
 784    {
 785       Srv->Close();
 786       if (LErrno == 0)
 787          return _error->Error("Error reading from server Remote end closed connection");
 788       errno = LErrno;
 789       return _error->Errno("read","Error reading from server");
 790    }
 791    else
 792    {
 793       Srv->In.Limit(-1);
 794
 795       // Nothing left in the buffer
 796       if (Srv->In.WriteSpace() == false)
 797          return false;
 798
 799       // We may have got multiple responses back in one packet..
 800       Srv->Close();
 801       return true;
 802    }
 803
 804    return false;
 805 }
 806                                                                         /*}}}*/
 807 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 808 // ---------------------------------------------------------------------
 809 /* We look at the header data we got back from the server and decide what
 810    to do. Returns
 811      0 - File is open,
 812      1 - IMS hit
 813      3 - Unrecoverable error
 814      4 - Error with error content page
 815      5 - Unrecoverable non-server error (close the connection) */
 816 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 817 {
 818    // Not Modified
 819    if (Srv->Result == 304)
 820    {
 821       unlink(Queue->DestFile.c_str());
 822       Res.IMSHit = true;
 823       Res.LastModified = Queue->LastModified;
 824       return 1;
 825    }
 826
 827    /* We have a reply we dont handle. This should indicate a perm server
 828       failure */
 829    if (Srv->Result < 200 || Srv->Result >= 300)
 830    {
 831       _error->Error("%u %s",Srv->Result,Srv->Code);
 832       if (Srv->HaveContent == true)
 833          return 4;
 834       return 3;
 835    }
 836
 837    // This is some sort of 2xx 'data follows' reply
 838    Res.LastModified = Srv->Date;
 839    Res.Size = Srv->Size;
 840
 841    // Open the file
 842    delete File;
 843    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 844    if (_error->PendingError() == true)
 845       return 5;
 846
 847    FailFile = Queue->DestFile;
 848    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 849    FailFd = File->Fd();
 850    FailTime = Srv->Date;
 851
 852    // Set the expected size
 853    if (Srv->StartPos >= 0)
 854    {
 855       Res.ResumePoint = Srv->StartPos;
 856       ftruncate(File->Fd(),Srv->StartPos);
 857    }
 858
 859    // Set the start point
 860    lseek(File->Fd(),0,SEEK_END);
 861
 862    delete Srv->In.MD5;
 863    Srv->In.MD5 = new MD5Summation;
 864
 865    // Fill the MD5 Hash if the file is non-empty (resume)
 866    if (Srv->StartPos > 0)
 867    {
 868       lseek(File->Fd(),0,SEEK_SET);
 869       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 870       {
 871          _error->Errno("read","Problem hashing file");
 872          return 5;
 873       }
 874       lseek(File->Fd(),0,SEEK_END);
 875    }
 876
 877    SetNonBlock(File->Fd(),true);
 878    return 0;
 879 }
 880                                                                         /*}}}*/
 881 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 882 // ---------------------------------------------------------------------
 883 /* This closes and timestamps the open file. This is neccessary to get
 884    resume behavoir on user abort */
 885 void HttpMethod::SigTerm(int)
 886 {
 887    if (FailFd == -1)
 888       exit(100);
 889    close(FailFd);
 890
 891    // Timestamp
 892    struct utimbuf UBuf;
 893    UBuf.actime = FailTime;
 894    UBuf.modtime = FailTime;
 895    utime(FailFile.c_str(),&UBuf);
 896
 897    exit(100);
 898 }
 899                                                                         /*}}}*/
 900 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 901 // ---------------------------------------------------------------------
 902 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 903    depth. */
 904 bool HttpMethod::Fetch(FetchItem *)
 905 {
 906    if (Server == 0)
 907       return true;
 908
 909    // Queue the requests
 910    int Depth = -1;
 911    bool Tail = false;
 912    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 913    {
 914       // Make sure we stick with the same server
 915       if (Server->Comp(I->Uri) == false)
 916          break;
 917       if (QueueBack == I)
 918          Tail = true;
 919       if (Tail == true)
 920       {
 921          QueueBack = I->Next;
 922          SendReq(I,Server->Out);
 923          continue;
 924       }
 925    }
 926
 927    return true;
 928 };
 929                                                                         /*}}}*/
 930 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 931 // ---------------------------------------------------------------------
 932 /* We stash the desired pipeline depth */
 933 bool HttpMethod::Configuration(string Message)
 934 {
 935    if (pkgAcqMethod::Configuration(Message) == false)
 936       return false;
 937
 938    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 939    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 940                                   PipelineDepth);
 941
 942    return true;
 943 }
 944                                                                         /*}}}*/
 945 // HttpMethod::Loop - Main loop                                         /*{{{*/
 946 // ---------------------------------------------------------------------
 947 /* */
 948 int HttpMethod::Loop()
 949 {
 950    signal(SIGTERM,SigTerm);
 951    signal(SIGINT,SigTerm);
 952
 953    Server = 0;
 954
 955    int FailCounter = 0;
 956    while (1)
 957    {
 958       // We have no commands, wait for some to arrive
 959       if (Queue == 0)
 960       {
 961          if (WaitFd(STDIN_FILENO) == false)
 962             return 0;
 963       }
 964
 965       // Run messages
 966       if (Run(true) != 0)
 967          return 100;
 968
 969       if (Queue == 0)
 970          continue;
 971
 972       // Connect to the server
 973       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 974       {
 975          delete Server;
 976          Server = new ServerState(Queue->Uri,this);
 977       }
 978
 979       // Reset the pipeline
 980       if (Server->ServerFd == -1)
 981          QueueBack = Queue;
 982
 983       // Connnect to the host
 984       if (Server->Open() == false)
 985       {
 986          Fail(true);
 987          delete Server;
 988          Server = 0;
 989          continue;
 990       }
 991
 992       // Fill the pipeline.
 993       Fetch(0);
 994
 995       // Fetch the next URL header data from the server.
 996       switch (Server->RunHeaders())
 997       {
 998          case 0:
 999          break;
1000
1001          // The header data is bad
1002          case 2:
1003          {
1004             _error->Error("Bad header Data");
1005             Fail(true);
1006             continue;
1007          }
1008
1009          // The server closed a connection during the header get..
1010          default:
1011          case 1:
1012          {
1013             FailCounter++;
1014             _error->Discard();
1015             Server->Close();
1016
1017             if (FailCounter >= 2)
1018             {
1019                Fail("Connection timed out",true);
1020                FailCounter = 0;
1021             }
1022
1023             continue;
1024          }
1025       };
1026
1027       // Decide what to do.
1028       FetchResult Res;
1029       Res.Filename = Queue->DestFile;
1030       switch (DealWithHeaders(Res,Server))
1031       {
1032          // Ok, the file is Open
1033          case 0:
1034          {
1035             URIStart(Res);
1036
1037             // Run the data
1038             bool Result =  Server->RunData();
1039
1040             // Close the file, destroy the FD object and timestamp it
1041             FailFd = -1;
1042             delete File;
1043             File = 0;
1044
1045             // Timestamp
1046             struct utimbuf UBuf;
1047             time(&UBuf.actime);
1048             UBuf.actime = Server->Date;
1049             UBuf.modtime = Server->Date;
1050             utime(Queue->DestFile.c_str(),&UBuf);
1051
1052             // Send status to APT
1053             if (Result == true)
1054             {
1055                Res.MD5Sum = Server->In.MD5->Result();
1056                URIDone(Res);
1057             }
1058             else
1059                Fail(true);
1060
1061             break;
1062          }
1063
1064          // IMS hit
1065          case 1:
1066          {
1067             URIDone(Res);
1068             break;
1069          }
1070
1071          // Hard server error, not found or something
1072          case 3:
1073          {
1074             Fail();
1075             break;
1076          }
1077
1078          // Hard internal error, kill the connection and fail
1079          case 5:
1080          {
1081             Fail();
1082             Server->Close();
1083             break;
1084          }
1085
1086          // We need to flush the data, the header is like a 404 w/ error text
1087          case 4:
1088          {
1089             Fail();
1090
1091             // Send to content to dev/null
1092             File = new FileFd("/dev/null",FileFd::WriteExists);
1093             Server->RunData();
1094             delete File;
1095             File = 0;
1096             break;
1097          }
1098
1099          default:
1100          Fail("Internal error");
1101          break;
1102       }
1103
1104       FailCounter = 0;
1105    }
1106
1107    return 0;
1108 }
1109                                                                         /*}}}*/
1110
1111 int main()
1112 {
1113    HttpMethod Mth;
1114
1115    return Mth.Loop();
1116 }