ftparchive/cachedb.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: cachedb.cc,v 1.7 2004/05/08 19:41:01 mdz Exp $
   4 /* ######################################################################
   5
   6    CacheDB
   7
   8    Simple uniform interface to a cache database.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include "cachedb.h"
  14
  15 #include <apti18n.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/md5.h>
  18 #include <apt-pkg/sha1.h>
  19 #include <apt-pkg/sha256.h>
  20 #include <apt-pkg/sha512.h>
  21 #include <apt-pkg/strutl.h>
  22 #include <apt-pkg/configuration.h>
  23
  24 #include <netinet/in.h>       // htonl, etc
  25                                                                         /*}}}*/
  26
  27 // CacheDB::ReadyDB - Ready the DB2                                     /*{{{*/
  28 // ---------------------------------------------------------------------
  29 /* This opens the DB2 file for caching package information */
  30 bool CacheDB::ReadyDB(string const &DB)
  31 {
  32    int err;
  33
  34    ReadOnly = _config->FindB("APT::FTPArchive::ReadOnlyDB",false);
  35
  36    // Close the old DB
  37    if (Dbp != 0)
  38       Dbp->close(Dbp,0);
  39
  40    /* Check if the DB was disabled while running and deal with a
  41       corrupted DB */
  42    if (DBFailed() == true)
  43    {
  44       _error->Warning(_("DB was corrupted, file renamed to %s.old"),DBFile.c_str());
  45       rename(DBFile.c_str(),(DBFile+".old").c_str());
  46    }
  47
  48    DBLoaded = false;
  49    Dbp = 0;
  50    DBFile = string();
  51
  52    if (DB.empty())
  53       return true;
  54
  55    db_create(&Dbp, NULL, 0);
  56    if ((err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_BTREE,
  57                         (ReadOnly?DB_RDONLY:DB_CREATE),
  58                         0644)) != 0)
  59    {
  60       if (err == DB_OLD_VERSION)
  61       {
  62           _error->Warning(_("DB is old, attempting to upgrade %s"),DBFile.c_str());
  63           err = Dbp->upgrade(Dbp, DB.c_str(), 0);
  64           if (!err)
  65              err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_HASH,
  66                             (ReadOnly?DB_RDONLY:DB_CREATE), 0644);
  67
  68       }
  69       // the database format has changed from DB_HASH to DB_BTREE in
  70       // apt 0.6.44
  71       if (err == EINVAL)
  72       {
  73          _error->Error(_("DB format is invalid. If you upgraded from an older version of apt, please remove and re-create the database."));
  74       }
  75       if (err)
  76       {
  77           Dbp = 0;
  78           return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
  79       }
  80    }
  81
  82    DBFile = DB;
  83    DBLoaded = true;
  84    return true;
  85 }
  86                                                                         /*}}}*/
  87 // CacheDB::OpenFile - Open the file                                    /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* */
  90 bool CacheDB::OpenFile()
  91 {
  92         Fd = new FileFd(FileName,FileFd::ReadOnly);
  93         if (_error->PendingError() == true)
  94         {
  95                 delete Fd;
  96                 Fd = NULL;
  97                 return false;
  98         }
  99         return true;
 100 }
 101                                                                         /*}}}*/
 102 // CacheDB::GetFileStat - Get stats from the file                       /*{{{*/
 103 // ---------------------------------------------------------------------
 104 /* This gets the size from the database if it's there.  If we need
 105  * to look at the file, also get the mtime from the file. */
 106 bool CacheDB::GetFileStat(bool const &doStat)
 107 {
 108         if ((CurStat.Flags & FlSize) == FlSize && doStat == false)
 109         {
 110                 /* Already worked out the file size */
 111         }
 112         else
 113         {
 114                 /* Get it from the file. */
 115                 if (Fd == NULL && OpenFile() == false)
 116                 {
 117                         return false;
 118                 }
 119                 // Stat the file
 120                 struct stat St;
 121                 if (fstat(Fd->Fd(),&St) != 0)
 122                 {
 123                         return _error->Errno("fstat",
 124                                 _("Failed to stat %s"),FileName.c_str());
 125                 }
 126                 CurStat.FileSize = St.st_size;
 127                 CurStat.mtime = htonl(St.st_mtime);
 128                 CurStat.Flags |= FlSize;
 129         }
 130         return true;
 131 }
 132                                                                         /*}}}*/
 133 // CacheDB::GetCurStat - Set the CurStat variable.                      /*{{{*/
 134 // ---------------------------------------------------------------------
 135 /* Sets the CurStat variable.  Either to 0 if no database is used
 136  * or to the value in the database if one is used */
 137 bool CacheDB::GetCurStat()
 138 {
 139    memset(&CurStat,0,sizeof(CurStat));
 140
 141         if (DBLoaded)
 142         {
 143                 /* First see if there is anything about it
 144                    in the database */
 145
 146                 /* Get the flags (and mtime) */
 147    InitQuery("st");
 148    // Ensure alignment of the returned structure
 149    Data.data = &CurStat;
 150    Data.ulen = sizeof(CurStat);
 151    Data.flags = DB_DBT_USERMEM;
 152                 if (Get() == false)
 153       {
 154          CurStat.Flags = 0;
 155       }
 156                 CurStat.Flags = ntohl(CurStat.Flags);
 157                 CurStat.FileSize = ntohl(CurStat.FileSize);
 158    }
 159         return true;
 160 }
 161                                                                         /*}}}*/
 162 // CacheDB::GetFileInfo - Get all the info about the file               /*{{{*/
 163 // ---------------------------------------------------------------------
 164 bool CacheDB::GetFileInfo(string const &FileName, bool const &DoControl, bool const &DoContents,
 165                                 bool const &GenContentsOnly, bool const &DoMD5, bool const &DoSHA1,
 166                                 bool const &DoSHA256,   bool const &DoSHA512,
 167                           bool const &checkMtime)
 168 {
 169         this->FileName = FileName;
 170
 171         if (GetCurStat() == false)
 172    {
 173                 return false;
 174    }
 175    OldStat = CurStat;
 176
 177         if (GetFileStat(checkMtime) == false)
 178         {
 179                 delete Fd;
 180                 Fd = NULL;
 181                 return false;
 182         }
 183
 184     /* if mtime changed, update CurStat from disk */
 185     if (checkMtime == true && OldStat.mtime != CurStat.mtime)
 186         CurStat.Flags = FlSize;
 187
 188         Stats.Bytes += CurStat.FileSize;
 189         Stats.Packages++;
 190
 191         if ((DoControl && LoadControl() == false)
 192                 || (DoContents && LoadContents(GenContentsOnly) == false)
 193                 || (DoMD5 && GetMD5(false) == false)
 194                 || (DoSHA1 && GetSHA1(false) == false)
 195                 || (DoSHA256 && GetSHA256(false) == false)
 196                 || (DoSHA512 && GetSHA512(false) == false)
 197            )
 198         {
 199                 delete Fd;
 200                 Fd = NULL;
 201                 delete DebFile;
 202                 DebFile = NULL;
 203                 return false;
 204         }
 205
 206         delete Fd;
 207         Fd = NULL;
 208         delete DebFile;
 209         DebFile = NULL;
 210
 211    return true;
 212 }
 213                                                                         /*}}}*/
 214 // CacheDB::LoadControl - Load Control information                      /*{{{*/
 215 // ---------------------------------------------------------------------
 216 /* */
 217 bool CacheDB::LoadControl()
 218 {
 219    // Try to read the control information out of the DB.
 220    if ((CurStat.Flags & FlControl) == FlControl)
 221    {
 222       // Lookup the control information
 223       InitQuery("cl");
 224       if (Get() == true && Control.TakeControl(Data.data,Data.size) == true)
 225             return true;
 226       CurStat.Flags &= ~FlControl;
 227    }
 228
 229    if (Fd == NULL && OpenFile() == false)
 230    {
 231       return false;
 232    }
 233    // Create a deb instance to read the archive
 234    if (DebFile == 0)
 235    {
 236       DebFile = new debDebFile(*Fd);
 237       if (_error->PendingError() == true)
 238          return false;
 239    }
 240
 241    Stats.Misses++;
 242    if (Control.Read(*DebFile) == false)
 243       return false;
 244
 245    if (Control.Control == 0)
 246       return _error->Error(_("Archive has no control record"));
 247
 248    // Write back the control information
 249    InitQuery("cl");
 250    if (Put(Control.Control,Control.Length) == true)
 251       CurStat.Flags |= FlControl;
 252    return true;
 253 }
 254                                                                         /*}}}*/
 255 // CacheDB::LoadContents - Load the File Listing                        /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* */
 258 bool CacheDB::LoadContents(bool const &GenOnly)
 259 {
 260    // Try to read the control information out of the DB.
 261    if ((CurStat.Flags & FlContents) == FlContents)
 262    {
 263       if (GenOnly == true)
 264          return true;
 265
 266       // Lookup the contents information
 267       InitQuery("cn");
 268       if (Get() == true)
 269       {
 270          if (Contents.TakeContents(Data.data,Data.size) == true)
 271             return true;
 272       }
 273
 274       CurStat.Flags &= ~FlContents;
 275    }
 276
 277    if (Fd == NULL && OpenFile() == false)
 278    {
 279       return false;
 280    }
 281    // Create a deb instance to read the archive
 282    if (DebFile == 0)
 283    {
 284       DebFile = new debDebFile(*Fd);
 285       if (_error->PendingError() == true)
 286          return false;
 287    }
 288
 289    if (Contents.Read(*DebFile) == false)
 290       return false;
 291
 292    // Write back the control information
 293    InitQuery("cn");
 294    if (Put(Contents.Data,Contents.CurSize) == true)
 295       CurStat.Flags |= FlContents;
 296    return true;
 297 }
 298                                                                         /*}}}*/
 299
 300 static string bytes2hex(uint8_t *bytes, size_t length) {
 301    char space[65];
 302    if (length * 2 > sizeof(space) - 1) length = (sizeof(space) - 1) / 2;
 303    for (size_t i = 0; i < length; i++)
 304       snprintf(&space[i*2], 3, "%02x", bytes[i]);
 305    return string(space);
 306 }
 307
 308 static inline unsigned char xdig2num(char const &dig) {
 309    if (isdigit(dig)) return dig - '0';
 310    if ('a' <= dig && dig <= 'f') return dig - 'a' + 10;
 311    if ('A' <= dig && dig <= 'F') return dig - 'A' + 10;
 312    return 0;
 313 }
 314
 315 static void hex2bytes(uint8_t *bytes, const char *hex, int length) {
 316    while (length-- > 0) {
 317       *bytes = 0;
 318       if (isxdigit(hex[0]) && isxdigit(hex[1])) {
 319           *bytes = xdig2num(hex[0]) * 16 + xdig2num(hex[1]);
 320           hex += 2;
 321       }
 322       bytes++;
 323    }
 324 }
 325
 326 // CacheDB::GetMD5 - Get the MD5 hash                                   /*{{{*/
 327 // ---------------------------------------------------------------------
 328 /* */
 329 bool CacheDB::GetMD5(bool const &GenOnly)
 330 {
 331    // Try to read the control information out of the DB.
 332    if ((CurStat.Flags & FlMD5) == FlMD5)
 333    {
 334       if (GenOnly == true)
 335          return true;
 336
 337       MD5Res = bytes2hex(CurStat.MD5, sizeof(CurStat.MD5));
 338          return true;
 339       }
 340
 341    Stats.MD5Bytes += CurStat.FileSize;
 342
 343    if (Fd == NULL && OpenFile() == false)
 344    {
 345       return false;
 346    }
 347    MD5Summation MD5;
 348    if (Fd->Seek(0) == false || MD5.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 349       return false;
 350
 351    MD5Res = MD5.Result();
 352    hex2bytes(CurStat.MD5, MD5Res.data(), sizeof(CurStat.MD5));
 353       CurStat.Flags |= FlMD5;
 354    return true;
 355 }
 356                                                                         /*}}}*/
 357 // CacheDB::GetSHA1 - Get the SHA1 hash                                 /*{{{*/
 358 // ---------------------------------------------------------------------
 359 /* */
 360 bool CacheDB::GetSHA1(bool const &GenOnly)
 361 {
 362    // Try to read the control information out of the DB.
 363    if ((CurStat.Flags & FlSHA1) == FlSHA1)
 364    {
 365       if (GenOnly == true)
 366          return true;
 367
 368       SHA1Res = bytes2hex(CurStat.SHA1, sizeof(CurStat.SHA1));
 369       return true;
 370    }
 371
 372    Stats.SHA1Bytes += CurStat.FileSize;
 373
 374    if (Fd == NULL && OpenFile() == false)
 375    {
 376       return false;
 377    }
 378    SHA1Summation SHA1;
 379    if (Fd->Seek(0) == false || SHA1.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 380       return false;
 381
 382    SHA1Res = SHA1.Result();
 383    hex2bytes(CurStat.SHA1, SHA1Res.data(), sizeof(CurStat.SHA1));
 384    CurStat.Flags |= FlSHA1;
 385    return true;
 386 }
 387                                                                         /*}}}*/
 388 // CacheDB::GetSHA256 - Get the SHA256 hash                             /*{{{*/
 389 // ---------------------------------------------------------------------
 390 /* */
 391 bool CacheDB::GetSHA256(bool const &GenOnly)
 392 {
 393    // Try to read the control information out of the DB.
 394    if ((CurStat.Flags & FlSHA256) == FlSHA256)
 395    {
 396       if (GenOnly == true)
 397          return true;
 398
 399       SHA256Res = bytes2hex(CurStat.SHA256, sizeof(CurStat.SHA256));
 400       return true;
 401    }
 402
 403    Stats.SHA256Bytes += CurStat.FileSize;
 404
 405    if (Fd == NULL && OpenFile() == false)
 406    {
 407       return false;
 408    }
 409    SHA256Summation SHA256;
 410    if (Fd->Seek(0) == false || SHA256.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 411       return false;
 412
 413    SHA256Res = SHA256.Result();
 414    hex2bytes(CurStat.SHA256, SHA256Res.data(), sizeof(CurStat.SHA256));
 415    CurStat.Flags |= FlSHA256;
 416    return true;
 417 }
 418                                                                         /*}}}*/
 419 // CacheDB::GetSHA256 - Get the SHA256 hash                             /*{{{*/
 420 // ---------------------------------------------------------------------
 421 /* */
 422 bool CacheDB::GetSHA512(bool const &GenOnly)
 423 {
 424    // Try to read the control information out of the DB.
 425    if ((CurStat.Flags & FlSHA512) == FlSHA512)
 426    {
 427       if (GenOnly == true)
 428          return true;
 429
 430       SHA512Res = bytes2hex(CurStat.SHA512, sizeof(CurStat.SHA512));
 431       return true;
 432    }
 433
 434    Stats.SHA512Bytes += CurStat.FileSize;
 435
 436    if (Fd == NULL && OpenFile() == false)
 437    {
 438       return false;
 439    }
 440    SHA512Summation SHA512;
 441    if (Fd->Seek(0) == false || SHA512.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 442       return false;
 443
 444    SHA512Res = SHA512.Result();
 445    hex2bytes(CurStat.SHA512, SHA512Res.data(), sizeof(CurStat.SHA512));
 446    CurStat.Flags |= FlSHA512;
 447    return true;
 448 }
 449                                                                         /*}}}*/
 450 // CacheDB::Finish - Write back the cache structure                     /*{{{*/
 451 // ---------------------------------------------------------------------
 452 /* */
 453 bool CacheDB::Finish()
 454 {
 455    // Optimize away some writes.
 456    if (CurStat.Flags == OldStat.Flags &&
 457        CurStat.mtime == OldStat.mtime)
 458       return true;
 459
 460    // Write the stat information
 461    CurStat.Flags = htonl(CurStat.Flags);
 462    CurStat.FileSize = htonl(CurStat.FileSize);
 463    InitQuery("st");
 464    Put(&CurStat,sizeof(CurStat));
 465    CurStat.Flags = ntohl(CurStat.Flags);
 466    CurStat.FileSize = ntohl(CurStat.FileSize);
 467
 468    return true;
 469 }
 470                                                                         /*}}}*/
 471 // CacheDB::Clean - Clean the Database                                  /*{{{*/
 472 // ---------------------------------------------------------------------
 473 /* Tidy the database by removing files that no longer exist at all. */
 474 bool CacheDB::Clean()
 475 {
 476    if (DBLoaded == false)
 477       return true;
 478
 479    /* I'm not sure what VERSION_MINOR should be here.. 2.4.14 certainly
 480       needs the lower one and 2.7.7 needs the upper.. */
 481    DBC *Cursor;
 482    if ((errno = Dbp->cursor(Dbp, NULL, &Cursor, 0)) != 0)
 483       return _error->Error(_("Unable to get a cursor"));
 484
 485    DBT Key;
 486    DBT Data;
 487    memset(&Key,0,sizeof(Key));
 488    memset(&Data,0,sizeof(Data));
 489    while ((errno = Cursor->c_get(Cursor,&Key,&Data,DB_NEXT)) == 0)
 490    {
 491       const char *Colon = (char*)memrchr(Key.data, ':', Key.size);
 492       if (Colon)
 493       {
 494          if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 ||
 495              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 ||
 496              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0)
 497          {
 498             if (FileExists(string((const char *)Key.data,Colon)) == true)
 499                 continue;
 500          }
 501       }
 502
 503       Cursor->c_del(Cursor,0);
 504    }
 505    Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL);
 506
 507    return true;
 508 }
 509                                                                         /*}}}*/