ftparchive/cachedb.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: cachedb.cc,v 1.7 2004/05/08 19:41:01 mdz Exp $
   4 /* ######################################################################
   5
   6    CacheDB
   7
   8    Simple uniform interface to a cache database.
   9
  10    ##################################################################### */
  11                                                                         /*}}}*/
  12 // Include Files                                                        /*{{{*/
  13 #include <config.h>
  14
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/md5.h>
  17 #include <apt-pkg/sha1.h>
  18 #include <apt-pkg/sha2.h>
  19 #include <apt-pkg/strutl.h>
  20 #include <apt-pkg/configuration.h>
  21
  22 #include <netinet/in.h>       // htonl, etc
  23
  24 #include <apti18n.h>
  25 #include "cachedb.h"
  26                                                                         /*}}}*/
  27
  28 // CacheDB::ReadyDB - Ready the DB2                                     /*{{{*/
  29 // ---------------------------------------------------------------------
  30 /* This opens the DB2 file for caching package information */
  31 bool CacheDB::ReadyDB(std::string const &DB)
  32 {
  33    int err;
  34
  35    ReadOnly = _config->FindB("APT::FTPArchive::ReadOnlyDB",false);
  36
  37    // Close the old DB
  38    if (Dbp != 0)
  39       Dbp->close(Dbp,0);
  40
  41    /* Check if the DB was disabled while running and deal with a
  42       corrupted DB */
  43    if (DBFailed() == true)
  44    {
  45       _error->Warning(_("DB was corrupted, file renamed to %s.old"),DBFile.c_str());
  46       rename(DBFile.c_str(),(DBFile+".old").c_str());
  47    }
  48
  49    DBLoaded = false;
  50    Dbp = 0;
  51    DBFile = std::string();
  52
  53    if (DB.empty())
  54       return true;
  55
  56    db_create(&Dbp, NULL, 0);
  57    if ((err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_BTREE,
  58                         (ReadOnly?DB_RDONLY:DB_CREATE),
  59                         0644)) != 0)
  60    {
  61       if (err == DB_OLD_VERSION)
  62       {
  63           _error->Warning(_("DB is old, attempting to upgrade %s"),DBFile.c_str());
  64           err = Dbp->upgrade(Dbp, DB.c_str(), 0);
  65           if (!err)
  66              err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_HASH,
  67                             (ReadOnly?DB_RDONLY:DB_CREATE), 0644);
  68
  69       }
  70       // the database format has changed from DB_HASH to DB_BTREE in
  71       // apt 0.6.44
  72       if (err == EINVAL)
  73       {
  74          _error->Error(_("DB format is invalid. If you upgraded from an older version of apt, please remove and re-create the database."));
  75       }
  76       if (err)
  77       {
  78           Dbp = 0;
  79           return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
  80       }
  81    }
  82
  83    DBFile = DB;
  84    DBLoaded = true;
  85    return true;
  86 }
  87                                                                         /*}}}*/
  88 // CacheDB::OpenFile - Open the file                                    /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 bool CacheDB::OpenFile()
  92 {
  93         Fd = new FileFd(FileName,FileFd::ReadOnly);
  94         if (_error->PendingError() == true)
  95         {
  96                 delete Fd;
  97                 Fd = NULL;
  98                 return false;
  99         }
 100         return true;
 101 }
 102                                                                         /*}}}*/
 103 // CacheDB::GetFileStat - Get stats from the file                       /*{{{*/
 104 // ---------------------------------------------------------------------
 105 /* This gets the size from the database if it's there.  If we need
 106  * to look at the file, also get the mtime from the file. */
 107 bool CacheDB::GetFileStat(bool const &doStat)
 108 {
 109         if ((CurStat.Flags & FlSize) == FlSize && doStat == false)
 110         {
 111                 /* Already worked out the file size */
 112         }
 113         else
 114         {
 115                 /* Get it from the file. */
 116                 if (Fd == NULL && OpenFile() == false)
 117                 {
 118                         return false;
 119                 }
 120                 // Stat the file
 121                 struct stat St;
 122                 if (fstat(Fd->Fd(),&St) != 0)
 123                 {
 124                         return _error->Errno("fstat",
 125                                 _("Failed to stat %s"),FileName.c_str());
 126                 }
 127                 CurStat.FileSize = St.st_size;
 128                 CurStat.mtime = htonl(St.st_mtime);
 129                 CurStat.Flags |= FlSize;
 130         }
 131         return true;
 132 }
 133                                                                         /*}}}*/
 134 // CacheDB::GetCurStat - Set the CurStat variable.                      /*{{{*/
 135 // ---------------------------------------------------------------------
 136 /* Sets the CurStat variable.  Either to 0 if no database is used
 137  * or to the value in the database if one is used */
 138 bool CacheDB::GetCurStat()
 139 {
 140    memset(&CurStat,0,sizeof(CurStat));
 141
 142         if (DBLoaded)
 143         {
 144                 /* First see if there is anything about it
 145                    in the database */
 146
 147                 /* Get the flags (and mtime) */
 148    InitQuery("st");
 149    // Ensure alignment of the returned structure
 150    Data.data = &CurStat;
 151    Data.ulen = sizeof(CurStat);
 152    Data.flags = DB_DBT_USERMEM;
 153                 if (Get() == false)
 154       {
 155          CurStat.Flags = 0;
 156       }
 157                 CurStat.Flags = ntohl(CurStat.Flags);
 158                 CurStat.FileSize = ntohl(CurStat.FileSize);
 159    }
 160         return true;
 161 }
 162                                                                         /*}}}*/
 163 // CacheDB::GetFileInfo - Get all the info about the file               /*{{{*/
 164 // ---------------------------------------------------------------------
 165 bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents,
 166                                 bool const &GenContentsOnly, bool const &DoMD5, bool const &DoSHA1,
 167                                 bool const &DoSHA256,   bool const &DoSHA512,
 168                           bool const &checkMtime)
 169 {
 170         this->FileName = FileName;
 171
 172         if (GetCurStat() == false)
 173    {
 174                 return false;
 175    }
 176    OldStat = CurStat;
 177
 178         if (GetFileStat(checkMtime) == false)
 179         {
 180                 delete Fd;
 181                 Fd = NULL;
 182                 return false;
 183         }
 184
 185     /* if mtime changed, update CurStat from disk */
 186     if (checkMtime == true && OldStat.mtime != CurStat.mtime)
 187         CurStat.Flags = FlSize;
 188
 189         Stats.Bytes += CurStat.FileSize;
 190         Stats.Packages++;
 191
 192         if ((DoControl && LoadControl() == false)
 193                 || (DoContents && LoadContents(GenContentsOnly) == false)
 194                 || (DoMD5 && GetMD5(false) == false)
 195                 || (DoSHA1 && GetSHA1(false) == false)
 196                 || (DoSHA256 && GetSHA256(false) == false)
 197                 || (DoSHA512 && GetSHA512(false) == false)
 198            )
 199         {
 200                 delete Fd;
 201                 Fd = NULL;
 202                 delete DebFile;
 203                 DebFile = NULL;
 204                 return false;
 205         }
 206
 207         delete Fd;
 208         Fd = NULL;
 209         delete DebFile;
 210         DebFile = NULL;
 211
 212    return true;
 213 }
 214                                                                         /*}}}*/
 215 // CacheDB::LoadControl - Load Control information                      /*{{{*/
 216 // ---------------------------------------------------------------------
 217 /* */
 218 bool CacheDB::LoadControl()
 219 {
 220    // Try to read the control information out of the DB.
 221    if ((CurStat.Flags & FlControl) == FlControl)
 222    {
 223       // Lookup the control information
 224       InitQuery("cl");
 225       if (Get() == true && Control.TakeControl(Data.data,Data.size) == true)
 226             return true;
 227       CurStat.Flags &= ~FlControl;
 228    }
 229
 230    if (Fd == NULL && OpenFile() == false)
 231    {
 232       return false;
 233    }
 234    // Create a deb instance to read the archive
 235    if (DebFile == 0)
 236    {
 237       DebFile = new debDebFile(*Fd);
 238       if (_error->PendingError() == true)
 239          return false;
 240    }
 241
 242    Stats.Misses++;
 243    if (Control.Read(*DebFile) == false)
 244       return false;
 245
 246    if (Control.Control == 0)
 247       return _error->Error(_("Archive has no control record"));
 248
 249    // Write back the control information
 250    InitQuery("cl");
 251    if (Put(Control.Control,Control.Length) == true)
 252       CurStat.Flags |= FlControl;
 253    return true;
 254 }
 255                                                                         /*}}}*/
 256 // CacheDB::LoadContents - Load the File Listing                        /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* */
 259 bool CacheDB::LoadContents(bool const &GenOnly)
 260 {
 261    // Try to read the control information out of the DB.
 262    if ((CurStat.Flags & FlContents) == FlContents)
 263    {
 264       if (GenOnly == true)
 265          return true;
 266
 267       // Lookup the contents information
 268       InitQuery("cn");
 269       if (Get() == true)
 270       {
 271          if (Contents.TakeContents(Data.data,Data.size) == true)
 272             return true;
 273       }
 274
 275       CurStat.Flags &= ~FlContents;
 276    }
 277
 278    if (Fd == NULL && OpenFile() == false)
 279    {
 280       return false;
 281    }
 282    // Create a deb instance to read the archive
 283    if (DebFile == 0)
 284    {
 285       DebFile = new debDebFile(*Fd);
 286       if (_error->PendingError() == true)
 287          return false;
 288    }
 289
 290    if (Contents.Read(*DebFile) == false)
 291       return false;
 292
 293    // Write back the control information
 294    InitQuery("cn");
 295    if (Put(Contents.Data,Contents.CurSize) == true)
 296       CurStat.Flags |= FlContents;
 297    return true;
 298 }
 299                                                                         /*}}}*/
 300
 301 static std::string bytes2hex(uint8_t *bytes, size_t length) {
 302    char space[65];
 303    if (length * 2 > sizeof(space) - 1) length = (sizeof(space) - 1) / 2;
 304    for (size_t i = 0; i < length; i++)
 305       snprintf(&space[i*2], 3, "%02x", bytes[i]);
 306    return std::string(space);
 307 }
 308
 309 static inline unsigned char xdig2num(char const &dig) {
 310    if (isdigit(dig)) return dig - '0';
 311    if ('a' <= dig && dig <= 'f') return dig - 'a' + 10;
 312    if ('A' <= dig && dig <= 'F') return dig - 'A' + 10;
 313    return 0;
 314 }
 315
 316 static void hex2bytes(uint8_t *bytes, const char *hex, int length) {
 317    while (length-- > 0) {
 318       *bytes = 0;
 319       if (isxdigit(hex[0]) && isxdigit(hex[1])) {
 320           *bytes = xdig2num(hex[0]) * 16 + xdig2num(hex[1]);
 321           hex += 2;
 322       }
 323       bytes++;
 324    }
 325 }
 326
 327 // CacheDB::GetMD5 - Get the MD5 hash                                   /*{{{*/
 328 // ---------------------------------------------------------------------
 329 /* */
 330 bool CacheDB::GetMD5(bool const &GenOnly)
 331 {
 332    // Try to read the control information out of the DB.
 333    if ((CurStat.Flags & FlMD5) == FlMD5)
 334    {
 335       if (GenOnly == true)
 336          return true;
 337
 338       MD5Res = bytes2hex(CurStat.MD5, sizeof(CurStat.MD5));
 339          return true;
 340       }
 341
 342    Stats.MD5Bytes += CurStat.FileSize;
 343
 344    if (Fd == NULL && OpenFile() == false)
 345    {
 346       return false;
 347    }
 348    MD5Summation MD5;
 349    if (Fd->Seek(0) == false || MD5.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 350       return false;
 351
 352    MD5Res = MD5.Result();
 353    hex2bytes(CurStat.MD5, MD5Res.data(), sizeof(CurStat.MD5));
 354       CurStat.Flags |= FlMD5;
 355    return true;
 356 }
 357                                                                         /*}}}*/
 358 // CacheDB::GetSHA1 - Get the SHA1 hash                                 /*{{{*/
 359 // ---------------------------------------------------------------------
 360 /* */
 361 bool CacheDB::GetSHA1(bool const &GenOnly)
 362 {
 363    // Try to read the control information out of the DB.
 364    if ((CurStat.Flags & FlSHA1) == FlSHA1)
 365    {
 366       if (GenOnly == true)
 367          return true;
 368
 369       SHA1Res = bytes2hex(CurStat.SHA1, sizeof(CurStat.SHA1));
 370       return true;
 371    }
 372
 373    Stats.SHA1Bytes += CurStat.FileSize;
 374
 375    if (Fd == NULL && OpenFile() == false)
 376    {
 377       return false;
 378    }
 379    SHA1Summation SHA1;
 380    if (Fd->Seek(0) == false || SHA1.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 381       return false;
 382
 383    SHA1Res = SHA1.Result();
 384    hex2bytes(CurStat.SHA1, SHA1Res.data(), sizeof(CurStat.SHA1));
 385    CurStat.Flags |= FlSHA1;
 386    return true;
 387 }
 388                                                                         /*}}}*/
 389 // CacheDB::GetSHA256 - Get the SHA256 hash                             /*{{{*/
 390 // ---------------------------------------------------------------------
 391 /* */
 392 bool CacheDB::GetSHA256(bool const &GenOnly)
 393 {
 394    // Try to read the control information out of the DB.
 395    if ((CurStat.Flags & FlSHA256) == FlSHA256)
 396    {
 397       if (GenOnly == true)
 398          return true;
 399
 400       SHA256Res = bytes2hex(CurStat.SHA256, sizeof(CurStat.SHA256));
 401       return true;
 402    }
 403
 404    Stats.SHA256Bytes += CurStat.FileSize;
 405
 406    if (Fd == NULL && OpenFile() == false)
 407    {
 408       return false;
 409    }
 410    SHA256Summation SHA256;
 411    if (Fd->Seek(0) == false || SHA256.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 412       return false;
 413
 414    SHA256Res = SHA256.Result();
 415    hex2bytes(CurStat.SHA256, SHA256Res.data(), sizeof(CurStat.SHA256));
 416    CurStat.Flags |= FlSHA256;
 417    return true;
 418 }
 419                                                                         /*}}}*/
 420 // CacheDB::GetSHA256 - Get the SHA256 hash                             /*{{{*/
 421 // ---------------------------------------------------------------------
 422 /* */
 423 bool CacheDB::GetSHA512(bool const &GenOnly)
 424 {
 425    // Try to read the control information out of the DB.
 426    if ((CurStat.Flags & FlSHA512) == FlSHA512)
 427    {
 428       if (GenOnly == true)
 429          return true;
 430
 431       SHA512Res = bytes2hex(CurStat.SHA512, sizeof(CurStat.SHA512));
 432       return true;
 433    }
 434
 435    Stats.SHA512Bytes += CurStat.FileSize;
 436
 437    if (Fd == NULL && OpenFile() == false)
 438    {
 439       return false;
 440    }
 441    SHA512Summation SHA512;
 442    if (Fd->Seek(0) == false || SHA512.AddFD(Fd->Fd(),CurStat.FileSize) == false)
 443       return false;
 444
 445    SHA512Res = SHA512.Result();
 446    hex2bytes(CurStat.SHA512, SHA512Res.data(), sizeof(CurStat.SHA512));
 447    CurStat.Flags |= FlSHA512;
 448    return true;
 449 }
 450                                                                         /*}}}*/
 451 // CacheDB::Finish - Write back the cache structure                     /*{{{*/
 452 // ---------------------------------------------------------------------
 453 /* */
 454 bool CacheDB::Finish()
 455 {
 456    // Optimize away some writes.
 457    if (CurStat.Flags == OldStat.Flags &&
 458        CurStat.mtime == OldStat.mtime)
 459       return true;
 460
 461    // Write the stat information
 462    CurStat.Flags = htonl(CurStat.Flags);
 463    CurStat.FileSize = htonl(CurStat.FileSize);
 464    InitQuery("st");
 465    Put(&CurStat,sizeof(CurStat));
 466    CurStat.Flags = ntohl(CurStat.Flags);
 467    CurStat.FileSize = ntohl(CurStat.FileSize);
 468
 469    return true;
 470 }
 471                                                                         /*}}}*/
 472 // CacheDB::Clean - Clean the Database                                  /*{{{*/
 473 // ---------------------------------------------------------------------
 474 /* Tidy the database by removing files that no longer exist at all. */
 475 bool CacheDB::Clean()
 476 {
 477    if (DBLoaded == false)
 478       return true;
 479
 480    /* I'm not sure what VERSION_MINOR should be here.. 2.4.14 certainly
 481       needs the lower one and 2.7.7 needs the upper.. */
 482    DBC *Cursor;
 483    if ((errno = Dbp->cursor(Dbp, NULL, &Cursor, 0)) != 0)
 484       return _error->Error(_("Unable to get a cursor"));
 485
 486    DBT Key;
 487    DBT Data;
 488    memset(&Key,0,sizeof(Key));
 489    memset(&Data,0,sizeof(Data));
 490    while ((errno = Cursor->c_get(Cursor,&Key,&Data,DB_NEXT)) == 0)
 491    {
 492       const char *Colon = (char*)memrchr(Key.data, ':', Key.size);
 493       if (Colon)
 494       {
 495          if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 ||
 496              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 ||
 497              stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0)
 498          {
 499             if (FileExists(std::string((const char *)Key.data,Colon)) == true)
 500                 continue;
 501          }
 502       }
 503
 504       Cursor->c_del(Cursor,0);
 505    }
 506    Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL);
 507
 508    return true;
 509 }
 510                                                                         /*}}}*/