nix/libutil/serialise.cc

   1 #include "serialise.hh"
   2 #include "util.hh"
   3
   4 #include <cstring>
   5 #include <cerrno>
   6
   7
   8 namespace nix {
   9
  10
  11 BufferedSink::~BufferedSink()
  12 {
  13     /* We can't call flush() here, because C++ for some insane reason
  14        doesn't allow you to call virtual methods from a destructor. */
  15     assert(!bufPos);
  16     delete[] buffer;
  17 }
  18
  19
  20 void BufferedSink::operator () (const unsigned char * data, size_t len)
  21 {
  22     if (!buffer) buffer = new unsigned char[bufSize];
  23
  24     while (len) {
  25         /* Optimisation: bypass the buffer if the data exceeds the
  26            buffer size. */
  27         if (bufPos + len >= bufSize) {
  28             flush();
  29             write(data, len);
  30             break;
  31         }
  32         /* Otherwise, copy the bytes to the buffer.  Flush the buffer
  33            when it's full. */
  34         size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
  35         memcpy(buffer + bufPos, data, n);
  36         data += n; bufPos += n; len -= n;
  37         if (bufPos == bufSize) flush();
  38     }
  39 }
  40
  41
  42 void BufferedSink::flush()
  43 {
  44     if (bufPos == 0) return;
  45     size_t n = bufPos;
  46     bufPos = 0; // don't trigger the assert() in ~BufferedSink()
  47     write(buffer, n);
  48 }
  49
  50
  51 FdSink::~FdSink()
  52 {
  53     try { flush(); } catch (...) { ignoreException(); }
  54 }
  55
  56
  57 size_t threshold = 256 * 1024 * 1024;
  58
  59 static void warnLargeDump()
  60 {
  61     printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory");
  62 }
  63
  64
  65 void FdSink::write(const unsigned char * data, size_t len)
  66 {
  67     static bool warned = false;
  68     if (warn && !warned) {
  69         written += len;
  70         if (written > threshold) {
  71             warnLargeDump();
  72             warned = true;
  73         }
  74     }
  75     writeFull(fd, data, len);
  76 }
  77
  78
  79 void Source::operator () (unsigned char * data, size_t len)
  80 {
  81     while (len) {
  82         size_t n = read(data, len);
  83         data += n; len -= n;
  84     }
  85 }
  86
  87
  88 BufferedSource::~BufferedSource()
  89 {
  90     delete[] buffer;
  91 }
  92
  93
  94 size_t BufferedSource::read(unsigned char * data, size_t len)
  95 {
  96     if (!buffer) buffer = new unsigned char[bufSize];
  97
  98     if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize);
  99
 100     /* Copy out the data in the buffer. */
 101     size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len;
 102     memcpy(data, buffer + bufPosOut, n);
 103     bufPosOut += n;
 104     if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0;
 105     return n;
 106 }
 107
 108
 109 bool BufferedSource::hasData()
 110 {
 111     return bufPosOut < bufPosIn;
 112 }
 113
 114
 115 size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
 116 {
 117     ssize_t n;
 118     do {
 119         checkInterrupt();
 120         n = ::read(fd, (char *) data, bufSize);
 121     } while (n == -1 && errno == EINTR);
 122     if (n == -1) throw SysError("reading from file");
 123     if (n == 0) throw EndOfFile("unexpected end-of-file");
 124     return n;
 125 }
 126
 127
 128 size_t StringSource::read(unsigned char * data, size_t len)
 129 {
 130     if (pos == s.size()) throw EndOfFile("end of string reached");
 131     size_t n = s.copy((char *) data, len, pos);
 132     pos += n;
 133     return n;
 134 }
 135
 136
 137 void writePadding(size_t len, Sink & sink)
 138 {
 139     if (len % 8) {
 140         unsigned char zero[8];
 141         memset(zero, 0, sizeof(zero));
 142         sink(zero, 8 - (len % 8));
 143     }
 144 }
 145
 146
 147 void writeInt(unsigned int n, Sink & sink)
 148 {
 149     unsigned char buf[8];
 150     memset(buf, 0, sizeof(buf));
 151     buf[0] = n & 0xff;
 152     buf[1] = (n >> 8) & 0xff;
 153     buf[2] = (n >> 16) & 0xff;
 154     buf[3] = (n >> 24) & 0xff;
 155     sink(buf, sizeof(buf));
 156 }
 157
 158
 159 void writeLongLong(unsigned long long n, Sink & sink)
 160 {
 161     unsigned char buf[8];
 162     buf[0] = n & 0xff;
 163     buf[1] = (n >> 8) & 0xff;
 164     buf[2] = (n >> 16) & 0xff;
 165     buf[3] = (n >> 24) & 0xff;
 166     buf[4] = (n >> 32) & 0xff;
 167     buf[5] = (n >> 40) & 0xff;
 168     buf[6] = (n >> 48) & 0xff;
 169     buf[7] = (n >> 56) & 0xff;
 170     sink(buf, sizeof(buf));
 171 }
 172
 173
 174 void writeString(const unsigned char * buf, size_t len, Sink & sink)
 175 {
 176     writeInt(len, sink);
 177     sink(buf, len);
 178     writePadding(len, sink);
 179 }
 180
 181
 182 void writeString(const string & s, Sink & sink)
 183 {
 184     writeString((const unsigned char *) s.data(), s.size(), sink);
 185 }
 186
 187
 188 template<class T> void writeStrings(const T & ss, Sink & sink)
 189 {
 190     writeInt(ss.size(), sink);
 191     foreach (typename T::const_iterator, i, ss)
 192         writeString(*i, sink);
 193 }
 194
 195 template void writeStrings(const Paths & ss, Sink & sink);
 196 template void writeStrings(const PathSet & ss, Sink & sink);
 197
 198
 199 void readPadding(size_t len, Source & source)
 200 {
 201     if (len % 8) {
 202         unsigned char zero[8];
 203         size_t n = 8 - (len % 8);
 204         source(zero, n);
 205         for (unsigned int i = 0; i < n; i++)
 206             if (zero[i]) throw SerialisationError("non-zero padding");
 207     }
 208 }
 209
 210
 211 unsigned int readInt(Source & source)
 212 {
 213     unsigned char buf[8];
 214     source(buf, sizeof(buf));
 215     if (buf[4] || buf[5] || buf[6] || buf[7])
 216         throw SerialisationError("implementation cannot deal with > 32-bit integers");
 217     return
 218         buf[0] |
 219         (buf[1] << 8) |
 220         (buf[2] << 16) |
 221         (buf[3] << 24);
 222 }
 223
 224
 225 unsigned long long readLongLong(Source & source)
 226 {
 227     unsigned char buf[8];
 228     source(buf, sizeof(buf));
 229     return
 230         ((unsigned long long) buf[0]) |
 231         ((unsigned long long) buf[1] << 8) |
 232         ((unsigned long long) buf[2] << 16) |
 233         ((unsigned long long) buf[3] << 24) |
 234         ((unsigned long long) buf[4] << 32) |
 235         ((unsigned long long) buf[5] << 40) |
 236         ((unsigned long long) buf[6] << 48) |
 237         ((unsigned long long) buf[7] << 56);
 238 }
 239
 240
 241 size_t readString(unsigned char * buf, size_t max, Source & source)
 242 {
 243     size_t len = readInt(source);
 244     if (len > max) throw Error("string is too long");
 245     source(buf, len);
 246     readPadding(len, source);
 247     return len;
 248 }
 249
 250
 251 string readString(Source & source)
 252 {
 253     size_t len = readInt(source);
 254     unsigned char * buf = new unsigned char[len];
 255     AutoDeleteArray<unsigned char> d(buf);
 256     source(buf, len);
 257     readPadding(len, source);
 258     return string((char *) buf, len);
 259 }
 260
 261
 262 template<class T> T readStrings(Source & source)
 263 {
 264     unsigned int count = readInt(source);
 265     T ss;
 266     while (count--)
 267         ss.insert(ss.end(), readString(source));
 268     return ss;
 269 }
 270
 271 template Paths readStrings(Source & source);
 272 template PathSet readStrings(Source & source);
 273
 274
 275 void StringSink::operator () (const unsigned char * data, size_t len)
 276 {
 277     static bool warned = false;
 278     if (!warned && s.size() > threshold) {
 279         warnLargeDump();
 280         warned = true;
 281     }
 282     s.append((const char *) data, len);
 283 }
 284
 285
 286 }