Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 1 | //===- lib/Support/Compressor.cpp -------------------------------*- C++ -*-===// |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 2 | // |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 3 | // The LLVM Compiler Infrastructure |
| 4 | // |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 5 | // This file was developed by Reid Spencer and is distributed under the |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 6 | // University of Illinois Open Source License. See LICENSE.TXT for details. |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 7 | // |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 8 | //===----------------------------------------------------------------------===// |
| 9 | // |
| 10 | // This file implements the llvm::Compressor class, an abstraction for memory |
| 11 | // block compression. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "llvm/Config/config.h" |
| 16 | #include "llvm/Support/Compressor.h" |
| 17 | #include "llvm/ADT/StringExtras.h" |
| 18 | #include <cassert> |
| 19 | #include <string> |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 20 | #include <ostream> |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 21 | #include "bzip2/bzlib.h" |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 22 | using namespace llvm; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 23 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 24 | enum CompressionTypes { |
| 25 | COMP_TYPE_NONE = '0', |
Chris Lattner | aa237256 | 2006-05-24 17:04:05 +0000 | [diff] [blame] | 26 | COMP_TYPE_BZIP2 = '2' |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 27 | }; |
| 28 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 29 | static int getdata(char*& buffer, size_t &size, |
Reid Spencer | 2e3cc54 | 2004-10-04 17:29:25 +0000 | [diff] [blame] | 30 | llvm::Compressor::OutputDataCallback* cb, void* context) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 31 | buffer = 0; |
| 32 | size = 0; |
Reid Spencer | 2e3cc54 | 2004-10-04 17:29:25 +0000 | [diff] [blame] | 33 | int result = (*cb)(buffer, size, context); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 34 | assert(buffer != 0 && "Invalid result from Compressor callback"); |
| 35 | assert(size != 0 && "Invalid result from Compressor callback"); |
| 36 | return result; |
| 37 | } |
| 38 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 39 | static int getdata_uns(char*& buffer, unsigned &size, |
Misha Brukman | 5191b4b | 2005-04-22 04:08:30 +0000 | [diff] [blame] | 40 | llvm::Compressor::OutputDataCallback* cb, void* context) |
| 41 | { |
Tanya Lattner | 238cf92 | 2005-01-29 23:08:01 +0000 | [diff] [blame] | 42 | size_t SizeOut; |
| 43 | int Res = getdata(buffer, SizeOut, cb, context); |
| 44 | size = SizeOut; |
| 45 | return Res; |
| 46 | } |
| 47 | |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 48 | //===----------------------------------------------------------------------===// |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 49 | //=== NULLCOMP - a compression like set of routines that just copies data |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 50 | //=== without doing any compression. This is provided so that if the |
| 51 | //=== configured environment doesn't have a compression library the |
| 52 | //=== program can still work, albeit using more data/memory. |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 53 | //===----------------------------------------------------------------------===// |
| 54 | |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 55 | struct NULLCOMP_stream { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 56 | // User provided fields |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 57 | char* next_in; |
| 58 | size_t avail_in; |
| 59 | char* next_out; |
| 60 | size_t avail_out; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 61 | |
| 62 | // Information fields |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 63 | size_t output_count; // Total count of output bytes |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 64 | }; |
| 65 | |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 66 | static void NULLCOMP_init(NULLCOMP_stream* s) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 67 | s->output_count = 0; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 68 | } |
| 69 | |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 70 | static bool NULLCOMP_compress(NULLCOMP_stream* s) { |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 71 | assert(s && "Invalid NULLCOMP_stream"); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 72 | assert(s->next_in != 0); |
| 73 | assert(s->next_out != 0); |
| 74 | assert(s->avail_in >= 1); |
| 75 | assert(s->avail_out >= 1); |
| 76 | |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 77 | if (s->avail_out >= s->avail_in) { |
| 78 | ::memcpy(s->next_out, s->next_in, s->avail_in); |
| 79 | s->output_count += s->avail_in; |
| 80 | s->avail_out -= s->avail_in; |
| 81 | s->next_in += s->avail_in; |
| 82 | s->avail_in = 0; |
| 83 | return true; |
| 84 | } else { |
| 85 | ::memcpy(s->next_out, s->next_in, s->avail_out); |
| 86 | s->output_count += s->avail_out; |
| 87 | s->avail_in -= s->avail_out; |
| 88 | s->next_in += s->avail_out; |
| 89 | s->avail_out = 0; |
| 90 | return false; |
| 91 | } |
| 92 | } |
| 93 | |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 94 | static bool NULLCOMP_decompress(NULLCOMP_stream* s) { |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 95 | assert(s && "Invalid NULLCOMP_stream"); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 96 | assert(s->next_in != 0); |
| 97 | assert(s->next_out != 0); |
| 98 | assert(s->avail_in >= 1); |
| 99 | assert(s->avail_out >= 1); |
| 100 | |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 101 | if (s->avail_out >= s->avail_in) { |
| 102 | ::memcpy(s->next_out, s->next_in, s->avail_in); |
| 103 | s->output_count += s->avail_in; |
| 104 | s->avail_out -= s->avail_in; |
| 105 | s->next_in += s->avail_in; |
| 106 | s->avail_in = 0; |
| 107 | return true; |
| 108 | } else { |
| 109 | ::memcpy(s->next_out, s->next_in, s->avail_out); |
| 110 | s->output_count += s->avail_out; |
| 111 | s->avail_in -= s->avail_out; |
| 112 | s->next_in += s->avail_out; |
| 113 | s->avail_out = 0; |
| 114 | return false; |
| 115 | } |
| 116 | } |
| 117 | |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 118 | static void NULLCOMP_end(NULLCOMP_stream* strm) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 119 | } |
| 120 | |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 121 | namespace { |
| 122 | |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 123 | /// This structure is only used when a bytecode file is compressed. |
| 124 | /// As bytecode is being decompressed, the memory buffer might need |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 125 | /// to be reallocated. The buffer allocation is handled in a callback |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 126 | /// and this structure is needed to retain information across calls |
| 127 | /// to the callback. |
| 128 | /// @brief An internal buffer object used for handling decompression |
| 129 | struct BufferContext { |
| 130 | char* buff; |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 131 | size_t size; |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 132 | BufferContext(size_t compressedSize) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 133 | // Null to indicate malloc of a new block |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 134 | buff = 0; |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 135 | |
| 136 | // Compute the initial length of the uncompression buffer. Note that this |
| 137 | // is twice the length of the compressed buffer and will be doubled again |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 138 | // in the callback for an initial allocation of 4x compressedSize. This |
| 139 | // calculation is based on the typical compression ratio of bzip2 on LLVM |
| 140 | // bytecode files which typically ranges in the 50%-75% range. Since we |
| 141 | // typically get at least 50%, doubling is insufficient. By using a 4x |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 142 | // multiplier on the first allocation, we minimize the impact of having to |
| 143 | // copy the buffer on reallocation. |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 144 | size = compressedSize*2; |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 145 | } |
| 146 | |
Chris Lattner | bb4384b | 2005-01-29 17:05:56 +0000 | [diff] [blame] | 147 | /// trimTo - Reduce the size of the buffer down to the specified amount. This |
| 148 | /// is useful after have read in the bytecode file to discard extra unused |
| 149 | /// memory. |
| 150 | /// |
| 151 | void trimTo(size_t NewSize) { |
| 152 | buff = (char*)::realloc(buff, NewSize); |
| 153 | size = NewSize; |
| 154 | } |
| 155 | |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 156 | /// This function handles allocation of the buffer used for decompression of |
| 157 | /// compressed bytecode files. It is called by Compressor::decompress which is |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 158 | /// called by BytecodeReader::ParseBytecode. |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 159 | static size_t callback(char*&buff, size_t &sz, void* ctxt){ |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 160 | // Case the context variable to our BufferContext |
| 161 | BufferContext* bc = reinterpret_cast<BufferContext*>(ctxt); |
| 162 | |
| 163 | // Compute the new, doubled, size of the block |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 164 | size_t new_size = bc->size * 2; |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 165 | |
| 166 | // Extend or allocate the block (realloc(0,n) == malloc(n)) |
| 167 | char* new_buff = (char*) ::realloc(bc->buff, new_size); |
| 168 | |
| 169 | // Figure out what to return to the Compressor. If this is the first call, |
| 170 | // then bc->buff will be null. In this case we want to return the entire |
| 171 | // buffer because there was no previous allocation. Otherwise, when the |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 172 | // buffer is reallocated, we save the new base pointer in the |
| 173 | // BufferContext.buff field but return the address of only the extension, |
| 174 | // mid-way through the buffer (since its size was doubled). Furthermore, |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 175 | // the sz result must be 1/2 the total size of the buffer. |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 176 | if (bc->buff == 0 ) { |
| 177 | buff = bc->buff = new_buff; |
| 178 | sz = new_size; |
| 179 | } else { |
| 180 | bc->buff = new_buff; |
| 181 | buff = new_buff + bc->size; |
| 182 | sz = bc->size; |
| 183 | } |
| 184 | |
| 185 | // Retain the size of the allocated block |
| 186 | bc->size = new_size; |
| 187 | |
| 188 | // Make sure we fail (return 1) if we didn't get any memory. |
| 189 | return (bc->buff == 0 ? 1 : 0); |
| 190 | } |
| 191 | }; |
| 192 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 193 | } // end anonymous namespace |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 194 | |
| 195 | |
| 196 | namespace { |
| 197 | |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 198 | // This structure retains the context when compressing the bytecode file. The |
| 199 | // WriteCompressedData function below uses it to keep track of the previously |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 200 | // filled chunk of memory (which it writes) and how many bytes have been |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 201 | // written. |
| 202 | struct WriterContext { |
| 203 | // Initialize the context |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 204 | WriterContext(std::ostream*OS, size_t CS) |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 205 | : chunk(0), sz(0), written(0), compSize(CS), Out(OS) {} |
| 206 | |
| 207 | // Make sure we clean up memory |
| 208 | ~WriterContext() { |
| 209 | if (chunk) |
| 210 | delete [] chunk; |
| 211 | } |
| 212 | |
| 213 | // Write the chunk |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 214 | void write(size_t size = 0) { |
| 215 | size_t write_size = (size == 0 ? sz : size); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 216 | Out->write(chunk,write_size); |
| 217 | written += write_size; |
| 218 | delete [] chunk; |
| 219 | chunk = 0; |
| 220 | sz = 0; |
| 221 | } |
| 222 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 223 | // This function is a callback used by the Compressor::compress function to |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 224 | // allocate memory for the compression buffer. This function fulfills that |
| 225 | // responsibility but also writes the previous (now filled) buffer out to the |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 226 | // stream. |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 227 | static size_t callback(char*& buffer, size_t &size, void* context) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 228 | // Cast the context to the structure it must point to. |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 229 | WriterContext* ctxt = reinterpret_cast<WriterContext*>(context); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 230 | |
| 231 | // If there's a previously allocated chunk, it must now be filled with |
| 232 | // compressed data, so we write it out and deallocate it. |
| 233 | if (ctxt->chunk != 0 && ctxt->sz > 0 ) { |
| 234 | ctxt->write(); |
| 235 | } |
| 236 | |
| 237 | // Compute the size of the next chunk to allocate. We attempt to allocate |
| 238 | // enough memory to handle the compression in a single memory allocation. In |
| 239 | // general, the worst we do on compression of bytecode is about 50% so we |
| 240 | // conservatively estimate compSize / 2 as the size needed for the |
| 241 | // compression buffer. compSize is the size of the compressed data, provided |
| 242 | // by WriteBytecodeToFile. |
| 243 | size = ctxt->sz = ctxt->compSize / 2; |
| 244 | |
| 245 | // Allocate the chunks |
| 246 | buffer = ctxt->chunk = new char [size]; |
| 247 | |
| 248 | // We must return 1 if the allocation failed so that the Compressor knows |
| 249 | // not to use the buffer pointer. |
| 250 | return (ctxt->chunk == 0 ? 1 : 0); |
| 251 | } |
| 252 | |
| 253 | char* chunk; // pointer to the chunk of memory filled by compression |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 254 | size_t sz; // size of chunk |
| 255 | size_t written; // aggregate total of bytes written in all chunks |
| 256 | size_t compSize; // size of the uncompressed buffer |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 257 | std::ostream* Out; // The stream we write the data to. |
| 258 | }; |
| 259 | |
Chris Lattner | 1733406 | 2005-01-29 16:53:02 +0000 | [diff] [blame] | 260 | } // end anonymous namespace |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 261 | |
| 262 | // Compress in one of three ways |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 263 | size_t Compressor::compress(const char* in, size_t size, |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 264 | OutputDataCallback* cb, void* context, |
| 265 | std::string* error ) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 266 | assert(in && "Can't compress null buffer"); |
| 267 | assert(size && "Can't compress empty buffer"); |
| 268 | assert(cb && "Can't compress without a callback function"); |
| 269 | |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 270 | size_t result = 0; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 271 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 272 | // For small files, we just don't bother compressing. bzip2 isn't very good |
| 273 | // with tiny files and can actually make the file larger, so we just avoid |
| 274 | // it altogether. |
Reid Spencer | 9572946 | 2004-11-30 07:13:34 +0000 | [diff] [blame] | 275 | if (size > 64*1024) { |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 276 | // Set up the bz_stream |
| 277 | bz_stream bzdata; |
| 278 | bzdata.bzalloc = 0; |
| 279 | bzdata.bzfree = 0; |
| 280 | bzdata.opaque = 0; |
| 281 | bzdata.next_in = (char*)in; |
| 282 | bzdata.avail_in = size; |
| 283 | bzdata.next_out = 0; |
| 284 | bzdata.avail_out = 0; |
| 285 | switch ( BZ2_bzCompressInit(&bzdata, 5, 0, 100) ) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 286 | case BZ_CONFIG_ERROR: |
| 287 | if (error) |
| 288 | *error = "bzip2 library mis-compiled"; |
| 289 | return result; |
| 290 | case BZ_PARAM_ERROR: |
| 291 | if (error) |
| 292 | *error = "Compressor internal error"; |
| 293 | return result; |
| 294 | case BZ_MEM_ERROR: |
| 295 | if (error) |
| 296 | *error = "Out of memory"; |
| 297 | return result; |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 298 | case BZ_OK: |
| 299 | default: |
| 300 | break; |
| 301 | } |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 302 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 303 | // Get a block of memory |
Tanya Lattner | 5ca41e2 | 2005-01-29 23:29:55 +0000 | [diff] [blame] | 304 | if (0 != getdata_uns(bzdata.next_out, bzdata.avail_out,cb,context)) { |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 305 | BZ2_bzCompressEnd(&bzdata); |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 306 | if (error) |
| 307 | *error = "Can't allocate output buffer"; |
| 308 | return result; |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 309 | } |
| 310 | |
| 311 | // Put compression code in first byte |
| 312 | (*bzdata.next_out++) = COMP_TYPE_BZIP2; |
| 313 | bzdata.avail_out--; |
| 314 | |
| 315 | // Compress it |
| 316 | int bzerr = BZ_FINISH_OK; |
| 317 | while (BZ_FINISH_OK == (bzerr = BZ2_bzCompress(&bzdata, BZ_FINISH))) { |
Tanya Lattner | 5ca41e2 | 2005-01-29 23:29:55 +0000 | [diff] [blame] | 318 | if (0 != getdata_uns(bzdata.next_out, bzdata.avail_out,cb,context)) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 319 | BZ2_bzCompressEnd(&bzdata); |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 320 | if (error) |
| 321 | *error = "Can't allocate output buffer"; |
| 322 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 323 | } |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 324 | } |
| 325 | switch (bzerr) { |
| 326 | case BZ_SEQUENCE_ERROR: |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 327 | case BZ_PARAM_ERROR: |
| 328 | if (error) |
| 329 | *error = "Param/Sequence error"; |
| 330 | return result; |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 331 | case BZ_FINISH_OK: |
| 332 | case BZ_STREAM_END: break; |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 333 | default: |
| 334 | if (error) |
| 335 | *error = "BZip2 Error: " + utostr(unsigned(bzerr)); |
| 336 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 337 | } |
| 338 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 339 | // Finish |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 340 | result = bzdata.total_out_lo32 + 1; |
| 341 | if (sizeof(size_t) == sizeof(uint64_t)) |
| 342 | result |= static_cast<uint64_t>(bzdata.total_out_hi32) << 32; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 343 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 344 | BZ2_bzCompressEnd(&bzdata); |
| 345 | } else { |
| 346 | // Do null compression, for small files |
| 347 | NULLCOMP_stream sdata; |
| 348 | sdata.next_in = (char*)in; |
| 349 | sdata.avail_in = size; |
| 350 | NULLCOMP_init(&sdata); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 351 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 352 | if (0 != getdata(sdata.next_out, sdata.avail_out,cb,context)) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 353 | if (error) |
| 354 | *error = "Can't allocate output buffer"; |
| 355 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 356 | } |
| 357 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 358 | *(sdata.next_out++) = COMP_TYPE_NONE; |
| 359 | sdata.avail_out--; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 360 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 361 | while (!NULLCOMP_compress(&sdata)) { |
Reid Spencer | 2e3cc54 | 2004-10-04 17:29:25 +0000 | [diff] [blame] | 362 | if (0 != getdata(sdata.next_out, sdata.avail_out,cb,context)) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 363 | if (error) |
| 364 | *error = "Can't allocate output buffer"; |
| 365 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 366 | } |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 367 | } |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 368 | |
| 369 | result = sdata.output_count + 1; |
| 370 | NULLCOMP_end(&sdata); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 371 | } |
| 372 | return result; |
| 373 | } |
| 374 | |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 375 | size_t Compressor::compressToNewBuffer(const char* in, size_t size, char*&out, |
| 376 | std::string* error) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 377 | BufferContext bc(size); |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 378 | size_t result = compress(in,size,BufferContext::callback,(void*)&bc,error); |
Chris Lattner | bb4384b | 2005-01-29 17:05:56 +0000 | [diff] [blame] | 379 | bc.trimTo(result); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 380 | out = bc.buff; |
| 381 | return result; |
| 382 | } |
| 383 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 384 | size_t |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 385 | Compressor::compressToStream(const char*in, size_t size, std::ostream& out, |
| 386 | std::string* error) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 387 | // Set up the context and writer |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 388 | WriterContext ctxt(&out, size / 2); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 389 | |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 390 | // Compress everything after the magic number (which we'll alter). |
| 391 | size_t zipSize = Compressor::compress(in,size, |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 392 | WriterContext::callback, (void*)&ctxt,error); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 393 | |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 394 | if (zipSize && ctxt.chunk) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 395 | ctxt.write(zipSize - ctxt.written); |
| 396 | } |
| 397 | return zipSize; |
| 398 | } |
| 399 | |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 400 | // Decompress in one of three ways |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 401 | size_t Compressor::decompress(const char *in, size_t size, |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 402 | OutputDataCallback* cb, void* context, |
| 403 | std::string* error) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 404 | assert(in && "Can't decompress null buffer"); |
| 405 | assert(size > 1 && "Can't decompress empty buffer"); |
| 406 | assert(cb && "Can't decompress without a callback function"); |
| 407 | |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 408 | size_t result = 0; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 409 | |
| 410 | switch (*in++) { |
| 411 | case COMP_TYPE_BZIP2: { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 412 | // Set up the bz_stream |
| 413 | bz_stream bzdata; |
| 414 | bzdata.bzalloc = 0; |
| 415 | bzdata.bzfree = 0; |
| 416 | bzdata.opaque = 0; |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 417 | bzdata.next_in = (char*)in; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 418 | bzdata.avail_in = size - 1; |
| 419 | bzdata.next_out = 0; |
| 420 | bzdata.avail_out = 0; |
| 421 | switch ( BZ2_bzDecompressInit(&bzdata, 0, 0) ) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 422 | case BZ_CONFIG_ERROR: |
| 423 | if (error) |
| 424 | *error = "bzip2 library mis-compiled"; |
| 425 | return result; |
| 426 | case BZ_PARAM_ERROR: |
| 427 | if (error) |
| 428 | *error = "Compressor internal error"; |
| 429 | return result; |
| 430 | case BZ_MEM_ERROR: |
| 431 | if (error) |
| 432 | *error = "Out of memory"; |
| 433 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 434 | case BZ_OK: |
| 435 | default: |
| 436 | break; |
| 437 | } |
| 438 | |
| 439 | // Get a block of memory |
Tanya Lattner | 5ca41e2 | 2005-01-29 23:29:55 +0000 | [diff] [blame] | 440 | if (0 != getdata_uns(bzdata.next_out, bzdata.avail_out,cb,context)) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 441 | BZ2_bzDecompressEnd(&bzdata); |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 442 | if (error) |
| 443 | *error = "Can't allocate output buffer"; |
| 444 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 445 | } |
| 446 | |
| 447 | // Decompress it |
| 448 | int bzerr = BZ_OK; |
Jeff Cohen | 5f4ef3c | 2005-07-27 06:12:32 +0000 | [diff] [blame] | 449 | while ( BZ_OK == (bzerr = BZ2_bzDecompress(&bzdata)) && |
Reid Spencer | 39ec7f0 | 2005-05-13 07:05:37 +0000 | [diff] [blame] | 450 | bzdata.avail_in != 0 ) { |
Tanya Lattner | 5ca41e2 | 2005-01-29 23:29:55 +0000 | [diff] [blame] | 451 | if (0 != getdata_uns(bzdata.next_out, bzdata.avail_out,cb,context)) { |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 452 | BZ2_bzDecompressEnd(&bzdata); |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 453 | if (error) |
| 454 | *error = "Can't allocate output buffer"; |
| 455 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 456 | } |
| 457 | } |
| 458 | |
| 459 | switch (bzerr) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 460 | BZ2_bzDecompressEnd(&bzdata); |
| 461 | case BZ_PARAM_ERROR: |
| 462 | if (error) |
| 463 | *error = "Compressor internal error"; |
| 464 | return result; |
| 465 | case BZ_MEM_ERROR: |
| 466 | BZ2_bzDecompressEnd(&bzdata); |
| 467 | if (error) |
| 468 | *error = "Out of memory"; |
| 469 | return result; |
| 470 | case BZ_DATA_ERROR: |
| 471 | BZ2_bzDecompressEnd(&bzdata); |
| 472 | if (error) |
| 473 | *error = "Data integrity error"; |
| 474 | return result; |
| 475 | case BZ_DATA_ERROR_MAGIC: |
| 476 | BZ2_bzDecompressEnd(&bzdata); |
| 477 | if (error) |
| 478 | *error = "Data is not BZIP2"; |
| 479 | return result; |
| 480 | case BZ_OK: |
| 481 | BZ2_bzDecompressEnd(&bzdata); |
| 482 | if (error) |
| 483 | *error = "Insufficient input for bzip2"; |
| 484 | return result; |
Reid Spencer | 39ec7f0 | 2005-05-13 07:05:37 +0000 | [diff] [blame] | 485 | case BZ_STREAM_END: break; |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 486 | default: |
| 487 | BZ2_bzDecompressEnd(&bzdata); |
| 488 | if (error) |
| 489 | *error = "Unknown result code from bzDecompress"; |
| 490 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 491 | } |
| 492 | |
| 493 | // Finish |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 494 | result = bzdata.total_out_lo32; |
| 495 | if (sizeof(size_t) == sizeof(uint64_t)) |
| 496 | result |= (static_cast<uint64_t>(bzdata.total_out_hi32) << 32); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 497 | BZ2_bzDecompressEnd(&bzdata); |
| 498 | break; |
Chris Lattner | b4abe32 | 2004-10-04 16:33:25 +0000 | [diff] [blame] | 499 | } |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 500 | |
Reid Spencer | fa8d2e0 | 2004-11-25 19:38:16 +0000 | [diff] [blame] | 501 | case COMP_TYPE_NONE: { |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 502 | NULLCOMP_stream sdata; |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 503 | sdata.next_in = (char*)in; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 504 | sdata.avail_in = size - 1; |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 505 | NULLCOMP_init(&sdata); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 506 | |
Reid Spencer | 2e3cc54 | 2004-10-04 17:29:25 +0000 | [diff] [blame] | 507 | if (0 != getdata(sdata.next_out, sdata.avail_out,cb,context)) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 508 | if (error) |
| 509 | *error = "Can't allocate output buffer"; |
| 510 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 511 | } |
| 512 | |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 513 | while (!NULLCOMP_decompress(&sdata)) { |
Reid Spencer | 2e3cc54 | 2004-10-04 17:29:25 +0000 | [diff] [blame] | 514 | if (0 != getdata(sdata.next_out, sdata.avail_out,cb,context)) { |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 515 | if (error) |
| 516 | *error = "Can't allocate output buffer"; |
| 517 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 518 | } |
| 519 | } |
| 520 | |
| 521 | result = sdata.output_count; |
Reid Spencer | 04f1e90 | 2004-10-04 17:45:44 +0000 | [diff] [blame] | 522 | NULLCOMP_end(&sdata); |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 523 | break; |
| 524 | } |
| 525 | |
| 526 | default: |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 527 | if (error) |
| 528 | *error = "Unknown type of compressed data"; |
| 529 | return result; |
Reid Spencer | c895037 | 2004-10-04 10:49:41 +0000 | [diff] [blame] | 530 | } |
| 531 | |
| 532 | return result; |
| 533 | } |
| 534 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 535 | size_t |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 536 | Compressor::decompressToNewBuffer(const char* in, size_t size, char*&out, |
| 537 | std::string* error) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 538 | BufferContext bc(size); |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 539 | size_t result = decompress(in,size,BufferContext::callback,(void*)&bc,error); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 540 | out = bc.buff; |
| 541 | return result; |
| 542 | } |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 543 | |
Misha Brukman | 10468d8 | 2005-04-21 22:55:34 +0000 | [diff] [blame] | 544 | size_t |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 545 | Compressor::decompressToStream(const char*in, size_t size, std::ostream& out, |
| 546 | std::string* error) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 547 | // Set up the context and writer |
| 548 | WriterContext ctxt(&out,size / 2); |
| 549 | |
Reid Spencer | 39ec7f0 | 2005-05-13 07:05:37 +0000 | [diff] [blame] | 550 | // Decompress everything after the magic number (which we'll alter) |
Chris Lattner | 800b724 | 2005-01-29 17:17:18 +0000 | [diff] [blame] | 551 | size_t zipSize = Compressor::decompress(in,size, |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 552 | WriterContext::callback, (void*)&ctxt,error); |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 553 | |
Chris Lattner | 4f1b929 | 2006-07-07 17:00:12 +0000 | [diff] [blame] | 554 | if (zipSize && ctxt.chunk) { |
Reid Spencer | cf602a8 | 2004-11-14 22:04:46 +0000 | [diff] [blame] | 555 | ctxt.write(zipSize - ctxt.written); |
| 556 | } |
| 557 | return zipSize; |
| 558 | } |