kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 1 | // Protocol Buffers - Google's data interchange format |
kenton@google.com | 80b1d62 | 2009-07-29 01:13:20 +0000 | [diff] [blame] | 2 | // Copyright 2008 Google Inc. All rights reserved. |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 3 | // http://code.google.com/p/protobuf/ |
| 4 | // |
| 5 | // Redistribution and use in source and binary forms, with or without |
| 6 | // modification, are permitted provided that the following conditions are |
| 7 | // met: |
| 8 | // |
| 9 | // * Redistributions of source code must retain the above copyright |
| 10 | // notice, this list of conditions and the following disclaimer. |
| 11 | // * Redistributions in binary form must reproduce the above |
| 12 | // copyright notice, this list of conditions and the following disclaimer |
| 13 | // in the documentation and/or other materials provided with the |
| 14 | // distribution. |
| 15 | // * Neither the name of Google Inc. nor the names of its |
| 16 | // contributors may be used to endorse or promote products derived from |
| 17 | // this software without specific prior written permission. |
| 18 | // |
| 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
| 31 | // Author: brianolson@google.com (Brian Olson) |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 32 | // |
| 33 | // This file contains the implementation of classes GzipInputStream and |
| 34 | // GzipOutputStream. |
| 35 | |
| 36 | #include "config.h" |
| 37 | |
| 38 | #if HAVE_ZLIB |
| 39 | #include <google/protobuf/io/gzip_stream.h> |
kenton@google.com | 80b1d62 | 2009-07-29 01:13:20 +0000 | [diff] [blame] | 40 | |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 41 | #include <google/protobuf/stubs/common.h> |
| 42 | |
| 43 | namespace google { |
| 44 | namespace protobuf { |
| 45 | namespace io { |
| 46 | |
| 47 | static const int kDefaultBufferSize = 65536; |
| 48 | |
| 49 | GzipInputStream::GzipInputStream( |
| 50 | ZeroCopyInputStream* sub_stream, Format format, int buffer_size) |
| 51 | : format_(format), sub_stream_(sub_stream), zerror_(Z_OK) { |
| 52 | zcontext_.zalloc = Z_NULL; |
| 53 | zcontext_.zfree = Z_NULL; |
| 54 | zcontext_.opaque = Z_NULL; |
| 55 | zcontext_.total_out = 0; |
| 56 | zcontext_.next_in = NULL; |
| 57 | zcontext_.avail_in = 0; |
| 58 | zcontext_.total_in = 0; |
| 59 | zcontext_.msg = NULL; |
| 60 | if (buffer_size == -1) { |
| 61 | output_buffer_length_ = kDefaultBufferSize; |
| 62 | } else { |
| 63 | output_buffer_length_ = buffer_size; |
| 64 | } |
| 65 | output_buffer_ = operator new(output_buffer_length_); |
| 66 | GOOGLE_CHECK(output_buffer_ != NULL); |
| 67 | zcontext_.next_out = static_cast<Bytef*>(output_buffer_); |
| 68 | zcontext_.avail_out = output_buffer_length_; |
| 69 | output_position_ = output_buffer_; |
| 70 | } |
| 71 | GzipInputStream::~GzipInputStream() { |
| 72 | operator delete(output_buffer_); |
| 73 | zerror_ = inflateEnd(&zcontext_); |
| 74 | } |
| 75 | |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 76 | static inline int internalInflateInit2( |
| 77 | z_stream* zcontext, GzipInputStream::Format format) { |
| 78 | int windowBitsFormat = 0; |
| 79 | switch (format) { |
| 80 | case GzipInputStream::GZIP: windowBitsFormat = 16; break; |
| 81 | case GzipInputStream::AUTO: windowBitsFormat = 32; break; |
| 82 | case GzipInputStream::ZLIB: windowBitsFormat = 0; break; |
| 83 | } |
| 84 | return inflateInit2(zcontext, /* windowBits */15 | windowBitsFormat); |
| 85 | } |
| 86 | |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 87 | int GzipInputStream::Inflate(int flush) { |
| 88 | if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) { |
| 89 | // previous inflate filled output buffer. don't change input params yet. |
| 90 | } else if (zcontext_.avail_in == 0) { |
| 91 | const void* in; |
| 92 | int in_size; |
| 93 | bool first = zcontext_.next_in == NULL; |
| 94 | bool ok = sub_stream_->Next(&in, &in_size); |
| 95 | if (!ok) { |
| 96 | zcontext_.next_out = NULL; |
| 97 | zcontext_.avail_out = 0; |
| 98 | return Z_STREAM_END; |
| 99 | } |
| 100 | zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in)); |
| 101 | zcontext_.avail_in = in_size; |
| 102 | if (first) { |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 103 | int error = internalInflateInit2(&zcontext_, format_); |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 104 | if (error != Z_OK) { |
| 105 | return error; |
| 106 | } |
| 107 | } |
| 108 | } |
| 109 | zcontext_.next_out = static_cast<Bytef*>(output_buffer_); |
| 110 | zcontext_.avail_out = output_buffer_length_; |
| 111 | output_position_ = output_buffer_; |
| 112 | int error = inflate(&zcontext_, flush); |
| 113 | return error; |
| 114 | } |
| 115 | |
| 116 | void GzipInputStream::DoNextOutput(const void** data, int* size) { |
| 117 | *data = output_position_; |
| 118 | *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_); |
| 119 | output_position_ = zcontext_.next_out; |
| 120 | } |
| 121 | |
| 122 | // implements ZeroCopyInputStream ---------------------------------- |
| 123 | bool GzipInputStream::Next(const void** data, int* size) { |
| 124 | bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) |
| 125 | || (zerror_ == Z_BUF_ERROR); |
| 126 | if ((!ok) || (zcontext_.next_out == NULL)) { |
| 127 | return false; |
| 128 | } |
| 129 | if (zcontext_.next_out != output_position_) { |
| 130 | DoNextOutput(data, size); |
| 131 | return true; |
| 132 | } |
| 133 | if (zerror_ == Z_STREAM_END) { |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 134 | if (zcontext_.next_out != NULL) { |
| 135 | // sub_stream_ may have concatenated streams to follow |
| 136 | zerror_ = inflateEnd(&zcontext_); |
| 137 | if (zerror_ != Z_OK) { |
| 138 | return false; |
| 139 | } |
| 140 | zerror_ = internalInflateInit2(&zcontext_, format_); |
| 141 | if (zerror_ != Z_OK) { |
| 142 | return false; |
| 143 | } |
| 144 | } else { |
| 145 | *data = NULL; |
| 146 | *size = 0; |
| 147 | return false; |
| 148 | } |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 149 | } |
| 150 | zerror_ = Inflate(Z_NO_FLUSH); |
| 151 | if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) { |
| 152 | // The underlying stream's Next returned false inside Inflate. |
| 153 | return false; |
| 154 | } |
| 155 | ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) |
| 156 | || (zerror_ == Z_BUF_ERROR); |
| 157 | if (!ok) { |
| 158 | return false; |
| 159 | } |
| 160 | DoNextOutput(data, size); |
| 161 | return true; |
| 162 | } |
| 163 | void GzipInputStream::BackUp(int count) { |
| 164 | output_position_ = reinterpret_cast<void*>( |
| 165 | reinterpret_cast<uintptr_t>(output_position_) - count); |
| 166 | } |
| 167 | bool GzipInputStream::Skip(int count) { |
| 168 | const void* data; |
| 169 | int size; |
| 170 | bool ok = Next(&data, &size); |
| 171 | while (ok && (size < count)) { |
| 172 | count -= size; |
| 173 | ok = Next(&data, &size); |
| 174 | } |
| 175 | if (size > count) { |
| 176 | BackUp(size - count); |
| 177 | } |
| 178 | return ok; |
| 179 | } |
| 180 | int64 GzipInputStream::ByteCount() const { |
| 181 | return zcontext_.total_out + |
| 182 | (((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_)); |
| 183 | } |
| 184 | |
| 185 | // ========================================================================= |
| 186 | |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 187 | GzipOutputStream::Options::Options() |
| 188 | : format(GZIP), |
| 189 | buffer_size(kDefaultBufferSize), |
| 190 | compression_level(Z_DEFAULT_COMPRESSION), |
| 191 | compression_strategy(Z_DEFAULT_STRATEGY) {} |
| 192 | |
| 193 | GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) { |
| 194 | Init(sub_stream, Options()); |
| 195 | } |
| 196 | |
| 197 | GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream, |
| 198 | const Options& options) { |
| 199 | Init(sub_stream, options); |
| 200 | } |
| 201 | |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 202 | GzipOutputStream::GzipOutputStream( |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 203 | ZeroCopyOutputStream* sub_stream, Format format, int buffer_size) { |
| 204 | Options options; |
| 205 | options.format = format; |
| 206 | if (buffer_size != -1) { |
| 207 | options.buffer_size = buffer_size; |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 208 | } |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 209 | Init(sub_stream, options); |
| 210 | } |
| 211 | |
| 212 | void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream, |
| 213 | const Options& options) { |
| 214 | sub_stream_ = sub_stream; |
| 215 | sub_data_ = NULL; |
| 216 | sub_data_size_ = 0; |
| 217 | |
| 218 | input_buffer_length_ = options.buffer_size; |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 219 | input_buffer_ = operator new(input_buffer_length_); |
| 220 | GOOGLE_CHECK(input_buffer_ != NULL); |
| 221 | |
| 222 | zcontext_.zalloc = Z_NULL; |
| 223 | zcontext_.zfree = Z_NULL; |
| 224 | zcontext_.opaque = Z_NULL; |
| 225 | zcontext_.next_out = NULL; |
| 226 | zcontext_.avail_out = 0; |
| 227 | zcontext_.total_out = 0; |
| 228 | zcontext_.next_in = NULL; |
| 229 | zcontext_.avail_in = 0; |
| 230 | zcontext_.total_in = 0; |
| 231 | zcontext_.msg = NULL; |
| 232 | // default to GZIP format |
| 233 | int windowBitsFormat = 16; |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 234 | if (options.format == ZLIB) { |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 235 | windowBitsFormat = 0; |
| 236 | } |
| 237 | zerror_ = deflateInit2( |
| 238 | &zcontext_, |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 239 | options.compression_level, |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 240 | Z_DEFLATED, |
| 241 | /* windowBits */15 | windowBitsFormat, |
| 242 | /* memLevel (default) */8, |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 243 | options.compression_strategy); |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 244 | } |
kenton@google.com | 253a850 | 2009-08-01 00:38:45 +0000 | [diff] [blame] | 245 | |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 246 | GzipOutputStream::~GzipOutputStream() { |
| 247 | Close(); |
| 248 | if (input_buffer_ != NULL) { |
| 249 | operator delete(input_buffer_); |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | // private |
| 254 | int GzipOutputStream::Deflate(int flush) { |
| 255 | int error = Z_OK; |
| 256 | do { |
| 257 | if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) { |
| 258 | bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_); |
| 259 | if (!ok) { |
| 260 | sub_data_ = NULL; |
| 261 | sub_data_size_ = 0; |
| 262 | return Z_BUF_ERROR; |
| 263 | } |
| 264 | GOOGLE_CHECK_GT(sub_data_size_, 0); |
| 265 | zcontext_.next_out = static_cast<Bytef*>(sub_data_); |
| 266 | zcontext_.avail_out = sub_data_size_; |
| 267 | } |
| 268 | error = deflate(&zcontext_, flush); |
| 269 | } while (error == Z_OK && zcontext_.avail_out == 0); |
liujisi@google.com | 33165fe | 2010-11-02 13:14:58 +0000 | [diff] [blame^] | 270 | if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) { |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 271 | // Notify lower layer of data. |
| 272 | sub_stream_->BackUp(zcontext_.avail_out); |
| 273 | // We don't own the buffer anymore. |
| 274 | sub_data_ = NULL; |
| 275 | sub_data_size_ = 0; |
| 276 | } |
| 277 | return error; |
| 278 | } |
| 279 | |
| 280 | // implements ZeroCopyOutputStream --------------------------------- |
| 281 | bool GzipOutputStream::Next(void** data, int* size) { |
| 282 | if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) { |
| 283 | return false; |
| 284 | } |
| 285 | if (zcontext_.avail_in != 0) { |
| 286 | zerror_ = Deflate(Z_NO_FLUSH); |
| 287 | if (zerror_ != Z_OK) { |
| 288 | return false; |
| 289 | } |
| 290 | } |
| 291 | if (zcontext_.avail_in == 0) { |
| 292 | // all input was consumed. reset the buffer. |
| 293 | zcontext_.next_in = static_cast<Bytef*>(input_buffer_); |
| 294 | zcontext_.avail_in = input_buffer_length_; |
| 295 | *data = input_buffer_; |
| 296 | *size = input_buffer_length_; |
| 297 | } else { |
| 298 | // The loop in Deflate should consume all avail_in |
| 299 | GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed"; |
| 300 | } |
| 301 | return true; |
| 302 | } |
| 303 | void GzipOutputStream::BackUp(int count) { |
| 304 | GOOGLE_CHECK_GE(zcontext_.avail_in, count); |
| 305 | zcontext_.avail_in -= count; |
| 306 | } |
| 307 | int64 GzipOutputStream::ByteCount() const { |
| 308 | return zcontext_.total_in + zcontext_.avail_in; |
| 309 | } |
| 310 | |
| 311 | bool GzipOutputStream::Flush() { |
| 312 | do { |
| 313 | zerror_ = Deflate(Z_FULL_FLUSH); |
| 314 | } while (zerror_ == Z_OK); |
| 315 | return zerror_ == Z_OK; |
| 316 | } |
| 317 | |
| 318 | bool GzipOutputStream::Close() { |
| 319 | if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) { |
| 320 | return false; |
| 321 | } |
| 322 | do { |
| 323 | zerror_ = Deflate(Z_FINISH); |
| 324 | } while (zerror_ == Z_OK); |
| 325 | zerror_ = deflateEnd(&zcontext_); |
| 326 | bool ok = zerror_ == Z_OK; |
| 327 | zerror_ = Z_STREAM_END; |
| 328 | return ok; |
| 329 | } |
| 330 | |
| 331 | } // namespace io |
| 332 | } // namespace protobuf |
kenton@google.com | 684d45b | 2009-12-19 04:50:00 +0000 | [diff] [blame] | 333 | } // namespace google |
kenton@google.com | e59427a | 2009-04-16 22:30:56 +0000 | [diff] [blame] | 334 | |
| 335 | #endif // HAVE_ZLIB |