Merge pull request #79 from szabadka/master
Add partial output support to the decoder.
diff --git a/dec/decode.c b/dec/decode.c
index b0374c5..674dbfb 100644
--- a/dec/decode.c
+++ b/dec/decode.c
@@ -191,7 +191,7 @@
const HuffmanCode* p = s->table;
uint8_t code_len;
if (!BrotliReadMoreInput(br)) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
BrotliFillBitWindow(br);
p += (br->val_ >> br->bit_pos_) & 31;
@@ -259,7 +259,7 @@
switch(s->sub_state[1]) {
case BROTLI_STATE_SUB_NONE:
if (!BrotliReadMoreInput(br)) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
s->code_lengths =
(uint8_t*)BrotliSafeMalloc((uint64_t)alphabet_size,
@@ -471,7 +471,7 @@
switch(s->sub_state[0]) {
case BROTLI_STATE_SUB_NONE:
if (!BrotliReadMoreInput(br)) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
*num_htrees = DecodeVarLenUint8(br) + 1;
@@ -512,7 +512,7 @@
while (s->context_index < context_map_size) {
int code;
if (!BrotliReadMoreInput(br)) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
code = ReadSymbol(s->context_map_table, br);
if (code == 0) {
@@ -647,6 +647,7 @@
int br_pos = s->br.pos_ & BROTLI_IBUF_MASK;
uint32_t remaining_bits;
int num_read;
+ int num_written;
/* State machine */
for (;;) {
@@ -691,13 +692,24 @@
rb_pos += s->nbytes;
s->meta_block_remaining_len -= s->nbytes;
+ s->partially_written = 0;
+ s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_1;
+ /* No break, continue to next state */
+ case BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_1:
/* If we wrote past the logical end of the ringbuffer, copy the tail of
the ringbuffer to its beginning and flush the ringbuffer to the
output. */
if (rb_pos >= rb_size) {
- if (BrotliWrite(output, s->ringbuffer, (size_t)rb_size) < rb_size) {
+ num_written = BrotliWrite(output,
+ s->ringbuffer + s->partially_written,
+ (size_t)(rb_size - s->partially_written));
+ if (num_written < 0) {
return BROTLI_RESULT_ERROR;
}
+ s->partially_written += num_written;
+ if (s->partially_written < rb_size) {
+ return BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ }
rb_pos -= rb_size;
s->meta_block_remaining_len += rb_size;
memcpy(s->ringbuffer, ringbuffer_end, (size_t)rb_pos);
@@ -707,37 +719,66 @@
case BROTLI_STATE_SUB_UNCOMPRESSED_SHORT:
while (s->meta_block_remaining_len > 0) {
if (!BrotliReadMoreInput(&s->br)) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
s->ringbuffer[rb_pos++] = (uint8_t)BrotliReadBits(&s->br, 8);
if (rb_pos == rb_size) {
- if (BrotliWrite(output, s->ringbuffer, (size_t)rb_size) < rb_size) {
- return BROTLI_RESULT_ERROR;
- }
- rb_pos = 0;
+ s->partially_written = 0;
+ s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_2;
+ break;
}
s->meta_block_remaining_len--;
}
- s->sub_state[0] = BROTLI_STATE_SUB_NONE;
- return BROTLI_RESULT_SUCCESS;
+ if (s->sub_state[0] == BROTLI_STATE_SUB_UNCOMPRESSED_SHORT) {
+ s->sub_state[0] = BROTLI_STATE_SUB_NONE;
+ return BROTLI_RESULT_SUCCESS;
+ }
+ /* No break, if state is updated, continue to next state */
+ case BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_2:
+ num_written = BrotliWrite(output, s->ringbuffer + s->partially_written,
+ (size_t)(rb_size - s->partially_written));
+ if (num_written < 0) {
+ return BROTLI_RESULT_ERROR;
+ }
+ s->partially_written += num_written;
+ if (s->partially_written < rb_size) {
+ return BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ }
+ rb_pos = 0;
+ s->meta_block_remaining_len--;
+ s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_SHORT;
+ break;
case BROTLI_STATE_SUB_UNCOMPRESSED_FILL:
/* If we have more to copy than the remaining size of the ringbuffer,
then we first fill the ringbuffer from the input and then flush the
ringbuffer to the output */
- while (rb_pos + s->meta_block_remaining_len >= rb_size) {
+ if (rb_pos + s->meta_block_remaining_len >= rb_size) {
s->nbytes = rb_size - rb_pos;
if (BrotliRead(s->br.input_, &s->ringbuffer[rb_pos],
(size_t)s->nbytes) < s->nbytes) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
- if (BrotliWrite(output, s->ringbuffer, (size_t)rb_size) < s->nbytes) {
- return BROTLI_RESULT_ERROR;
- }
- s->meta_block_remaining_len -= s->nbytes;
- rb_pos = 0;
+ s->partially_written = 0;
+ s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_3;
+ } else {
+ s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_COPY;
+ break;
}
- s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_COPY;
/* No break, continue to next state */
+ case BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_3:
+ num_written = BrotliWrite(output, s->ringbuffer + s->partially_written,
+ (size_t)(rb_size - s->partially_written));
+ if (num_written < 0) {
+ return BROTLI_RESULT_ERROR;
+ }
+ s->partially_written += num_written;
+ if (s->partially_written < rb_size) {
+ return BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ }
+ s->meta_block_remaining_len -= s->nbytes;
+ rb_pos = 0;
+ s->sub_state[0] = BROTLI_STATE_SUB_UNCOMPRESSED_FILL;
+ break;
case BROTLI_STATE_SUB_UNCOMPRESSED_COPY:
/* Copy straight from the input onto the ringbuffer. The ringbuffer will
be flushed to the output at a later time. */
@@ -745,7 +786,7 @@
(size_t)s->meta_block_remaining_len);
s->meta_block_remaining_len -= num_read;
if (s->meta_block_remaining_len > 0) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
/* Restore the state of the bit reader. */
@@ -754,7 +795,7 @@
/* No break, continue to next state */
case BROTLI_STATE_SUB_UNCOMPRESSED_WARMUP:
if (!BrotliWarmupBitReader(&s->br)) {
- return BROTLI_RESULT_PARTIAL;
+ return BROTLI_RESULT_NEEDS_MORE_INPUT;
}
s->sub_state[0] = BROTLI_STATE_SUB_NONE;
return BROTLI_RESULT_SUCCESS;
@@ -845,7 +886,7 @@
BrotliResult result;
BrotliStateInit(&s);
result = BrotliDecompressStreaming(input, output, 1, &s);
- if (result == BROTLI_RESULT_PARTIAL) {
+ if (result == BROTLI_RESULT_NEEDS_MORE_INPUT) {
/* Not ok: it didn't finish even though this is a non-streaming function. */
result = BROTLI_RESULT_ERROR;
}
@@ -889,6 +930,7 @@
BrotliBitReader* br = &s->br;
int initial_remaining_len;
int bytes_copied;
+ int num_written;
/* We need the slack region for the following reasons:
- always doing two 8-byte copies for fast backward copying
@@ -901,7 +943,7 @@
/* State machine */
for (;;) {
if (result != BROTLI_RESULT_SUCCESS) {
- if (result == BROTLI_RESULT_PARTIAL && finish) {
+ if (result == BROTLI_RESULT_NEEDS_MORE_INPUT && finish) {
printf("Unexpected end of input. State: %d\n", s->state);
result = BROTLI_RESULT_ERROR;
}
@@ -930,7 +972,7 @@
/* No break, continue to next state */
case BROTLI_STATE_BITREADER_WARMUP:
if (!BrotliWarmupBitReader(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
/* Decode window size. */
@@ -961,10 +1003,11 @@
/* No break, continue to next state */
case BROTLI_STATE_METABLOCK_BEGIN:
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
if (s->input_end) {
+ s->partially_written = 0;
s->state = BROTLI_STATE_DONE;
break;
}
@@ -1002,7 +1045,7 @@
/* No break, continue to next state */
case BROTLI_STATE_METABLOCK_HEADER_1:
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
BROTLI_LOG_UINT(pos);
@@ -1042,6 +1085,9 @@
initial_remaining_len = s->meta_block_remaining_len;
/* pos is given as argument since s->pos is only updated at the end. */
result = CopyUncompressedBlockToOutput(output, pos, s);
+ if (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {
+ break;
+ }
bytes_copied = initial_remaining_len - s->meta_block_remaining_len;
pos += bytes_copied;
if (bytes_copied > 0) {
@@ -1055,7 +1101,7 @@
case BROTLI_STATE_METADATA:
for (; s->meta_block_remaining_len > 0; --s->meta_block_remaining_len) {
if (!BrotliReadMoreInput(&s->br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
/* Read one byte and ignore it. */
@@ -1104,7 +1150,7 @@
break;
case BROTLI_STATE_METABLOCK_HEADER_2:
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
s->distance_postfix_bits = (int)BrotliReadBits(br, 2);
@@ -1178,7 +1224,7 @@
/* Block decoding is the inner loop, jumping with goto makes it 3% faster */
BlockBegin:
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
if (s->meta_block_remaining_len <= 0) {
@@ -1227,7 +1273,7 @@
if (s->trivial_literal_context) {
while (i < s->insert_length) {
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
if (s->block_length[0] == 0) {
@@ -1241,19 +1287,19 @@
BROTLI_LOG_UINT(s->literal_htree_index);
BROTLI_LOG_ARRAY_INDEX(s->ringbuffer, pos & s->ringbuffer_mask);
if ((pos & s->ringbuffer_mask) == s->ringbuffer_mask) {
- if (BrotliWrite(output, s->ringbuffer,
- (size_t)s->ringbuffer_size) < 0) {
- result = BROTLI_RESULT_ERROR;
- break;
- }
+ s->partially_written = 0;
+ s->state = BROTLI_STATE_BLOCK_INNER_WRITE;
+ break;
}
+ /* Modifications to this code shold be reflected in
+ BROTLI_STATE_BLOCK_INNER_WRITE case */
++pos;
++i;
}
} else {
while (i < s->insert_length) {
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
if (s->block_length[0] == 0) {
@@ -1273,18 +1319,18 @@
BROTLI_LOG_UINT(s->literal_htree_index);
BROTLI_LOG_ARRAY_INDEX(s->ringbuffer, pos & s->ringbuffer_mask);
if ((pos & s->ringbuffer_mask) == s->ringbuffer_mask) {
- if (BrotliWrite(output, s->ringbuffer,
- (size_t)s->ringbuffer_size) < 0) {
- result = BROTLI_RESULT_ERROR;
- break;
- }
+ s->partially_written = 0;
+ s->state = BROTLI_STATE_BLOCK_INNER_WRITE;
+ break;
}
+ /* Modifications to this code shold be reflected in
+ BROTLI_STATE_BLOCK_INNER_WRITE case */
++pos;
++i;
}
}
-
- if (result != BROTLI_RESULT_SUCCESS) break;
+ if (result != BROTLI_RESULT_SUCCESS ||
+ s->state == BROTLI_STATE_BLOCK_INNER_WRITE) break;
s->meta_block_remaining_len -= s->insert_length;
if (s->meta_block_remaining_len <= 0) {
@@ -1299,7 +1345,7 @@
/* No break, go to next state */
case BROTLI_STATE_BLOCK_DISTANCE:
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
assert(s->distance_code < 0);
@@ -1337,7 +1383,7 @@
/* No break, go to next state */
case BROTLI_STATE_BLOCK_POST:
if (!BrotliReadMoreInput(br)) {
- result = BROTLI_RESULT_PARTIAL;
+ result = BROTLI_RESULT_NEEDS_MORE_INPUT;
break;
}
/* Convert the distance code to the actual distance by possibly */
@@ -1377,11 +1423,21 @@
pos += len;
s->meta_block_remaining_len -= len;
if (s->copy_dst >= s->ringbuffer_end) {
- if (BrotliWrite(output, s->ringbuffer,
- (size_t)s->ringbuffer_size) < 0) {
+ s->partially_written = 0;
+ num_written = BrotliWrite(output, s->ringbuffer,
+ (size_t)s->ringbuffer_size);
+ if (num_written < 0) {
result = BROTLI_RESULT_ERROR;
break;
}
+ s->partially_written += num_written;
+ if (s->partially_written < s->ringbuffer_size) {
+ result = BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ s->state = BROTLI_STATE_BLOCK_POST_WRITE_1;
+ break;
+ }
+ /* Modifications to this code shold be reflected in
+ BROTLI_STATE_BLOCK_POST_WRITE_1 case */
memcpy(s->ringbuffer, s->ringbuffer_end,
(size_t)(s->copy_dst - s->ringbuffer_end));
}
@@ -1431,22 +1487,35 @@
s->copy_length = 0;
}
#endif
-
+ /* Modifications to this loop shold be reflected in
+ BROTLI_STATE_BLOCK_POST_WRITE_2 case */
for (i = 0; i < s->copy_length; ++i) {
s->ringbuffer[pos & s->ringbuffer_mask] =
s->ringbuffer[(pos - s->distance) & s->ringbuffer_mask];
if ((pos & s->ringbuffer_mask) == s->ringbuffer_mask) {
- if (BrotliWrite(output, s->ringbuffer,
- (size_t)s->ringbuffer_size) < 0) {
+ s->partially_written = 0;
+ num_written = BrotliWrite(output, s->ringbuffer,
+ (size_t)s->ringbuffer_size);
+ if (num_written < 0) {
result = BROTLI_RESULT_ERROR;
break;
}
+ s->partially_written += num_written;
+ if (s->partially_written < s->ringbuffer_size) {
+ result = BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ s->state = BROTLI_STATE_BLOCK_POST_WRITE_2;
+ break;
+ }
}
++pos;
--s->meta_block_remaining_len;
}
+ if (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {
+ break;
+ }
}
-
+ /* No break, continue to next state */
+ case BROTLI_STATE_BLOCK_POST_CONTINUE:
/* When we get here, we must have inserted at least one literal and */
/* made a copy of at least length two, therefore accessing the last 2 */
/* bytes is valid. */
@@ -1454,6 +1523,62 @@
s->prev_byte2 = s->ringbuffer[(pos - 2) & s->ringbuffer_mask];
s->state = BROTLI_STATE_BLOCK_BEGIN;
goto BlockBegin;
+ case BROTLI_STATE_BLOCK_INNER_WRITE:
+ case BROTLI_STATE_BLOCK_POST_WRITE_1:
+ case BROTLI_STATE_BLOCK_POST_WRITE_2:
+ num_written = BrotliWrite(
+ output, s->ringbuffer + s->partially_written,
+ (size_t)(s->ringbuffer_size - s->partially_written));
+ if (num_written < 0) {
+ result = BROTLI_RESULT_ERROR;
+ break;
+ }
+ s->partially_written += num_written;
+ if (s->partially_written < s->ringbuffer_size) {
+ result = BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ break;
+ }
+ if (s->state == BROTLI_STATE_BLOCK_POST_WRITE_1) {
+ memcpy(s->ringbuffer, s->ringbuffer_end,
+ (size_t)(s->copy_dst - s->ringbuffer_end));
+ s->state = BROTLI_STATE_BLOCK_POST_CONTINUE;
+ } else if (s->state == BROTLI_STATE_BLOCK_POST_WRITE_2) {
+ /* The tail of "i < s->copy_length" loop. */
+ ++pos;
+ --s->meta_block_remaining_len;
+ ++i;
+ /* Reenter the loop. */
+ for (; i < s->copy_length; ++i) {
+ s->ringbuffer[pos & s->ringbuffer_mask] =
+ s->ringbuffer[(pos - s->distance) & s->ringbuffer_mask];
+ if ((pos & s->ringbuffer_mask) == s->ringbuffer_mask) {
+ s->partially_written = 0;
+ num_written = BrotliWrite(output, s->ringbuffer,
+ (size_t)s->ringbuffer_size);
+ if (num_written < 0) {
+ result = BROTLI_RESULT_ERROR;
+ break;
+ }
+ s->partially_written += num_written;
+ if (s->partially_written < s->ringbuffer_size) {
+ result = BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ break;
+ }
+ }
+ ++pos;
+ --s->meta_block_remaining_len;
+ }
+ if (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {
+ break;
+ }
+ s->state = BROTLI_STATE_BLOCK_POST_CONTINUE;
+ } else { /* BROTLI_STATE_BLOCK_INNER_WRITE */
+ /* The tail of "i < s->insert_length" loop. */
+ ++pos;
+ ++i;
+ s->state = BROTLI_STATE_BLOCK_INNER;
+ }
+ break;
case BROTLI_STATE_METABLOCK_DONE:
if (s->context_modes != 0) {
free(s->context_modes);
@@ -1476,10 +1601,17 @@
break;
case BROTLI_STATE_DONE:
if (s->ringbuffer != 0) {
- if (BrotliWrite(output, s->ringbuffer,
- (size_t)(pos & s->ringbuffer_mask)) < 0) {
+ num_written = BrotliWrite(
+ output, s->ringbuffer + s->partially_written,
+ (size_t)((pos & s->ringbuffer_mask) - s->partially_written));
+ if (num_written < 0) {
result = BROTLI_RESULT_ERROR;
}
+ s->partially_written += num_written;
+ if (s->partially_written < (pos & s->ringbuffer_mask)) {
+ result = BROTLI_RESULT_NEEDS_MORE_OUTPUT;
+ break;
+ }
}
if (!JumpToByteBoundary(&s->br)) {
result = BROTLI_RESULT_ERROR;
diff --git a/dec/decode.h b/dec/decode.h
index 9efd34a..834a7b5 100644
--- a/dec/decode.h
+++ b/dec/decode.h
@@ -32,7 +32,9 @@
/* Successfully completely done */
BROTLI_RESULT_SUCCESS = 1,
/* Partially done, but must be called again with more input */
- BROTLI_RESULT_PARTIAL = 2
+ BROTLI_RESULT_NEEDS_MORE_INPUT = 2,
+ /* Partially done, but must be called again with more output */
+ BROTLI_RESULT_NEEDS_MORE_OUTPUT = 3
} BrotliResult;
/* Sets *decoded_size to the decompressed size of the given encoded stream. */
diff --git a/dec/state.h b/dec/state.h
index aa5b43a..e5d6b5a 100644
--- a/dec/state.h
+++ b/dec/state.h
@@ -40,7 +40,11 @@
BROTLI_STATE_BLOCK_POST = 16,
BROTLI_STATE_UNCOMPRESSED = 17,
BROTLI_STATE_METADATA = 18,
+ BROTLI_STATE_BLOCK_INNER_WRITE = 19,
BROTLI_STATE_METABLOCK_DONE = 20,
+ BROTLI_STATE_BLOCK_POST_WRITE_1 = 21,
+ BROTLI_STATE_BLOCK_POST_WRITE_2 = 22,
+ BROTLI_STATE_BLOCK_POST_CONTINUE = 23,
BROTLI_STATE_HUFFMAN_CODE_0 = 30,
BROTLI_STATE_HUFFMAN_CODE_1 = 31,
BROTLI_STATE_HUFFMAN_CODE_2 = 32,
@@ -52,6 +56,9 @@
BROTLI_STATE_SUB_UNCOMPRESSED_FILL = 52,
BROTLI_STATE_SUB_UNCOMPRESSED_COPY = 53,
BROTLI_STATE_SUB_UNCOMPRESSED_WARMUP = 54,
+ BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_1 = 55,
+ BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_2 = 56,
+ BROTLI_STATE_SUB_UNCOMPRESSED_WRITE_3 = 57,
BROTLI_STATE_SUB_HUFFMAN_LENGTH_BEGIN = 60,
BROTLI_STATE_SUB_HUFFMAN_LENGTH_SYMBOLS = 61,
BROTLI_STATE_SUB_HUFFMAN_DONE = 62,
@@ -133,6 +140,9 @@
/* For CopyUncompressedBlockToOutput */
int nbytes;
+ /* For partial write operations */
+ int partially_written;
+
/* For HuffmanTreeGroupDecode */
int htrees_decoded;
diff --git a/dec/streams.c b/dec/streams.c
index 623d417..b33f7a4 100644
--- a/dec/streams.c
+++ b/dec/streams.c
@@ -51,8 +51,9 @@
int BrotliMemOutputFunction(void* data, const uint8_t* buf, size_t count) {
BrotliMemOutput* output = (BrotliMemOutput*)data;
- if (output->pos + count > output->length) {
- return -1;
+ size_t limit = output->length - output->pos;
+ if (count > limit) {
+ count = limit;
}
memcpy(output->buffer + output->pos, buf, count);
output->pos += count;