changed streaming decoder behavior : now, when all compressed frame is consumed, it means decompression is completed, with regenerated data fully flushed.
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 6acb259..c6bb532 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1286,6 +1286,7 @@
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
+ U32 hostageByte;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */
@@ -1349,6 +1350,7 @@
zds->dictSize = dictSize;
}
zds->legacyVersion = 0;
+ zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix;
}
@@ -1371,11 +1373,11 @@
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{
- return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize;
+ return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize + zds->dictSize;
}
-/* *** Decompression *** */
+/* ***** Decompression ***** */
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
@@ -1445,7 +1447,7 @@
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
- /* Frame header instruct buffer sizes */
+ /* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize;
zds->blockSize = blockSize;
@@ -1479,7 +1481,7 @@
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
- zds->outEnd = zds->outStart + decodedSize;
+ zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush;
break;
}
@@ -1522,7 +1524,7 @@
zds->outStart = zds->outEnd = 0;
break;
}
- /* cannot flush everything */
+ /* cannot complete flush */
someMoreWork = 0;
break;
}
@@ -1533,8 +1535,21 @@
input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd);
- if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */
- nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block);
+ if (!nextSrcSizeHint) { /* frame fully decoded */
+ if (zds->outEnd == zds->outStart) { /* output fully flushed */
+ if (zds->hostageByte) {
+ if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */
+ input->pos++; /* release hostage */
+ }
+ return 0;
+ }
+ if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
+ input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
+ zds->hostageByte=1;
+ }
+ return 1;
+ }
+ nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint;
diff --git a/lib/zstd.h b/lib/zstd.h
index 10312fe..5cc40c6 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -252,15 +252,13 @@
*
* Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update both `pos` fields.
-* If `input.pos < input.size`, some input is not consumed.
+* If `input.pos < input.size`, some input has not been consumed.
* It's up to the caller to present again remaining data.
-* If `output.pos == output.size`, there is probably some more data to flush, still stored inside internal buffers.
+* If `output.pos < output.size`, decoder has flushed everything it could.
* @return : 0 when a frame is completely decoded and fully flushed,
* an error code, which can be tested using ZSTD_isError(),
-* any value > 0, which means there is still some work to do to complete the frame.
-* In general, the return value is a suggested next input size (merely a hint, to help latency).
-* 1 is a special value, which means either "there is still some data to flush", or "need 1 more byte as input".
-* In which case, start by flushing. When flush is completed, if return value is still `1`, it means "need 1 more byte".
+* any other value > 0, which means there is still some work to do to complete the frame.
+* The return value is a suggested next input size (just an hint, to help latency).
* *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 64b3fd9..21e98bf 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -45,7 +45,9 @@
$ECHO "\n**** simple tests **** "
./datagen > tmp
+$ECHO "test : basic compression "
$ZSTD -f tmp # trivial compression case, creates tmp.zst
+$ECHO "test : basic decompression"
$ZSTD -df tmp.zst # trivial decompression case (overwrites tmp)
$ECHO "test : too large compression level (must fail)"
$ZSTD -99 -f tmp # too large compression level, automatic sized down
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 3053638..97fbaa1 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -58,7 +58,7 @@
if ((FUZ_GetClockSpan(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = clock(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stdout); } }
-static const clock_t g_refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
static clock_t g_displayClock = 0;
static clock_t g_clockTime = 0;
@@ -118,8 +118,7 @@
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
{
- int testResult = 0;
- size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
+ size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize);
size_t const skippableFrameSize = 11;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
@@ -127,6 +126,7 @@
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
size_t cSize;
+ int testResult = 0;
U32 testNb=0;
ZSTD_CStream* zc = ZSTD_createCStream_advanced(customMem);
ZSTD_DStream* zd = ZSTD_createDStream_advanced(customMem);
@@ -437,7 +437,7 @@
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
- dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
+ dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
/* random dictionary selection */
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
@@ -446,7 +446,7 @@
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
{ size_t const initError = ZSTD_initCStream_advanced(zc, dict, dictSize, params, 0);
- CHECK (ZSTD_isError(initError),"init error : %s", ZSTD_getErrorName(initError));
+ CHECK (ZSTD_isError(initError),"ZSTD_initCStream_advanced error : %s", ZSTD_getErrorName(initError));
} } }
/* multi-segments compression test */