Added ZSTD_initCStream_srcSize().
Added relevant test cases in zstreamtest
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 6e52ec8..3d10fbd 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2852,6 +2852,8 @@
ZSTD_cStreamStage stage;
U32 checksum;
U32 frameEnded;
+ U64 pledgedSrcSize;
+ U64 inputProcessed;
ZSTD_parameters params;
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -2909,6 +2911,8 @@
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->frameEnded = 0;
+ zcs->pledgedSrcSize = pledgedSrcSize;
+ zcs->inputProcessed = 0;
return 0; /* ready to go */
}
@@ -2961,6 +2965,12 @@
return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0);
}
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
+{
+ ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
+ return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
+}
+
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
{
return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel);
@@ -3057,6 +3067,7 @@
*srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart;
+ zcs->inputProcessed += *srcSizePtr;
if (zcs->frameEnded) return 0;
{ size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos;
if (hintInSize==0) hintInSize = zcs->blockSize;
@@ -3101,6 +3112,9 @@
BYTE* const oend = (BYTE*)(output->dst) + output->size;
BYTE* op = ostart;
+ if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize))
+ return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */
+
if (zcs->stage != zcss_final) {
/* flush whatever remains */
size_t srcSize = 0;
diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h
index 8296dc6..85f9735 100644
--- a/lib/deprecated/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -27,7 +27,7 @@
* Dependencies
***************************************/
#include <stddef.h> /* size_t */
-#include <zstd.h> /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
+#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */
/* ***************************************************************
@@ -170,7 +170,6 @@
#ifdef ZBUFF_STATIC_LINKING_ONLY
-
#ifndef ZBUFF_STATIC_H_30298098432
#define ZBUFF_STATIC_H_30298098432
@@ -203,7 +202,6 @@
#endif /* ZBUFF_STATIC_H_30298098432 */
-
#endif /* ZBUFF_STATIC_LINKING_ONLY */
diff --git a/lib/zstd.h b/lib/zstd.h
index 44aa995..a4766e3 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -237,20 +237,20 @@
*
* Start a new compression by initializing ZSTD_CStream.
* Use ZSTD_initCStream() to start a new compression operation.
-* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
+* Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section)
*
* Use ZSTD_compressStream() repetitively to consume input stream.
* The function will automatically update both `pos` fields.
* Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data.
* @return : a size hint, preferred nb of bytes to use as input for next function call
-* (it's just a hint, to help latency a little, any other value will work fine)
-* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() )
* or an error code, which can be tested using ZSTD_isError().
+* Note 1 : it's just a hint, to help latency a little, any other value will work fine.
+* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize()
*
-* At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().
+* At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream().
* `output->pos` will be updated.
-* Note some content might still be left within internal buffer if `output->size` is too small.
+* Note that some content might still be left within internal buffer if `output->size` is too small.
* @return : nb of bytes still present within internal buffer (0 if it's empty)
* or an error code, which can be tested using ZSTD_isError().
*
@@ -259,15 +259,15 @@
* The epilogue is required for decoders to consider a frame completed.
* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small.
* In which case, call again ZSTD_endStream() to complete the flush.
-* @return : nb of bytes still present within internal buffer (0 if it's empty)
+* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed)
* or an error code, which can be tested using ZSTD_isError().
*
* *******************************************************************/
-/*===== Streaming compression functions ======*/
typedef struct ZSTD_CStream_s ZSTD_CStream;
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -300,10 +300,10 @@
* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame.
* *******************************************************************************/
-/*===== Streaming decompression functions =====*/
typedef struct ZSTD_DStream_s ZSTD_DStream;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
@@ -490,6 +490,7 @@
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
@@ -602,7 +603,7 @@
Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
This information is not required to properly decode a frame.
- == Special case : skippable frames ==
+ == Special case : skippable frames ==
Skippable frames allow integration of user-defined data into a flow of concatenated frames.
Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
diff --git a/tests/Makefile b/tests/Makefile
index 6110465..d7d2502 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -100,9 +100,9 @@
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
-zbufftest : CFLAGS += -Wno-deprecated-declarations
+zbufftest : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
- $(CC) $(FLAGS) $^ -o $@$(EXT) # flag required to silence deprecation warnings
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
diff --git a/tests/zbufftest.c b/tests/zbufftest.c
index 9dc164e..14b7392 100644
--- a/tests/zbufftest.c
+++ b/tests/zbufftest.c
@@ -28,8 +28,8 @@
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd.h" /* ZSTD_compressBound */
-#define ZBUFF_STATIC_LINKING_ONLY
-#include "zbuff.h" /* ZBUFF_createCCtx_advanced */
+#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
+#include "zbuff.h" /* ZBUFF_isError */
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64_* */
@@ -265,11 +265,11 @@
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
BYTE* cNoiseBuffer[5];
- size_t srcBufferSize = (size_t)1<<maxSrcLog;
+ size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
- size_t copyBufferSize= srcBufferSize + (1<<maxSampleLog);
+ size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
- size_t cBufferSize = ZSTD_compressBound(srcBufferSize);
+ size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index c21b9de..7783fe1 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -28,6 +28,7 @@
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem */
#include "zstd.h" /* ZSTD_compressBound */
+#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH64_* */
@@ -88,16 +89,6 @@
return rand32 >> 5;
}
-/*
-static unsigned FUZ_highbit32(U32 v32)
-{
- unsigned nbBits = 0;
- if (v32==0) return 0;
- for ( ; v32 ; v32>>=1) nbBits++;
- return nbBits;
-}
-*/
-
static void* allocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
@@ -254,6 +245,38 @@
} }
DISPLAYLEVEL(4, "OK \n");
+ /* _srcSize compression test */
+ DISPLAYLEVEL(4, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
+ ZSTD_initCStream_srcSize(zc, 1, CNBufferSize);
+ outBuff.dst = (char*)(compressedBuffer)+cSize;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = CNBufferSize;
+ inBuff.pos = 0;
+ { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+ if (ZSTD_isError(r)) goto _output_error; }
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ if (r != 0) goto _output_error; } /* error, or some data not flushed */
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+
+ /* wrong _srcSize compression test */
+ DISPLAYLEVEL(4, "test%3i : wrong srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
+ ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1);
+ outBuff.dst = (char*)(compressedBuffer)+cSize;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = CNBufferSize;
+ inBuff.pos = 0;
+ { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+ if (ZSTD_isError(r)) goto _output_error; }
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
+ DISPLAYLEVEL(4, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
+
/* Complex context re-use scenario */
DISPLAYLEVEL(4, "test%3i : context re-use : ", testNb++);
ZSTD_freeCStream(zc);
@@ -519,7 +542,7 @@
{ ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
U32 n;
for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) {
- /* compress random chunk into random size dst buffer */
+ /* compress random chunks into randomly sized dst buffers */
{ size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);