first sketch for ZSTD_initCStream_usingCDict()
diff --git a/NEWS b/NEWS
index 27bf47a..2e61186 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,7 @@
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption
New : doc/zstd_manual.html, by Przemyslaw Skibinski
+Improved : slightly better compression ratio at --ultra levels (>= 20)
Changed : zstd_errors.h is now part of include installation
Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 3dbaff5..5409c23 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -122,6 +122,11 @@
return &(ctx->seqStore);
}
+static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
+{
+ return cctx->params;
+}
+
/** ZSTD_checkParams() :
ensure param values remain within authorized range.
@@ -2779,6 +2784,10 @@
}
}
+static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
+ return ZSTD_getParamsFromCCtx(cdict->refContext);
+}
+
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
@@ -2815,7 +2824,8 @@
struct ZSTD_CStream_s {
ZSTD_CCtx* cctx;
- ZSTD_CDict* cdict;
+ ZSTD_CDict* cdictLocal;
+ const ZSTD_CDict* cdict;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
@@ -2858,7 +2868,7 @@
if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->cctx);
- ZSTD_freeCDict(zcs->cdict);
+ ZSTD_freeCDict(zcs->cdictLocal);
ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem);
@@ -2906,15 +2916,25 @@
if (zcs->outBuff == NULL) return ERROR(memory_allocation);
}
- ZSTD_freeCDict(zcs->cdict);
- zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
- if (zcs->cdict == NULL) return ERROR(memory_allocation);
+ ZSTD_freeCDict(zcs->cdictLocal);
+ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
+ if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
+ zcs->cdict = zcs->cdictLocal;
zcs->checksum = params.fParams.checksumFlag > 0;
return ZSTD_resetCStream(zcs, pledgedSrcSize);
}
+/* note : cdict must outlive compression session */
+size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
+{
+ ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
+ size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
+ zcs->cdict = cdict;
+ return initError;
+}
+
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
diff --git a/lib/zstd.h b/lib/zstd.h
index e0d4ce9..89b6c82 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -459,6 +459,7 @@
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
+ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
@@ -468,6 +469,7 @@
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
+ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 8486013..963078d 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -77,7 +77,7 @@
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
-# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
@@ -281,6 +281,28 @@
}
DISPLAYLEVEL(4, "OK \n");
+ /* CDict scenario */
+ DISPLAYLEVEL(4, "test%3i : digested dictionary : ", testNb++);
+ { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 128 KB, 1);
+ size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
+ if (ZSTD_isError(initError)) goto _output_error;
+ cSize = 0;
+ outBuff.dst = compressedBuffer;
+ outBuff.size = compressedBufferSize;
+ outBuff.pos = 0;
+ inBuff.src = CNBuffer;
+ inBuff.size = CNBufferSize;
+ inBuff.pos = 0;
+ { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
+ if (ZSTD_isError(r)) goto _output_error; }
+ if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
+ { size_t const r = ZSTD_endStream(zc, &outBuff);
+ if (r != 0) goto _output_error; } /* error, or some data not flushed */
+ cSize = outBuff.pos;
+ ZSTD_freeCDict(cdict);
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBufferSize*100);
+ }
+
DISPLAYLEVEL(4, "test%3i : check CStream size : ", testNb++);
{ size_t const s = ZSTD_sizeof_CStream(zc);
if (ZSTD_isError(s)) goto _output_error;
@@ -511,7 +533,7 @@
/* multi - fragments decompression test */
if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
- CHECK( ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
+ CHECK (ZSTD_isError(ZSTD_resetDStream(zd)), "ZSTD_resetDStream failed");
} else
ZSTD_initDStream_usingDict(zd, dict, dictSize);
{ size_t decompressionResult = 1;