Merge pull request #2412 from animalize/dict_compressionlevel
use ZSTD_CLEVEL_DEFAULT in zdict.c
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 270a36c..f6faca7 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -4441,26 +4441,42 @@
/* compression stage */
#ifdef ZSTD_MULTITHREAD
if (cctx->appliedParams.nbWorkers > 0) {
- int const forceMaxProgress = (endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
size_t flushMin;
- assert(forceMaxProgress || endOp == ZSTD_e_continue /* Protection for a new flush type */);
if (cctx->cParamsChanged) {
ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams);
cctx->cParamsChanged = 0;
}
- do {
+ for (;;) {
+ size_t const ipos = input->pos;
+ size_t const opos = output->pos;
flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp);
if ( ZSTD_isError(flushMin)
|| (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */
ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
}
FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed");
- } while (forceMaxProgress && flushMin != 0 && output->pos < output->size);
+
+ if (endOp == ZSTD_e_continue) {
+ /* We only require some progress with ZSTD_e_continue, not maximal progress.
+ * We're done if we've consumed or produced any bytes, or either buffer is
+ * full.
+ */
+ if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size)
+ break;
+ } else {
+ assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end);
+ /* We require maximal progress. We're done when the flush is complete or the
+ * output buffer is full.
+ */
+ if (flushMin == 0 || output->pos == output->size)
+ break;
+ }
+ }
DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic");
/* Either we don't require maximum forward progress, we've finished the
* flush, or we are out of output space.
*/
- assert(!forceMaxProgress || flushMin == 0 || output->pos == output->size);
+ assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size);
ZSTD_setBufferExpectations(cctx, output, input);
return flushMin;
}
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index 1855b4d..fa18ea4 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -2299,6 +2299,7 @@
/* Ensure maximal forward progress for determinism */
forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
} while (forwardProgress);
+ assert(inBuff.pos == inBuff.size);
XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 28a53b6..d596802 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -19,13 +19,13 @@
#include <stdlib.h>
#include <stdio.h> /* vsprintf */
#include <stdarg.h> /* va_list, for z_gzprintf */
+#include <string.h>
#define NO_DUMMY_DECL
#define ZLIB_CONST
#include <zlib.h> /* without #define Z_PREFIX */
#include "zstd_zlibwrapper.h"
-#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_isFrame, ZSTD_MAGICNUMBER */
+#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_isFrame, ZSTD_MAGICNUMBER, ZSTD_customMem */
#include "zstd.h"
-#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */
/* === Constants === */
@@ -42,6 +42,45 @@
#define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; }
#define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; }
+/* === Utility === */
+
+#define MIN(x,y) ((x) < (y) ? (x) : (y))
+
+static unsigned ZWRAP_isLittleEndian(void)
+{
+ const union { unsigned u; char c[4]; } one = { 1 }; /* don't use static : performance detrimental */
+ return one.c[0];
+}
+
+#ifndef __has_builtin
+# define __has_builtin(x) 0
+#endif
+
+static unsigned ZWRAP_swap32(unsigned in)
+{
+#if defined(_MSC_VER) /* Visual Studio */
+ return _byteswap_ulong(in);
+#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \
+ || (defined(__clang__) && __has_builtin(__builtin_bswap32))
+ return __builtin_bswap32(in);
+#else
+ return ((in << 24) & 0xff000000 ) |
+ ((in << 8) & 0x00ff0000 ) |
+ ((in >> 8) & 0x0000ff00 ) |
+ ((in >> 24) & 0x000000ff );
+#endif
+}
+
+static unsigned ZWRAP_readLE32(const void* ptr)
+{
+ unsigned value;
+ memcpy(&value, ptr, sizeof(value));
+ if (ZWRAP_isLittleEndian())
+ return value;
+ else
+ return ZWRAP_swap32(value);
+}
+
/* === Wrapper === */
static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
@@ -64,8 +103,6 @@
ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
-
-
static void* ZWRAP_allocFunction(void* opaque, size_t size)
{
z_streamp strm = (z_streamp) opaque;
@@ -81,6 +118,35 @@
/* if (address) LOG_WRAPPERC("ZWRAP free %p \n", address); */
}
+static void* ZWRAP_customMalloc(size_t size, ZSTD_customMem customMem)
+{
+ if (customMem.customAlloc)
+ return customMem.customAlloc(customMem.opaque, size);
+ return malloc(size);
+}
+
+static void* ZWRAP_customCalloc(size_t size, ZSTD_customMem customMem)
+{
+ if (customMem.customAlloc) {
+ /* calloc implemented as malloc+memset;
+ * not as efficient as calloc, but next best guess for custom malloc */
+ void* const ptr = customMem.customAlloc(customMem.opaque, size);
+ memset(ptr, 0, size);
+ return ptr;
+ }
+ return calloc(1, size);
+}
+
+static void ZWRAP_customFree(void* ptr, ZSTD_customMem customMem)
+{
+ if (ptr!=NULL) {
+ if (customMem.customFree)
+ customMem.customFree(customMem.opaque, ptr);
+ else
+ free(ptr);
+ }
+}
+
/* === Compression === */
@@ -107,7 +173,7 @@
{
if (zwc==NULL) return 0; /* support free on NULL */
ZSTD_freeCStream(zwc->zbc);
- ZSTD_customFree(zwc, zwc->customMem);
+ ZWRAP_customFree(zwc, zwc->customMem);
return 0;
}
@@ -115,20 +181,19 @@
static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm)
{
ZWRAP_CCtx* zwc;
-
+ ZSTD_customMem customMem = { NULL, NULL, NULL };
+
if (strm->zalloc && strm->zfree) {
- zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx));
- if (zwc==NULL) return NULL;
- memset(zwc, 0, sizeof(ZWRAP_CCtx));
- memcpy(&zwc->allocFunc, strm, sizeof(z_stream));
- { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
- ZWRAP_customMem.opaque = &zwc->allocFunc;
- zwc->customMem = ZWRAP_customMem;
- }
- } else {
- zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc));
- if (zwc==NULL) return NULL;
+ customMem.customAlloc = ZWRAP_allocFunction;
+ customMem.customFree = ZWRAP_freeFunction;
}
+ customMem.opaque = strm;
+
+ zwc = (ZWRAP_CCtx*)ZWRAP_customCalloc(sizeof(ZWRAP_CCtx), customMem);
+ if (zwc == NULL) return NULL;
+ zwc->allocFunc = *strm;
+ customMem.opaque = &zwc->allocFunc;
+ zwc->customMem = customMem;
return zwc;
}
@@ -457,21 +522,19 @@
static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{
ZWRAP_DCtx* zwd;
- DEBUG_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_HEADERSIZE); /* check static buffer size condition */
+ ZSTD_customMem customMem = { NULL, NULL, NULL };
if (strm->zalloc && strm->zfree) {
- zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx));
- if (zwd==NULL) return NULL;
- memset(zwd, 0, sizeof(ZWRAP_DCtx));
- zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */
- { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL };
- ZWRAP_customMem.opaque = &zwd->allocFunc;
- zwd->customMem = ZWRAP_customMem;
- }
- } else {
- zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd));
- if (zwd==NULL) return NULL;
+ customMem.customAlloc = ZWRAP_allocFunction;
+ customMem.customFree = ZWRAP_freeFunction;
}
+ customMem.opaque = strm;
+
+ zwd = (ZWRAP_DCtx*)ZWRAP_customCalloc(sizeof(ZWRAP_DCtx), customMem);
+ if (zwd == NULL) return NULL;
+ zwd->allocFunc = *strm;
+ customMem.opaque = &zwd->allocFunc;
+ zwd->customMem = customMem;
ZWRAP_initDCtx(zwd);
return zwd;
@@ -481,8 +544,8 @@
{
if (zwd==NULL) return 0; /* support free on null */
ZSTD_freeDStream(zwd->zbd);
- ZSTD_customFree(zwd->version, zwd->customMem);
- ZSTD_customFree(zwd, zwd->customMem);
+ ZWRAP_customFree(zwd->version, zwd->customMem);
+ ZWRAP_customFree(zwd, zwd->customMem);
return 0;
}
@@ -524,7 +587,7 @@
LOG_WRAPPERD("- inflateInit\n");
if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
- zwd->version = (char*)ZSTD_customMalloc(strlen(version)+1, zwd->customMem);
+ zwd->version = (char*)ZWRAP_customMalloc(strlen(version)+1, zwd->customMem);
if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
strcpy(zwd->version, version);
@@ -670,7 +733,7 @@
if (zwd->totalInBytes < ZLIB_HEADERSIZE) {
if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
- if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
+ if (ZWRAP_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
{ int const initErr = (zwd->windowBits) ?
inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) :
inflateInit_(strm, zwd->version, zwd->stream_size);
@@ -698,7 +761,7 @@
strm->avail_in -= srcSize;
if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK;
- if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
+ if (ZWRAP_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
z_stream strm2;
strm2.next_in = strm->next_in;
strm2.avail_in = strm->avail_in;