Merge remote-tracking branch 'refs/remotes/facebook/dev' into zlibWrapper
diff --git a/.travis.yml b/.travis.yml
index 7a3664a..ff6ab0f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@
packages:
- gcc-4.8
- g++-4.8
- env: PLATFORM="Ubuntu 12.04 container" CMD="make -C tests test-zstd_nolegacy && make clean && make zlibwrapper && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
+ env: PLATFORM="Ubuntu 12.04 container" CMD="make zlibwrapper && make clean && make -C tests test-zstd_nolegacy && make clean && make clean && make cmaketest && make clean && make -C contrib/pzstd pzstd && make -C contrib/pzstd googletest && make -C contrib/pzstd test && make -C contrib/pzstd clean"
- os: linux
sudo: false
env: PLATFORM="Ubuntu 12.04 container" CMD="make usan"
@@ -55,14 +55,15 @@
packages:
- libc6-dev-i386
- gcc-multilib
+ # Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
+ dist: trusty
sudo: required
- env: PLATFORM="Ubuntu 12.04" CMD="make -C tests valgrindTest"
+ env: PLATFORM="Ubuntu 14.04" CMD='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
addons:
apt:
packages:
- valgrind
- # Ubuntu 14.04 LTS Server Edition 64 bit
- os: linux
dist: trusty
sudo: required
@@ -90,7 +91,7 @@
- os: linux
dist: trusty
sudo: required
- env: PLATFORM="Ubuntu 14.04" CMD="make zlibwrapper && make clean && make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
+ env: PLATFORM="Ubuntu 14.04" CMD="make gcc5test && make clean && make gcc6test && sudo apt-get install -y -q qemu-system-ppc binfmt-support qemu-user-static gcc-powerpc-linux-gnu && make clean && make ppctest"
addons:
apt:
sources:
diff --git a/Makefile b/Makefile
index 2e5eada..ac0c583 100644
--- a/Makefile
+++ b/Makefile
@@ -34,8 +34,7 @@
cp $(PRGDIR)/zstd .
zlibwrapper:
- $(MAKE) -C $(ZSTDDIR) all
- $(MAKE) -C $(ZWRAPDIR) all
+ $(MAKE) -C $(ZWRAPDIR) test
test:
$(MAKE) -C $(TESTDIR) $@
diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore
index bf3f3d8..f197c8c 100644
--- a/zlibWrapper/.gitignore
+++ b/zlibWrapper/.gitignore
@@ -26,4 +26,4 @@
# Misc files
*.bat
*.zip
-examples/example2.c
+*.txt
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index 9ad1c01..ea025b9 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -1,54 +1,71 @@
# Makefile for example of using zstd wrapper for zlib
#
-# make - compiles statically and dynamically linked examples/example.c
-# make test testdll - compiles and runs statically and dynamically linked examples/example.c
-# make LOC=-DZWRAP_USE_ZSTD=1 - compiles statically and dynamically linked examples/example.c with zstd compression turned on
+# make - compiles examples
+# make LOC=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on
+# make test - runs examples
# Paths to static and dynamic zlib and zstd libraries
-# Use "make ZLIBDIR=path/to/zlib" to select a path to library
-ifdef ZLIBDIR
-STATICLIB = $(ZLIBDIR)/libz.a ../lib/libzstd.a
-IMPLIB = $(ZLIBDIR)/libz.dll.a ../lib/libzstd.a
-else
-STATICLIB = -static -lz ../lib/libzstd.a
-IMPLIB = -lz ../lib/libzstd.a
-endif
+# Use "make ZLIB_LIBRARY=path/to/zlib" to select a path to library
+ZLIB_LIBRARY ?= -lz
+ZSTDLIBDIR = ../lib
+ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
ZLIBWRAPPER_PATH = .
EXAMPLE_PATH = examples
+PROGRAMS_PATH = ../programs
CC ?= gcc
-CFLAGS = $(LOC) -I../lib -I../lib/common -I$(ZLIBDIR) -I$(ZLIBWRAPPER_PATH) -O3 -std=gnu90
+CFLAGS ?= -O3
+CFLAGS += $(LOC) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) -std=gnu90
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
LDFLAGS = $(LOC)
RM = rm -f
-all: clean test testzstd
+all: clean fitblk example zwrapbench
-test: example
+test: example fitblk example_zstd fitblk_zstd zwrapbench
./example
-
-testdll: example_d
- ./example_d
-
-testzstd: example_zstd
./example_zstd
+ ./fitblk 10240 <../zstd_compression_format.md
+ ./fitblk 40960 <../zstd_compression_format.md
+ ./fitblk_zstd 10240 <../zstd_compression_format.md
+ ./fitblk_zstd 40960 <../zstd_compression_format.md
+ ./zwrapbench -qb1e5 ../zstd_compression_format.md
+ ./zwrapbench -qb1e5B1K ../zstd_compression_format.md
+
+#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so
+valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1
+valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench
+ @echo "\n ---- valgrind tests ----"
+ $(VALGRIND) ./example
+ $(VALGRIND) ./example_zstd
+ $(VALGRIND) ./fitblk 10240 <../zstd_compression_format.md
+ $(VALGRIND) ./fitblk 40960 <../zstd_compression_format.md
+ $(VALGRIND) ./fitblk_zstd 10240 <../zstd_compression_format.md
+ $(VALGRIND) ./fitblk_zstd 40960 <../zstd_compression_format.md
+ $(VALGRIND) ./zwrapbench -qb1e5 ../zstd_compression_format.md
+ $(VALGRIND) ./zwrapbench -qb1e5B1K ../zstd_compression_format.md
.c.o:
$(CC) $(CFLAGS) -c -o $@ $<
+
+example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
+ $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
-example: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
- $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(STATICLIB)
+example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY)
+ $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
-example_d: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o
- $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(IMPLIB)
+fitblk: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
+ $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
-example_zstd: $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o
- $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/example.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(STATICLIB)
+fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(ZSTDLIBRARY)
+ $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/fitblk.o $(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
-$(EXAMPLE_PATH)/example.o: $(EXAMPLE_PATH)/example.c
- $(CC) $(CFLAGS) -I. -c -o $@ $(EXAMPLE_PATH)/example.c
+zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY)
+ $(CC) $(LDFLAGS) -o $@ $(EXAMPLE_PATH)/zwrapbench.o $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) $(ZLIB_LIBRARY)
+
+$(EXAMPLE_PATH)/zwrapbench.o: $(EXAMPLE_PATH)/zwrapbench.c
$(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
@@ -56,6 +73,12 @@
$(ZLIBWRAPPER_PATH)/zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h
$(CC) $(CFLAGS) -DZWRAP_USE_ZSTD=1 -I. -c -o $@ $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c
+$(ZSTDLIBDIR)/libzstd.a:
+ $(MAKE) -C $(ZSTDLIBDIR) all
+
+$(ZSTDLIBDIR)/libzstd.so:
+ $(MAKE) -C $(ZSTDLIBDIR) all
+
clean:
- -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_d example_zstd
+ -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o *.exe foo.gz example example_zstd fitblk fitblk_zstd zwrapbench
@echo Cleaning completed
diff --git a/zlibWrapper/README.md b/zlibWrapper/README.md
index 3a39f00..70cfb0e 100644
--- a/zlibWrapper/README.md
+++ b/zlibWrapper/README.md
@@ -23,10 +23,10 @@
To compile the zstd wrapper with your project you have to do the following:
- change all references with ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```
-- compile your project with zlib_wrapper.c and a static or dynamic zstd library
+- compile your project with `zstd_zlibwrapper.c` and a static or dynamic zstd library
The linking should be changed to:
-```gcc project.o zlib_wrapper.o -lz -lzstd```
+```gcc project.o zstd_zlibwrapper.o -lz -lzstd```
#### Enabling zstd compression within your project
@@ -34,8 +34,61 @@
After embedding the zstd wrapper within your project the zstd library is turned off by default.
Your project should work as before with zlib. There are two options to enable zstd compression:
- compilation with ```-DZWRAP_USE_ZSTD=1``` (or using ```#define ZWRAP_USE_ZSTD 1``` before ```#include "zstd_zlibwrapper.h"```)
-- using the ```void useZSTD(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```)
-There is no switch for zstd decompression because zlib and zstd streams are automatically detected and decompressed using a proper library.
+- using the ```void ZWRAP_useZSTDcompression(int turn_on)``` function (declared in ```#include "zstd_zlibwrapper.h"```)
+
+During decompression zlib and zstd streams are automatically detected and decompressed using a proper library.
+This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)` what will make zlib decompression slightly faster.
+
+
+#### The measurement of performace of Zstandard wrapper for zlib
+
+The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper.
+The benchmark is conducted using given filenames or synthetic data if filenames are not provided.
+The files are read into memory and joined together.
+It makes benchmark more precise as it eliminates I/O overhead.
+Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option.
+One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels.
+With `-B` option bigger files can be divided into smaller, independently compressed blocks.
+The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Makefile](Makefile).
+
+
+#### Improving speed of streaming compression
+
+During streaming compression the compressor never knows how big is data to compress.
+Zstandard compression can be improved by providing size of source data to the compressor. By default streaming compressor assumes that data is bigger than 256 KB but it can hurt compression speed on smaller data.
+The zstd wrapper provides the `ZWRAP_setPledgedSrcSize()` function that allows to change a pledged source size for a given compression stream.
+The function will change zstd compression parameters what may improve compression speed and/or ratio.
+It should be called just after `deflateInit()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` immediately followed by `deflate(strm, Z_FINISH)`
+as this case is automatically detected.
+
+
+#### Reusing contexts
+
+The ordinary zlib compression of two files/streams allocates two contexts:
+- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
+- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `defalateEnd`
+
+The speed of compression can be improved with reusing a single context with following steps:
+- initialize the context with `deflateInit`
+- for the 1st file call `deflate`, `...`, `deflate`
+- for the 2nd file call `deflateReset`, `deflate`, `...`, `deflate`
+- free the context with `deflateEnd`
+
+To check the difference we made experiments using `zwrapbench` with zstd and zlib compression (both at level 3) with 4 KB blocks.
+The input data was git repository downloaded from https://github.com/git/git/archive/master.zip and converted to uncompressed tarball.
+The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd.
+In our example (the last 2 lines) is gives 15% better compression speed and 6% better decompression speed.
+
+| Compression type | Compression | Decompress.| Compr. size | Ratio |
+| ------------------------------------------------- | ------------| -----------| ----------- | ----- |
+| zlib 1.2.8 | 54.77 MB/s | 176.2 MB/s | 8928115 | 2.910 |
+| zlib 1.2.8 not reusing a context | 54.98 MB/s | 174.6 MB/s | 8928115 | 2.910 |
+| zlib 1.2.8 with zlibWrapper and reusing a context | 55.16 MB/s | 176.8 MB/s | 8928115 | 2.910 |
+| zlib 1.2.8 with zlibWrapper not reusing a context | 54.11 MB/s | 174.5 MB/s | 8928115 | 2.910 |
+| zstd 1.1.0 using ZSTD_CCtx | 108.03 MB/s | 319.8 MB/s | 8962336 | 2.899 |
+| zstd 1.1.0 using ZSTD_CStream | 107.34 MB/s | 307.3 MB/s | 8981368 | 2.893 |
+| zstd 1.1.0 with zlibWrapper and reusing a context | 107.52 MB/s | 297.3 MB/s | 8981368 | 2.893 |
+| zstd 1.1.0 with zlibWrapper not reusing a context | 91.45 MB/s | 279.8 MB/s | 8981368 | 2.893 |
#### Example
@@ -52,7 +105,7 @@
inflate with dictionary: hello, hello!
```
Then we have changed ```#include "zlib.h"``` to ```#include "zstd_zlibwrapper.h"```, compiled the [example.c](examples/example.c) file
-with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zlib_wrapper.o -lzstd```.
+with ```-DZWRAP_USE_ZSTD=1``` and linked with additional ```zstd_zlibwrapper.o -lzstd```.
We were forced to turn off the following functions: ```test_gzio```, ```test_flush```, ```test_sync``` which use currently unsupported features.
After running it shows the following results:
```
@@ -66,17 +119,20 @@
#### Compatibility issues
-After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they print error message and return an error value.
+After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into strm->msg and return Z_STREAM_ERROR.
Supported methods:
- deflateInit
-- deflate (with exception of Z_FULL_FLUSH)
+- deflate (with exception of Z_FULL_FLUSH, Z_BLOCK, and Z_TREES)
- deflateSetDictionary
- deflateEnd
+- deflateReset
- deflateBound
- inflateInit
- inflate
- inflateSetDictionary
+- inflateReset
+- inflateReset2
- compress
- compress2
- compressBound
@@ -88,15 +144,13 @@
Unsupported methods:
- gzip file access functions
- deflateCopy
-- deflateReset
- deflateTune
- deflatePending
- deflatePrime
- deflateSetHeader
- inflateGetDictionary
- inflateCopy
-- inflateReset
-- inflateReset2
+- inflateSync
- inflatePrime
- inflateMark
- inflateGetHeader
diff --git a/zlibWrapper/examples/example.c b/zlibWrapper/examples/example.c
index bbb2cd5..20ed81d 100644
--- a/zlibWrapper/examples/example.c
+++ b/zlibWrapper/examples/example.c
@@ -45,12 +45,12 @@
} \
}
-z_const char hello[] = "hello, hello!";
+z_const char hello[] = "hello, hello! I said hello, hello!";
/* "hello world" would be more standard, but the repeated "hello"
* stresses the compression code better, sorry...
*/
-const char dictionary[] = "hello";
+const char dictionary[] = "hello, hello!";
uLong dictId; /* Adler32 value of the dictionary */
void test_deflate OF((Byte *compr, uLong comprLen));
@@ -156,7 +156,7 @@
fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err));
exit(1);
}
- if (gzprintf(file, ", %s!", "hello") != 8) {
+ if (gzprintf(file, ", %s! I said hello, hello!", "hello") != 8+21) {
fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err));
exit(1);
}
@@ -182,7 +182,7 @@
}
pos = gzseek(file, -8L, SEEK_CUR);
- if (pos != 6 || gztell(file) != pos) {
+ if (pos != 6+21 || gztell(file) != pos) {
fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n",
(long)pos, (long)gztell(file));
exit(1);
@@ -203,7 +203,7 @@
fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err));
exit(1);
}
- if (strcmp((char*)uncompr, hello + 6)) {
+ if (strcmp((char*)uncompr, hello + 6+21)) {
fprintf(stderr, "bad gzgets after gzseek\n");
exit(1);
} else {
@@ -583,7 +583,7 @@
printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n",
ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags());
- if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion());
+ if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
compr = (Byte*)calloc((uInt)comprLen, 1);
uncompr = (Byte*)calloc((uInt)uncomprLen, 1);
@@ -600,7 +600,7 @@
#else
test_compress(compr, comprLen, uncompr, uncomprLen);
- if (!isUsingZSTD())
+ if (!ZWRAP_isUsingZSTDcompression())
test_gzio((argc > 1 ? argv[1] : TESTFILE),
uncompr, uncomprLen);
#endif
@@ -611,7 +611,7 @@
test_large_deflate(compr, comprLen, uncompr, uncomprLen);
test_large_inflate(compr, comprLen, uncompr, uncomprLen);
- if (!isUsingZSTD()) {
+ if (!ZWRAP_isUsingZSTDcompression()) {
test_flush(compr, &comprLen);
test_sync(compr, comprLen, uncompr, uncomprLen);
}
diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c
new file mode 100644
index 0000000..e3fda3c
--- /dev/null
+++ b/zlibWrapper/examples/fitblk.c
@@ -0,0 +1,253 @@
+/* fitblk.c: example of fitting compressed output to a specified size
+ Not copyrighted -- provided to the public domain
+ Version 1.1 25 November 2004 Mark Adler */
+
+/* Version history:
+ 1.0 24 Nov 2004 First version
+ 1.1 25 Nov 2004 Change deflateInit2() to deflateInit()
+ Use fixed-size, stack-allocated raw buffers
+ Simplify code moving compression to subroutines
+ Use assert() for internal errors
+ Add detailed description of approach
+ */
+
+/* Approach to just fitting a requested compressed size:
+
+ fitblk performs three compression passes on a portion of the input
+ data in order to determine how much of that input will compress to
+ nearly the requested output block size. The first pass generates
+ enough deflate blocks to produce output to fill the requested
+ output size plus a specfied excess amount (see the EXCESS define
+ below). The last deflate block may go quite a bit past that, but
+ is discarded. The second pass decompresses and recompresses just
+ the compressed data that fit in the requested plus excess sized
+ buffer. The deflate process is terminated after that amount of
+ input, which is less than the amount consumed on the first pass.
+ The last deflate block of the result will be of a comparable size
+ to the final product, so that the header for that deflate block and
+ the compression ratio for that block will be about the same as in
+ the final product. The third compression pass decompresses the
+ result of the second step, but only the compressed data up to the
+ requested size minus an amount to allow the compressed stream to
+ complete (see the MARGIN define below). That will result in a
+ final compressed stream whose length is less than or equal to the
+ requested size. Assuming sufficient input and a requested size
+ greater than a few hundred bytes, the shortfall will typically be
+ less than ten bytes.
+
+ If the input is short enough that the first compression completes
+ before filling the requested output size, then that compressed
+ stream is return with no recompression.
+
+ EXCESS is chosen to be just greater than the shortfall seen in a
+ two pass approach similar to the above. That shortfall is due to
+ the last deflate block compressing more efficiently with a smaller
+ header on the second pass. EXCESS is set to be large enough so
+ that there is enough uncompressed data for the second pass to fill
+ out the requested size, and small enough so that the final deflate
+ block of the second pass will be close in size to the final deflate
+ block of the third and final pass. MARGIN is chosen to be just
+ large enough to assure that the final compression has enough room
+ to complete in all cases.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+//#include "zlib.h"
+#include "zstd_zlibwrapper.h"
+
+#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/
+#define local static
+
+/* print nastygram and leave */
+local void quit(char *why)
+{
+ fprintf(stderr, "fitblk abort: %s\n", why);
+ exit(1);
+}
+
+#define RAWLEN 4096 /* intermediate uncompressed buffer size */
+
+/* compress from file to def until provided buffer is full or end of
+ input reached; return last deflate() return value, or Z_ERRNO if
+ there was read error on the file */
+local int partcompress(FILE *in, z_streamp def)
+{
+ int ret, flush;
+ unsigned char raw[RAWLEN];
+
+ flush = Z_SYNC_FLUSH;
+ do {
+ def->avail_in = fread(raw, 1, RAWLEN, in);
+ if (ferror(in))
+ return Z_ERRNO;
+ def->next_in = raw;
+ if (feof(in))
+ flush = Z_FINISH;
+ LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+ ret = deflate(def, flush);
+ LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+ assert(ret != Z_STREAM_ERROR);
+ } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH);
+ return ret;
+}
+
+/* recompress from inf's input to def's output; the input for inf and
+ the output for def are set in those structures before calling;
+ return last deflate() return value, or Z_MEM_ERROR if inflate()
+ was not able to allocate enough memory when it needed to */
+local int recompress(z_streamp inf, z_streamp def)
+{
+ int ret, flush;
+ unsigned char raw[RAWLEN];
+
+ flush = Z_NO_FLUSH;
+ do {
+ /* decompress */
+ inf->avail_out = RAWLEN;
+ inf->next_out = raw;
+ LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
+ ret = inflate(inf, Z_NO_FLUSH);
+ LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out);
+ assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
+ ret != Z_NEED_DICT);
+ if (ret == Z_MEM_ERROR)
+ return ret;
+
+ /* compress what was decompresed until done or no room */
+ def->avail_in = RAWLEN - inf->avail_out;
+ def->next_in = raw;
+ if (inf->avail_out != 0)
+ flush = Z_FINISH;
+ LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+ ret = deflate(def, flush);
+ LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out);
+ assert(ret != Z_STREAM_ERROR);
+ } while (ret != Z_STREAM_END && def->avail_out != 0);
+ return ret;
+}
+
+#define EXCESS 256 /* empirically determined stream overage */
+#define MARGIN 8 /* amount to back off for completion */
+
+/* compress from stdin to fixed-size block on stdout */
+int main(int argc, char **argv)
+{
+ int ret; /* return code */
+ unsigned size; /* requested fixed output block size */
+ unsigned have; /* bytes written by deflate() call */
+ unsigned char *blk; /* intermediate and final stream */
+ unsigned char *tmp; /* close to desired size stream */
+ z_stream def, inf; /* zlib deflate and inflate states */
+
+ /* get requested output size */
+ if (argc != 2)
+ quit("need one argument: size of output block");
+ ret = strtol(argv[1], argv + 1, 10);
+ if (argv[1][0] != 0)
+ quit("argument must be a number");
+ if (ret < 8) /* 8 is minimum zlib stream size */
+ quit("need positive size of 8 or greater");
+ size = (unsigned)ret;
+
+ printf("zlib version %s\n", ZLIB_VERSION);
+ if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion());
+
+ /* allocate memory for buffers and compression engine */
+ blk = malloc(size + EXCESS);
+ def.zalloc = Z_NULL;
+ def.zfree = Z_NULL;
+ def.opaque = Z_NULL;
+ ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
+ if (ret != Z_OK || blk == NULL)
+ quit("out of memory");
+ ret = ZWRAP_setPledgedSrcSize(&def, 1<<16);
+ if (ret != Z_OK)
+ quit("ZWRAP_setPledgedSrcSize");
+
+ /* compress from stdin until output full, or no more input */
+ def.avail_out = size + EXCESS;
+ def.next_out = blk;
+ LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
+ ret = partcompress(stdin, &def);
+ printf("partcompress total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out);
+ if (ret == Z_ERRNO)
+ quit("error reading input");
+
+ /* if it all fit, then size was undersubscribed -- done! */
+ if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
+ /* write block to stdout */
+ have = size + EXCESS - def.avail_out;
+ // if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
+ // quit("error writing output");
+
+ /* clean up and print results to stderr */
+ ret = deflateEnd(&def);
+ assert(ret != Z_STREAM_ERROR);
+ free(blk);
+ fprintf(stderr,
+ "%u bytes unused out of %u requested (all input)\n",
+ size - have, size);
+ return 0;
+ }
+
+ /* it didn't all fit -- set up for recompression */
+ inf.zalloc = Z_NULL;
+ inf.zfree = Z_NULL;
+ inf.opaque = Z_NULL;
+ inf.avail_in = 0;
+ inf.next_in = Z_NULL;
+ ret = inflateInit(&inf);
+ tmp = malloc(size + EXCESS);
+ if (ret != Z_OK || tmp == NULL)
+ quit("out of memory");
+ ret = deflateReset(&def);
+ assert(ret != Z_STREAM_ERROR);
+
+ /* do first recompression close to the right amount */
+ inf.avail_in = size + EXCESS;
+ inf.next_in = blk;
+ def.avail_out = size + EXCESS;
+ def.next_out = tmp;
+ LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
+ ret = recompress(&inf, &def);
+ LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
+ if (ret == Z_MEM_ERROR)
+ quit("out of memory");
+
+ /* set up for next reocmpression */
+ ret = inflateReset(&inf);
+ assert(ret != Z_STREAM_ERROR);
+ ret = deflateReset(&def);
+ assert(ret != Z_STREAM_ERROR);
+
+ /* do second and final recompression (third compression) */
+ inf.avail_in = size - MARGIN; /* assure stream will complete */
+ inf.next_in = tmp;
+ def.avail_out = size;
+ def.next_out = blk;
+ LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
+ ret = recompress(&inf, &def);
+ LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out);
+ if (ret == Z_MEM_ERROR)
+ quit("out of memory");
+ assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
+
+ /* done -- write block to stdout */
+ have = size - def.avail_out;
+// if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
+// quit("error writing output");
+
+ /* clean up and print results to stderr */
+ free(tmp);
+ ret = inflateEnd(&inf);
+ assert(ret != Z_STREAM_ERROR);
+ ret = deflateEnd(&def);
+ assert(ret != Z_STREAM_ERROR);
+ free(blk);
+ fprintf(stderr,
+ "%u bytes unused out of %u requested (%lu input)\n",
+ size - have, size, def.total_in);
+ return 0;
+}
diff --git a/zlibWrapper/examples/fitblk_original.c b/zlibWrapper/examples/fitblk_original.c
new file mode 100644
index 0000000..c61de5c
--- /dev/null
+++ b/zlibWrapper/examples/fitblk_original.c
@@ -0,0 +1,233 @@
+/* fitblk.c: example of fitting compressed output to a specified size
+ Not copyrighted -- provided to the public domain
+ Version 1.1 25 November 2004 Mark Adler */
+
+/* Version history:
+ 1.0 24 Nov 2004 First version
+ 1.1 25 Nov 2004 Change deflateInit2() to deflateInit()
+ Use fixed-size, stack-allocated raw buffers
+ Simplify code moving compression to subroutines
+ Use assert() for internal errors
+ Add detailed description of approach
+ */
+
+/* Approach to just fitting a requested compressed size:
+
+ fitblk performs three compression passes on a portion of the input
+ data in order to determine how much of that input will compress to
+ nearly the requested output block size. The first pass generates
+ enough deflate blocks to produce output to fill the requested
+ output size plus a specfied excess amount (see the EXCESS define
+ below). The last deflate block may go quite a bit past that, but
+ is discarded. The second pass decompresses and recompresses just
+ the compressed data that fit in the requested plus excess sized
+ buffer. The deflate process is terminated after that amount of
+ input, which is less than the amount consumed on the first pass.
+ The last deflate block of the result will be of a comparable size
+ to the final product, so that the header for that deflate block and
+ the compression ratio for that block will be about the same as in
+ the final product. The third compression pass decompresses the
+ result of the second step, but only the compressed data up to the
+ requested size minus an amount to allow the compressed stream to
+ complete (see the MARGIN define below). That will result in a
+ final compressed stream whose length is less than or equal to the
+ requested size. Assuming sufficient input and a requested size
+ greater than a few hundred bytes, the shortfall will typically be
+ less than ten bytes.
+
+ If the input is short enough that the first compression completes
+ before filling the requested output size, then that compressed
+ stream is return with no recompression.
+
+ EXCESS is chosen to be just greater than the shortfall seen in a
+ two pass approach similar to the above. That shortfall is due to
+ the last deflate block compressing more efficiently with a smaller
+ header on the second pass. EXCESS is set to be large enough so
+ that there is enough uncompressed data for the second pass to fill
+ out the requested size, and small enough so that the final deflate
+ block of the second pass will be close in size to the final deflate
+ block of the third and final pass. MARGIN is chosen to be just
+ large enough to assure that the final compression has enough room
+ to complete in all cases.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "zlib.h"
+
+#define local static
+
+/* print nastygram and leave */
+local void quit(char *why)
+{
+ fprintf(stderr, "fitblk abort: %s\n", why);
+ exit(1);
+}
+
+#define RAWLEN 4096 /* intermediate uncompressed buffer size */
+
+/* compress from file to def until provided buffer is full or end of
+ input reached; return last deflate() return value, or Z_ERRNO if
+ there was read error on the file */
+local int partcompress(FILE *in, z_streamp def)
+{
+ int ret, flush;
+ unsigned char raw[RAWLEN];
+
+ flush = Z_NO_FLUSH;
+ do {
+ def->avail_in = fread(raw, 1, RAWLEN, in);
+ if (ferror(in))
+ return Z_ERRNO;
+ def->next_in = raw;
+ if (feof(in))
+ flush = Z_FINISH;
+ ret = deflate(def, flush);
+ assert(ret != Z_STREAM_ERROR);
+ } while (def->avail_out != 0 && flush == Z_NO_FLUSH);
+ return ret;
+}
+
+/* recompress from inf's input to def's output; the input for inf and
+ the output for def are set in those structures before calling;
+ return last deflate() return value, or Z_MEM_ERROR if inflate()
+ was not able to allocate enough memory when it needed to */
+local int recompress(z_streamp inf, z_streamp def)
+{
+ int ret, flush;
+ unsigned char raw[RAWLEN];
+
+ flush = Z_NO_FLUSH;
+ do {
+ /* decompress */
+ inf->avail_out = RAWLEN;
+ inf->next_out = raw;
+ ret = inflate(inf, Z_NO_FLUSH);
+ assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR &&
+ ret != Z_NEED_DICT);
+ if (ret == Z_MEM_ERROR)
+ return ret;
+
+ /* compress what was decompresed until done or no room */
+ def->avail_in = RAWLEN - inf->avail_out;
+ def->next_in = raw;
+ if (inf->avail_out != 0)
+ flush = Z_FINISH;
+ ret = deflate(def, flush);
+ assert(ret != Z_STREAM_ERROR);
+ } while (ret != Z_STREAM_END && def->avail_out != 0);
+ return ret;
+}
+
+#define EXCESS 256 /* empirically determined stream overage */
+#define MARGIN 8 /* amount to back off for completion */
+
+/* compress from stdin to fixed-size block on stdout */
+int main(int argc, char **argv)
+{
+ int ret; /* return code */
+ unsigned size; /* requested fixed output block size */
+ unsigned have; /* bytes written by deflate() call */
+ unsigned char *blk; /* intermediate and final stream */
+ unsigned char *tmp; /* close to desired size stream */
+ z_stream def, inf; /* zlib deflate and inflate states */
+
+ /* get requested output size */
+ if (argc != 2)
+ quit("need one argument: size of output block");
+ ret = strtol(argv[1], argv + 1, 10);
+ if (argv[1][0] != 0)
+ quit("argument must be a number");
+ if (ret < 8) /* 8 is minimum zlib stream size */
+ quit("need positive size of 8 or greater");
+ size = (unsigned)ret;
+
+ /* allocate memory for buffers and compression engine */
+ blk = malloc(size + EXCESS);
+ def.zalloc = Z_NULL;
+ def.zfree = Z_NULL;
+ def.opaque = Z_NULL;
+ ret = deflateInit(&def, Z_DEFAULT_COMPRESSION);
+ if (ret != Z_OK || blk == NULL)
+ quit("out of memory");
+
+ /* compress from stdin until output full, or no more input */
+ def.avail_out = size + EXCESS;
+ def.next_out = blk;
+ ret = partcompress(stdin, &def);
+ if (ret == Z_ERRNO)
+ quit("error reading input");
+
+ /* if it all fit, then size was undersubscribed -- done! */
+ if (ret == Z_STREAM_END && def.avail_out >= EXCESS) {
+ /* write block to stdout */
+ have = size + EXCESS - def.avail_out;
+ if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
+ quit("error writing output");
+
+ /* clean up and print results to stderr */
+ ret = deflateEnd(&def);
+ assert(ret != Z_STREAM_ERROR);
+ free(blk);
+ fprintf(stderr,
+ "%u bytes unused out of %u requested (all input)\n",
+ size - have, size);
+ return 0;
+ }
+
+ /* it didn't all fit -- set up for recompression */
+ inf.zalloc = Z_NULL;
+ inf.zfree = Z_NULL;
+ inf.opaque = Z_NULL;
+ inf.avail_in = 0;
+ inf.next_in = Z_NULL;
+ ret = inflateInit(&inf);
+ tmp = malloc(size + EXCESS);
+ if (ret != Z_OK || tmp == NULL)
+ quit("out of memory");
+ ret = deflateReset(&def);
+ assert(ret != Z_STREAM_ERROR);
+
+ /* do first recompression close to the right amount */
+ inf.avail_in = size + EXCESS;
+ inf.next_in = blk;
+ def.avail_out = size + EXCESS;
+ def.next_out = tmp;
+ ret = recompress(&inf, &def);
+ if (ret == Z_MEM_ERROR)
+ quit("out of memory");
+
+ /* set up for next reocmpression */
+ ret = inflateReset(&inf);
+ assert(ret != Z_STREAM_ERROR);
+ ret = deflateReset(&def);
+ assert(ret != Z_STREAM_ERROR);
+
+ /* do second and final recompression (third compression) */
+ inf.avail_in = size - MARGIN; /* assure stream will complete */
+ inf.next_in = tmp;
+ def.avail_out = size;
+ def.next_out = blk;
+ ret = recompress(&inf, &def);
+ if (ret == Z_MEM_ERROR)
+ quit("out of memory");
+ assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */
+
+ /* done -- write block to stdout */
+ have = size - def.avail_out;
+ if (fwrite(blk, 1, have, stdout) != have || ferror(stdout))
+ quit("error writing output");
+
+ /* clean up and print results to stderr */
+ free(tmp);
+ ret = inflateEnd(&inf);
+ assert(ret != Z_STREAM_ERROR);
+ ret = deflateEnd(&def);
+ assert(ret != Z_STREAM_ERROR);
+ free(blk);
+ fprintf(stderr,
+ "%u bytes unused out of %u requested (%lu input)\n",
+ size - have, size, def.total_in);
+ return 0;
+}
diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c
new file mode 100644
index 0000000..f0ae8cd
--- /dev/null
+++ b/zlibWrapper/examples/zwrapbench.c
@@ -0,0 +1,976 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Przemyslaw Skibinski, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+
+/* *************************************
+* Includes
+***************************************/
+#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */
+#include <stdlib.h> /* malloc, free */
+#include <string.h> /* memset */
+#include <stdio.h> /* fprintf, fopen, ftello64 */
+#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
+#include <ctype.h> /* toupper */
+
+#include "mem.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#include "datagen.h" /* RDG_genBuffer */
+#include "xxhash.h"
+
+#include "zstd_zlibwrapper.h"
+
+
+
+/*-************************************
+* Tuning parameters
+**************************************/
+#ifndef ZSTDCLI_CLEVEL_DEFAULT
+# define ZSTDCLI_CLEVEL_DEFAULT 3
+#endif
+
+
+
+/*-************************************
+* Constants
+**************************************/
+#define COMPRESSOR_NAME "Zstandard wrapper for zlib command line interface"
+#ifndef ZSTD_VERSION
+# define ZSTD_VERSION "v" ZSTD_VERSION_STRING
+#endif
+#define AUTHOR "Yann Collet"
+#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
+
+#ifndef ZSTD_GIT_COMMIT
+# define ZSTD_GIT_COMMIT_STRING ""
+#else
+# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT)
+#endif
+
+#define NBLOOPS 3
+#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */
+#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */
+#define COOLPERIOD_SEC 10
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31));
+
+static U32 g_compressibilityDefault = 50;
+
+
+/* *************************************
+* console display
+***************************************/
+#define DEFAULT_DISPLAY_LEVEL 2
+#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
+static FILE* displayOut;
+
+#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
+ if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \
+ { g_time = clock(); DISPLAY(__VA_ARGS__); \
+ if (g_displayLevel>=4) fflush(stdout); } }
+static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100;
+static clock_t g_time = 0;
+
+
+/* *************************************
+* Exceptions
+***************************************/
+#ifndef DEBUG
+# define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...) \
+{ \
+ DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+ DISPLAYLEVEL(1, "Error %i : ", error); \
+ DISPLAYLEVEL(1, __VA_ARGS__); \
+ DISPLAYLEVEL(1, "\n"); \
+ exit(error); \
+}
+
+
+/* *************************************
+* Benchmark Parameters
+***************************************/
+static U32 g_nbIterations = NBLOOPS;
+static size_t g_blockSize = 0;
+int g_additionalParam = 0;
+
+void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; }
+
+void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; }
+
+void BMK_SetNbIterations(unsigned nbLoops)
+{
+ g_nbIterations = nbLoops;
+ DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations);
+}
+
+void BMK_SetBlockSize(size_t blockSize)
+{
+ g_blockSize = blockSize;
+ DISPLAYLEVEL(2, "using blocks of size %u KB \n", (U32)(blockSize>>10));
+}
+
+
+/* ********************************************************
+* Bench functions
+**********************************************************/
+typedef struct
+{
+ const char* srcPtr;
+ size_t srcSize;
+ char* cPtr;
+ size_t cRoom;
+ size_t cSize;
+ char* resPtr;
+ size_t resSize;
+} blockParam_t;
+
+typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor;
+
+
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+#define MAX(a,b) ((a)>(b) ? (a) : (b))
+
+static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+ const char* displayName, int cLevel,
+ const size_t* fileSizes, U32 nbFiles,
+ const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor)
+{
+ size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ;
+ size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles));
+ U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
+ blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
+ size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */
+ void* const compressedBuffer = malloc(maxCompressedSize);
+ void* const resultBuffer = malloc(srcSize);
+ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ U32 nbBlocks;
+ UTIL_time_t ticksPerSecond;
+
+ /* checks */
+ if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx)
+ EXM_THROW(31, "allocation error : not enough memory");
+
+ /* init */
+ if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
+ UTIL_initTimer(&ticksPerSecond);
+
+ /* Init blockTable data */
+ { const char* srcPtr = (const char*)srcBuffer;
+ char* cPtr = (char*)compressedBuffer;
+ char* resPtr = (char*)resultBuffer;
+ U32 fileNb;
+ for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) {
+ size_t remaining = fileSizes[fileNb];
+ U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize);
+ U32 const blockEnd = nbBlocks + nbBlocksforThisFile;
+ for ( ; nbBlocks<blockEnd; nbBlocks++) {
+ size_t const thisBlockSize = MIN(remaining, blockSize);
+ blockTable[nbBlocks].srcPtr = srcPtr;
+ blockTable[nbBlocks].cPtr = cPtr;
+ blockTable[nbBlocks].resPtr = resPtr;
+ blockTable[nbBlocks].srcSize = thisBlockSize;
+ blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
+ srcPtr += thisBlockSize;
+ cPtr += blockTable[nbBlocks].cRoom;
+ resPtr += thisBlockSize;
+ remaining -= thisBlockSize;
+ } } }
+
+ /* warmimg up memory */
+ RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1);
+
+ /* Bench */
+ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL);
+ U64 const crcOrig = XXH64(srcBuffer, srcSize, 0);
+ UTIL_time_t coolTime;
+ U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100;
+ U64 totalCTime=0, totalDTime=0;
+ U32 cCompleted=0, dCompleted=0;
+# define NB_MARKS 4
+ const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" };
+ U32 markNb = 0;
+ size_t cSize = 0;
+ double ratio = 0.;
+
+ UTIL_getTime(&coolTime);
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ while (!cCompleted | !dCompleted) {
+ UTIL_time_t clockStart;
+ U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1;
+
+ /* overheat protection */
+ if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) {
+ DISPLAYLEVEL(2, "\rcooling down ... \r");
+ UTIL_sleep(COOLPERIOD_SEC);
+ UTIL_getTime(&coolTime);
+ }
+
+ /* Compression */
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize);
+ if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */
+
+ UTIL_sleepMilli(1); /* give processor time to other processes */
+ UTIL_waitForNextTick(ticksPerSecond);
+ UTIL_getTime(&clockStart);
+
+ if (!cCompleted) { /* still some time to do compression tests */
+ U32 nbLoops = 0;
+ if (compressor == BMK_ZSTD) {
+ ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
+ ZSTD_customMem const cmem = { NULL, NULL, NULL };
+ ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, zparams, cmem);
+ if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure");
+
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ size_t const rSize = ZSTD_compress_usingCDict(ctx,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cRoom,
+ blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize,
+ cdict);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize));
+ blockTable[blockNb].cSize = rSize;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeCDict(cdict);
+ } else if (compressor == BMK_ZSTD_STREAM) {
+ ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize);
+ ZSTD_inBuffer inBuffer;
+ ZSTD_outBuffer outBuffer;
+ ZSTD_CStream* zbc = ZSTD_createCStream();
+ size_t rSize;
+ if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure");
+ rSize = ZSTD_initCStream_advanced(zbc, NULL, 0, zparams, avgSize);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize));
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ rSize = ZSTD_resetCStream(zbc, blockTable[blockNb].srcSize);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetCStream() failed : %s", ZSTD_getErrorName(rSize));
+ inBuffer.src = blockTable[blockNb].srcPtr;
+ inBuffer.size = blockTable[blockNb].srcSize;
+ inBuffer.pos = 0;
+ outBuffer.dst = blockTable[blockNb].cPtr;
+ outBuffer.size = blockTable[blockNb].cRoom;
+ outBuffer.pos = 0;
+ rSize = ZSTD_compressStream(zbc, &outBuffer, &inBuffer);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compressStream() failed : %s", ZSTD_getErrorName(rSize));
+ rSize = ZSTD_endStream(zbc, &outBuffer);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_endStream() failed : %s", ZSTD_getErrorName(rSize));
+ blockTable[blockNb].cSize = outBuffer.pos;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeCStream(zbc);
+ } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
+ z_stream def;
+ int ret;
+ if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) ZWRAP_useZSTDcompression(0);
+ else ZWRAP_useZSTDcompression(1);
+ def.zalloc = Z_NULL;
+ def.zfree = Z_NULL;
+ def.opaque = Z_NULL;
+ ret = deflateInit(&def, cLevel);
+ if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
+ /* if (ZWRAP_isUsingZSTDcompression()) {
+ ret = ZWRAP_setPledgedSrcSize(&def, avgSize);
+ if (ret != Z_OK) EXM_THROW(1, "ZWRAP_setPledgedSrcSize failure");
+ } */
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ ret = deflateReset(&def);
+ if (ret != Z_OK) EXM_THROW(1, "deflateReset failure");
+ def.next_in = (const void*) blockTable[blockNb].srcPtr;
+ def.avail_in = blockTable[blockNb].srcSize;
+ def.total_in = 0;
+ def.next_out = (void*) blockTable[blockNb].cPtr;
+ def.avail_out = blockTable[blockNb].cRoom;
+ def.total_out = 0;
+ ret = deflate(&def, Z_FINISH);
+ if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize);
+ blockTable[blockNb].cSize = def.total_out;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ret = deflateEnd(&def);
+ if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
+ } else {
+ z_stream def;
+ if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) ZWRAP_useZSTDcompression(0);
+ else ZWRAP_useZSTDcompression(1);
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ int ret;
+ def.zalloc = Z_NULL;
+ def.zfree = Z_NULL;
+ def.opaque = Z_NULL;
+ ret = deflateInit(&def, cLevel);
+ if (ret != Z_OK) EXM_THROW(1, "deflateInit failure");
+ def.next_in = (const void*) blockTable[blockNb].srcPtr;
+ def.avail_in = blockTable[blockNb].srcSize;
+ def.total_in = 0;
+ def.next_out = (void*) blockTable[blockNb].cPtr;
+ def.avail_out = blockTable[blockNb].cRoom;
+ def.total_out = 0;
+ ret = deflate(&def, Z_FINISH);
+ if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure");
+ ret = deflateEnd(&def);
+ if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure");
+ blockTable[blockNb].cSize = def.total_out;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ }
+ { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+ if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops;
+ totalCTime += clockSpan;
+ cCompleted = totalCTime>maxTime;
+ } }
+
+ cSize = 0;
+ { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; }
+ ratio = (double)srcSize / (double)cSize;
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+ (double)srcSize / fastestC );
+
+ (void)fastestD; (void)crcOrig; /* unused when decompression disabled */
+#if 1
+ /* Decompression */
+ if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */
+
+ UTIL_sleepMilli(1); /* give processor time to other processes */
+ UTIL_waitForNextTick(ticksPerSecond);
+ UTIL_getTime(&clockStart);
+
+ if (!dCompleted) {
+ U32 nbLoops = 0;
+ if (compressor == BMK_ZSTD) {
+ ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize);
+ if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure");
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ size_t const regenSize = ZSTD_decompress_usingDDict(dctx,
+ blockTable[blockNb].resPtr, blockTable[blockNb].srcSize,
+ blockTable[blockNb].cPtr, blockTable[blockNb].cSize,
+ ddict);
+ if (ZSTD_isError(regenSize)) {
+ DISPLAY("ZSTD_decompress_usingDDict() failed on block %u : %s \n",
+ blockNb, ZSTD_getErrorName(regenSize));
+ clockLoop = 0; /* force immediate test end */
+ break;
+ }
+ blockTable[blockNb].resSize = regenSize;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeDDict(ddict);
+ } else if (compressor == BMK_ZSTD_STREAM) {
+ ZSTD_inBuffer inBuffer;
+ ZSTD_outBuffer outBuffer;
+ ZSTD_DStream* zbd = ZSTD_createDStream();
+ size_t rSize;
+ if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure");
+ rSize = ZSTD_initDStream(zbd);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize));
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ rSize = ZSTD_resetDStream(zbd);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetDStream() failed : %s", ZSTD_getErrorName(rSize));
+ inBuffer.src = blockTable[blockNb].cPtr;
+ inBuffer.size = blockTable[blockNb].cSize;
+ inBuffer.pos = 0;
+ outBuffer.dst = blockTable[blockNb].resPtr;
+ outBuffer.size = blockTable[blockNb].srcSize;
+ outBuffer.pos = 0;
+ rSize = ZSTD_decompressStream(zbd, &outBuffer, &inBuffer);
+ if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_decompressStream() failed : %s", ZSTD_getErrorName(rSize));
+ blockTable[blockNb].resSize = outBuffer.pos;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ZSTD_freeDStream(zbd);
+ } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) {
+ z_stream inf;
+ int ret;
+ if (compressor == BMK_ZLIB_REUSE) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB);
+ else ZWRAP_setDecompressionType(ZWRAP_AUTO);
+ inf.zalloc = Z_NULL;
+ inf.zfree = Z_NULL;
+ inf.opaque = Z_NULL;
+ ret = inflateInit(&inf);
+ if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ ret = inflateReset(&inf);
+ if (ret != Z_OK) EXM_THROW(1, "inflateReset failure");
+ inf.next_in = (const void*) blockTable[blockNb].cPtr;
+ inf.avail_in = blockTable[blockNb].cSize;
+ inf.total_in = 0;
+ inf.next_out = (void*) blockTable[blockNb].resPtr;
+ inf.avail_out = blockTable[blockNb].srcSize;
+ inf.total_out = 0;
+ ret = inflate(&inf, Z_FINISH);
+ if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
+ blockTable[blockNb].resSize = inf.total_out;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ ret = inflateEnd(&inf);
+ if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
+ } else {
+ z_stream inf;
+ if (compressor == BMK_ZLIB) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB);
+ else ZWRAP_setDecompressionType(ZWRAP_AUTO);
+ do {
+ U32 blockNb;
+ for (blockNb=0; blockNb<nbBlocks; blockNb++) {
+ int ret;
+ inf.zalloc = Z_NULL;
+ inf.zfree = Z_NULL;
+ inf.opaque = Z_NULL;
+ ret = inflateInit(&inf);
+ if (ret != Z_OK) EXM_THROW(1, "inflateInit failure");
+ inf.next_in = (const void*) blockTable[blockNb].cPtr;
+ inf.avail_in = blockTable[blockNb].cSize;
+ inf.total_in = 0;
+ inf.next_out = (void*) blockTable[blockNb].resPtr;
+ inf.avail_out = blockTable[blockNb].srcSize;
+ inf.total_out = 0;
+ ret = inflate(&inf, Z_FINISH);
+ if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure");
+ ret = inflateEnd(&inf);
+ if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure");
+ blockTable[blockNb].resSize = inf.total_out;
+ }
+ nbLoops++;
+ } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop);
+ }
+ { U64 const clockSpan = UTIL_clockSpanMicro(clockStart, ticksPerSecond);
+ if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops;
+ totalDTime += clockSpan;
+ dCompleted = totalDTime>maxTime;
+ } }
+
+ markNb = (markNb+1) % NB_MARKS;
+ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r",
+ marks[markNb], displayName, (U32)srcSize, (U32)cSize, ratio,
+ (double)srcSize / fastestC,
+ (double)srcSize / fastestD );
+
+ /* CRC Checking */
+ { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0);
+ if (crcOrig!=crcCheck) {
+ size_t u;
+ DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
+ for (u=0; u<srcSize; u++) {
+ if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) {
+ U32 segNb, bNb, pos;
+ size_t bacc = 0;
+ DISPLAY("Decoding error at pos %u ", (U32)u);
+ for (segNb = 0; segNb < nbBlocks; segNb++) {
+ if (bacc + blockTable[segNb].srcSize > u) break;
+ bacc += blockTable[segNb].srcSize;
+ }
+ pos = (U32)(u - bacc);
+ bNb = pos / (128 KB);
+ DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos);
+ break;
+ }
+ if (u==srcSize-1) { /* should never happen */
+ DISPLAY("no difference detected\n");
+ } }
+ break;
+ } } /* CRC Checking */
+#endif
+ } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */
+
+ if (g_displayLevel == 1) {
+ double cSpeed = (double)srcSize / fastestC;
+ double dSpeed = (double)srcSize / fastestD;
+ if (g_additionalParam)
+ DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam);
+ else
+ DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName);
+ }
+ DISPLAYLEVEL(2, "%2i#\n", cLevel);
+ } /* Bench */
+
+ /* clean up */
+ free(blockTable);
+ free(compressedBuffer);
+ free(resultBuffer);
+ ZSTD_freeCCtx(ctx);
+ ZSTD_freeDCtx(dctx);
+ return 0;
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+ size_t const step = 64 MB;
+ BYTE* testmem = NULL;
+
+ requiredMem = (((requiredMem >> 26) + 1) << 26);
+ requiredMem += step;
+ if (requiredMem > maxMemory) requiredMem = maxMemory;
+
+ do {
+ testmem = (BYTE*)malloc((size_t)requiredMem);
+ requiredMem -= step;
+ } while (!testmem);
+
+ free(testmem);
+ return (size_t)(requiredMem);
+}
+
+static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+ const char* displayName, int cLevel, int cLevelLast,
+ const size_t* fileSizes, unsigned nbFiles,
+ const void* dictBuffer, size_t dictBufferSize)
+{
+ int l;
+
+ const char* pch = strrchr(displayName, '\\'); /* Windows */
+ if (!pch) pch = strrchr(displayName, '/'); /* Linux */
+ if (pch) displayName = pch+1;
+
+ SET_HIGH_PRIORITY;
+
+ if (g_displayLevel == 1 && !g_additionalParam)
+ DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10));
+
+ if (cLevelLast < cLevel) cLevelLast = cLevel;
+
+ DISPLAY("benchmarking zstd %s (using ZSTD_CCtx)\n", ZSTD_VERSION_STRING);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZSTD);
+ }
+
+ DISPLAY("benchmarking zstd %s (using ZSTD_CStream)\n", ZSTD_VERSION_STRING);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZSTD_STREAM);
+ }
+
+ DISPLAY("benchmarking zstd %s (using zlibWrapper)\n", ZSTD_VERSION_STRING);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD_REUSE);
+ }
+
+ DISPLAY("benchmarking zstd %s (zlibWrapper not reusing a context)\n", ZSTD_VERSION_STRING);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD);
+ }
+
+
+ if (cLevelLast > Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION;
+
+ DISPLAY("\n");
+ DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZLIB_REUSE);
+ }
+
+ DISPLAY("benchmarking zlib %s (zlib not reusing a context)\n", ZLIB_VERSION);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZLIB);
+ }
+
+ DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE);
+ }
+
+ DISPLAY("benchmarking zlib %s (zlibWrapper not reusing a context)\n", ZLIB_VERSION);
+ for (l=cLevel; l <= cLevelLast; l++) {
+ BMK_benchMem(srcBuffer, benchedSize,
+ displayName, l,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB);
+ }
+}
+
+
+/*! BMK_loadFiles() :
+ Loads `buffer` with content of files listed within `fileNamesTable`.
+ At most, fills `buffer` entirely */
+static void BMK_loadFiles(void* buffer, size_t bufferSize,
+ size_t* fileSizes,
+ const char** fileNamesTable, unsigned nbFiles)
+{
+ size_t pos = 0, totalSize = 0;
+ unsigned n;
+ for (n=0; n<nbFiles; n++) {
+ FILE* f;
+ U64 fileSize = UTIL_getFileSize(fileNamesTable[n]);
+ if (UTIL_isDirectory(fileNamesTable[n])) {
+ DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]);
+ fileSizes[n] = 0;
+ continue;
+ }
+ f = fopen(fileNamesTable[n], "rb");
+ if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+ DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
+ if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */
+ { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f);
+ if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ pos += readSize; }
+ fileSizes[n] = (size_t)fileSize;
+ totalSize += (size_t)fileSize;
+ fclose(f);
+ }
+
+ if (totalSize == 0) EXM_THROW(12, "no data to bench");
+}
+
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles,
+ const char* dictFileName, int cLevel, int cLevelLast)
+{
+ void* srcBuffer;
+ size_t benchedSize;
+ void* dictBuffer = NULL;
+ size_t dictBufferSize = 0;
+ size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+ U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
+ char mfName[20] = {0};
+
+ if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
+
+ /* Load dictionary */
+ if (dictFileName != NULL) {
+ U64 dictFileSize = UTIL_getFileSize(dictFileName);
+ if (dictFileSize > 64 MB) EXM_THROW(10, "dictionary file %s too large", dictFileName);
+ dictBufferSize = (size_t)dictFileSize;
+ dictBuffer = malloc(dictBufferSize);
+ if (dictBuffer==NULL) EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (U32)dictBufferSize);
+ BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1);
+ }
+
+ /* Memory allocation & restrictions */
+ benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
+ if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+ if (benchedSize < totalSizeToLoad)
+ DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+ srcBuffer = malloc(benchedSize);
+ if (!srcBuffer) EXM_THROW(12, "not enough memory");
+
+ /* Load input buffer */
+ BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
+
+ /* Bench */
+ snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+ { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
+ BMK_benchCLevel(srcBuffer, benchedSize,
+ displayName, cLevel, cLevelLast,
+ fileSizes, nbFiles,
+ dictBuffer, dictBufferSize);
+ }
+
+ /* clean up */
+ free(srcBuffer);
+ free(dictBuffer);
+ free(fileSizes);
+}
+
+
+static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility)
+{
+ char name[20] = {0};
+ size_t benchedSize = 10000000;
+ void* const srcBuffer = malloc(benchedSize);
+
+ /* Memory allocation */
+ if (!srcBuffer) EXM_THROW(21, "not enough memory");
+
+ /* Fill input buffer */
+ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
+
+ /* Bench */
+ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
+ BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0);
+
+ /* clean up */
+ free(srcBuffer);
+}
+
+
+int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
+ const char* dictFileName, int cLevel, int cLevelLast)
+{
+ double const compressibility = (double)g_compressibilityDefault / 100;
+
+ if (nbFiles == 0)
+ BMK_syntheticTest(cLevel, cLevelLast, compressibility);
+ else
+ BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast);
+ return 0;
+}
+
+
+
+
+/*-************************************
+* Command Line
+**************************************/
+static int usage(const char* programName)
+{
+ DISPLAY(WELCOME_MESSAGE);
+ DISPLAY( "Usage :\n");
+ DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName);
+ DISPLAY( "\n");
+ DISPLAY( "FILE : a filename\n");
+ DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
+ DISPLAY( "Arguments :\n");
+ DISPLAY( " -D file: use `file` as Dictionary \n");
+ DISPLAY( " -h/-H : display help/long help and exit\n");
+ DISPLAY( " -V : display Version number and exit\n");
+ DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL);
+ DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
+#ifdef UTIL_HAS_CREATEFILELIST
+ DISPLAY( " -r : operate recursively on directories\n");
+#endif
+ DISPLAY( "\n");
+ DISPLAY( "Benchmark arguments :\n");
+ DISPLAY( " -b# : benchmark file(s), using # compression level (default : %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
+ DISPLAY( " -e# : test all compression levels from -bX to # (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT);
+ DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
+ DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
+ return 0;
+}
+
+static int badusage(const char* programName)
+{
+ DISPLAYLEVEL(1, "Incorrect parameters\n");
+ if (g_displayLevel >= 1) usage(programName);
+ return 1;
+}
+
+static void waitEnter(void)
+{
+ int unused;
+ DISPLAY("Press enter to continue...\n");
+ unused = getchar();
+ (void)unused;
+}
+
+/*! readU32FromChar() :
+ @return : unsigned integer value reach from input in `char` format
+ Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+ Note : this function can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+ unsigned result = 0;
+ while ((**stringPtr >='0') && (**stringPtr <='9'))
+ result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+ return result;
+}
+
+
+#define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
+
+int main(int argCount, char** argv)
+{
+ int argNb,
+ main_pause=0,
+ nextEntryIsDictionary=0,
+ operationResult=0,
+ nextArgumentIsFile=0;
+ int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
+ int cLevelLast = 1;
+ unsigned recursive = 0;
+ const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*)); /* argCount >= 1 */
+ unsigned filenameIdx = 0;
+ const char* programName = argv[0];
+ const char* dictFileName = NULL;
+ char* dynNameSpace = NULL;
+#ifdef UTIL_HAS_CREATEFILELIST
+ const char** fileNamesTable = NULL;
+ char* fileNamesBuf = NULL;
+ unsigned fileNamesNb;
+#endif
+
+ /* init */
+ if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
+ displayOut = stderr;
+
+ /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
+ { size_t pos;
+ for (pos = (int)strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } }
+ programName += pos;
+ }
+
+ /* command switches */
+ for(argNb=1; argNb<argCount; argNb++) {
+ const char* argument = argv[argNb];
+ if(!argument) continue; /* Protection if argument empty */
+
+ if (nextArgumentIsFile==0) {
+
+ /* long commands (--long-word) */
+ if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; }
+ if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
+ if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage(programName)); }
+ if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
+ if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
+
+ /* Decode commands (note : aggregated commands are allowed) */
+ if (argument[0]=='-') {
+ argument++;
+
+ while (argument[0]!=0) {
+ switch(argument[0])
+ {
+ /* Display help */
+ case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */
+ case 'H':
+ case 'h': displayOut=stdout; CLEAN_RETURN(usage(programName));
+
+ /* Use file content as dictionary */
+ case 'D': nextEntryIsDictionary = 1; argument++; break;
+
+ /* Verbose mode */
+ case 'v': g_displayLevel++; argument++; break;
+
+ /* Quiet mode */
+ case 'q': g_displayLevel--; argument++; break;
+
+#ifdef UTIL_HAS_CREATEFILELIST
+ /* recursive */
+ case 'r': recursive=1; argument++; break;
+#endif
+
+ /* Benchmark */
+ case 'b':
+ /* first compression Level */
+ argument++;
+ cLevel = readU32FromChar(&argument);
+ break;
+
+ /* range bench (benchmark only) */
+ case 'e':
+ /* last compression Level */
+ argument++;
+ cLevelLast = readU32FromChar(&argument);
+ break;
+
+ /* Modify Nb Iterations (benchmark only) */
+ case 'i':
+ argument++;
+ { U32 const iters = readU32FromChar(&argument);
+ BMK_setNotificationLevel(g_displayLevel);
+ BMK_SetNbIterations(iters);
+ }
+ break;
+
+ /* cut input into blocks (benchmark only) */
+ case 'B':
+ argument++;
+ { size_t bSize = readU32FromChar(&argument);
+ if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */
+ if (toupper(*argument)=='M') bSize<<=20, argument++;
+ if (toupper(*argument)=='B') argument++;
+ BMK_setNotificationLevel(g_displayLevel);
+ BMK_SetBlockSize(bSize);
+ }
+ break;
+
+ /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
+ case 'p': argument++;
+ if ((*argument>='0') && (*argument<='9')) {
+ BMK_setAdditionalParam(readU32FromChar(&argument));
+ } else
+ main_pause=1;
+ break;
+ /* unknown command */
+ default : CLEAN_RETURN(badusage(programName));
+ }
+ }
+ continue;
+ } /* if (argument[0]=='-') */
+
+ } /* if (nextArgumentIsAFile==0) */
+
+ if (nextEntryIsDictionary) {
+ nextEntryIsDictionary = 0;
+ dictFileName = argument;
+ continue;
+ }
+
+ /* add filename to list */
+ filenameTable[filenameIdx++] = argument;
+ }
+
+ /* Welcome message (if verbose) */
+ DISPLAYLEVEL(3, WELCOME_MESSAGE);
+
+#ifdef UTIL_HAS_CREATEFILELIST
+ if (recursive) {
+ fileNamesTable = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb);
+ if (fileNamesTable) {
+ unsigned u;
+ for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, fileNamesTable[u]);
+ free((void*)filenameTable);
+ filenameTable = fileNamesTable;
+ filenameIdx = fileNamesNb;
+ }
+ }
+#endif
+
+ BMK_setNotificationLevel(g_displayLevel);
+ BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast);
+
+_end:
+ if (main_pause) waitEnter();
+ free(dynNameSpace);
+#ifdef UTIL_HAS_CREATEFILELIST
+ if (fileNamesTable)
+ UTIL_freeFileList(fileNamesTable, fileNamesBuf);
+ else
+#endif
+ free((void*)filenameTable);
+ return operationResult;
+}
diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c
index 9467950..e4906a2 100644
--- a/zlibWrapper/zstd_zlibwrapper.c
+++ b/zlibWrapper/zstd_zlibwrapper.c
@@ -14,50 +14,48 @@
#include "zstd_zlibwrapper.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_MAGICNUMBER */
#include "zstd.h"
-#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
-#include "zbuff.h"
#include "zstd_internal.h" /* defaultCustomMem */
#define Z_INFLATE_SYNC 8
-#define ZWRAP_HEADERSIZE 4
+#define ZLIB_HEADERSIZE 4
+#define ZSTD_HEADERSIZE ZSTD_frameHeaderSize_min
#define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */
-#define LOG_WRAPPER(...) /* printf(__VA_ARGS__) */
+#define LOG_WRAPPERC(...) /* printf(__VA_ARGS__) */
+#define LOG_WRAPPERD(...) /* printf(__VA_ARGS__) */
+
+#define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; }
+#define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; }
-#define FINISH_WITH_GZ_ERR(msg) { \
- (void)msg; \
- return Z_MEM_ERROR; \
-}
-
-#define FINISH_WITH_ERR(strm, message) { \
- strm->msg = message; \
- return Z_MEM_ERROR; \
-}
-
-#define FINISH_WITH_NULL_ERR(msg) { \
- (void)msg; \
- return NULL; \
-}
#ifndef ZWRAP_USE_ZSTD
#define ZWRAP_USE_ZSTD 0
#endif
-static int g_useZSTD = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
+static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */
+
+void ZWRAP_useZSTDcompression(int turn_on) { g_ZWRAP_useZSTDcompression = turn_on; }
+
+int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; }
-void useZSTD(int turn_on) { g_useZSTD = turn_on; }
+static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO;
-int isUsingZSTD(void) { return g_useZSTD; }
+void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; };
+
+ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; }
+
+
const char * zstdVersion(void) { return ZSTD_VERSION_STRING; }
ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); }
+
static void* ZWRAP_allocFunction(void* opaque, size_t size)
{
z_streamp strm = (z_streamp) opaque;
@@ -78,18 +76,20 @@
/* *** Compression *** */
typedef struct {
- ZBUFF_CCtx* zbc;
- size_t bytesLeft;
+ ZSTD_CStream* zbc;
int compressionLevel;
ZSTD_customMem customMem;
z_stream allocFunc; /* copy of zalloc, zfree, opaque */
+ ZSTD_inBuffer inBuffer;
+ ZSTD_outBuffer outBuffer;
+ unsigned long long pledgedSrcSize;
} ZWRAP_CCtx;
size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc)
{
if (zwc==NULL) return 0; /* support free on NULL */
- ZBUFF_freeCCtx(zwc->zbc);
+ if (zwc->zbc) ZSTD_freeCStream(zwc->zbc);
zwc->customMem.customFree(zwc->customMem.opaque, zwc);
return 0;
}
@@ -114,36 +114,81 @@
memcpy(&zwc->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
- zwc->zbc = ZBUFF_createCCtx_advanced(zwc->customMem);
- if (zwc->zbc == NULL) { ZWRAP_freeCCtx(zwc); return NULL; }
return zwc;
}
+int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, unsigned long long pledgedSrcSize)
+{
+ LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc);
+ if (zwc == NULL) return Z_STREAM_ERROR;
+
+ if (zwc->zbc == NULL) {
+ zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem);
+ if (zwc->zbc == NULL) return Z_STREAM_ERROR;
+
+ if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize;
+ { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, 0);
+ size_t errorCode;
+ LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d searchLength=%d strategy=%d\n", (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.searchLength, params.cParams.strategy);
+ errorCode = ZSTD_initCStream_advanced(zwc->zbc, NULL, 0, params, pledgedSrcSize);
+ if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; }
+ }
+
+ return Z_OK;
+}
+
+
+int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error)
+{
+ LOG_WRAPPERC("- ZWRAPC_finishWithError=%d\n", error);
+ if (zwc) ZWRAP_freeCCtx(zwc);
+ if (strm) strm->state = NULL;
+ return (error) ? error : Z_STREAM_ERROR;
+}
+
+
+int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message)
+{
+ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
+ strm->msg = message;
+ if (zwc == NULL) return Z_STREAM_ERROR;
+
+ return ZWRAPC_finishWithError(zwc, strm, 0);
+}
+
+
+int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize)
+{
+ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
+ if (zwc == NULL) return Z_STREAM_ERROR;
+
+ zwc->pledgedSrcSize = pledgedSrcSize;
+ return Z_OK;
+}
+
+
ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level,
const char *version, int stream_size))
{
ZWRAP_CCtx* zwc;
- if (!g_useZSTD) {
- LOG_WRAPPER("- deflateInit level=%d\n", level);
+ LOG_WRAPPERC("- deflateInit level=%d\n", level);
+ if (!g_ZWRAP_useZSTDcompression) {
return deflateInit_((strm), (level), version, stream_size);
}
- LOG_WRAPPER("- deflateInit level=%d\n", level);
zwc = ZWRAP_createCCtx(strm);
if (zwc == NULL) return Z_MEM_ERROR;
if (level == Z_DEFAULT_COMPRESSION)
level = ZWRAP_DEFAULT_CLEVEL;
- { size_t const errorCode = ZBUFF_compressInit(zwc->zbc, level);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
-
zwc->compressionLevel = level;
strm->state = (struct internal_state*) zwc; /* use state which in not used by user */
strm->total_in = 0;
strm->total_out = 0;
+ strm->adler = 0;
return Z_OK;
}
@@ -153,24 +198,44 @@
int strategy, const char *version,
int stream_size))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size);
return z_deflateInit_ (strm, level, version, stream_size);
}
+ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
+{
+ LOG_WRAPPERC("- deflateReset\n");
+ if (!g_ZWRAP_useZSTDcompression)
+ return deflateReset(strm);
+
+ strm->total_in = 0;
+ strm->total_out = 0;
+ strm->adler = 0;
+ return Z_OK;
+}
+
+
ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm,
const Bytef *dictionary,
uInt dictLength))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression) {
+ LOG_WRAPPERC("- deflateSetDictionary\n");
return deflateSetDictionary(strm, dictionary, dictLength);
+ }
{ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
- LOG_WRAPPER("- deflateSetDictionary level=%d\n", (int)strm->data_type);
- { size_t const errorCode = ZBUFF_compressInitDictionary(zwc->zbc, dictionary, dictLength, zwc->compressionLevel);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR; }
+ LOG_WRAPPERC("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel);
+ if (!zwc) return Z_STREAM_ERROR;
+ if (zwc->zbc == NULL) {
+ int res = ZWRAP_initializeCStream(zwc, 0);
+ if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
+ }
+ { size_t const errorCode = ZSTD_initCStream_usingDict(zwc->zbc, dictionary, dictLength, zwc->compressionLevel);
+ if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0); }
}
return Z_OK;
@@ -181,74 +246,92 @@
{
ZWRAP_CCtx* zwc;
- if (!g_useZSTD) {
- int res = deflate(strm, flush);
- LOG_WRAPPER("- avail_in=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->total_in, (int)strm->total_out);
+ if (!g_ZWRAP_useZSTDcompression) {
+ int res;
+ LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+ res = deflate(strm, flush);
return res;
}
zwc = (ZWRAP_CCtx*) strm->state;
+ if (zwc == NULL) { LOG_WRAPPERC("zwc == NULL\n"); return Z_STREAM_ERROR; }
- LOG_WRAPPER("deflate flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
- if (strm->avail_in > 0) {
- size_t dstCapacity = strm->avail_out;
- size_t srcSize = strm->avail_in;
- size_t const errorCode = ZBUFF_compressContinue(zwc->zbc, strm->next_out, &dstCapacity, strm->next_in, &srcSize);
- LOG_WRAPPER("ZBUFF_compressContinue srcSize=%d dstCapacity=%d\n", (int)srcSize, (int)dstCapacity);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
- strm->next_out += dstCapacity;
- strm->total_out += dstCapacity;
- strm->avail_out -= dstCapacity;
- strm->total_in += srcSize;
- strm->next_in += srcSize;
- strm->avail_in -= srcSize;
+ if (zwc->zbc == NULL) {
+ int res = ZWRAP_initializeCStream(zwc, (flush == Z_FINISH) ? strm->avail_in : 0);
+ if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res);
+ } else {
+ if (strm->total_in == 0) {
+ size_t const errorCode = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize);
+ if (ZSTD_isError(errorCode)) { LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); return ZWRAPC_finishWithError(zwc, strm, 0); }
+ }
}
- if (flush == Z_FULL_FLUSH) FINISH_WITH_ERR(strm, "Z_FULL_FLUSH is not supported!");
+ LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+ if (strm->avail_in > 0) {
+ zwc->inBuffer.src = strm->next_in;
+ zwc->inBuffer.size = strm->avail_in;
+ zwc->inBuffer.pos = 0;
+ zwc->outBuffer.dst = strm->next_out;
+ zwc->outBuffer.size = strm->avail_out;
+ zwc->outBuffer.pos = 0;
+ { size_t const errorCode = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer);
+ LOG_WRAPPERC("deflate ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size);
+ if (ZSTD_isError(errorCode)) return ZWRAPC_finishWithError(zwc, strm, 0);
+ }
+ strm->next_out += zwc->outBuffer.pos;
+ strm->total_out += zwc->outBuffer.pos;
+ strm->avail_out -= zwc->outBuffer.pos;
+ strm->total_in += zwc->inBuffer.pos;
+ strm->next_in += zwc->inBuffer.pos;
+ strm->avail_in -= zwc->inBuffer.pos;
+ }
+
+ if (flush == Z_FULL_FLUSH || flush == Z_BLOCK || flush == Z_TREES) return ZWRAPC_finishWithErrorMsg(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!");
if (flush == Z_FINISH) {
size_t bytesLeft;
- size_t dstCapacity = strm->avail_out;
- if (zwc->bytesLeft) {
- bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity);
- LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft);
- } else {
- bytesLeft = ZBUFF_compressEnd(zwc->zbc, strm->next_out, &dstCapacity);
- LOG_WRAPPER("ZBUFF_compressEnd dstCapacity=%d bytesLeft=%d\n", (int)dstCapacity, (int)bytesLeft);
- }
- if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR;
- strm->next_out += dstCapacity;
- strm->total_out += dstCapacity;
- strm->avail_out -= dstCapacity;
- if (flush == Z_FINISH && bytesLeft == 0) return Z_STREAM_END;
- zwc->bytesLeft = bytesLeft;
+ zwc->outBuffer.dst = strm->next_out;
+ zwc->outBuffer.size = strm->avail_out;
+ zwc->outBuffer.pos = 0;
+ bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer);
+ LOG_WRAPPERC("deflate ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft);
+ if (ZSTD_isError(bytesLeft)) return ZWRAPC_finishWithError(zwc, strm, 0);
+ strm->next_out += zwc->outBuffer.pos;
+ strm->total_out += zwc->outBuffer.pos;
+ strm->avail_out -= zwc->outBuffer.pos;
+ if (bytesLeft == 0) { LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; }
}
-
- if (flush == Z_SYNC_FLUSH) {
+ else
+ if (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH) {
size_t bytesLeft;
- size_t dstCapacity = strm->avail_out;
- bytesLeft = ZBUFF_compressFlush(zwc->zbc, strm->next_out, &dstCapacity);
- LOG_WRAPPER("ZBUFF_compressFlush avail_out=%d dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)dstCapacity, (int)bytesLeft);
- if (ZSTD_isError(bytesLeft)) return Z_MEM_ERROR;
- strm->next_out += dstCapacity;
- strm->total_out += dstCapacity;
- strm->avail_out -= dstCapacity;
- zwc->bytesLeft = bytesLeft;
+ zwc->outBuffer.dst = strm->next_out;
+ zwc->outBuffer.size = strm->avail_out;
+ zwc->outBuffer.pos = 0;
+ bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer);
+ LOG_WRAPPERC("deflate ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft);
+ if (ZSTD_isError(bytesLeft)) return ZWRAPC_finishWithError(zwc, strm, 0);
+ strm->next_out += zwc->outBuffer.pos;
+ strm->total_out += zwc->outBuffer.pos;
+ strm->avail_out -= zwc->outBuffer.pos;
}
+ LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
return Z_OK;
}
ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm))
{
- if (!g_useZSTD) {
- LOG_WRAPPER("- deflateEnd\n");
+ if (!g_ZWRAP_useZSTDcompression) {
+ LOG_WRAPPERC("- deflateEnd\n");
return deflateEnd(strm);
}
- LOG_WRAPPER("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
- { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
- size_t const errorCode = ZWRAP_freeCCtx(zwc);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
+ LOG_WRAPPERC("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
+ { size_t errorCode;
+ ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state;
+ if (zwc == NULL) return Z_OK; /* structures are already freed */
+ strm->state = NULL;
+ errorCode = ZWRAP_freeCCtx(zwc);
+ if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
}
return Z_OK;
}
@@ -257,7 +340,7 @@
ZEXTERN uLong ZEXPORT z_deflateBound OF((z_streamp strm,
uLong sourceLen))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflateBound(strm, sourceLen);
return ZSTD_compressBound(sourceLen);
@@ -268,8 +351,8 @@
int level,
int strategy))
{
- if (!g_useZSTD) {
- LOG_WRAPPER("- deflateParams level=%d strategy=%d\n", level, strategy);
+ if (!g_ZWRAP_useZSTDcompression) {
+ LOG_WRAPPERC("- deflateParams level=%d strategy=%d\n", level, strategy);
return deflateParams(strm, level, strategy);
}
@@ -283,9 +366,12 @@
/* *** Decompression *** */
typedef struct {
- ZBUFF_DCtx* zbd;
- char headerBuf[ZWRAP_HEADERSIZE];
+ ZSTD_DStream* zbd;
+ char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */
int errorCount;
+ int decompState;
+ ZSTD_inBuffer inBuffer;
+ ZSTD_outBuffer outBuffer;
/* zlib params */
int stream_size;
@@ -296,6 +382,14 @@
} ZWRAP_DCtx;
+void ZWRAP_initDCtx(ZWRAP_DCtx* zwd)
+{
+ zwd->errorCount = zwd->decompState = 0;
+ zwd->outBuffer.pos = 0;
+ zwd->outBuffer.size = 0;
+}
+
+
ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm)
{
ZWRAP_DCtx* zwd;
@@ -315,6 +409,7 @@
memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem));
}
+ ZWRAP_initDCtx(zwd);
return zwd;
}
@@ -322,22 +417,46 @@
size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd)
{
if (zwd==NULL) return 0; /* support free on null */
- ZBUFF_freeDCtx(zwd->zbd);
+ if (zwd->zbd) ZSTD_freeDStream(zwd->zbd);
if (zwd->version) zwd->customMem.customFree(zwd->customMem.opaque, zwd->version);
zwd->customMem.customFree(zwd->customMem.opaque, zwd);
return 0;
}
+int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error)
+{
+ LOG_WRAPPERD("- ZWRAPD_finishWithError=%d\n", error);
+ if (zwd) ZWRAP_freeDCtx(zwd);
+ if (strm) strm->state = NULL;
+ return (error) ? error : Z_STREAM_ERROR;
+}
+
+
+int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message)
+{
+ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+ strm->msg = message;
+ if (zwd == NULL) return Z_STREAM_ERROR;
+
+ return ZWRAPD_finishWithError(zwd, strm, 0);
+}
+
+
ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm,
const char *version, int stream_size))
{
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
+ return inflateInit(strm);
+ }
+
+ {
ZWRAP_DCtx* zwd = ZWRAP_createDCtx(strm);
- LOG_WRAPPER("- inflateInit\n");
- if (zwd == NULL) { strm->state = NULL; return Z_MEM_ERROR; }
+ LOG_WRAPPERD("- inflateInit\n");
+ if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
zwd->version = zwd->customMem.customAlloc(zwd->customMem.opaque, strlen(version) + 1);
- if (zwd->version == NULL) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; }
+ if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0);
strcpy(zwd->version, version);
zwd->stream_size = stream_size;
@@ -345,6 +464,8 @@
strm->total_in = 0;
strm->total_out = 0;
strm->reserved = 1; /* mark as unknown steam */
+ strm->adler = 0;
+ }
return Z_OK;
}
@@ -353,38 +474,89 @@
ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits,
const char *version, int stream_size))
{
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) {
+ return inflateInit2_(strm, windowBits, version, stream_size);
+ }
+
+ {
int ret = z_inflateInit_ (strm, version, stream_size);
if (ret == Z_OK) {
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state;
+ if (zwd == NULL) return Z_STREAM_ERROR;
zwd->windowBits = windowBits;
}
return ret;
+ }
}
+ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
+{
+ LOG_WRAPPERD("- inflateReset\n");
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
+ return inflateReset(strm);
+
+ { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+ if (zwd == NULL) return Z_STREAM_ERROR;
+ if (zwd->zbd) {
+ size_t const errorCode = ZSTD_resetDStream(zwd->zbd);
+ if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
+ }
+ ZWRAP_initDCtx(zwd);
+ }
+
+ strm->total_in = 0;
+ strm->total_out = 0;
+ return Z_OK;
+}
+
+
+#if ZLIB_VERNUM >= 0x1240
+ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
+ int windowBits))
+{
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
+ return inflateReset2(strm, windowBits);
+
+ { int ret = z_inflateReset (strm);
+ if (ret == Z_OK) {
+ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*)strm->state;
+ if (zwd == NULL) return Z_STREAM_ERROR;
+ zwd->windowBits = windowBits;
+ }
+ return ret;
+ }
+}
+#endif
+
+
ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm,
const Bytef *dictionary,
uInt dictLength))
{
- if (!strm->reserved)
+ LOG_WRAPPERD("- inflateSetDictionary\n");
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateSetDictionary(strm, dictionary, dictLength);
- LOG_WRAPPER("- inflateSetDictionary\n");
{ size_t errorCode;
ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
- if (strm->state == NULL) return Z_MEM_ERROR;
- errorCode = ZBUFF_decompressInitDictionary(zwd->zbd, dictionary, dictLength);
- if (ZSTD_isError(errorCode)) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return Z_MEM_ERROR; }
+ if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR;
+ errorCode = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength);
+ if (ZSTD_isError(errorCode)) return ZWRAPD_finishWithError(zwd, strm, 0);
+ zwd->decompState = Z_NEED_DICT;
- if (strm->total_in == ZSTD_frameHeaderSize_min) {
- size_t dstCapacity = 0;
- size_t srcSize = strm->total_in;
- errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
- LOG_WRAPPER("ZBUFF_decompressContinue3 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
- if (dstCapacity > 0 || ZSTD_isError(errorCode)) {
- LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode));
- ZWRAP_freeDCtx(zwd); strm->state = NULL;
- return Z_MEM_ERROR;
+ if (strm->total_in == ZSTD_HEADERSIZE) {
+ zwd->inBuffer.src = zwd->headerBuf;
+ zwd->inBuffer.size = strm->total_in;
+ zwd->inBuffer.pos = 0;
+ zwd->outBuffer.dst = strm->next_out;
+ zwd->outBuffer.size = 0;
+ zwd->outBuffer.pos = 0;
+ errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+ LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
+ if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) {
+ LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode));
+ return ZWRAPD_finishWithError(zwd, strm, 0);
}
}
}
@@ -395,141 +567,201 @@
ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush))
{
- if (!strm->reserved)
- return inflate(strm, flush);
+ ZWRAP_DCtx* zwd;
+ int res;
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
+ LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+ res = inflate(strm, flush);
+ LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+ return res;
+ }
- if (strm->avail_in > 0) {
- size_t errorCode, dstCapacity, srcSize;
- ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
- if (strm->state == NULL) return Z_MEM_ERROR;
- LOG_WRAPPER("inflate avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
- if (strm->total_in < ZWRAP_HEADERSIZE)
- {
- srcSize = MIN(strm->avail_in, ZWRAP_HEADERSIZE - strm->total_in);
- memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
- strm->total_in += srcSize;
- strm->next_in += srcSize;
- strm->avail_in -= srcSize;
- if (strm->total_in < ZWRAP_HEADERSIZE) return Z_OK;
+ if (strm->avail_in <= 0) return Z_OK;
- if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
- z_stream strm2;
- strm2.next_in = strm->next_in;
- strm2.avail_in = strm->avail_in;
- strm2.next_out = strm->next_out;
- strm2.avail_out = strm->avail_out;
+ { size_t errorCode, srcSize;
+ zwd = (ZWRAP_DCtx*) strm->state;
+ LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
- if (zwd->windowBits)
- errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
- else
- errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
- LOG_WRAPPER("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
- if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; }
+ if (zwd == NULL) return Z_STREAM_ERROR;
+ if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END;
- /* inflate header */
- strm->next_in = (unsigned char*)zwd->headerBuf;
- strm->avail_in = ZWRAP_HEADERSIZE;
- strm->avail_out = 0;
- errorCode = inflate(strm, Z_NO_FLUSH);
- LOG_WRAPPER("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
- if (errorCode != Z_OK) { ZWRAP_freeDCtx(zwd); strm->state = NULL; return errorCode; }
- if (strm->avail_in > 0) goto error;
+ if (strm->total_in < ZLIB_HEADERSIZE) {
+ if (strm->total_in == 0 && strm->avail_in >= ZLIB_HEADERSIZE) {
+ if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) {
+ if (zwd->windowBits)
+ errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
+ else
+ errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
- strm->next_in = strm2.next_in;
- strm->avail_in = strm2.avail_in;
- strm->next_out = strm2.next_out;
- strm->avail_out = strm2.avail_out;
+ strm->reserved = 0; /* mark as zlib stream */
+ errorCode = ZWRAP_freeDCtx(zwd);
+ if (ZSTD_isError(errorCode)) goto error;
- strm->reserved = 0; /* mark as zlib stream */
- errorCode = ZWRAP_freeDCtx(zwd);
- if (ZSTD_isError(errorCode)) goto error;
+ if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
+ else res = inflate(strm, flush);
+ LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+ return res;
+ }
+ } else {
+ srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - strm->total_in);
+ memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
+ strm->total_in += srcSize;
+ strm->next_in += srcSize;
+ strm->avail_in -= srcSize;
+ if (strm->total_in < ZLIB_HEADERSIZE) return Z_OK;
- if (flush == Z_INFLATE_SYNC) return inflateSync(strm);
- return inflate(strm, flush);
+ if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) {
+ z_stream strm2;
+ strm2.next_in = strm->next_in;
+ strm2.avail_in = strm->avail_in;
+ strm2.next_out = strm->next_out;
+ strm2.avail_out = strm->avail_out;
+
+ if (zwd->windowBits)
+ errorCode = inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size);
+ else
+ errorCode = inflateInit_(strm, zwd->version, zwd->stream_size);
+ LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", (int)errorCode);
+ if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
+
+ /* inflate header */
+ strm->next_in = (unsigned char*)zwd->headerBuf;
+ strm->avail_in = ZLIB_HEADERSIZE;
+ strm->avail_out = 0;
+ errorCode = inflate(strm, Z_NO_FLUSH);
+ LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", (int)errorCode, (int)strm->avail_in);
+ if (errorCode != Z_OK) return ZWRAPD_finishWithError(zwd, strm, (int)errorCode);
+ if (strm->avail_in > 0) goto error;
+
+ strm->next_in = strm2.next_in;
+ strm->avail_in = strm2.avail_in;
+ strm->next_out = strm2.next_out;
+ strm->avail_out = strm2.avail_out;
+
+ strm->reserved = 0; /* mark as zlib stream */
+ errorCode = ZWRAP_freeDCtx(zwd);
+ if (ZSTD_isError(errorCode)) goto error;
+
+ if (flush == Z_INFLATE_SYNC) res = inflateSync(strm);
+ else res = inflate(strm, flush);
+ LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res);
+ return res;
+ }
}
-
- zwd->zbd = ZBUFF_createDCtx_advanced(zwd->customMem);
- if (zwd->zbd == NULL) goto error;
-
- errorCode = ZBUFF_decompressInit(zwd->zbd);
- if (ZSTD_isError(errorCode)) goto error;
-
- srcSize = ZWRAP_HEADERSIZE;
- dstCapacity = 0;
- errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, zwd->headerBuf, &srcSize);
- LOG_WRAPPER("ZBUFF_decompressContinue1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
- if (ZSTD_isError(errorCode)) {
- LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode));
- goto error;
- }
- if (strm->avail_in == 0) return Z_OK;
}
- srcSize = strm->avail_in;
- dstCapacity = strm->avail_out;
- errorCode = ZBUFF_decompressContinue(zwd->zbd, strm->next_out, &dstCapacity, strm->next_in, &srcSize);
- LOG_WRAPPER("ZBUFF_decompressContinue2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)srcSize, (int)dstCapacity);
+ if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; }
+
+ if (!zwd->zbd) {
+ zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem);
+ if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; }
+ }
+
+ if (strm->total_in < ZSTD_HEADERSIZE)
+ {
+ if (strm->total_in == 0 && strm->avail_in >= ZSTD_HEADERSIZE) {
+ if (zwd->decompState != Z_NEED_DICT) {
+ errorCode = ZSTD_initDStream(zwd->zbd);
+ if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+ }
+ } else {
+ srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - strm->total_in);
+ memcpy(zwd->headerBuf+strm->total_in, strm->next_in, srcSize);
+ strm->total_in += srcSize;
+ strm->next_in += srcSize;
+ strm->avail_in -= srcSize;
+ if (strm->total_in < ZSTD_HEADERSIZE) return Z_OK;
+
+ errorCode = ZSTD_initDStream(zwd->zbd);
+ if (ZSTD_isError(errorCode)) { LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", ZSTD_getErrorName(errorCode)); goto error; }
+
+ zwd->inBuffer.src = zwd->headerBuf;
+ zwd->inBuffer.size = ZSTD_HEADERSIZE;
+ zwd->inBuffer.pos = 0;
+ zwd->outBuffer.dst = strm->next_out;
+ zwd->outBuffer.size = 0;
+ zwd->outBuffer.pos = 0;
+ errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+ LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size);
+ if (ZSTD_isError(errorCode)) {
+ LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode));
+ goto error;
+ }
+ if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */
+ }
+ }
+
+ zwd->inBuffer.src = strm->next_in;
+ zwd->inBuffer.size = strm->avail_in;
+ zwd->inBuffer.pos = 0;
+ zwd->outBuffer.dst = strm->next_out;
+ zwd->outBuffer.size = strm->avail_out;
+ zwd->outBuffer.pos = 0;
+ errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer);
+ LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out);
if (ZSTD_isError(errorCode)) {
- LOG_WRAPPER("ERROR: ZBUFF_decompressContinue %s\n", ZSTD_getErrorName(errorCode));
zwd->errorCount++;
+ LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount);
if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error;
}
- strm->next_out += dstCapacity;
- strm->total_out += dstCapacity;
- strm->avail_out -= dstCapacity;
- strm->total_in += srcSize;
- strm->next_in += srcSize;
- strm->avail_in -= srcSize;
- if (errorCode == 0) return Z_STREAM_END;
- return Z_OK;
-error:
- ZWRAP_freeDCtx(zwd);
- strm->state = NULL;
- return Z_MEM_ERROR;
+ LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size);
+ strm->next_out += zwd->outBuffer.pos;
+ strm->total_out += zwd->outBuffer.pos;
+ strm->avail_out -= zwd->outBuffer.pos;
+ strm->total_in += zwd->inBuffer.pos;
+ strm->next_in += zwd->inBuffer.pos;
+ strm->avail_in -= zwd->inBuffer.pos;
+ if (errorCode == 0) {
+ LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out);
+ zwd->decompState = Z_STREAM_END;
+ return Z_STREAM_END;
+ }
}
+ LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK);
return Z_OK;
+
+error:
+ return ZWRAPD_finishWithError(zwd, strm, 0);
}
ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm))
{
- int ret = Z_OK;
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateEnd(strm);
- LOG_WRAPPER("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
- { ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
- size_t const errorCode = ZWRAP_freeDCtx(zwd);
+ LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out));
+ { size_t errorCode;
+ ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state;
+ if (zwd == NULL) return Z_OK; /* structures are already freed */
strm->state = NULL;
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
+ errorCode = ZWRAP_freeDCtx(zwd);
+ if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
}
- return ret;
+ return Z_OK;
}
ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm))
{
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) {
+ return inflateSync(strm);
+ }
+
return z_inflate(strm, Z_INFLATE_SYNC);
}
+
/* Advanced compression functions */
ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest,
z_streamp source))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflateCopy(dest, source);
- FINISH_WITH_ERR(source, "deflateCopy is not supported!");
-}
-
-
-ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm))
-{
- if (!g_useZSTD)
- return deflateReset(strm);
- FINISH_WITH_ERR(strm, "deflateReset is not supported!");
+ return ZWRAPC_finishWithErrorMsg(source, "deflateCopy is not supported!");
}
@@ -539,9 +771,9 @@
int nice_length,
int max_chain))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflateTune(strm, good_length, max_lazy, nice_length, max_chain);
- FINISH_WITH_ERR(strm, "deflateTune is not supported!");
+ return ZWRAPC_finishWithErrorMsg(strm, "deflateTune is not supported!");
}
@@ -550,9 +782,9 @@
unsigned *pending,
int *bits))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflatePending(strm, pending, bits);
- FINISH_WITH_ERR(strm, "deflatePending is not supported!");
+ return ZWRAPC_finishWithErrorMsg(strm, "deflatePending is not supported!");
}
#endif
@@ -561,32 +793,32 @@
int bits,
int value))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflatePrime(strm, bits, value);
- FINISH_WITH_ERR(strm, "deflatePrime is not supported!");
+ return ZWRAPC_finishWithErrorMsg(strm, "deflatePrime is not supported!");
}
ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm,
gz_headerp head))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return deflateSetHeader(strm, head);
- FINISH_WITH_ERR(strm, "deflateSetHeader is not supported!");
+ return ZWRAPC_finishWithErrorMsg(strm, "deflateSetHeader is not supported!");
}
-/* Advanced compression functions */
+/* Advanced decompression functions */
#if ZLIB_VERNUM >= 0x1280
ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm,
Bytef *dictionary,
uInt *dictLength))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateGetDictionary(strm, dictionary, dictLength);
- FINISH_WITH_ERR(strm, "inflateGetDictionary is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflateGetDictionary is not supported!");
}
#endif
@@ -594,37 +826,18 @@
ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest,
z_streamp source))
{
- if (!g_useZSTD)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !source->reserved)
return inflateCopy(dest, source);
- FINISH_WITH_ERR(source, "inflateCopy is not supported!");
+ return ZWRAPD_finishWithErrorMsg(source, "inflateCopy is not supported!");
}
-ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm))
-{
- if (!strm->reserved)
- return inflateReset(strm);
- FINISH_WITH_ERR(strm, "inflateReset is not supported!");
-}
-
-
-#if ZLIB_VERNUM >= 0x1240
-ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm,
- int windowBits))
-{
- if (!strm->reserved)
- return inflateReset2(strm, windowBits);
- FINISH_WITH_ERR(strm, "inflateReset2 is not supported!");
-}
-#endif
-
-
#if ZLIB_VERNUM >= 0x1240
ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateMark(strm);
- FINISH_WITH_ERR(strm, "inflateMark is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflateMark is not supported!");
}
#endif
@@ -633,18 +846,18 @@
int bits,
int value))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflatePrime(strm, bits, value);
- FINISH_WITH_ERR(strm, "inflatePrime is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflatePrime is not supported!");
}
ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm,
gz_headerp head))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateGetHeader(strm, head);
- FINISH_WITH_ERR(strm, "inflateGetHeader is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflateGetHeader is not supported!");
}
@@ -653,9 +866,9 @@
const char *version,
int stream_size))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateBackInit_(strm, windowBits, window, version, stream_size);
- FINISH_WITH_ERR(strm, "inflateBackInit is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflateBackInit is not supported!");
}
@@ -663,17 +876,17 @@
in_func in, void FAR *in_desc,
out_func out, void FAR *out_desc))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateBack(strm, in, in_desc, out, out_desc);
- FINISH_WITH_ERR(strm, "inflateBack is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflateBack is not supported!");
}
ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm))
{
- if (!strm->reserved)
+ if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved)
return inflateBackEnd(strm);
- FINISH_WITH_ERR(strm, "inflateBackEnd is not supported!");
+ return ZWRAPD_finishWithErrorMsg(strm, "inflateBackEnd is not supported!");
}
@@ -687,13 +900,13 @@
ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen,
const Bytef *source, uLong sourceLen))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return compress(dest, destLen, source, sourceLen);
{ size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, ZWRAP_DEFAULT_CLEVEL);
- LOG_WRAPPER("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
+ LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n", (int)sourceLen, (int)dstCapacity);
+ if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
*destLen = errorCode;
}
return Z_OK;
@@ -704,12 +917,12 @@
const Bytef *source, uLong sourceLen,
int level))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return compress2(dest, destLen, source, sourceLen, level);
{ size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_compress(dest, dstCapacity, source, sourceLen, level);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
+ if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
*destLen = errorCode;
}
return Z_OK;
@@ -718,7 +931,7 @@
ZEXTERN uLong ZEXPORT z_compressBound OF((uLong sourceLen))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return compressBound(sourceLen);
return ZSTD_compressBound(sourceLen);
@@ -733,7 +946,7 @@
{ size_t dstCapacity = *destLen;
size_t const errorCode = ZSTD_decompress(dest, dstCapacity, source, sourceLen);
- if (ZSTD_isError(errorCode)) return Z_MEM_ERROR;
+ if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR;
*destLen = errorCode;
}
return Z_OK;
@@ -744,7 +957,7 @@
/* gzip file access functions */
ZEXTERN gzFile ZEXPORT z_gzopen OF((const char *path, const char *mode))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzopen(path, mode);
FINISH_WITH_NULL_ERR("gzopen is not supported!");
}
@@ -752,7 +965,7 @@
ZEXTERN gzFile ZEXPORT z_gzdopen OF((int fd, const char *mode))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzdopen(fd, mode);
FINISH_WITH_NULL_ERR("gzdopen is not supported!");
}
@@ -761,7 +974,7 @@
#if ZLIB_VERNUM >= 0x1240
ZEXTERN int ZEXPORT z_gzbuffer OF((gzFile file, unsigned size))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzbuffer(file, size);
FINISH_WITH_GZ_ERR("gzbuffer is not supported!");
}
@@ -769,7 +982,7 @@
ZEXTERN z_off_t ZEXPORT z_gzoffset OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzoffset(file);
FINISH_WITH_GZ_ERR("gzoffset is not supported!");
}
@@ -777,7 +990,7 @@
ZEXTERN int ZEXPORT z_gzclose_r OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzclose_r(file);
FINISH_WITH_GZ_ERR("gzclose_r is not supported!");
}
@@ -785,7 +998,7 @@
ZEXTERN int ZEXPORT z_gzclose_w OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzclose_w(file);
FINISH_WITH_GZ_ERR("gzclose_w is not supported!");
}
@@ -794,7 +1007,7 @@
ZEXTERN int ZEXPORT z_gzsetparams OF((gzFile file, int level, int strategy))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzsetparams(file, level, strategy);
FINISH_WITH_GZ_ERR("gzsetparams is not supported!");
}
@@ -802,7 +1015,7 @@
ZEXTERN int ZEXPORT z_gzread OF((gzFile file, voidp buf, unsigned len))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzread(file, buf, len);
FINISH_WITH_GZ_ERR("gzread is not supported!");
}
@@ -811,7 +1024,7 @@
ZEXTERN int ZEXPORT z_gzwrite OF((gzFile file,
voidpc buf, unsigned len))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzwrite(file, buf, len);
FINISH_WITH_GZ_ERR("gzwrite is not supported!");
}
@@ -823,7 +1036,7 @@
ZEXTERN int ZEXPORTVA z_gzprintf OF((gzFile file, const char *format, ...))
#endif
{
- if (!g_useZSTD) {
+ if (!g_ZWRAP_useZSTDcompression) {
int ret;
char buf[1024];
va_list args;
@@ -840,7 +1053,7 @@
ZEXTERN int ZEXPORT z_gzputs OF((gzFile file, const char *s))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzputs(file, s);
FINISH_WITH_GZ_ERR("gzputs is not supported!");
}
@@ -848,7 +1061,7 @@
ZEXTERN char * ZEXPORT z_gzgets OF((gzFile file, char *buf, int len))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzgets(file, buf, len);
FINISH_WITH_NULL_ERR("gzgets is not supported!");
}
@@ -856,7 +1069,7 @@
ZEXTERN int ZEXPORT z_gzputc OF((gzFile file, int c))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzputc(file, c);
FINISH_WITH_GZ_ERR("gzputc is not supported!");
}
@@ -868,7 +1081,7 @@
ZEXTERN int ZEXPORT z_gzgetc OF((gzFile file))
#endif
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzgetc(file);
FINISH_WITH_GZ_ERR("gzgetc is not supported!");
}
@@ -876,7 +1089,7 @@
ZEXTERN int ZEXPORT z_gzungetc OF((int c, gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzungetc(c, file);
FINISH_WITH_GZ_ERR("gzungetc is not supported!");
}
@@ -884,7 +1097,7 @@
ZEXTERN int ZEXPORT z_gzflush OF((gzFile file, int flush))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzflush(file, flush);
FINISH_WITH_GZ_ERR("gzflush is not supported!");
}
@@ -892,7 +1105,7 @@
ZEXTERN z_off_t ZEXPORT z_gzseek OF((gzFile file, z_off_t offset, int whence))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzseek(file, offset, whence);
FINISH_WITH_GZ_ERR("gzseek is not supported!");
}
@@ -900,7 +1113,7 @@
ZEXTERN int ZEXPORT z_gzrewind OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzrewind(file);
FINISH_WITH_GZ_ERR("gzrewind is not supported!");
}
@@ -908,7 +1121,7 @@
ZEXTERN z_off_t ZEXPORT z_gztell OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gztell(file);
FINISH_WITH_GZ_ERR("gztell is not supported!");
}
@@ -916,7 +1129,7 @@
ZEXTERN int ZEXPORT z_gzeof OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzeof(file);
FINISH_WITH_GZ_ERR("gzeof is not supported!");
}
@@ -924,7 +1137,7 @@
ZEXTERN int ZEXPORT z_gzdirect OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzdirect(file);
FINISH_WITH_GZ_ERR("gzdirect is not supported!");
}
@@ -932,7 +1145,7 @@
ZEXTERN int ZEXPORT z_gzclose OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzclose(file);
FINISH_WITH_GZ_ERR("gzclose is not supported!");
}
@@ -940,7 +1153,7 @@
ZEXTERN const char * ZEXPORT z_gzerror OF((gzFile file, int *errnum))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
return gzerror(file, errnum);
FINISH_WITH_NULL_ERR("gzerror is not supported!");
}
@@ -948,7 +1161,7 @@
ZEXTERN void ZEXPORT z_gzclearerr OF((gzFile file))
{
- if (!g_useZSTD)
+ if (!g_ZWRAP_useZSTDcompression)
gzclearerr(file);
}
diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index 24247b2..258cb23 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -26,11 +26,36 @@
#endif
#endif
-void useZSTD(int turn_on);
-int isUsingZSTD(void);
+/* returns a string with version of zstd library */
const char * zstdVersion(void);
+/* COMPRESSION */
+/* enables/disables zstd compression during runtime */
+void ZWRAP_useZSTDcompression(int turn_on);
+
+/* check if zstd compression is turned on */
+int ZWRAP_isUsingZSTDcompression(void);
+
+/* Changes a pledged source size for a given compression stream.
+ It will change ZSTD compression parameters what may improve compression speed and/or ratio.
+ The function should be called just after deflateInit(). */
+int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize);
+
+
+/* DECOMPRESSION */
+typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type;
+
+/* enables/disables automatic recognition of zstd/zlib compressed data during runtime */
+void ZWRAP_setDecompressionType(ZWRAP_decompress_type type);
+
+/* check zstd decompression type */
+ZWRAP_decompress_type ZWRAP_getDecompressionType(void);
+
+
+
+
+
#if defined (__cplusplus)
}
#endif