Merge remote-tracking branch 'refs/remotes/origin/dev' into dev070
# Conflicts:
# .travis.yml
# Makefile
# lib/common/zstd_static.h
# programs/Makefile
# projects/VS2008/zstd/zstd.vcproj
# projects/VS2008/zstdlib/zstdlib.vcproj
# projects/cmake/lib/CMakeLists.txt
# projects/cmake/programs/CMakeLists.txt
diff --git a/.gitattributes b/.gitattributes
index f314af9..da0f7a5 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -12,6 +12,7 @@
# Visual Studio
*.sln text eol=crlf
*.vcxproj* text eol=crlf
+*.vcproj* text eol=crlf
*.suo binary
*.rc binary
diff --git a/.travis.yml b/.travis.yml
index f130978..e6a32c2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -94,3 +94,4 @@
script:
- make $MAKE_PARAM
+
diff --git a/Makefile b/Makefile
index 3a585da..3c600f5 100644
--- a/Makefile
+++ b/Makefile
@@ -2,19 +2,19 @@
# zstd - Makefile
# Copyright (C) Yann Collet 2014-2016
# All rights reserved.
-#
+#
# BSD license
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
-#
+#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
-#
+#
# * Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
-#
+#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -25,7 +25,7 @@
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
+#
# You can contact the author at :
# - zstd homepage : http://www.zstd.net/
# ################################################################
@@ -45,7 +45,7 @@
default: zstdprogram
-all:
+all:
$(MAKE) -C $(ZSTDDIR) $@
$(MAKE) -C $(PRGDIR) $@
@@ -82,6 +82,9 @@
$(MAKE) install PREFIX=~/install_test_dir
cmaketest:
+ cmake --version
+ rm -rf projects/cmake/build
+ mkdir projects/cmake/build
cd projects/cmake/build ; cmake .. ; $(MAKE)
clangtest: clean
@@ -128,7 +131,7 @@
ppctest: clean
$(MAKE) -C $(PRGDIR) datagen # use native, faster
- $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
+ $(MAKE) -C $(PRGDIR) test CC=powerpc-linux-gnu-gcc ZSTDRTTEST= MOREFLAGS="-Werror -static"
# for Travis CI
ppcinstall: clean
diff --git a/NEWS b/NEWS
index c100700..beb6ae3 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,10 @@
-v0.6.2
+v0.7.0
+New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski
New : Support for Sparse File-systems (do not use space for zero-filled sectors)
-New : Support pass-through mode, when using `-df`
+New : Support pass-through mode (when using `-df`)
+New : API : dictionary files from custom content, by Giuseppe Ottaviano
+New : API support for custom malloc/free functions
+New : controllable Dictionary ID
v0.6.1
New : zlib wrapper API, thanks to Przemyslaw Skibinski
@@ -63,7 +67,7 @@
removed `zstd.c`
v0.4.0
-Command line utility compatible with high compression levels
+Command line utility compatible with high compression levels
Removed zstdhc => merged into zstd
Added : ZBUFF API (see zstd_buffered.h)
Rolling buffer support
@@ -93,7 +97,7 @@
Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier
v0.2.1
-Fix : Read errors, advanced fuzzer tests, by Hanno Böck
+Fix : Read errors, advanced fuzzer tests, by Hanno Böck
v0.2.0
**Breaking format change**
@@ -112,4 +116,3 @@
git@github.com:Cyan4973/zstd.git
v0.1.0
first release
-
diff --git a/lib/Makefile b/lib/Makefile
index 9043669..75b09e5 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -2,19 +2,19 @@
# ZSTD library - Makefile
# Copyright (C) Yann Collet 2015-2016
# All rights reserved.
-#
+#
# BSD license
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
-#
+#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
-#
+#
# * Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
-#
+#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -25,7 +25,7 @@
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
+#
# You can contact the author at :
# - ZSTD homepage : http://www.zstd.net
# ################################################################
@@ -43,24 +43,21 @@
DESTDIR?=
PREFIX ?= /usr/local
-CPPFLAGS= -I./common
+LIBDIR ?= $(PREFIX)/lib
+INCLUDEDIR=$(PREFIX)/include
+
+CPPFLAGS= -I./common -DXXH_NAMESPACE=ZSTD_
CFLAGS ?= -O3
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
-LIBDIR ?= $(PREFIX)/lib
-INCLUDEDIR=$(PREFIX)/include
-ZSTDCOMP_FILES := compress/zstd_compress.c compress/fse_compress.c compress/huf_compress.c compress/zbuff_compress.c
-ZSTDDECOMP_FILES := decompress/zstd_decompress.c common/fse_decompress.c decompress/huf_decompress.c decompress/zbuff_decompress.c
-ZSTDDICT_FILES := dictBuilder/zdict.c dictBuilder/divsufsort.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) common/entropy_common.c common/zstd_common.c $(ZSTDCOMP_FILES) $(ZSTDDICT_FILES)
-ZSTD_LEGACY:= legacy/zstd_v01.c legacy/zstd_v02.c legacy/zstd_v03.c legacy/zstd_v04.c legacy/zstd_v05.c
+ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
else
-ZSTD_FILES+= $(ZSTD_LEGACY)
+ZSTD_FILES+= legacy/*.c
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
endif
diff --git a/lib/common/error_private.h b/lib/common/error_private.h
index 3f039ae..6b243c0 100644
--- a/lib/common/error_private.h
+++ b/lib/common/error_private.h
@@ -106,6 +106,7 @@
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+ case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
diff --git a/lib/common/error_public.h b/lib/common/error_public.h
index 6fcf802..660b2d3 100644
--- a/lib/common/error_public.h
+++ b/lib/common/error_public.h
@@ -58,6 +58,7 @@
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,
ZSTD_error_dictionary_corrupted,
+ ZSTD_error_dictionary_wrong,
ZSTD_error_maxCode
} ZSTD_ErrorCode;
diff --git a/programs/xxhash.c b/lib/common/xxhash.c
similarity index 62%
rename from programs/xxhash.c
rename to lib/common/xxhash.c
index 92df902..e175ae9 100644
--- a/programs/xxhash.c
+++ b/lib/common/xxhash.c
@@ -1,41 +1,42 @@
/*
-xxHash - Fast Hash algorithm
-Copyright (C) 2012-2016, Yann Collet
-
-BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-* Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- xxHash source repository : https://github.com/Cyan4973/xxHash
+* xxHash - Fast Hash algorithm
+* Copyright (C) 2012-2016, Yann Collet
+*
+* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+*
+* Redistribution and use in source and binary forms, with or without
+* modification, are permitted provided that the following conditions are
+* met:
+*
+* * Redistributions of source code must retain the above copyright
+* notice, this list of conditions and the following disclaimer.
+* * Redistributions in binary form must reproduce the above
+* copyright notice, this list of conditions and the following disclaimer
+* in the documentation and/or other materials provided with the
+* distribution.
+*
+* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*
+* You can contact the author at :
+* - xxHash homepage: http://www.xxhash.com
+* - xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* *************************************
* Tuning parameters
***************************************/
-/*!XXH_FORCE_MEMORY_ACCESS
+/*!XXH_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
@@ -65,24 +66,47 @@
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
/*!XXH_FORCE_NATIVE_FORMAT :
- * By default, xxHash library provides endian-independent Hash values, based on little-endian convention.
+ * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
* Results are therefore identical for little-endian and big-endian CPU.
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
* Should endian-independance be of no importance for your application, you may set the #define below to 1,
* to improve speed for Big-endian CPU.
* This option has no impact on Little_Endian CPU.
*/
-#define XXH_FORCE_NATIVE_FORMAT 0
-
-/*!XXH_USELESS_ALIGN_BRANCH :
- * This is a minor performance trick, only useful with lots of very small keys.
- * It means : don't check for aligned/unaligned input, because performance will be the same.
- * It saves one initial branch per hash.
- */
-#if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
-# define XXH_USELESS_ALIGN_BRANCH 1
+#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
+# define XXH_FORCE_NATIVE_FORMAT 0
#endif
+/*!XXH_FORCE_ALIGN_CHECK :
+ * This is a minor performance trick, only useful with lots of very small keys.
+ * It means : check for aligned/unaligned input.
+ * The check costs one initial branch per hash; set to 0 when the input data
+ * is guaranteed to be aligned.
+ */
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
+# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+# define XXH_FORCE_ALIGN_CHECK 0
+# else
+# define XXH_FORCE_ALIGN_CHECK 1
+# endif
+#endif
+
+
+/* *************************************
+* Includes & Memory related functions
+***************************************/
+/* Modify the local functions below should you wish to use some other memory routines */
+/* for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void XXH_free (void* p) { free(p); }
+/* for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"
+
/* *************************************
* Compiler Specific Options
@@ -104,26 +128,11 @@
/* *************************************
-* Includes & Memory related functions
-***************************************/
-/* Modify the local functions below should you wish to use some other memory routines */
-/* for malloc(), free() */
-#include <stdlib.h>
-static void* XXH_malloc(size_t s) { return malloc(s); }
-static void XXH_free (void* p) { free(p); }
-/* for memcpy() */
-#include <string.h>
-static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
-
-#include "xxhash.h"
-
-
-/* *************************************
* Basic Types
***************************************/
#ifndef MEM_MODULE
# define MEM_MODULE
-# if !defined (__VMS) && ( defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ )
+# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
@@ -250,6 +259,11 @@
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
+static U32 XXH_readBE32(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
+}
+
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
@@ -263,6 +277,11 @@
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
+static U64 XXH_readBE64(const void* ptr)
+{
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
+}
+
/* *************************************
* Macros
@@ -273,17 +292,17 @@
/* *************************************
* Constants
***************************************/
-#define PRIME32_1 2654435761U
-#define PRIME32_2 2246822519U
-#define PRIME32_3 3266489917U
-#define PRIME32_4 668265263U
-#define PRIME32_5 374761393U
+static const U32 PRIME32_1 = 2654435761U;
+static const U32 PRIME32_2 = 2246822519U;
+static const U32 PRIME32_3 = 3266489917U;
+static const U32 PRIME32_4 = 668265263U;
+static const U32 PRIME32_5 = 374761393U;
-#define PRIME64_1 11400714785074694791ULL
-#define PRIME64_2 14029467366897019727ULL
-#define PRIME64_3 1609587929392839161ULL
-#define PRIME64_4 9650029242287828579ULL
-#define PRIME64_5 2870177450012600261ULL
+static const U64 PRIME64_1 = 11400714785074694791ULL;
+static const U64 PRIME64_2 = 14029467366897019727ULL;
+static const U64 PRIME64_3 = 1609587929392839161ULL;
+static const U64 PRIME64_4 = 9650029242287828579ULL;
+static const U64 PRIME64_5 = 2870177450012600261ULL;
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
@@ -291,6 +310,15 @@
/* ***************************
* Simple Hash Functions
*****************************/
+
+static U32 XXH32_round(U32 seed, U32 input)
+{
+ seed += input * PRIME32_2;
+ seed = XXH_rotl32(seed, 13);
+ seed *= PRIME32_1;
+ return seed;
+}
+
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
@@ -299,60 +327,40 @@
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL)
- {
+ if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)16;
}
#endif
- if (len>=16)
- {
+ if (len>=16) {
const BYTE* const limit = bEnd - 16;
U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0;
U32 v4 = seed - PRIME32_1;
- do
- {
- v1 += XXH_get32bits(p) * PRIME32_2;
- v1 = XXH_rotl32(v1, 13);
- v1 *= PRIME32_1;
- p+=4;
- v2 += XXH_get32bits(p) * PRIME32_2;
- v2 = XXH_rotl32(v2, 13);
- v2 *= PRIME32_1;
- p+=4;
- v3 += XXH_get32bits(p) * PRIME32_2;
- v3 = XXH_rotl32(v3, 13);
- v3 *= PRIME32_1;
- p+=4;
- v4 += XXH_get32bits(p) * PRIME32_2;
- v4 = XXH_rotl32(v4, 13);
- v4 *= PRIME32_1;
- p+=4;
- }
- while (p<=limit);
+ do {
+ v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
+ v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
+ v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
+ v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
+ } while (p<=limit);
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
- }
- else
- {
+ } else {
h32 = seed + PRIME32_5;
}
h32 += (U32) len;
- while (p+4<=bEnd)
- {
+ while (p+4<=bEnd) {
h32 += XXH_get32bits(p) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
p++;
@@ -372,22 +380,20 @@
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
- XXH32_state_t state;
- XXH32_reset(&state, seed);
- XXH32_update(&state, input, len);
- return XXH32_digest(&state);
+ XXH32_CREATESTATE_STATIC(state);
+ XXH32_reset(state, seed);
+ XXH32_update(state, input, len);
+ return XXH32_digest(state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-# if !defined(XXH_USELESS_ALIGN_BRANCH)
- if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */
- {
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- }
-# endif
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ } }
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
@@ -396,103 +402,77 @@
#endif
}
+
+static U64 XXH64_round(U64 acc, U64 input)
+{
+ acc += input * PRIME64_2;
+ acc = XXH_rotl64(acc, 31);
+ acc *= PRIME64_1;
+ return acc;
+}
+
+static U64 XXH64_mergeRound(U64 acc, U64 val)
+{
+ val = XXH64_round(0, val);
+ acc ^= val;
+ acc = acc * PRIME64_1 + PRIME64_4;
+ return acc;
+}
+
FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
- const BYTE* bEnd = p + len;
+ const BYTE* const bEnd = p + len;
U64 h64;
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
- if (p==NULL)
- {
+ if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)32;
}
#endif
- if (len>=32)
- {
+ if (len>=32) {
const BYTE* const limit = bEnd - 32;
U64 v1 = seed + PRIME64_1 + PRIME64_2;
U64 v2 = seed + PRIME64_2;
U64 v3 = seed + 0;
U64 v4 = seed - PRIME64_1;
- do
- {
- v1 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- v2 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- v3 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- v4 += XXH_get64bits(p) * PRIME64_2;
- p+=8;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- }
- while (p<=limit);
+ do {
+ v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
+ v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
+ v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
+ v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
+ } while (p<=limit);
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
- v1 *= PRIME64_2;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- h64 ^= v1;
- h64 = h64 * PRIME64_1 + PRIME64_4;
-
- v2 *= PRIME64_2;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- h64 ^= v2;
- h64 = h64 * PRIME64_1 + PRIME64_4;
-
- v3 *= PRIME64_2;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- h64 ^= v3;
- h64 = h64 * PRIME64_1 + PRIME64_4;
-
- v4 *= PRIME64_2;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- h64 ^= v4;
- h64 = h64 * PRIME64_1 + PRIME64_4;
- }
- else
- {
+ } else {
h64 = seed + PRIME64_5;
}
h64 += (U64) len;
- while (p+8<=bEnd)
- {
- U64 k1 = XXH_get64bits(p);
- k1 *= PRIME64_2;
- k1 = XXH_rotl64(k1,31);
- k1 *= PRIME64_1;
+ while (p+8<=bEnd) {
+ U64 const k1 = XXH64_round(0, XXH_get64bits(p));
h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
- if (p+4<=bEnd)
- {
+ if (p+4<=bEnd) {
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
@@ -512,22 +492,20 @@
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
- XXH64_state_t state;
- XXH64_reset(&state, seed);
- XXH64_update(&state, input, len);
- return XXH64_digest(&state);
+ XXH64_CREATESTATE_STATIC(state);
+ XXH64_reset(state, seed);
+ XXH64_update(state, input, len);
+ return XXH64_digest(state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
-# if !defined(XXH_USELESS_ALIGN_BRANCH)
- if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */
- {
- if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
- return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
- else
- return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
- }
-# endif
+ if (XXH_FORCE_ALIGN_CHECK) {
+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
+ if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+ return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+ else
+ return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+ } }
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
@@ -536,39 +514,13 @@
#endif
}
+
/* **************************************************
* Advanced Hash Functions
****************************************************/
-/*** Allocation ***/
-struct XXH32_state_s
-{
- U64 total_len;
- U32 seed;
- U32 v1;
- U32 v2;
- U32 v3;
- U32 v4;
- U32 mem32[4]; /* defined as U32 for alignment */
- U32 memsize;
-}; /* typedef'd to XXH32_state_t within xxhash.h */
-
-struct XXH64_state_s
-{
- U64 total_len;
- U64 seed;
- U64 v1;
- U64 v2;
- U64 v3;
- U64 v4;
- U64 mem64[4]; /* defined as U64 for alignment */
- U32 memsize;
-}; /* typedef'd to XXH64_state_t within xxhash.h */
-
-
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
{
- XXH_STATIC_ASSERT(sizeof(XXH32_stateBody_t) >= sizeof(XXH32_state_t)); /* A compilation error here means XXH32_state_t is not large enough */
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
@@ -579,7 +531,6 @@
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
{
- XXH_STATIC_ASSERT(sizeof(XXH64_stateBody_t) >= sizeof(XXH64_state_t)); /* A compilation error here means XXH64_state_t is not large enough */
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
@@ -630,67 +581,37 @@
state->total_len += len;
- if (state->memsize + len < 16) /* fill in tmp buffer */
- {
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
- if (state->memsize) /* some data left from previous update */
- {
+ if (state->memsize) { /* some data left from previous update */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
- {
- const U32* p32 = state->mem32;
- state->v1 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v1 = XXH_rotl32(state->v1, 13);
- state->v1 *= PRIME32_1;
- p32++;
- state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v2 = XXH_rotl32(state->v2, 13);
- state->v2 *= PRIME32_1;
- p32++;
- state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v3 = XXH_rotl32(state->v3, 13);
- state->v3 *= PRIME32_1;
- p32++;
- state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
- state->v4 = XXH_rotl32(state->v4, 13);
- state->v4 *= PRIME32_1;
- p32++;
+ { const U32* p32 = state->mem32;
+ state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
+ state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
+ state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
+ state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
}
p += 16-state->memsize;
state->memsize = 0;
}
- if (p <= bEnd-16)
- {
+ if (p <= bEnd-16) {
const BYTE* const limit = bEnd - 16;
U32 v1 = state->v1;
U32 v2 = state->v2;
U32 v3 = state->v3;
U32 v4 = state->v4;
- do
- {
- v1 += XXH_readLE32(p, endian) * PRIME32_2;
- v1 = XXH_rotl32(v1, 13);
- v1 *= PRIME32_1;
- p+=4;
- v2 += XXH_readLE32(p, endian) * PRIME32_2;
- v2 = XXH_rotl32(v2, 13);
- v2 *= PRIME32_1;
- p+=4;
- v3 += XXH_readLE32(p, endian) * PRIME32_2;
- v3 = XXH_rotl32(v3, 13);
- v3 *= PRIME32_1;
- p+=4;
- v4 += XXH_readLE32(p, endian) * PRIME32_2;
- v4 = XXH_rotl32(v4, 13);
- v4 *= PRIME32_1;
- p+=4;
- }
- while (p<=limit);
+ do {
+ v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
+ v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
+ v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
+ v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
+ } while (p<=limit);
state->v1 = v1;
state->v2 = v2;
@@ -698,8 +619,7 @@
state->v4 = v4;
}
- if (p < bEnd)
- {
+ if (p < bEnd) {
XXH_memcpy(state->mem32, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
@@ -722,31 +642,26 @@
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{
const BYTE * p = (const BYTE*)state->mem32;
- const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
+ const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
- if (state->total_len >= 16)
- {
+ if (state->total_len >= 16) {
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
- }
- else
- {
- h32 = state->seed + PRIME32_5;
+ } else {
+ h32 = state->seed + PRIME32_5;
}
h32 += (U32) state->total_len;
- while (p+4<=bEnd)
- {
+ while (p+4<=bEnd) {
h32 += XXH_readLE32(p, endian) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h32 += (*p) * PRIME32_5;
- h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+ h32 = XXH_rotl32(h32, 11) * PRIME32_1;
p++;
}
@@ -771,6 +686,9 @@
}
+
+/* **** XXH64 **** */
+
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
const BYTE* p = (const BYTE*)input;
@@ -782,67 +700,35 @@
state->total_len += len;
- if (state->memsize + len < 32) /* fill in tmp buffer */
- {
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
- if (state->memsize) /* some data left from previous update */
- {
+ if (state->memsize) { /* tmp buffer is full */
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
- {
- const U64* p64 = state->mem64;
- state->v1 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v1 = XXH_rotl64(state->v1, 31);
- state->v1 *= PRIME64_1;
- p64++;
- state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v2 = XXH_rotl64(state->v2, 31);
- state->v2 *= PRIME64_1;
- p64++;
- state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v3 = XXH_rotl64(state->v3, 31);
- state->v3 *= PRIME64_1;
- p64++;
- state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
- state->v4 = XXH_rotl64(state->v4, 31);
- state->v4 *= PRIME64_1;
- p64++;
- }
+ state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
+ state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
+ state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
+ state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
p += 32-state->memsize;
state->memsize = 0;
}
- if (p+32 <= bEnd)
- {
+ if (p+32 <= bEnd) {
const BYTE* const limit = bEnd - 32;
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
- do
- {
- v1 += XXH_readLE64(p, endian) * PRIME64_2;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- p+=8;
- v2 += XXH_readLE64(p, endian) * PRIME64_2;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- p+=8;
- v3 += XXH_readLE64(p, endian) * PRIME64_2;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- p+=8;
- v4 += XXH_readLE64(p, endian) * PRIME64_2;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- p+=8;
- }
- while (p<=limit);
+ do {
+ v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
+ v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
+ v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
+ v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
+ } while (p<=limit);
state->v1 = v1;
state->v2 = v2;
@@ -850,8 +736,7 @@
state->v4 = v4;
}
- if (p < bEnd)
- {
+ if (p < bEnd) {
XXH_memcpy(state->mem64, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
@@ -874,71 +759,42 @@
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{
const BYTE * p = (const BYTE*)state->mem64;
- const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
+ const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
- if (state->total_len >= 32)
- {
- U64 v1 = state->v1;
- U64 v2 = state->v2;
- U64 v3 = state->v3;
- U64 v4 = state->v4;
+ if (state->total_len >= 32) {
+ U64 const v1 = state->v1;
+ U64 const v2 = state->v2;
+ U64 const v3 = state->v3;
+ U64 const v4 = state->v4;
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
-
- v1 *= PRIME64_2;
- v1 = XXH_rotl64(v1, 31);
- v1 *= PRIME64_1;
- h64 ^= v1;
- h64 = h64*PRIME64_1 + PRIME64_4;
-
- v2 *= PRIME64_2;
- v2 = XXH_rotl64(v2, 31);
- v2 *= PRIME64_1;
- h64 ^= v2;
- h64 = h64*PRIME64_1 + PRIME64_4;
-
- v3 *= PRIME64_2;
- v3 = XXH_rotl64(v3, 31);
- v3 *= PRIME64_1;
- h64 ^= v3;
- h64 = h64*PRIME64_1 + PRIME64_4;
-
- v4 *= PRIME64_2;
- v4 = XXH_rotl64(v4, 31);
- v4 *= PRIME64_1;
- h64 ^= v4;
- h64 = h64*PRIME64_1 + PRIME64_4;
- }
- else
- {
+ h64 = XXH64_mergeRound(h64, v1);
+ h64 = XXH64_mergeRound(h64, v2);
+ h64 = XXH64_mergeRound(h64, v3);
+ h64 = XXH64_mergeRound(h64, v4);
+ } else {
h64 = state->seed + PRIME64_5;
}
h64 += (U64) state->total_len;
- while (p+8<=bEnd)
- {
- U64 k1 = XXH_readLE64(p, endian);
- k1 *= PRIME64_2;
- k1 = XXH_rotl64(k1,31);
- k1 *= PRIME64_1;
+ while (p+8<=bEnd) {
+ U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
h64 ^= k1;
- h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+ h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
- if (p+4<=bEnd)
- {
+ if (p+4<=bEnd) {
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
- h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
- while (p<bEnd)
- {
+ while (p<bEnd) {
h64 ^= (*p) * PRIME64_5;
- h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+ h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
@@ -963,3 +819,36 @@
}
+/* **************************
+* Canonical representation
+****************************/
+
+/*! Default XXH result types are basic unsigned 32 and 64 bits.
+* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
+* These functions allow transformation of hash result into and from its canonical format.
+* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
+*/
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
+ memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
+{
+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
+ memcpy(dst, &hash, sizeof(*dst));
+}
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
+{
+ return XXH_readBE32(src);
+}
+
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
+{
+ return XXH_readBE64(src);
+}
diff --git a/programs/xxhash.h b/lib/common/xxhash.h
similarity index 71%
rename from programs/xxhash.h
rename to lib/common/xxhash.h
index 4b1e1dc..d654871 100644
--- a/programs/xxhash.h
+++ b/lib/common/xxhash.h
@@ -64,7 +64,8 @@
XXH32 6.8 GB/s 6.0 GB/s
*/
-#pragma once
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
#if defined (__cplusplus)
extern "C" {
@@ -138,7 +139,7 @@
* Version
***************************************/
#define XXH_VERSION_MAJOR 0
-#define XXH_VERSION_MINOR 5
+#define XXH_VERSION_MINOR 6
#define XXH_VERSION_RELEASE 0
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
XXH_PUBLIC_API unsigned XXH_versionNumber (void);
@@ -147,9 +148,11 @@
/* ****************************
* Simple Hash Functions
******************************/
+typedef unsigned int XXH32_hash_t;
+typedef unsigned long long XXH64_hash_t;
-XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t length, unsigned int seed);
-XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
+XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
/*!
XXH32() :
@@ -165,23 +168,13 @@
/* ****************************
-* Advanced Hash Functions
+* Streaming Hash Functions
******************************/
-typedef struct XXH32_state_s XXH32_state_t; /* incomplete */
-typedef struct XXH64_state_s XXH64_state_t; /* incomplete */
+typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
-
-/*!Static allocation
- For static linking only, do not use in the context of DLL ! */
-typedef struct { long long ll[ 6]; } XXH32_stateBody_t;
-typedef struct { long long ll[11]; } XXH64_stateBody_t;
-
-#define XXH32_CREATESTATE_STATIC(name) XXH32_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH32_state_t* name = (XXH32_state_t*)(name##xxhvoid) /* no final ; */
-#define XXH64_CREATESTATE_STATIC(name) XXH64_stateBody_t name##xxhbody; void* name##xxhvoid = &(name##xxhbody); XXH64_state_t* name = (XXH64_state_t*)(name##xxhvoid) /* no final ; */
-
-
-/*!Dynamic allocation
- To be preferred in the context of DLL */
+/*! Dynamic allocation of states
+ Compatible with dynamic libraries */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
@@ -194,11 +187,11 @@
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* statePtr);
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
-XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
-XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
-XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
+XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
/*!
These functions generate the xxHash of an input provided in multiple segments,
@@ -213,14 +206,68 @@
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, a hash value can be produced anytime, by using XXHnn_digest().
-This function returns the nn-bits hash.
-It's nonetheless possible to continue inserting input into the hash state
+This function returns the nn-bits hash as an int or long long.
+
+It's still possible to continue inserting input into the hash state after a digest,
and later on generate some new hashes, by calling again XXHnn_digest().
When done, free XXH state space if it was allocated dynamically.
*/
+/* **************************
+* Canonical representation
+****************************/
+typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
+typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
+
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
+
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
+
+/*! Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+* These functions allow transformation of hash result into and from its canonical format.
+* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
+
+
+#ifdef XXH_STATIC_LINKING_ONLY
+
+/* This part contains definition which shall only be used with static linking.
+ The prototypes / types defined here are not guaranteed to remain stable.
+ They could change in a future version, becoming incompatible with a different version of the library */
+
+ struct XXH32_state_s {
+ unsigned long long total_len;
+ unsigned seed;
+ unsigned v1;
+ unsigned v2;
+ unsigned v3;
+ unsigned v4;
+ unsigned mem32[4]; /* buffer defined as U32 for alignment */
+ unsigned memsize;
+ }; /* typedef'd to XXH32_state_t */
+
+ struct XXH64_state_s {
+ unsigned long long total_len;
+ unsigned long long seed;
+ unsigned long long v1;
+ unsigned long long v2;
+ unsigned long long v3;
+ unsigned long long v4;
+ unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
+ unsigned memsize;
+ }; /* typedef'd to XXH64_state_t */
+
+
+#endif
+
+
#if defined (__cplusplus)
}
#endif
+
+#endif /* XXHASH_H_5627135585666179 */
diff --git a/lib/common/zstd.h b/lib/common/zstd.h
index dbcf62b..3ac4056 100644
--- a/lib/common/zstd.h
+++ b/lib/common/zstd.h
@@ -60,8 +60,8 @@
* Version
***************************************/
#define ZSTD_VERSION_MAJOR 0
-#define ZSTD_VERSION_MINOR 6
-#define ZSTD_VERSION_RELEASE 2
+#define ZSTD_VERSION_MINOR 7
+#define ZSTD_VERSION_RELEASE 0
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 8812c03..2a2c39d 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -34,7 +34,7 @@
* Dependencies
***************************************/
#include "error_private.h"
-#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName */
+#include "zstd_static.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode */
#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index b2fcd32..63e56aa 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -63,7 +63,7 @@
#endif
#define ZSTD_OPT_NUM (1<<12)
-#define ZSTD_DICT_MAGIC 0xEC30A436
+#define ZSTD_DICT_MAGIC 0xEC30A437
#define ZSTD_REP_NUM 3
#define ZSTD_REP_INIT ZSTD_REP_NUM
@@ -82,6 +82,7 @@
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 12
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 };
+static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
diff --git a/lib/common/zstd_static.h b/lib/common/zstd_static.h
index 126d6cf..596ca03 100644
--- a/lib/common/zstd_static.h
+++ b/lib/common/zstd_static.h
@@ -88,6 +88,7 @@
typedef struct {
U32 contentSizeFlag; /* 1: content size will be in frame header (if known). */
+ U32 noDictIDFlag; /* 1: no dict ID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters;
typedef struct {
@@ -104,11 +105,11 @@
* Advanced functions
***************************************/
/*! ZSTD_createCCtx_advanced() :
- * Create a ZSTD compression context using external alloc and free functions */
+ * Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_createDCtx_advanced() :
- * Create a ZSTD decompression context using external alloc and free functions */
+ * Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API unsigned ZSTD_maxCLevel (void);
@@ -192,10 +193,14 @@
You can then reuse ZSTD_CCtx to compress some new frame.
*/
-typedef struct { U64 frameContentSize; U32 windowLog; } ZSTD_frameParams;
+typedef struct {
+ U64 frameContentSize;
+ U32 windowLog;
+ U32 dictID;
+} ZSTD_frameParams;
-#define ZSTD_FRAMEHEADERSIZE_MAX 13 /* for static allocation */
-static const size_t ZSTD_frameHeaderSize_min = 5;
+#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
+static const size_t ZSTD_frameHeaderSize_min = 6;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 91f6bb7..c7ba030 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -99,6 +99,7 @@
U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd;
U32 stage; /* 0: created; 1: init,dictLoad; 2:started */
+ U32 dictID;
ZSTD_parameters params;
void* workSpace;
size_t workSpaceSize;
@@ -144,7 +145,7 @@
if (!ctx) return NULL;
memset(ctx, 0, sizeof(ZSTD_CCtx));
- ctx->customAlloc = customMem.customAlloc;
+ ctx->customAlloc = customMem.customAlloc;
ctx->customFree = customMem.customFree;
return ctx;
}
@@ -229,6 +230,7 @@
{
ZSTD_CCtx* zc = ZSTD_createCCtx();
ZSTD_parameters params;
+ memset(¶ms, 0, sizeof(params));
params.cParams = cParams;
params.fParams.contentSizeFlag = 1;
ZSTD_compressBegin_advanced(zc, NULL, 0, params, 0);
@@ -300,6 +302,7 @@
zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
zc->stage = 1;
+ zc->dictID = 0;
zc->loadedDictEnd = 0;
return 0;
@@ -328,7 +331,7 @@
memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace);
}
- /* copy dictionary pointers */
+ /* copy dictionary offsets */
dstCCtx->nextToUpdate = srcCCtx->nextToUpdate;
dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3;
dstCCtx->nextSrc = srcCCtx->nextSrc;
@@ -337,6 +340,7 @@
dstCCtx->dictLimit = srcCCtx->dictLimit;
dstCCtx->lowLimit = srcCCtx->lowLimit;
dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd;
+ dstCCtx->dictID = srcCCtx->dictID;
/* copy entropy tables */
dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
@@ -2092,27 +2096,40 @@
static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity,
- ZSTD_parameters params, U64 pledgedSrcSize)
+ ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID)
{ BYTE* const op = (BYTE*)dst;
U32 const fcsId = params.fParams.contentSizeFlag ?
(pledgedSrcSize>0) + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) : /* 0-3 */
0;
- BYTE const fdescriptor = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
+ BYTE const fAllocByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) /* windowLog : 4 KB - 128 MB */
| (fcsId << 6) );
- size_t const hSize = ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId];
- if (hSize > dstCapacity) return ERROR(dstSize_tooSmall);
+ U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */
+ BYTE const fCheckByte = (BYTE)(dictIDSizeCode&3);
+ size_t pos;
+
+ if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall);
MEM_writeLE32(dst, ZSTD_MAGICNUMBER);
- op[4] = fdescriptor;
+ op[4] = fAllocByte;
+ op[5] = fCheckByte;
+ pos = 6;
+ switch(dictIDSizeCode)
+ {
+ default: /* impossible */
+ case 0 : break;
+ case 1 : op[pos] = (BYTE)(dictID); pos++; break;
+ case 2 : MEM_writeLE16(op+pos, (U16)(dictID)); pos+=2; break;
+ case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break;
+ }
switch(fcsId)
{
default: /* impossible */
case 0 : break;
- case 1 : op[5] = (BYTE)(pledgedSrcSize); break;
- case 2 : MEM_writeLE16(op+5, (U16)(pledgedSrcSize-256)); break;
- case 3 : MEM_writeLE64(op+5, (U64)(pledgedSrcSize)); break;
+ case 1 : op[pos] = (BYTE)(pledgedSrcSize); pos++; break;
+ case 2 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break;
+ case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break;
}
- return hSize;
+ return pos;
}
@@ -2126,7 +2143,7 @@
if (zc->stage==0) return ERROR(stage_wrong);
if (frame && (zc->stage==1)) { /* copy saved header */
- fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize);
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, srcSize, zc->dictID);
if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize;
dst = (char*)dst + fhSize;
@@ -2284,13 +2301,14 @@
* @return : 0, or an error code */
static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize)
{
- if ((dict==NULL) || (dictSize<=4)) return 0;
+ if ((dict==NULL) || (dictSize<=8)) return 0;
/* default : dict is pure content */
if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
+ zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4);
/* known magic number : dict is parsed for entropy stats and content */
- { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+4 /* skip magic */, dictSize-4) + 4;
+ { size_t const eSize = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8) + 8;
if (ZSTD_isError(eSize)) return eSize;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
}
@@ -2303,7 +2321,7 @@
const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize)
{
- { U32 const hashLog3 = (pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
+ { U32 const hashLog3 = (!pledgedSrcSize || pledgedSrcSize >= 8192) ? ZSTD_HASHLOG3_MAX : ((pledgedSrcSize >= 2048) ? ZSTD_HASHLOG3_MIN + 1 : ZSTD_HASHLOG3_MIN);
zc->hashLog3 = (params.cParams.searchLength==3) ? hashLog3 : 0; }
{ size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, 1);
@@ -2330,8 +2348,8 @@
size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* zc, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters params;
+ memset(¶ms, 0, sizeof(params));
params.cParams = ZSTD_getCParams(compressionLevel, 0, dictSize);
- params.fParams.contentSizeFlag = 0;
ZSTD_adjustCParams(¶ms.cParams, 0, dictSize);
ZSTD_LOG_BLOCK("%p: ZSTD_compressBegin_usingDict compressionLevel=%d\n", zc->base, compressionLevel);
return ZSTD_compressBegin_internal(zc, dict, dictSize, params, 0);
@@ -2358,7 +2376,7 @@
/* special case : empty frame */
if (zc->stage==1) {
- fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0);
+ fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, zc->params, 0, 0);
if (ZSTD_isError(fhSize)) return fhSize;
dstCapacity -= fhSize;
op += fhSize;
@@ -2434,6 +2452,7 @@
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters params;
+ memset(¶ms, 0, sizeof(params));
ZSTD_LOG_BLOCK("%p: ZSTD_compress_usingDict srcSize=%d dictSize=%d compressionLevel=%d\n", ctx->base, (int)srcSize, (int)dictSize, compressionLevel);
params.cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize);
params.fParams.contentSizeFlag = 1;
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 75e91e8..da00799 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -121,6 +121,7 @@
ZSTD_freeFunction customFree;
blockType_t bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
ZSTD_dStage stage;
+ U32 dictID;
U32 flagRepeatTable;
const BYTE* litPtr;
size_t litBufSize;
@@ -141,6 +142,7 @@
dctx->dictEnd = NULL;
dctx->hufTableX4[0] = HufLog;
dctx->flagRepeatTable = 0;
+ dctx->dictID = 0;
return 0;
}
@@ -154,8 +156,7 @@
{
ZSTD_DCtx* dctx;
- if (!customMem.customAlloc && !customMem.customFree)
- {
+ if (!customMem.customAlloc && !customMem.customFree) {
dctx = (ZSTD_DCtx*) malloc(sizeof(ZSTD_DCtx));
if (!dctx) return NULL;
dctx->customAlloc = malloc;
@@ -170,7 +171,7 @@
dctx = (ZSTD_DCtx*) customMem.customAlloc(sizeof(ZSTD_DCtx));
if (!dctx) return NULL;
- dctx->customAlloc = customMem.customAlloc;
+ dctx->customAlloc = customMem.customAlloc;
dctx->customFree = customMem.customFree;
ZSTD_decompressBegin(dctx);
@@ -212,12 +213,23 @@
/* Frame descriptor
- 1 byte, using :
+ 1 byte - Alloc :
bit 0-3 : windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN (see zstd_internal.h)
- bit 4 : minmatch 4(0) or 3(1)
+ bit 4 : reserved for windowLog (must be zero)
bit 5 : reserved (must be zero)
bit 6-7 : Frame content size : unknown, 1 byte, 2 bytes, 8 bytes
+ 1 byte - checker :
+ bit 0-1 : dictID (0, 1, 2 or 4 bytes)
+ bit 2-7 : reserved (must be zero)
+
+ Optional : dictID (0, 1, 2 or 4 bytes)
+ Automatic adaptation
+ 0 : no dictID
+ 1 : 1 - 255
+ 2 : 256 - 65535
+ 4 : all other values
+
Optional : content size (0, 1, 2 or 8 bytes)
0 : unknown
1 : 0-255 bytes
@@ -298,8 +310,10 @@
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong);
- { U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
- return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId]; }
+ { U32 const fcsId = (((const BYTE*)src)[4]) >> 6;
+ U32 const dictID =(((const BYTE*)src)[5]) & 3;
+ return ZSTD_frameHeaderSize_min + ZSTD_fcs_fieldSize[fcsId] + ZSTD_did_fieldSize[dictID];
+ }
}
@@ -328,16 +342,27 @@
if (srcSize < fhsize) return fhsize; }
memset(fparamsPtr, 0, sizeof(*fparamsPtr));
- { BYTE const frameDesc = ip[4];
- fparamsPtr->windowLog = (frameDesc & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
- if ((frameDesc & 0x20) != 0) return ERROR(frameParameter_unsupported); /* reserved 1 bit */
- switch(frameDesc >> 6) /* fcsId */
+ { BYTE const allocByte = ip[4];
+ BYTE const checkByte = ip[5];
+ size_t pos = ZSTD_frameHeaderSize_min;
+ U32 const dictIDSizeCode = checkByte&3;
+ fparamsPtr->windowLog = (allocByte & 0xF) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
+ if ((allocByte & 0x30) != 0) return ERROR(frameParameter_unsupported); /* reserved bits */
+ switch(dictIDSizeCode) /* fcsId */
+ {
+ default: /* impossible */
+ case 0 : fparamsPtr->dictID = 0; break;
+ case 1 : fparamsPtr->dictID = ip[pos]; pos++; break;
+ case 2 : fparamsPtr->dictID = MEM_readLE16(ip+pos); pos+=2; break;
+ case 3 : fparamsPtr->dictID = MEM_readLE32(ip+pos); pos+=4; break;
+ }
+ switch(allocByte >> 6) /* fcsId */
{
default: /* impossible */
case 0 : fparamsPtr->frameContentSize = 0; break;
- case 1 : fparamsPtr->frameContentSize = ip[5]; break;
- case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+5)+256; break;
- case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+5); break;
+ case 1 : fparamsPtr->frameContentSize = ip[pos]; break;
+ case 2 : fparamsPtr->frameContentSize = MEM_readLE16(ip+pos)+256; break;
+ case 3 : fparamsPtr->frameContentSize = MEM_readLE64(ip+pos); break;
} }
return 0;
}
@@ -346,10 +371,11 @@
/** ZSTD_decodeFrameHeader() :
* `srcSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
-static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* zc, const void* src, size_t srcSize)
+static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize)
{
- size_t const result = ZSTD_getFrameParams(&(zc->fParams), src, srcSize);
- if ((MEM_32bits()) && (zc->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
+ size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize);
+ if ((MEM_32bits()) && (dctx->fParams.windowLog > 25)) return ERROR(frameParameter_unsupportedBy32bits);
+ if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
return result;
}
@@ -1151,25 +1177,29 @@
static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{
- size_t eSize;
- U32 const magic = MEM_readLE32(dict);
- if (magic != ZSTD_DICT_MAGIC) {
- /* pure content mode */
+ if (dictSize < 8) return ERROR(dictionary_corrupted);
+ { U32 const magic = MEM_readLE32(dict);
+ if (magic != ZSTD_DICT_MAGIC) {
+ /* pure content mode */
+ ZSTD_refDictContent(dctx, dict, dictSize);
+ return 0;
+ }
+ dctx->dictID = MEM_readLE32((const char*)dict + 4);
+
+ /* load entropy tables */
+ dict = (const char*)dict + 8;
+ dictSize -= 8;
+ { size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+ if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
+ dict = (const char*)dict + eSize;
+ dictSize -= eSize;
+ }
+
+ /* reference dictionary content */
ZSTD_refDictContent(dctx, dict, dictSize);
+
return 0;
}
- /* load entropy tables */
- dict = (const char*)dict + 4;
- dictSize -= 4;
- eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
- if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
-
- /* reference dictionary content */
- dict = (const char*)dict + eSize;
- dictSize -= eSize;
- ZSTD_refDictContent(dctx, dict, dictSize);
-
- return 0;
}
@@ -1185,4 +1215,3 @@
return 0;
}
-
diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c
index 95d291f..09249e9 100644
--- a/lib/dictBuilder/zdict.c
+++ b/lib/dictBuilder/zdict.c
@@ -61,11 +61,11 @@
#include "fse.h"
#include "huf_static.h"
#include "zstd_internal.h"
+#include "xxhash.h"
#include "divsufsort.h"
#include "zdict_static.h"
-
/*-*************************************
* Constants
***************************************/
@@ -104,8 +104,7 @@
{
const BYTE* const b = (const BYTE*)ptr;
size_t u;
- for (u=0; u<length; u++)
- {
+ for (u=0; u<length; u++) {
BYTE c = b[u];
if (c<32 || c>126) c = '.'; /* non-printable char */
DISPLAYLEVEL(dlevel, "%c", c);
@@ -198,8 +197,12 @@
{
const char* const pStart = (const char*)pIn;
for (;;) {
- size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
- if (!diff) { pIn = (const char*)pIn+sizeof(size_t); pMatch = (const char*)pMatch+sizeof(size_t); continue; }
+ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
+ if (!diff) {
+ pIn = (const char*)pIn+sizeof(size_t);
+ pMatch = (const char*)pMatch+sizeof(size_t);
+ continue;
+ }
pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff);
return (size_t)((const char*)pIn - pStart);
}
@@ -346,9 +349,8 @@
maxLength = i;
/* reduce maxLength in case of final into repetitive data */
- {
- U32 l = (U32)maxLength;
- BYTE c = b[pos + maxLength-1];
+ { U32 l = (U32)maxLength;
+ BYTE const c = b[pos + maxLength-1];
while (b[pos+l-2]==c) l--;
maxLength = l;
}
@@ -367,12 +369,10 @@
solution.savings = savings[maxLength];
/* mark positions done */
- {
- U32 id;
- U32 testedPos;
+ { U32 id;
for (id=start; id<end; id++) {
U32 p, pEnd;
- testedPos = suffix[id];
+ U32 const testedPos = suffix[id];
if (testedPos == pos)
length = solution.length;
else {
@@ -439,7 +439,7 @@
static void ZDICT_removeDictItem(dictItem* table, U32 id)
{
/* convention : first element is nb of elts */
- U32 max = table->pos;
+ U32 const max = table->pos;
U32 u;
if (!id) return; /* protection, should never happen */
for (u=id; u<max-1; u++)
@@ -463,8 +463,7 @@
}
/* insert */
- {
- U32 current;
+ { U32 current;
U32 nextElt = table->pos;
if (nextElt >= maxSize) nextElt = maxSize-1;
current = nextElt-1;
@@ -530,8 +529,7 @@
DISPLAYLEVEL(2, "finding patterns ... \n");
DISPLAYLEVEL(3, "minimum ratio : %u \n", minRatio);
- {
- U32 cursor; for (cursor=0; cursor < bufferSize; ) {
+ { U32 cursor; for (cursor=0; cursor < bufferSize; ) {
dictItem solution;
if (doneMarks[cursor]) { cursor++; continue; }
solution = ZDICT_analyzePos(doneMarks, suffix, reverseSuffix[cursor], buffer, minRatio);
@@ -542,8 +540,7 @@
} }
/* limit dictionary size */
- {
- U32 max = dictList->pos; /* convention : nb of useful elts within dictList */
+ { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */
U32 currentSize = 0;
U32 n; for (n=1; n<max; n++) {
currentSize += dictList[n].length;
@@ -785,7 +782,7 @@
{
char* dstPtr = (char*)dictBuffer + dictSize;
const char* srcPtr = (const char*)samplesBuffer;
- size_t nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
+ size_t const nbSegments = dictSize / DIB_FASTSEGMENTSIZE;
size_t segNb, interSize;
if (nbSegments <= 2) return ERROR(srcSize_wrong);
@@ -810,6 +807,33 @@
return nbSegments * DIB_FASTSEGMENTSIZE;
}
+size_t ZDICT_addEntropyTablesFromBuffer_advanced(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t params)
+{
+ size_t hSize;
+ unsigned const compressionLevel = (params.compressionLevel == 0) ? g_compressionLevel_default : params.compressionLevel;
+
+ /* dictionary header */
+ MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
+ { U64 const randomID = XXH64((char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize, 0);
+ U32 const dictID = params.dictID ? params.dictID : (U32)(randomID>>11);
+ MEM_writeLE32((char*)dictBuffer+4, dictID);
+ }
+ hSize = 8;
+
+ /* entropy tables */
+ DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
+ DISPLAYLEVEL(2, "statistics ... \n");
+ hSize += ZDICT_analyzeEntropy((char*)dictBuffer+hSize, dictBufferCapacity-hSize,
+ compressionLevel,
+ samplesBuffer, samplesSizes, nbSamples,
+ (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
+
+ if (hSize + dictContentSize < dictBufferCapacity)
+ memmove((char*)dictBuffer + hSize, (char*)dictBuffer + dictBufferCapacity - dictContentSize, dictContentSize);
+ return MIN(dictBufferCapacity, hSize+dictContentSize);
+}
#define DIB_MINSAMPLESSIZE (DIB_FASTSEGMENTSIZE*3)
/*! ZDICT_trainFromBuffer_unsafe() :
@@ -818,14 +842,13 @@
*/
size_t ZDICT_trainFromBuffer_unsafe(
void* dictBuffer, size_t maxDictSize,
- const void* samplesBuffer, const size_t* sampleSizes, unsigned nbSamples,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
ZDICT_params_t params)
{
U32 const dictListSize = MAX( MAX(DICTLISTSIZE, nbSamples), (U32)(maxDictSize/16));
- dictItem* dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
+ dictItem* const dictList = (dictItem*)malloc(dictListSize * sizeof(*dictList));
unsigned selectivity = params.selectivityLevel;
- unsigned compressionLevel = params.compressionLevel;
- size_t targetDictSize = maxDictSize;
+ size_t const targetDictSize = maxDictSize;
size_t sBuffSize;
size_t dictSize = 0;
@@ -834,18 +857,17 @@
if (!dictList) return ERROR(memory_allocation);
/* init */
- { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += sampleSizes[u]; }
+ { unsigned u; for (u=0, sBuffSize=0; u<nbSamples; u++) sBuffSize += samplesSizes[u]; }
if (sBuffSize < DIB_MINSAMPLESSIZE) return 0; /* not enough source to create dictionary */
ZDICT_initDictItem(dictList);
g_displayLevel = params.notificationLevel;
if (selectivity==0) selectivity = g_selectivity_default;
- if (compressionLevel==0) compressionLevel = g_compressionLevel_default;
/* build dictionary */
if (selectivity>1) { /* selectivity == 1 => fast mode */
ZDICT_trainBuffer(dictList, dictListSize,
samplesBuffer, sBuffSize,
- sampleSizes, nbSamples,
+ samplesSizes, nbSamples,
selectivity, (U32)targetDictSize);
/* display best matches */
@@ -867,18 +889,16 @@
/* create dictionary */
{ U32 dictContentSize = ZDICT_dictSize(dictList);
- size_t hSize;
- BYTE* ptr;
- U32 u;
/* build dict content */
- ptr = (BYTE*)dictBuffer + maxDictSize;
- for (u=1; u<dictList->pos; u++) {
- U32 l = dictList[u].length;
- ptr -= l;
- if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
- memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
- }
+ { U32 u;
+ BYTE* ptr = (BYTE*)dictBuffer + maxDictSize;
+ for (u=1; u<dictList->pos; u++) {
+ U32 l = dictList[u].length;
+ ptr -= l;
+ if (ptr<(BYTE*)dictBuffer) return ERROR(GENERIC); /* should not happen */
+ memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l);
+ } }
/* fast mode dict content */
if (selectivity==1) { /* note could also be used to complete a dictionary, but not necessarily better */
@@ -888,21 +908,9 @@
samplesBuffer, sBuffSize);
}
- /* dictionary header */
- MEM_writeLE32(dictBuffer, ZSTD_DICT_MAGIC);
- hSize = 4;
-
- /* entropic tables */
- DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */
- DISPLAYLEVEL(2, "statistics ... \n");
- hSize += ZDICT_analyzeEntropy((char*)dictBuffer+4, maxDictSize-4,
- compressionLevel,
- samplesBuffer, sampleSizes, nbSamples,
- (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
-
- if (hSize + dictContentSize < maxDictSize)
- memmove((char*)dictBuffer + hSize, (char*)dictBuffer + maxDictSize - dictContentSize, dictContentSize);
- dictSize = MIN(maxDictSize, hSize+dictContentSize);
+ dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize,
+ samplesBuffer, samplesSizes, nbSamples,
+ params);
}
/* clean up */
@@ -914,8 +922,8 @@
/* issue : samplesBuffer need to be followed by a noisy guard band.
* work around : duplicate the buffer, and add the noise */
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
- const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
- ZDICT_params_t params)
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples,
+ ZDICT_params_t params)
{
void* newBuff;
size_t sBuffSize;
@@ -947,3 +955,12 @@
params);
}
+size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples)
+{
+ ZDICT_params_t params;
+ memset(¶ms, 0, sizeof(params));
+ return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity,
+ samplesBuffer, samplesSizes, nbSamples,
+ params);
+}
diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h
index 2ca190c..3a724d0 100644
--- a/lib/dictBuilder/zdict.h
+++ b/lib/dictBuilder/zdict.h
@@ -52,6 +52,21 @@
size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity,
const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+/*! ZDICT_addEntropyTablesFromBuffer() :
+
+ Given a content-only dictionary (built for example from common strings in
+ the input), add entropy tables computed from the memory buffer
+ `samplesBuffer`, where `nbSamples` samples have been stored concatenated.
+ Each sample size is provided into an orderly table `samplesSizes`.
+
+ The input dictionary is the last `dictContentSize` bytes of `dictBuffer`. The
+ resulting dictionary with added entropy tables will written back to
+ `dictBuffer`.
+ @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`).
+*/
+size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity,
+ const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples);
+
/*-*************************************
* Helper functions
diff --git a/lib/dictBuilder/zdict_static.h b/lib/dictBuilder/zdict_static.h
index e5f909a..e34e6c0 100644
--- a/lib/dictBuilder/zdict_static.h
+++ b/lib/dictBuilder/zdict_static.h
@@ -54,7 +54,8 @@
unsigned selectivityLevel; /* 0 means default; larger => bigger selection => larger dictionary */
unsigned compressionLevel; /* 0 means default; target a specific zstd compression level */
unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */
- unsigned reserved[3]; /* space for future parameters */
+ unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */
+ unsigned reserved[2]; /* space for future parameters */
} ZDICT_params_t;
@@ -65,7 +66,7 @@
Same as ZDICT_trainFromBuffer() with control over more parameters.
`parameters` is optional and can be provided with values set to 0 to mean "default".
@return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`)
- or an error code, which can be tested by DiB_isError().
+ or an error code, which can be tested by ZDICT_isError().
note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using ZDICT_setNotificationLevel()
*/
size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity,
diff --git a/programs/.clang_complete b/programs/.clang_complete
new file mode 100644
index 0000000..658aa00
--- /dev/null
+++ b/programs/.clang_complete
@@ -0,0 +1,3 @@
+-I../lib/common
+-I../lib/legacy
+-I./legacy
diff --git a/programs/.gitignore b/programs/.gitignore
index 886af77..f7061d3 100644
--- a/programs/.gitignore
+++ b/programs/.gitignore
@@ -9,6 +9,7 @@
zbufftest32
datagen
paramgrill
+roundTripCrash
# Object files
*.o
@@ -37,6 +38,8 @@
dictionary
grillResults.txt
_*
+tmp*
+*.zst
# fuzzer
afl
diff --git a/programs/Makefile b/programs/Makefile
index c57d8ec..1e8a7f2 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -33,29 +33,31 @@
DESTDIR?=
PREFIX ?= /usr/local
-CPPFLAGS= -I../lib/common -I../lib/dictBuilder
+BINDIR = $(PREFIX)/bin
+MANDIR = $(PREFIX)/share/man/man1
+
+ZSTDDIR = ../lib
+
+CPPFLAGS= -I$(ZSTDDIR)/common -I$(ZSTDDIR)/dictBuilder -DXXH_NAMESPACE=ZSTD_
CFLAGS ?= -O3 # -falign-loops=32 # not always beneficial
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wstrict-prototypes -Wundef
FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS)
-BINDIR = $(PREFIX)/bin
-MANDIR = $(PREFIX)/share/man/man1
-ZSTDDIR = ../lib
-ZSTDCOMP_FILES := $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/common/fse_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/entropy_common.c
-ZDICT_FILES := $(ZSTDDIR)/dictBuilder/zdict.c $(ZSTDDIR)/dictBuilder/divsufsort.c
+ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/huf_decompress.c $(ZSTDDIR)/decompress/zstd_decompress.c
+ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMP_FILES)
+ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
-ZSTD_FILES_LEGACY:=
+ZSTDLEGACY_FILES:=
else
ZSTD_LEGACY_SUPPORT:=1
-CPPFLAGS += -I../lib/legacy -I./legacy
-ZSTD_FILES_LEGACY:= $(ZSTDDIR)/legacy/zstd_v01.c $(ZSTDDIR)/legacy/zstd_v02.c $(ZSTDDIR)/legacy/zstd_v03.c \
- $(ZSTDDIR)/legacy/zstd_v04.c $(ZSTDDIR)/legacy/zstd_v05.c legacy/fileio_legacy.c
+CPPFLAGS += -I$(ZSTDDIR)/legacy -I./legacy
+ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c legacy/fileio_legacy.c
endif
@@ -78,12 +80,12 @@
all: zstd fullbench fuzzer zbufftest paramgrill datagen zstd32 fullbench32 fuzzer32 zbufftest32
-zstd : $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \
- zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c
+zstd : $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
+ zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
-zstd32: $(ZSTD_FILES) $(ZSTD_FILES_LEGACY) $(ZBUFF_FILES) $(ZDICT_FILES) \
- zstdcli.c fileio.c bench.c xxhash.c datagen.c dibio.c
+zstd32: $(ZSTD_FILES) $(ZSTDLEGACY_FILES) $(ZBUFF_FILES) $(ZDICT_FILES) \
+ zstdcli.c fileio.c bench.c datagen.c dibio.c
$(CC) -m32 $(FLAGS) -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT) $^ -o $@$(EXT)
zstd_nolegacy :
@@ -103,14 +105,16 @@
zstd-frugal: $(ZSTD_FILES) $(ZBUFF_FILES) zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_LEGACY_SUPPORT=0 $^ -o zstd$(EXT)
-zstd-compress: $(ZSTDCOMP_FILES) $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
+zstd-compress: $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) \
+ $(ZSTDDIR)/compress/zbuff_compress.c zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NODECOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
-zstd-decompress: $(ZSTDDECOMP_FILES) $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
+zstd-decompress: $(ZSTDCOMMON_FILES) $(ZSTDDECOMP_FILES) \
+ $(ZSTDDIR)/decompress/zbuff_decompress.c zstdcli.c fileio.c
$(CC) $(FLAGS) -DZSTD_NOBENCH -DZSTD_NODICT -DZSTD_NOCOMPRESS -DZSTD_LEGACY_SUPPORT=0 $^ -o $@$(EXT)
-zstd-small: clean
- CFLAGS="-Os -s" $(MAKE) zstd-frugal
+zstd-small: clean
+ CFLAGS="-Os -s" $(MAKE) zstd-frugal
fullbench : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -118,35 +122,35 @@
fullbench32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c fullbench.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-fuzzer : $(ZSTD_FILES) \
- datagen.c xxhash.c fuzzer.c
+fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
+fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
-fuzzer32: $(ZSTD_FILES) \
- datagen.c xxhash.c fuzzer.c
+fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
+fuzzer32: $(ZSTD_FILES) $(ZDICT_FILES) datagen.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) \
- datagen.c xxhash.c zbufftest.c
+zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
-zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) \
- datagen.c xxhash.c zbufftest.c
+zbufftest32: $(ZSTD_FILES) $(ZBUFF_FILES) datagen.c zbufftest.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-paramgrill : $(ZSTD_FILES) \
- datagen.c xxhash.c paramgrill.c
+paramgrill : $(ZSTD_FILES) datagen.c paramgrill.c
$(CC) $(FLAGS) $^ -lm -o $@$(EXT)
datagen : datagen.c datagencli.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
+roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
+
clean:
@rm -f core *.o tmp* result* *.gcda dictionary *.zst \
zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
- datagen$(EXT) paramgrill$(EXT)
+ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
@echo Cleaning completed
diff --git a/programs/bench.h b/programs/bench.h
index 74ac20f..1a31564 100644
--- a/programs/bench.h
+++ b/programs/bench.h
@@ -21,10 +21,11 @@
You can contact the author at :
- ZSTD homepage : http://www.zstd.net/
*/
-#pragma once
+#ifndef BENCH_H_121279284357
+#define BENCH_H_121279284357
+#include <stddef.h>
-/* Main function */
int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,
const char* dictFileName, int cLevel, int cLevelLast);
@@ -34,3 +35,4 @@
void BMK_setAdditionalParam(int additionalParam);
void BMK_setNotificationLevel(unsigned level);
+ #endif /* BENCH_H_121279284357 */
diff --git a/programs/dibio.c b/programs/dibio.c
index 23f3c81..d23476e 100644
--- a/programs/dibio.c
+++ b/programs/dibio.c
@@ -101,27 +101,30 @@
/* ********************************************************
* File related operations
**********************************************************/
-static void DiB_loadFiles(void* buffer, size_t bufferSize,
- size_t* fileSizes,
- const char** fileNamesTable, unsigned nbFiles)
+/** DiB_loadFiles() :
+* @return : nb of files effectively loaded into `buffer` */
+static unsigned DiB_loadFiles(void* buffer, size_t bufferSize,
+ size_t* fileSizes,
+ const char** fileNamesTable, unsigned nbFiles)
{
- char* buff = (char*)buffer;
+ char* const buff = (char*)buffer;
size_t pos = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
- size_t readSize;
- unsigned long long fileSize = UTIL_getFileSize(fileNamesTable[n]);
- FILE* f = fopen(fileNamesTable[n], "rb");
+ unsigned long long const fs64 = UTIL_getFileSize(fileNamesTable[n]);
+ size_t const fileSize = (size_t)(fs64 > bufferSize-pos ? 0 : fs64);
+ FILE* const f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]);
- if (fileSize > bufferSize-pos) fileSize = 0; /* stop there, not enough memory to load all files */
- readSize = fread(buff+pos, 1, (size_t)fileSize, f);
- if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
- pos += readSize;
- fileSizes[n] = (size_t)fileSize;
+ { size_t const readSize = fread(buff+pos, 1, fileSize, f);
+ if (readSize != fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+ pos += readSize; }
+ fileSizes[n] = fileSize;
fclose(f);
+ if (fileSize == 0) break; /* stop there, not enough memory to load all files */
}
+ return n;
}
@@ -130,7 +133,7 @@
**********************************************************/
static size_t DiB_findMaxMem(unsigned long long requiredMem)
{
- size_t step = 8 MB;
+ size_t const step = 8 MB;
void* testmem = NULL;
requiredMem = (((requiredMem >> 23) + 1) << 23);
@@ -162,7 +165,7 @@
static void DiB_saveDict(const char* dictFileName,
const void* buff, size_t buffSize)
{
- FILE* f = fopen(dictFileName, "wb");
+ FILE* const f = fopen(dictFileName, "wb");
if (f==NULL) EXM_THROW(3, "cannot open %s ", dictFileName);
{ size_t const n = fwrite(buff, 1, buffSize, f);
@@ -185,47 +188,44 @@
ZDICT_params_t parameters);
+#define MIN(a,b) ((a)<(b)?(a):(b))
int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize,
const char** fileNamesTable, unsigned nbFiles,
ZDICT_params_t params)
{
- void* srcBuffer;
- size_t benchedSize;
- size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
- unsigned long long totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
- void* dictBuffer = malloc(maxDictSize);
- size_t dictSize;
+ void* const dictBuffer = malloc(maxDictSize);
+ size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
+ unsigned long long const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
+ size_t const maxMem = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
+ size_t const benchedSize = MIN (maxMem, (size_t)totalSizeToLoad);
+ void* const srcBuffer = malloc(benchedSize+NOISELENGTH);
int result = 0;
+ /* Checks */
+ if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
+
/* init */
g_displayLevel = params.notificationLevel;
- benchedSize = DiB_findMaxMem(totalSizeToLoad * MEMMULT) / MEMMULT;
- if ((unsigned long long)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(benchedSize >> 20));
- /* Memory allocation & restrictions */
- srcBuffer = malloc(benchedSize+NOISELENGTH); /* + noise */
- if ((!fileSizes) || (!srcBuffer) || (!dictBuffer)) EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */
-
/* Load input buffer */
- DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
+ nbFiles = DiB_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
DiB_fillNoise((char*)srcBuffer + benchedSize, NOISELENGTH); /* guard band, for end of buffer condition */
- /* call buffer version */
- dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
- srcBuffer, fileSizes, nbFiles,
- params);
- if (ZDICT_isError(dictSize)) {
- DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
- result = 1;
- goto _cleanup;
+ { size_t const dictSize = ZDICT_trainFromBuffer_unsafe(dictBuffer, maxDictSize,
+ srcBuffer, fileSizes, nbFiles,
+ params);
+ if (ZDICT_isError(dictSize)) {
+ DISPLAYLEVEL(1, "dictionary training failed : %s \n", ZDICT_getErrorName(dictSize)); /* should not happen */
+ result = 1;
+ goto _cleanup;
+ }
+ /* save dict */
+ DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
+ DiB_saveDict(dictFileName, dictBuffer, dictSize);
}
- /* save dict */
- DISPLAYLEVEL(2, "Save dictionary of size %u into file %s \n", (U32)dictSize, dictFileName);
- DiB_saveDict(dictFileName, dictBuffer, dictSize);
-
/* clean up */
_cleanup:
free(srcBuffer);
diff --git a/programs/fileio.c b/programs/fileio.c
index b333ded..55e36cc 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -97,7 +97,7 @@
#define CACHELINE 64
-#define MAX_DICT_SIZE (1 MB) /* protection against large input (attack scenario) ; can be changed */
+#define MAX_DICT_SIZE (8 MB) /* protection against large input (attack scenario) */
#define FNSPACE 30
@@ -133,6 +133,8 @@
void FIO_setMaxWLog(unsigned maxWLog) { g_maxWLog = maxWLog; }
static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport=sparse; }
+static U32 g_dictIDFlag = 1;
+void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; }
/*-*************************************
@@ -186,7 +188,7 @@
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
}
} else {
- if (!g_overwrite) { /* Check if destination file already exists */
+ if (!g_overwrite && strcmp (dstFileName, nulmark)) { /* Check if destination file already exists */
f = fopen( dstFileName, "rb" );
if (f != 0) { /* dest file exists, prompt for overwrite authorization */
fclose(f);
@@ -212,12 +214,12 @@
/*! FIO_loadFile() :
* creates a buffer, pointed by `*bufferPtr`,
* loads `filename` content into it,
-* up to MAX_DICT_SIZE bytes
+* up to MAX_DICT_SIZE bytes.
+* @return : loaded size
*/
static size_t FIO_loadFile(void** bufferPtr, const char* fileName)
{
FILE* fileHandle;
- size_t readSize;
U64 fileSize;
*bufferPtr = NULL;
@@ -237,8 +239,8 @@
}
*bufferPtr = (BYTE*)malloc((size_t)fileSize);
if (*bufferPtr==NULL) EXM_THROW(34, "Allocation error : not enough memory for dictBuffer");
- readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
- if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName);
+ { size_t const readSize = fread(*bufferPtr, 1, (size_t)fileSize, fileHandle);
+ if (readSize!=fileSize) EXM_THROW(35, "Error reading dictionary file %s", fileName); }
fclose(fileHandle);
return (size_t)fileSize;
}
@@ -308,12 +310,14 @@
/* init */
{ ZSTD_parameters params;
+ memset(¶ms, 0, sizeof(params));
params.cParams = ZSTD_getCParams(cLevel, fileSize, ress.dictBufferSize);
params.fParams.contentSizeFlag = 1;
+ params.fParams.noDictIDFlag = !g_dictIDFlag;
if (g_maxWLog) if (params.cParams.windowLog > g_maxWLog) params.cParams.windowLog = g_maxWLog;
- { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
- if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode)); }
- }
+ { size_t const errorCode = ZBUFF_compressInit_advanced(ress.ctx, ress.dictBuffer, ress.dictBufferSize, params, fileSize);
+ if (ZBUFF_isError(errorCode)) EXM_THROW(21, "Error initializing compression : %s", ZBUFF_getErrorName(errorCode));
+ } }
/* Main compression loop */
readsize = 0;
@@ -409,14 +413,10 @@
int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel)
{
- clock_t start;
- cRess_t ress;
+ clock_t const start = clock();
+ cRess_t const ress = FIO_createCResources(dictFileName);
int issueWithSrcFile = 0;
- /* Init */
- start = clock();
- ress = FIO_createCResources(dictFileName);
-
issueWithSrcFile += FIO_compressFilename_extRess(ress, dstFileName, srcFileName, compressionLevel);
FIO_freeCResources(ress);
@@ -701,7 +701,7 @@
/* Final Status */
DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "Successfully decoded %llu bytes \n", filesize);
+ DISPLAYLEVEL(2, "%-20.20s: %llu bytes \n", srcFileName, filesize);
/* Close */
fclose(srcFile);
diff --git a/programs/fileio.h b/programs/fileio.h
index 6e79123..5a9cdc1 100644
--- a/programs/fileio.h
+++ b/programs/fileio.h
@@ -48,6 +48,7 @@
void FIO_setNotificationLevel(unsigned level);
void FIO_setMaxWLog(unsigned maxWLog); /**< if `maxWLog` == 0, no max enforced */
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
+void FIO_setDictIDFlag(unsigned dictIDFlag);
/*-*************************************
diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index e664bf4..7fbf906 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -28,7 +28,7 @@
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
-# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
+# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
@@ -40,10 +40,12 @@
#include <sys/timeb.h> /* timeb */
#include <string.h> /* strcmp */
#include <time.h> /* clock_t */
-#include "zstd_static.h" /* ZSTD_VERSION_STRING */
+#include "zstd_static.h" /* ZSTD_VERSION_STRING, ZSTD_getErrorCode */
+#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
-#include "xxhash.h" /* XXH64 */
#include "mem.h"
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h" /* XXH64 */
/*-************************************
@@ -53,7 +55,6 @@
#define MB *(1U<<20)
#define GB *(1U<<30)
-static const size_t COMPRESSIBLE_NOISE_LENGTH = 10 MB; /* capital, used to be a macro */
static const U32 FUZ_compressibility_default = 50;
static const U32 nbTestsDefault = 30000;
@@ -77,7 +78,6 @@
* Fuzzer functions
*********************************************************/
#define MIN(a,b) ((a)<(b)?(a):(b))
-#define MAX(a,b) ((a)>(b)?(a):(b))
static clock_t FUZ_clockSpan(clock_t cStart)
{
@@ -85,8 +85,8 @@
}
-# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
-unsigned int FUZ_rand(unsigned int* src)
+#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+static unsigned FUZ_rand(unsigned* src)
{
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
@@ -103,142 +103,131 @@
{
unsigned nbBits = 0;
if (v32==0) return 0;
- while (v32) {
- v32 >>= 1;
- nbBits ++;
- }
+ while (v32) v32 >>= 1, nbBits++;
return nbBits;
}
+#define CHECKTEST(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
+#define CHECK(fn) { CHECKTEST(err, fn); }
+#define CHECKPLUS(var, fn, more) { CHECKTEST(var, fn); more; }
static int basicUnitTests(U32 seed, double compressibility)
{
+ size_t const CNBuffSize = 5 MB;
+ void* const CNBuffer = malloc(CNBuffSize);
+ void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
+ void* const decodedBuffer = malloc(CNBuffSize);
int testResult = 0;
- void* CNBuffer;
- void* compressedBuffer;
- void* decodedBuffer;
- U32 randState = seed;
- size_t result, cSize;
U32 testNb=0;
+ size_t cSize;
- /* Create compressible test buffer */
- CNBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
- compressedBuffer = malloc(ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH));
- decodedBuffer = malloc(COMPRESSIBLE_NOISE_LENGTH);
+ /* Create compressible noise */
if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
DISPLAY("Not enough memory, aborting\n");
testResult = 1;
goto _end;
}
- RDG_genBuffer(CNBuffer, COMPRESSIBLE_NOISE_LENGTH, compressibility, 0., randState);
+ RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
/* Basic tests */
- DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH);
- result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), CNBuffer, COMPRESSIBLE_NOISE_LENGTH, 1);
- if (ZSTD_isError(result)) goto _output_error;
- cSize = result;
- DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+ DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, (U32)CNBuffSize);
+ CHECKPLUS(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize, 1),
+ cSize=r );
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
- DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)COMPRESSIBLE_NOISE_LENGTH);
- result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != COMPRESSIBLE_NOISE_LENGTH) goto _output_error;
+ DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, (U32)CNBuffSize);
+ CHECKPLUS( r , ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize),
+ if (r != CNBuffSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
- { size_t i;
- DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
- for (i=0; i<COMPRESSIBLE_NOISE_LENGTH; i++) {
- if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;;
- }
- DISPLAYLEVEL(4, "OK \n");
- }
+ DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
+ { size_t u;
+ for (u=0; u<CNBuffSize; u++) {
+ if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;;
+ } }
+ DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
- result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize-1);
- if (!ZSTD_isError(result)) goto _output_error;
- if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
+ if (!ZSTD_isError(r)) goto _output_error;
+ if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : decompress with 1 too much byte : ", testNb++);
- result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize+1);
- if (!ZSTD_isError(result)) goto _output_error;
- if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
+ if (!ZSTD_isError(r)) goto _output_error;
+ if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* Dictionary and CCtx Duplication tests */
- { ZSTD_CCtx* ctxOrig = ZSTD_createCCtx();
- ZSTD_CCtx* ctxDuplicated = ZSTD_createCCtx();
- ZSTD_DCtx* dctx = ZSTD_createDCtx();
- size_t const dictSize = 500;
+ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
+ ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ static const size_t dictSize = 551;
DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
{ size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
- if (!ZSTD_isError(copyResult)) goto _output_error; } /* error should be detected */
+ if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++);
- { size_t const initResult = ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2);
- if (ZSTD_isError(initResult)) goto _output_error; }
- { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
- if (ZSTD_isError(copyResult)) goto _output_error; }
+ CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
+ CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
DISPLAYLEVEL(4, "OK \n");
- DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
+ DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++);
cSize = 0;
- result = ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
- if (ZSTD_isError(result)) goto _output_error;
- cSize += result;
- result = ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
- if (ZSTD_isError(result)) goto _output_error;
- cSize += result;
- DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+ CHECKPLUS(r, ZSTD_compressContinue(ctxOrig, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
+ cSize += r);
+ CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
+ cSize += r);
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
- DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
- result = ZSTD_decompress_usingDict(dctx,
- decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
- compressedBuffer, cSize,
- CNBuffer, dictSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
+ DISPLAYLEVEL(4, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ CNBuffer, dictSize),
+ if (r != CNBuffSize - dictSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : compress with duplicated context : ", testNb++);
{ size_t const cSizeOrig = cSize;
cSize = 0;
- result = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
- if (ZSTD_isError(result)) goto _output_error;
- cSize += result;
- result = ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH)-cSize);
- if (ZSTD_isError(result)) goto _output_error;
- cSize += result;
- if (cSize != cSizeOrig) goto _output_error; /* should be identical == have same size */
+ CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
+ cSize += r);
+ CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, (char*)compressedBuffer+cSize, ZSTD_compressBound(CNBuffSize)-cSize),
+ cSize += r);
+ if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> have same size */
}
- DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
DISPLAYLEVEL(4, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
- result = ZSTD_decompress_usingDict(dctx,
- decodedBuffer, COMPRESSIBLE_NOISE_LENGTH,
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
compressedBuffer, cSize,
- CNBuffer, dictSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != COMPRESSIBLE_NOISE_LENGTH - dictSize) goto _output_error;
+ CNBuffer, dictSize),
+ if (r != CNBuffSize - dictSize) goto _output_error);
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
- { size_t const testSize = COMPRESSIBLE_NOISE_LENGTH / 3;
- { ZSTD_parameters p;
- p.cParams = ZSTD_getCParams(2, testSize, dictSize);
- p.fParams.contentSizeFlag = 1;
- { size_t const initResult = ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1);
- if (ZSTD_isError(initResult)) goto _output_error;
- } }
- { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig);
- if (ZSTD_isError(copyResult)) goto _output_error; }
- cSize = ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), (const char*)CNBuffer + dictSize, COMPRESSIBLE_NOISE_LENGTH - dictSize);
- if (ZSTD_isError(cSize)) goto _output_error;
+ { size_t const testSize = CNBuffSize / 3;
+ { ZSTD_compressionParameters const cPar = ZSTD_getCParams(2, testSize, dictSize);
+ ZSTD_frameParameters const fPar = { 1 , 0 };
+ ZSTD_parameters p;
+ p.cParams = cPar; p.fParams = fPar;
+ CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) );
+ }
+ CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig) );
+
+ CHECKPLUS(r, ZSTD_compressContinue(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
+ (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
+ cSize = r);
{ ZSTD_frameParams fp;
- size_t const gfpResult = ZSTD_getFrameParams(&fp, compressedBuffer, cSize);
- if (gfpResult!=0) goto _output_error;
+ if (ZSTD_getFrameParams(&fp, compressedBuffer, cSize)) goto _output_error;
if ((fp.frameContentSize != testSize) && (fp.frameContentSize != 0)) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
@@ -248,55 +237,113 @@
ZSTD_freeDCtx(dctx);
}
+ /* Dictionary and dictBuilder tests */
+ { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+ size_t dictSize = 16 KB;
+ void* dictBuffer = malloc(dictSize);
+ size_t const totalSampleSize = 1 MB;
+ size_t const sampleUnitSize = 8 KB;
+ U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
+ size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
+
+ if (dictBuffer==NULL || samplesSizes==NULL) {
+ free(dictBuffer);
+ free(samplesSizes);
+ goto _output_error;
+ }
+
+ DISPLAYLEVEL(4, "test%3i : dictBuilder : ", testNb++);
+ { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
+ dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
+ CNBuffer, samplesSizes, nbSamples);
+ if (ZDICT_isError(dictSize)) goto _output_error;
+ DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
+
+ DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
+ cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize,
+ dictBuffer, dictSize, 4);
+ if (ZSTD_isError(cSize)) goto _output_error;
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+ DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : compress without dictID : ", testNb++);
+ { ZSTD_frameParameters const fParams = { 0, 1 /*NoDictID*/ };
+ ZSTD_compressionParameters const cParams = ZSTD_getCParams(3, CNBuffSize, dictSize);
+ ZSTD_parameters p;
+ p.cParams = cParams; p.fParams = fParams;
+ cSize = ZSTD_compress_advanced(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
+ CNBuffer, CNBuffSize,
+ dictBuffer, dictSize, p);
+ if (ZSTD_isError(cSize)) goto _output_error;
+ }
+ DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+
+ DISPLAYLEVEL(4, "test%3i : frame built without dictID should be decompressible : ", testNb++);
+ CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
+ decodedBuffer, CNBuffSize,
+ compressedBuffer, cSize,
+ dictBuffer, dictSize),
+ if (r != CNBuffSize) goto _output_error);
+ DISPLAYLEVEL(4, "OK \n");
+
+ ZSTD_freeCCtx(cctx);
+ ZSTD_freeDCtx(dctx);
+ free(dictBuffer);
+ free(samplesSizes);
+ }
+
/* Decompression defense tests */
DISPLAYLEVEL(4, "test%3i : Check input length for magic number : ", testNb++);
- result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 3);
- if (!ZSTD_isError(result)) goto _output_error;
- if (result != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error;
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3);
+ if (!ZSTD_isError(r)) goto _output_error;
+ if (r != (size_t)-ZSTD_error_srcSize_wrong) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Check magic Number : ", testNb++);
((char*)(CNBuffer))[0] = 1;
- result = ZSTD_decompress(decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, CNBuffer, 4);
- if (!ZSTD_isError(result)) goto _output_error;
+ { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
+ if (!ZSTD_isError(r)) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* block API tests */
{ ZSTD_CCtx* const cctx = ZSTD_createCCtx();
ZSTD_DCtx* const dctx = ZSTD_createDCtx();
- const size_t blockSize = 100 KB;
- const size_t dictSize = 16 KB;
+ static const size_t blockSize = 100 KB;
+ static const size_t dictSize = 16 KB;
/* basic block compression */
DISPLAYLEVEL(4, "test%3i : Block compression test : ", testNb++);
- result = ZSTD_compressBegin(cctx, 5);
- if (ZSTD_isError(result)) goto _output_error;
+ CHECK( ZSTD_compressBegin(cctx, 5) );
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Block decompression test : ", testNb++);
- result = ZSTD_decompressBegin(dctx);
- if (ZSTD_isError(result)) goto _output_error;
- result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != blockSize) goto _output_error;
+ CHECK( ZSTD_decompressBegin(dctx) );
+ { CHECKTEST(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+ if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* dictionary block compression */
DISPLAYLEVEL(4, "test%3i : Dictionary Block compression test : ", testNb++);
- result = ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5);
- if (ZSTD_isError(result)) goto _output_error;
+ CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize);
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : Dictionary Block decompression test : ", testNb++);
- result = ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize);
- if (ZSTD_isError(result)) goto _output_error;
- result = ZSTD_decompressBlock(dctx, decodedBuffer, COMPRESSIBLE_NOISE_LENGTH, compressedBuffer, cSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != blockSize) goto _output_error;
+ CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
+ { CHECKTEST( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
+ if (r != blockSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
ZSTD_freeCCtx(cctx);
@@ -306,32 +353,29 @@
/* long rle test */
{ size_t sampleSize = 0;
DISPLAYLEVEL(4, "test%3i : Long RLE test : ", testNb++);
- RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., randState);
+ RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1);
memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
sampleSize += 256 KB - 1;
- RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., randState);
+ RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2);
sampleSize += 96 KB;
cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
if (ZSTD_isError(cSize)) goto _output_error;
- result = ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result!=sampleSize) goto _output_error;
+ { CHECKTEST(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
+ if (regenSize!=sampleSize) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
}
- /* All zeroes test (#137 verif) */
+ /* All zeroes test (test bug #137) */
#define ZEROESLENGTH 100
DISPLAYLEVEL(4, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
memset(CNBuffer, 0, ZEROESLENGTH);
- result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1);
- if (ZSTD_isError(result)) goto _output_error;
- cSize = result;
+ { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
+ cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/ZEROESLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
- result = ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != ZEROESLENGTH) goto _output_error;
+ { CHECKTEST(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
+ if (r != ZEROESLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
/* nbSeq limit test */
@@ -351,23 +395,21 @@
}}
/* randomly fills CNBuffer with prepared 3-bytes sequences */
- { int i; for (i=0; i < _3BYTESTESTLENGTH; ) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
- U32 id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
+ { int i; for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
+ U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
- i += 3;
} }}
DISPLAYLEVEL(4, "test%3i : compress lots 3-bytes sequences : ", testNb++);
- result = ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), CNBuffer, _3BYTESTESTLENGTH, 19);
- if (ZSTD_isError(result)) goto _output_error;
- cSize = result;
+ { CHECKTEST(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
+ CNBuffer, _3BYTESTESTLENGTH, 19) );
+ cSize = r; }
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
DISPLAYLEVEL(4, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
- result = ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize);
- if (ZSTD_isError(result)) goto _output_error;
- if (result != _3BYTESTESTLENGTH) goto _output_error;
+ { CHECKTEST(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
+ if (r != _3BYTESTESTLENGTH) goto _output_error; }
DISPLAYLEVEL(4, "OK \n");
_end:
@@ -387,11 +429,11 @@
{
const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2;
- size_t i;
- for (i=0; i<max; i++) {
- if (b1[i] != b2[i]) break;
+ size_t u;
+ for (u=0; u<max; u++) {
+ if (b1[u] != b2[u]) break;
}
- return i;
+ return u;
}
@@ -407,6 +449,7 @@
return FUZ_rLogLength(seed, logLength);
}
+#undef CHECK
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
@@ -414,35 +457,29 @@
{
static const U32 maxSrcLog = 23;
static const U32 maxSampleLog = 22;
+ size_t const srcBufferSize = (size_t)1<<maxSrcLog;
+ size_t const dstBufferSize = (size_t)1<<maxSampleLog;
+ size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
BYTE* cNoiseBuffer[5];
- BYTE* srcBuffer;
- BYTE* cBuffer;
- BYTE* dstBuffer;
- BYTE* mirrorBuffer;
- size_t srcBufferSize = (size_t)1<<maxSrcLog;
- size_t dstBufferSize = (size_t)1<<maxSampleLog;
- size_t cBufferSize = ZSTD_compressBound(dstBufferSize);
+ BYTE* srcBuffer; /* jumping pointer */
+ BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
+ BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
+ BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
+ ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
+ ZSTD_CCtx* const ctx = ZSTD_createCCtx();
+ ZSTD_DCtx* const dctx = ZSTD_createDCtx();
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed, lseed = 0;
- ZSTD_CCtx* refCtx;
- ZSTD_CCtx* ctx;
- ZSTD_DCtx* dctx;
- clock_t startClock = clock();
+ clock_t const startClock = clock();
clock_t const maxClockSpan = maxDurationS * CLOCKS_PER_SEC;
/* allocation */
- refCtx = ZSTD_createCCtx();
- ctx = ZSTD_createCCtx();
- dctx= ZSTD_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
- dstBuffer = (BYTE*)malloc (dstBufferSize);
- mirrorBuffer = (BYTE*)malloc (dstBufferSize);
- cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
|| !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
"Not enough memory, fuzzer tests cancelled");
@@ -460,10 +497,9 @@
/* main test loop */
for ( ; (testNb <= nbTests) || (FUZ_clockSpan(startClock) < maxClockSpan); testNb++ ) {
- size_t sampleSize, sampleStart, maxTestSize, totalTestSize;
- size_t cSize, dSize, totalCSize, totalGenSize;
- U32 sampleSizeLog, nbChunks, n;
- XXH64_CREATESTATE_STATIC(xxh64);
+ size_t sampleSize, maxTestSize, totalTestSize;
+ size_t cSize, totalCSize, totalGenSize;
+ XXH64_state_t xxhState;
U64 crcOrig;
BYTE* sampleBuffer;
const BYTE* dict;
@@ -492,18 +528,17 @@
}
/* select src segment */
- sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
- sampleSize = FUZ_rLogLength(&lseed, sampleSizeLog);
- sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+ sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
/* create sample buffer (to catch read error with valgrind & sanitizers) */
sampleBuffer = (BYTE*)malloc(sampleSize);
- CHECK (sampleBuffer==NULL, "not enough memory for sample buffer");
- memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize);
+ CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
+ { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+ memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
crcOrig = XXH64(sampleBuffer, sampleSize, 0);
/* compression tests */
- { int const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (sampleSizeLog/3))) + 1;
+ { unsigned const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize)/3))) + 1;
cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed");
@@ -517,9 +552,7 @@
CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (U32)tooSmallSize, (U32)cSize); }
{ U32 endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, 4);
CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow"); }
- }
- }
-
+ } }
/* frame header decompression test */
{ ZSTD_frameParams dParams;
@@ -530,7 +563,7 @@
/* successful decompression test */
{ size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
- dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
+ size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (U32)sampleSize, (U32)cSize);
{ U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (U32)findDiff(sampleBuffer, dstBuffer, sampleSize), (U32)sampleSize);
@@ -541,7 +574,7 @@
/* truncated src decompression test */
{ size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
size_t const tooSmallSize = cSize - missing;
- void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch overflows */
+ void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
CHECK(cBufferTooSmall == NULL, "not enough memory !");
memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
{ size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
@@ -574,13 +607,12 @@
}
if (pos <= cSize) break;
/* add noise */
- { U32 nbBits = FUZ_rand(&lseed) % maxNbBits;
- size_t mask, noiseStart, noiseLength;
- if (nbBits>0) nbBits--;
- mask = (1<<nbBits) - 1;
- noiseLength = (FUZ_rand(&lseed) & mask) + 1;
- if ( pos+noiseLength > cSize ) noiseLength = cSize-pos;
- noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
+ { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
+ U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
+ size_t const mask = (1<<nbBits) - 1;
+ size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
+ size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
+ size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
pos += noiseLength;
} } }
@@ -604,44 +636,53 @@
maxTestSize = FUZ_rLogLength(&lseed, testLog);
if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
- sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
- sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
- dict = srcBuffer + sampleStart;
- dictSize = sampleSize;
+ dictSize = FUZ_randomLength(&lseed, maxSampleLog); /* needed also for decompression */
+ dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
- { size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
- CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
+ if (FUZ_rand(&lseed) & 15) {
+ size_t const errorCode = ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel);
+ CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode));
+ } else {
+ ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, 0, dictSize);
+ ZSTD_frameParameters const fpar = { FUZ_rand(&lseed)&1, FUZ_rand(&lseed)&1 }; /* note : since dictionary is fake, dictIDflag has no impact */
+ ZSTD_parameters p;
+ size_t errorCode;
+ p.cParams = cPar; p.fParams = fpar;
+ errorCode = ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0);
+ CHECK (ZSTD_isError(errorCode), "ZSTD_compressBegin_advanced error : %s", ZSTD_getErrorName(errorCode));
+ }
{ size_t const errorCode = ZSTD_copyCCtx(ctx, refCtx);
CHECK (ZSTD_isError(errorCode), "ZSTD_copyCCtx error : %s", ZSTD_getErrorName(errorCode)); }
}
- XXH64_reset(xxh64, 0);
- nbChunks = (FUZ_rand(&lseed) & 127) + 2;
- for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
- sampleSizeLog = FUZ_rand(&lseed) % maxSampleLog;
- sampleSize = (size_t)1 << sampleSizeLog;
- sampleSize += FUZ_rand(&lseed) & (sampleSize-1);
- sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
+ XXH64_reset(&xxhState, 0);
+ { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
+ U32 n;
+ for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
+ size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
+ size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
- if (cBufferSize-cSize < ZSTD_compressBound(sampleSize)) break; /* avoid invalid dstBufferTooSmall */
- if (totalTestSize+sampleSize > maxTestSize) break;
+ if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
+ if (totalTestSize+segmentSize > maxTestSize) break;
- { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+sampleStart, sampleSize);
- CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
- cSize += compressResult;
- }
- XXH64_update(xxh64, srcBuffer+sampleStart, sampleSize);
- memcpy(mirrorBuffer + totalTestSize, srcBuffer+sampleStart, sampleSize);
- totalTestSize += sampleSize;
- }
+ { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
+ CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
+ cSize += compressResult;
+ }
+ XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
+ memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
+ totalTestSize += segmentSize;
+ } }
+
{ size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize);
CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
cSize += flushResult;
}
- crcOrig = XXH64_digest(xxh64);
+ crcOrig = XXH64_digest(&xxhState);
/* streaming decompression test */
+ if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dict, dictSize);
- CHECK (ZSTD_isError(errorCode), "cannot init DCtx : %s", ZSTD_getErrorName(errorCode)); }
+ CHECK (ZSTD_isError(errorCode), "ZSTD_decompressBegin_usingDict error : %s", ZSTD_getErrorName(errorCode)); }
totalCSize = 0;
totalGenSize = 0;
while (totalCSize < cSize) {
@@ -652,12 +693,12 @@
totalCSize += inSize;
}
CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
- CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
+ CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) {
size_t const errorPos = findDiff(mirrorBuffer, dstBuffer, totalTestSize);
- CHECK (crcDest!=crcOrig, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
+ CHECK (1, "streaming decompressed data corrupted : byte %u / %u (%02X!=%02X)",
(U32)errorPos, (U32)totalTestSize, dstBuffer[errorPos], mirrorBuffer[errorPos]);
} }
} /* for ( ; (testNb <= nbTests) */
diff --git a/programs/playTests.sh b/programs/playTests.sh
old mode 100755
new mode 100644
index 189d169..46e2fb6
--- a/programs/playTests.sh
+++ b/programs/playTests.sh
@@ -47,13 +47,13 @@
$ECHO "test : too large compression level (must fail)"
$ZSTD -99 tmp && die "too large compression level undetected"
$ECHO "test : compress to stdout"
-$ZSTD tmp -c > tmpCompressed
+$ZSTD tmp -c > tmpCompressed
$ZSTD tmp --stdout > tmpCompressed # long command format
$ECHO "test : null-length file roundtrip"
$ECHO -n '' | $ZSTD - --stdout | $ZSTD -d --stdout
$ECHO "test : decompress file with wrong suffix (must fail)"
$ZSTD -d tmpCompressed && die "wrong suffix error not detected!"
-$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
+$ZSTD -d tmpCompressed -c > tmpResult # decompression using stdout
$ZSTD --decompress tmpCompressed -c > tmpResult
$ZSTD --decompress tmpCompressed --stdout > tmpResult
if [ "$isWindows" = false ] ; then
@@ -122,7 +122,7 @@
$ZSTD -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse*
diff tmpSparse2M tmpSparseRegenerated
-# rm tmpSparse*
+rm tmpSparse*
$ECHO "\n**** dictionary tests **** "
@@ -131,10 +131,23 @@
./datagen -g1M | $MD5SUM > tmp1
./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
diff -q tmp1 tmp2
-$ZSTD --train *.c *.h -o tmpDict
-$ZSTD xxhash.c -D tmpDict -of tmp
-$ZSTD -d tmp -D tmpDict -of result
-diff xxhash.c result
+$ECHO "Create first dictionary"
+$ZSTD --train *.c -o tmpDict
+cp zstdcli.c tmp
+$ZSTD -f tmp -D tmpDict
+$ZSTD -d tmp.zst -D tmpDict -of result
+diff zstdcli.c result
+$ECHO "Create second (different) dictionary"
+$ZSTD --train *.c *.h -o tmpDictC
+$ZSTD -d tmp.zst -D tmpDictC -of result && die "wrong dictionary not detected!"
+$ECHO "Create dictionary with short dictID"
+$ZSTD --train *.c --dictID 1 -o tmpDict1
+cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
+$ECHO "Compress without dictID"
+$ZSTD -f tmp -D tmpDict1 --no-dictID
+$ZSTD -d tmp.zst -D tmpDict -of result
+diff zstdcli.c result
+rm tmp*
$ECHO "\n**** multiple files tests **** "
@@ -179,7 +192,7 @@
roundTripTest -g255K # TableID==1
roundTripTest -g513K # TableID==0
roundTripTest -g512K 6 # greedy, hash chain
-roundTripTest -g512K 16 # btlazy2
+roundTripTest -g512K 16 # btlazy2
roundTripTest -g512K 19 # btopt
rm tmp*
@@ -218,4 +231,3 @@
roundTripTest -g6000000000 -P99 1
rm tmp*
-
diff --git a/programs/roundTripCrash.c b/programs/roundTripCrash.c
new file mode 100644
index 0000000..1b6e1d7
--- /dev/null
+++ b/programs/roundTripCrash.c
@@ -0,0 +1,185 @@
+/*
+ roundTripCrash
+ Copyright (C) Yann Collet 2013-2016
+
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - zstd homepage : http://www.zstd.net
+*/
+/*
+ This program takes a file in input,
+ performs a zstd round-trip test (compression - decompress)
+ compares the result with original
+ and generates a crash (double free) on corruption detection.
+*/
+
+/*===========================================
+* Dependencies
+*==========================================*/
+#include <stddef.h> /* size_t */
+#include <stdlib.h> /* malloc, free, exit */
+#include <stdio.h> /* fprintf */
+#include <sys/types.h> /* stat */
+#include <sys/stat.h> /* stat */
+#include "zstd.h"
+
+/** roundTripTest() :
+* Compresses `srcBuff` into `compressedBuff`,
+* then decompresses `compressedBuff` into `resultBuff`.
+* Compression level used is derived from first content byte.
+* @return : result of decompression, which should be == `srcSize`
+* or an error code if either compression or decompression fails.
+* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)`
+* for compression to be guaranteed to work */
+static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity,
+ void* compressedBuff, size_t compressedBuffCapacity,
+ const void* srcBuff, size_t srcBuffSize)
+{
+ static const int maxClevel = 19;
+ int const cLevel = (!srcBuffSize) ? 1 : (*(const unsigned char*)srcBuff) % maxClevel;
+ size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel);
+ if (ZSTD_isError(cSize)) {
+ fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize));
+ return cSize;
+ }
+ return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize);
+}
+
+
+static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize)
+{
+ const char* ip1 = (const char*)buff1;
+ const char* ip2 = (const char*)buff2;
+ size_t pos;
+
+ for (pos=0; pos<buffSize; pos++)
+ if (ip1[pos]!=ip2[pos])
+ break;
+
+ return pos;
+}
+
+
+static void roundTripCheck(const void* srcBuff, size_t srcBuffSize)
+{
+ size_t const cBuffSize = ZSTD_compressBound(srcBuffSize);
+ void* cBuff = malloc(cBuffSize);
+ void* rBuff = malloc(cBuffSize);
+ #define CRASH { free(cBuff); free(cBuff); } /* double free, to crash program */
+
+ if (!cBuff || !rBuff) {
+ fprintf(stderr, "not enough memory ! \n");
+ exit (1);
+ }
+
+ { size_t const result = roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize);
+ if (ZSTD_isError(result)) {
+ fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result));
+ CRASH;
+ }
+ if (result != srcBuffSize) {
+ fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize);
+ CRASH;
+ }
+ if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) {
+ fprintf(stderr, "Silent decoding corruption !!!");
+ CRASH;
+ }
+ }
+
+ free(cBuff);
+ free(rBuff);
+}
+
+
+static size_t getFileSize(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+#endif
+ return (size_t)statbuf.st_size;
+}
+
+
+static int isDirectory(const char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+ if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+ if (!r && S_ISDIR(statbuf.st_mode)) return 1;
+#endif
+ return 0;
+}
+
+
+/** loadFile() :
+* requirement : `buffer` size >= `fileSize` */
+static void loadFile(void* buffer, const char* fileName, size_t fileSize)
+{
+ FILE* const f = fopen(fileName, "rb");
+ if (isDirectory(fileName)) {
+ fprintf(stderr, "Ignoring %s directory \n", fileName);
+ exit(2);
+ }
+ if (f==NULL) {
+ fprintf(stderr, "Impossible to open %s \n", fileName);
+ exit(3);
+ }
+ { size_t const readSize = fread(buffer, 1, fileSize, f);
+ if (readSize != fileSize) {
+ fprintf(stderr, "Error reading %s \n", fileName);
+ exit(5);
+ } }
+ fclose(f);
+}
+
+
+static void fileCheck(const char* fileName)
+{
+ size_t const fileSize = getFileSize(fileName);
+ void* buffer = malloc(fileSize);
+ if (!buffer) {
+ fprintf(stderr, "not enough memory \n");
+ exit(4);
+ }
+ loadFile(buffer, fileName, fileSize);
+ roundTripCheck(buffer, fileSize);
+ free (buffer);
+}
+
+int main(int argCount, const char** argv) {
+ if (argCount < 2) {
+ fprintf(stderr, "Error : no argument : need input file \n");
+ exit(9);
+ }
+ fileCheck(argv[1]);
+ fprintf(stderr, "no pb detected\n");
+ return 0;
+}
diff --git a/programs/zbufftest.c b/programs/zbufftest.c
index 552d092..278339d 100644
--- a/programs/zbufftest.c
+++ b/programs/zbufftest.c
@@ -44,6 +44,7 @@
#include "zstd_static.h" /* ZSTD_compressBound(), ZSTD_maxCLevel() */
#include "zbuff_static.h" /* ZBUFF_createCCtx_advanced */
#include "datagen.h" /* RDG_genBuffer */
+#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64 */
@@ -335,7 +336,7 @@
size_t cSize, totalTestSize, totalCSize, totalGenSize;
size_t errorCode;
U32 n, nbChunks;
- XXH64_CREATESTATE_STATIC(xxh64);
+ XXH64_state_t xxhState;
U64 crcOrig;
/* init */
@@ -379,7 +380,7 @@
} }
/* multi-segments compression test */
- XXH64_reset(xxh64, 0);
+ XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */
@@ -391,7 +392,7 @@
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError));
- XXH64_update(xxh64, srcBuffer+srcStart, readChunkSize);
+ XXH64_update(&xxhState, srcBuffer+srcStart, readChunkSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
cSize += dstBuffSize;
totalTestSize += readChunkSize;
@@ -412,7 +413,7 @@
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
}
- crcOrig = XXH64_digest(xxh64);
+ crcOrig = XXH64_digest(&xxhState);
/* multi - fragments decompression test */
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
diff --git a/programs/zstd.1 b/programs/zstd.1
index 27d607f..1bab57a 100644
--- a/programs/zstd.1
+++ b/programs/zstd.1
@@ -18,11 +18,11 @@
.PP
.B unzstd
is equivalent to
-.BR "zstd \-d"
+.BR "zstd \-d"
.br
.B zstdcat
is equivalent to
-.BR "zstd \-dc"
+.BR "zstd \-dc"
.br
.SH DESCRIPTION
@@ -90,7 +90,15 @@
dictionary saved into `file` (default: dictionary)
.TP
.B \--maxdict #
- limit dictionary to specified size (default : 112640)
+ limit dictionary to specified size (default : 112640)
+.TP
+.B \--dictID #
+ A dictionary ID is a locally unique ID that a decoder can use to verify it is using the right dictionary.
+ By default, zstd will create a 4-bytes random number ID.
+ It's possible to give a precise number instead.
+ Short numbers have an advantage : an ID < 256 will only need 1 byte in the compressed frame header,
+ and an ID < 65536 will only need 2 bytes. This compares favorably to 4 bytes default.
+ However, it's up to the dictionary manager to not assign twice the same ID to 2 different dictionaries.
.TP
.B \-s#
dictionary selectivity level (default: 9)
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index 18a81d6..24bba89 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -125,7 +125,6 @@
DISPLAY( "\n");
DISPLAY( "Advanced arguments :\n");
DISPLAY( " -V : display Version number and exit\n");
- DISPLAY( " -t : test compressed file integrity \n");
DISPLAY( " -v : verbose mode\n");
DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
DISPLAY( " -c : force write to standard output, even if it is the console\n");
@@ -134,8 +133,12 @@
#endif
#ifndef ZSTD_NOCOMPRESS
DISPLAY( "--ultra : enable ultra modes (requires more memory to decompress)\n");
+ DISPLAY( "--no-dictID:don't write dictID into header (dictionary compression)\n");
#endif
+#ifndef ZSTD_NODECOMPRESS
+ DISPLAY( " -t : test compressed file integrity \n");
DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
+#endif
#ifndef ZSTD_NODICT
DISPLAY( "\n");
DISPLAY( "Dictionary builder :\n");
@@ -143,6 +146,7 @@
DISPLAY( " -o file: `file` is dictionary name (default: %s) \n", g_defaultDictName);
DISPLAY( "--maxdict:limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize);
DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel);
+ DISPLAY( "--dictID: force dictionary ID to specified value (default: random)\n");
#endif
#ifndef ZSTD_NOBENCH
DISPLAY( "\n");
@@ -185,7 +189,8 @@
operationResult=0,
dictBuild=0,
nextArgumentIsOutFileName=0,
- nextArgumentIsMaxDict=0;
+ nextArgumentIsMaxDict=0,
+ nextArgumentIsDictID=0;
unsigned cLevel = 1;
unsigned cLevelLast = 1;
unsigned recursive = 0;
@@ -196,6 +201,7 @@
const char* dictFileName = NULL;
char* dynNameSpace = NULL;
unsigned maxDictSize = g_defaultMaxDictSize;
+ unsigned dictID = 0;
unsigned dictCLevel = g_defaultDictCLevel;
unsigned dictSelect = g_defaultSelectivityLevel;
#ifdef UTIL_HAS_CREATEFILELIST
@@ -233,13 +239,15 @@
if (!strcmp(argument, "--verbose")) { displayLevel=4; continue; }
if (!strcmp(argument, "--quiet")) { displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; displayLevel=1; continue; }
+ if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
+ if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
+ if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
+ if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
if (!strcmp(argument, "--test")) { decode=1; outFileName=nulmark; FIO_overwriteMode(); continue; }
if (!strcmp(argument, "--train")) { dictBuild=1; outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; continue; }
+ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; continue; }
if (!strcmp(argument, "--keep")) { continue; } /* does nothing, since preserving input is default; for gzip/xz compatibility */
- if (!strcmp(argument, "--ultra")) { FIO_setMaxWLog(0); continue; }
- if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
- if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
/* '-' means stdin/stdout */
if (!strcmp(argument, "-")){
@@ -296,7 +304,7 @@
case 'k': argument++; break;
/* test compressed file */
- case 't': decode=1; outFileName=nulmark; FIO_overwriteMode(); argument++; break;
+ case 't': decode=1; outFileName=nulmark; argument++; break;
/* dictionary name */
case 'o': nextArgumentIsOutFileName=1; argument++; break;
@@ -393,6 +401,14 @@
continue;
}
+ if (nextArgumentIsDictID) {
+ nextArgumentIsDictID = 0;
+ dictID = 0;
+ while ((*argument>='0') && (*argument<='9'))
+ dictID = dictID * 10 + (*argument - '0'), argument++;
+ continue;
+ }
+
/* add filename to list */
filenameTable[filenameIdx++] = argument;
}
@@ -429,6 +445,7 @@
dictParams.compressionLevel = dictCLevel;
dictParams.selectivityLevel = dictSelect;
dictParams.notificationLevel = displayLevel;
+ dictParams.dictID = dictID;
DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, dictParams);
#endif
goto _end;
diff --git a/projects/.gitignore b/projects/.gitignore
new file mode 100644
index 0000000..378eac2
--- /dev/null
+++ b/projects/.gitignore
@@ -0,0 +1 @@
+build
diff --git a/projects/VS2008/fuzzer/fuzzer.vcproj b/projects/VS2008/fuzzer/fuzzer.vcproj
index 9e572da..ab0bab2 100644
--- a/projects/VS2008/fuzzer/fuzzer.vcproj
+++ b/projects/VS2008/fuzzer/fuzzer.vcproj
@@ -44,7 +44,7 @@
<Tool
Name="VCCLCompilerTool"
Optimization="0"
- AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+ AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
@@ -93,80 +93,6 @@
/>
</Configuration>
<Configuration
- Name="Debug|x64"
- OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
- IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
- ConfigurationType="1"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- TargetEnvironment="3"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="3"
- UsePrecompiledHeader="0"
- WarningLevel="4"
- WarnAsError="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- LinkIncremental="2"
- GenerateDebugInformation="true"
- SubSystem="1"
- TargetMachine="17"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
@@ -194,7 +120,7 @@
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
- AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+ AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
@@ -243,6 +169,80 @@
/>
</Configuration>
<Configuration
+ Name="Debug|x64"
+ OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+ IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+ ConfigurationType="1"
+ CharacterSet="2"
+ >
+ <Tool
+ Name="VCPreBuildEventTool"
+ />
+ <Tool
+ Name="VCCustomBuildTool"
+ />
+ <Tool
+ Name="VCXMLDataGeneratorTool"
+ />
+ <Tool
+ Name="VCWebServiceProxyGeneratorTool"
+ />
+ <Tool
+ Name="VCMIDLTool"
+ TargetEnvironment="3"
+ />
+ <Tool
+ Name="VCCLCompilerTool"
+ Optimization="0"
+ AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ MinimalRebuild="true"
+ BasicRuntimeChecks="3"
+ RuntimeLibrary="3"
+ UsePrecompiledHeader="0"
+ WarningLevel="4"
+ WarnAsError="true"
+ DebugInformationFormat="3"
+ />
+ <Tool
+ Name="VCManagedResourceCompilerTool"
+ />
+ <Tool
+ Name="VCResourceCompilerTool"
+ />
+ <Tool
+ Name="VCPreLinkEventTool"
+ />
+ <Tool
+ Name="VCLinkerTool"
+ LinkIncremental="2"
+ GenerateDebugInformation="true"
+ SubSystem="1"
+ TargetMachine="17"
+ />
+ <Tool
+ Name="VCALinkTool"
+ />
+ <Tool
+ Name="VCManifestTool"
+ />
+ <Tool
+ Name="VCXDCMakeTool"
+ />
+ <Tool
+ Name="VCBscMakeTool"
+ />
+ <Tool
+ Name="VCFxCopTool"
+ />
+ <Tool
+ Name="VCAppVerifierTool"
+ />
+ <Tool
+ Name="VCPostBuildEventTool"
+ />
+ </Configuration>
+ <Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
@@ -271,7 +271,7 @@
Optimization="2"
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
- AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
+ AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy"
PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
@@ -333,6 +333,10 @@
>
</File>
<File
+ RelativePath="..\..\..\lib\dictBuilder\divsufsort.c"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\entropy_common.c"
>
</File>
@@ -357,7 +361,11 @@
>
</File>
<File
- RelativePath="..\..\..\programs\xxhash.c"
+ RelativePath="..\..\..\lib\common\xxhash.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\dictBuilder\zdict.c"
>
</File>
<File
@@ -383,6 +391,10 @@
>
</File>
<File
+ RelativePath="..\..\..\lib\dictBuilder\divsufsort.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\error_private.h"
>
</File>
@@ -411,10 +423,22 @@
>
</File>
<File
+ RelativePath="..\..\..\lib\common\xxhash.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\zbuff_static.h"
>
</File>
<File
+ RelativePath="..\..\..\lib\dictBuilder\zdict.h"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\dictBuilder\zdict_static.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\zstd.h"
>
</File>
diff --git a/projects/VS2008/zstd/zstd.vcproj b/projects/VS2008/zstd/zstd.vcproj
index f84fbc9..38b7693 100644
--- a/projects/VS2008/zstd/zstd.vcproj
+++ b/projects/VS2008/zstd/zstd.vcproj
@@ -94,81 +94,6 @@
/>
</Configuration>
<Configuration
- Name="Debug|x64"
- OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
- IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
- ConfigurationType="1"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- TargetEnvironment="3"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="3"
- UsePrecompiledHeader="0"
- WarningLevel="4"
- WarnAsError="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- AdditionalDependencies="setargv.obj"
- LinkIncremental="2"
- GenerateDebugInformation="true"
- SubSystem="1"
- TargetMachine="17"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
@@ -246,6 +171,81 @@
/>
</Configuration>
<Configuration
+ Name="Debug|x64"
+ OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+ IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+ ConfigurationType="1"
+ CharacterSet="2"
+ >
+ <Tool
+ Name="VCPreBuildEventTool"
+ />
+ <Tool
+ Name="VCCustomBuildTool"
+ />
+ <Tool
+ Name="VCXMLDataGeneratorTool"
+ />
+ <Tool
+ Name="VCWebServiceProxyGeneratorTool"
+ />
+ <Tool
+ Name="VCMIDLTool"
+ TargetEnvironment="3"
+ />
+ <Tool
+ Name="VCCLCompilerTool"
+ Optimization="0"
+ AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+ PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ MinimalRebuild="true"
+ BasicRuntimeChecks="3"
+ RuntimeLibrary="3"
+ UsePrecompiledHeader="0"
+ WarningLevel="4"
+ WarnAsError="true"
+ DebugInformationFormat="3"
+ />
+ <Tool
+ Name="VCManagedResourceCompilerTool"
+ />
+ <Tool
+ Name="VCResourceCompilerTool"
+ />
+ <Tool
+ Name="VCPreLinkEventTool"
+ />
+ <Tool
+ Name="VCLinkerTool"
+ AdditionalDependencies="setargv.obj"
+ LinkIncremental="2"
+ GenerateDebugInformation="true"
+ SubSystem="1"
+ TargetMachine="17"
+ />
+ <Tool
+ Name="VCALinkTool"
+ />
+ <Tool
+ Name="VCManifestTool"
+ />
+ <Tool
+ Name="VCXDCMakeTool"
+ />
+ <Tool
+ Name="VCBscMakeTool"
+ />
+ <Tool
+ Name="VCFxCopTool"
+ />
+ <Tool
+ Name="VCAppVerifierTool"
+ />
+ <Tool
+ Name="VCPostBuildEventTool"
+ />
+ </Configuration>
+ <Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
@@ -377,7 +377,7 @@
>
</File>
<File
- RelativePath="..\..\..\programs\xxhash.c"
+ RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
@@ -471,6 +471,10 @@
>
</File>
<File
+ RelativePath="..\..\..\lib\common\xxhash.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
diff --git a/projects/VS2008/zstdlib/zstdlib.vcproj b/projects/VS2008/zstdlib/zstdlib.vcproj
index 7c16034..2051da5 100644
--- a/projects/VS2008/zstdlib/zstdlib.vcproj
+++ b/projects/VS2008/zstdlib/zstdlib.vcproj
@@ -93,80 +93,6 @@
/>
</Configuration>
<Configuration
- Name="Debug|x64"
- OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
- IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
- ConfigurationType="2"
- CharacterSet="2"
- >
- <Tool
- Name="VCPreBuildEventTool"
- />
- <Tool
- Name="VCCustomBuildTool"
- />
- <Tool
- Name="VCXMLDataGeneratorTool"
- />
- <Tool
- Name="VCWebServiceProxyGeneratorTool"
- />
- <Tool
- Name="VCMIDLTool"
- TargetEnvironment="3"
- />
- <Tool
- Name="VCCLCompilerTool"
- Optimization="0"
- AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
- MinimalRebuild="true"
- BasicRuntimeChecks="3"
- RuntimeLibrary="3"
- UsePrecompiledHeader="0"
- WarningLevel="4"
- WarnAsError="true"
- DebugInformationFormat="3"
- />
- <Tool
- Name="VCManagedResourceCompilerTool"
- />
- <Tool
- Name="VCResourceCompilerTool"
- />
- <Tool
- Name="VCPreLinkEventTool"
- />
- <Tool
- Name="VCLinkerTool"
- LinkIncremental="2"
- GenerateDebugInformation="true"
- SubSystem="1"
- TargetMachine="17"
- />
- <Tool
- Name="VCALinkTool"
- />
- <Tool
- Name="VCManifestTool"
- />
- <Tool
- Name="VCXDCMakeTool"
- />
- <Tool
- Name="VCBscMakeTool"
- />
- <Tool
- Name="VCFxCopTool"
- />
- <Tool
- Name="VCAppVerifierTool"
- />
- <Tool
- Name="VCPostBuildEventTool"
- />
- </Configuration>
- <Configuration
Name="Release|Win32"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
@@ -243,6 +169,80 @@
/>
</Configuration>
<Configuration
+ Name="Debug|x64"
+ OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
+ IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+ ConfigurationType="2"
+ CharacterSet="2"
+ >
+ <Tool
+ Name="VCPreBuildEventTool"
+ />
+ <Tool
+ Name="VCCustomBuildTool"
+ />
+ <Tool
+ Name="VCXMLDataGeneratorTool"
+ />
+ <Tool
+ Name="VCWebServiceProxyGeneratorTool"
+ />
+ <Tool
+ Name="VCMIDLTool"
+ TargetEnvironment="3"
+ />
+ <Tool
+ Name="VCCLCompilerTool"
+ Optimization="0"
+ AdditionalIncludeDirectories="$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+ MinimalRebuild="true"
+ BasicRuntimeChecks="3"
+ RuntimeLibrary="3"
+ UsePrecompiledHeader="0"
+ WarningLevel="4"
+ WarnAsError="true"
+ DebugInformationFormat="3"
+ />
+ <Tool
+ Name="VCManagedResourceCompilerTool"
+ />
+ <Tool
+ Name="VCResourceCompilerTool"
+ />
+ <Tool
+ Name="VCPreLinkEventTool"
+ />
+ <Tool
+ Name="VCLinkerTool"
+ LinkIncremental="2"
+ GenerateDebugInformation="true"
+ SubSystem="1"
+ TargetMachine="17"
+ />
+ <Tool
+ Name="VCALinkTool"
+ />
+ <Tool
+ Name="VCManifestTool"
+ />
+ <Tool
+ Name="VCXDCMakeTool"
+ />
+ <Tool
+ Name="VCBscMakeTool"
+ />
+ <Tool
+ Name="VCFxCopTool"
+ />
+ <Tool
+ Name="VCAppVerifierTool"
+ />
+ <Tool
+ Name="VCPostBuildEventTool"
+ />
+ </Configuration>
+ <Configuration
Name="Release|x64"
OutputDirectory="$(SolutionDir)bin\$(PlatformName)\$(ConfigurationName)"
IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
@@ -353,6 +353,10 @@
>
</File>
<File
+ RelativePath="..\..\..\lib\common\xxhash.c"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\compress\zbuff_compress.c"
>
</File>
@@ -419,6 +423,10 @@
>
</File>
<File
+ RelativePath="..\..\..\lib\common\xxhash.h"
+ >
+ </File>
+ <File
RelativePath="..\..\..\lib\common\zbuff.h"
>
</File>
diff --git a/projects/VS2010/fuzzer/fuzzer.vcxproj b/projects/VS2010/fuzzer/fuzzer.vcxproj
index 047d5fd..bdda5f3 100644
--- a/projects/VS2010/fuzzer/fuzzer.vcxproj
+++ b/projects/VS2010/fuzzer/fuzzer.vcxproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
@@ -66,24 +66,24 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis>
- <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis>
- <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis>
- <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
<IntDir>$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<RunCodeAnalysis>false</RunCodeAnalysis>
- <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(UniversalCRT_IncludePath);</IncludePath>
+ <IncludePath>$(IncludePath);$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\dictBuilder;$(UniversalCRT_IncludePath);</IncludePath>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
@@ -158,30 +158,35 @@
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
+ <ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
<ClCompile Include="..\..\..\lib\compress\huf_compress.c" />
<ClCompile Include="..\..\..\lib\compress\zstd_compress.c" />
<ClCompile Include="..\..\..\lib\decompress\huf_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
+ <ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
+ <ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
<ClCompile Include="..\..\..\programs\datagen.c" />
<ClCompile Include="..\..\..\programs\fuzzer.c" />
- <ClCompile Include="..\..\..\programs\xxhash.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
+ <ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
<ClInclude Include="..\..\..\lib\common\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_static.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
+ <ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
+ <ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
+ <ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
- <ClInclude Include="..\..\..\programs\xxhash.h" />
<ClInclude Include="..\..\..\programs\util.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
diff --git a/projects/VS2010/zstd/zstd.vcxproj b/projects/VS2010/zstd/zstd.vcxproj
index 8744922..40cb20d 100644
--- a/projects/VS2010/zstd/zstd.vcxproj
+++ b/projects/VS2010/zstd/zstd.vcxproj
@@ -20,6 +20,7 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
+ <ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
@@ -41,10 +42,10 @@
<ClCompile Include="..\..\..\programs\dibio.c" />
<ClCompile Include="..\..\..\programs\fileio.c" />
<ClCompile Include="..\..\..\programs\legacy\fileio_legacy.c" />
- <ClCompile Include="..\..\..\programs\xxhash.c" />
<ClCompile Include="..\..\..\programs\zstdcli.c" />
</ItemGroup>
<ItemGroup>
+ <ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict_static.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
@@ -69,7 +70,6 @@
<ClInclude Include="..\..\..\programs\dibio.h" />
<ClInclude Include="..\..\..\programs\fileio.h" />
<ClInclude Include="..\..\..\programs\legacy\fileio_legacy.h" />
- <ClInclude Include="..\..\..\programs\xxhash.h" />
<ClInclude Include="..\..\..\programs\util.h" />
</ItemGroup>
<PropertyGroup Label="Globals">
diff --git a/projects/VS2010/zstdlib/zstdlib.vcxproj b/projects/VS2010/zstdlib/zstdlib.vcxproj
index 9e49781..7990050 100644
--- a/projects/VS2010/zstdlib/zstdlib.vcxproj
+++ b/projects/VS2010/zstdlib/zstdlib.vcxproj
@@ -20,6 +20,7 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\..\lib\common\entropy_common.c" />
+ <ClCompile Include="..\..\..\lib\common\xxhash.c" />
<ClCompile Include="..\..\..\lib\common\zstd_common.c" />
<ClCompile Include="..\..\..\lib\common\fse_decompress.c" />
<ClCompile Include="..\..\..\lib\compress\fse_compress.c" />
@@ -39,6 +40,7 @@
<ClInclude Include="..\..\..\lib\common\fse_static.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\huf_static.h" />
+ <ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
<ClInclude Include="..\..\..\lib\common\zbuff_static.h" />
<ClInclude Include="..\..\..\lib\common\zstd.h" />
diff --git a/projects/cmake/.gitignore b/projects/cmake/.gitignore
new file mode 100644
index 0000000..98f29c7
--- /dev/null
+++ b/projects/cmake/.gitignore
@@ -0,0 +1,6 @@
+# cmake producted
+CMakeCache.txt
+CMakeFiles
+Makefile
+cmake_install.cmake
+cmake_uninstall.cmake
diff --git a/projects/cmake/build/.keep b/projects/cmake/build/.keep
deleted file mode 100644
index e69de29..0000000
--- a/projects/cmake/build/.keep
+++ /dev/null
diff --git a/projects/cmake/lib/CMakeLists.txt b/projects/cmake/lib/CMakeLists.txt
index 4058887..716ccbf 100644
--- a/projects/cmake/lib/CMakeLists.txt
+++ b/projects/cmake/lib/CMakeLists.txt
@@ -59,6 +59,7 @@
SET(Sources
${LIBRARY_DIR}/common/entropy_common.c
${LIBRARY_DIR}/common/zstd_common.c
+ ${LIBRARY_DIR}/common/xxhash.c
${LIBRARY_DIR}/common/fse_decompress.c
${LIBRARY_DIR}/compress/fse_compress.c
${LIBRARY_DIR}/compress/huf_compress.c
diff --git a/projects/cmake/programs/.gitignore b/projects/cmake/programs/.gitignore
new file mode 100644
index 0000000..81eec34
--- /dev/null
+++ b/projects/cmake/programs/.gitignore
@@ -0,0 +1,8 @@
+# produced by make
+datagen
+fullbench
+fuzzer
+paramgrill
+zbufftest
+zstd
+zstd-frugal
diff --git a/projects/cmake/programs/CMakeLists.txt b/projects/cmake/programs/CMakeLists.txt
index 1cf1b69..c8fe5d2 100644
--- a/projects/cmake/programs/CMakeLists.txt
+++ b/projects/cmake/programs/CMakeLists.txt
@@ -42,20 +42,19 @@
SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR}/common ${LIBRARY_DIR}/dictBuilder)
-
IF (ZSTD_LEGACY_SUPPORT)
SET(PROGRAMS_LEGACY_DIR ${PROGRAMS_DIR}/legacy)
INCLUDE_DIRECTORIES(${PROGRAMS_LEGACY_DIR} ${LIBRARY_DIR}/legacy)
SET(ZSTD_FILEIO_LEGACY ${PROGRAMS_LEGACY_DIR}/fileio_legacy.c)
ENDIF (ZSTD_LEGACY_SUPPORT)
-ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
+ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${ZSTD_FILEIO_LEGACY})
TARGET_LINK_LIBRARIES(zstd libzstd_static)
ADD_EXECUTABLE(fullbench ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fullbench.c)
TARGET_LINK_LIBRARIES(fullbench libzstd_static)
-ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/fuzzer.c)
+ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/fuzzer.c)
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
IF (UNIX)
@@ -63,10 +62,10 @@
TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
- ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/zbufftest.c)
+ ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/zbufftest.c)
TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
- ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/xxhash.c ${PROGRAMS_DIR}/paramgrill.c)
+ ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/paramgrill.c)
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
ADD_EXECUTABLE(datagen ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/datagencli.c)
diff --git a/versionsTest/test-zstd-versions.py b/versionsTest/test-zstd-versions.py
index b18c3d3..f7428bc 100644
--- a/versionsTest/test-zstd-versions.py
+++ b/versionsTest/test-zstd-versions.py
@@ -1,14 +1,14 @@
#!/usr/bin/env python3
-
+"""Test zstd interoperability between versions"""
# Based on LZ4 version test script, by Takayuki Matsuoka
-import glob
-import subprocess
import filecmp
+import glob
+import hashlib
import os
import shutil
+import subprocess
import sys
-import hashlib
repo_url = 'https://github.com/Cyan4973/zstd.git'
tmp_dir_name = 'versionsTest/zstdtest'
@@ -18,6 +18,7 @@
test_dat = 'test_dat'
head = 'vdevel'
+
def proc(cmd_args, pipe=True, dummy=False):
if dummy:
return
@@ -29,46 +30,52 @@
subproc = subprocess.Popen(cmd_args)
return subproc.communicate()
+
def make(args, pipe=True):
return proc([make_cmd] + args, pipe)
+
def git(args, pipe=True):
return proc([git_cmd] + args, pipe)
+
def get_git_tags():
stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]'])
tags = stdout.decode('utf-8').split()
return tags
+
def compress_sample(tag, sample):
try:
- from subprocess import DEVNULL # py3k
+ from subprocess import DEVNULL # py3k
except ImportError:
DEVNULL = open(os.devnull, 'wb')
- if subprocess.call(['./zstd.' + tag, '-f' , sample], stderr=DEVNULL)==0:
+ if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_01_64_' + tag + '.zst')
- if subprocess.call(['./zstd.' + tag, '-5f' , sample], stderr=DEVNULL)==0:
+ if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_05_64_' + tag + '.zst')
- if subprocess.call(['./zstd.' + tag, '-9f' , sample], stderr=DEVNULL)==0 :
+ if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_09_64_' + tag + '.zst')
- if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL)==0 :
+ if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_15_64_' + tag + '.zst')
- if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL)==0:
+ if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_18_64_' + tag + '.zst')
# zstdFiles = glob.glob("*.zst*")
# print(zstdFiles)
+
# http://stackoverflow.com/a/19711609/2132223
def sha1_of_file(filepath):
with open(filepath, 'rb') as f:
return hashlib.sha1(f.read()).hexdigest()
+
def remove_duplicates():
list_of_zst = sorted(glob.glob('*.zst'))
for i, ref_zst in enumerate(list_of_zst):
if not os.path.isfile(ref_zst):
continue
- for j in range(i+1, len(list_of_zst)):
+ for j in range(i + 1, len(list_of_zst)):
compared_zst = list_of_zst[j]
if not os.path.isfile(compared_zst):
continue
@@ -76,18 +83,19 @@
os.remove(compared_zst)
print('duplicated : {} == {}'.format(ref_zst, compared_zst))
+
def decompress_zst(tag):
dec_error = 0
list_zst = sorted(glob.glob('*.zst'))
try:
- from subprocess import DEVNULL # py3k
+ from subprocess import DEVNULL # py3k
except ImportError:
DEVNULL = open(os.devnull, 'wb')
for file_zst in list_zst:
- print(file_zst, end=" ")
- print(tag, end=" ")
+ print(file_zst, end=' ')
+ print(tag, end=' ')
file_dec = file_zst + '_d64_' + tag + '.dec'
- if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL)==0:
+ if subprocess.call(['./zstd.' + tag, '-df', file_zst, '-o', file_dec], stderr=DEVNULL) == 0:
if not filecmp.cmp(file_dec, test_dat):
print('ERR !! ')
dec_error = 1
@@ -116,12 +124,12 @@
print('Retrieve all release tags :')
os.chdir(clone_dir)
tags = get_git_tags() + [head]
- print(tags);
+ print(tags)
# Build all release zstd
for tag in tags:
os.chdir(base_dir)
- dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd.<TAG>
+ dst_zstd = '{}/zstd.{}' .format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/zstd.<TAG>
if not os.path.isfile(dst_zstd) or tag == head:
if tag != head:
r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/test/zstdtest/<TAG>
@@ -139,17 +147,18 @@
os.chdir(tmp_dir)
for compressed in glob.glob("*.zst"):
os.remove(compressed)
- for dec in glob.glob("*.dec"):
+ for dec in glob.glob("*.dec"):
os.remove(dec)
print('Compress test.dat by all released zstd')
- error_code = 0;
+ error_code = 0
for tag in tags:
print(tag)
compress_sample(tag, test_dat)
remove_duplicates()
- error_code += decompress_zst(tag)
+ if tag >= 'v0.5.1':
+ error_code += decompress_zst(tag)
print('')
print('Enumerate different compressed files')