Merge pull request #786 from terrelln/squashfs-tools
[linux-kernel] Update patches for v4 and v5
diff --git a/Makefile b/Makefile
index 5e88736..1dceec8 100644
--- a/Makefile
+++ b/Makefile
@@ -146,6 +146,11 @@
gcc-6 -v
CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
+.PHONY: gcc7build
+gcc7build: clean
+ gcc-7 -v
+ CC=gcc-7 $(MAKE) all MOREFLAGS="-Werror"
+
.PHONY: clangbuild
clangbuild: clean
clang -v
diff --git a/circle.yml b/circle.yml
index 218e33b..8c2bd30 100644
--- a/circle.yml
+++ b/circle.yml
@@ -3,7 +3,7 @@
- sudo dpkg --add-architecture i386
- sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
- sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev
+ - sudo apt-get -y install libstdc++-7-dev clang gcc g++ gcc-5 gcc-6 gcc-7 zlib1g-dev liblzma-dev
- sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
test:
@@ -45,7 +45,7 @@
parallel: true
- ? |
if [[ "$CIRCLE_NODE_INDEX" == "0" ]] ; then make ppc64build && make clean; fi &&
- if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true && make clean; fi #could add another test here
+ if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc7build && make clean; fi #could add another test here
:
parallel: true
- ? |
@@ -64,7 +64,7 @@
#- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean
#- make uasan && make clean
#- make asan32 && make clean
- #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
+ #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu"
# Valgrind tests
#- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean
#- make -C tests valgrindTest && make clean
diff --git a/contrib/adaptive-compression/Makefile b/contrib/adaptive-compression/Makefile
index 9bc19ee..c64fce9 100644
--- a/contrib/adaptive-compression/Makefile
+++ b/contrib/adaptive-compression/Makefile
@@ -6,6 +6,7 @@
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+MULTITHREAD_LDFLAGS = -pthread
DEBUGFLAGS= -g -DZSTD_DEBUG=1
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
@@ -17,7 +18,7 @@
-Wredundant-decls
CFLAGS += $(DEBUGFLAGS)
CFLAGS += $(MOREFLAGS)
-FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
+FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
all: adapt datagen
@@ -28,7 +29,7 @@
$(CC) $(FLAGS) -DDEBUG_MODE=2 $^ -o adapt
datagen : $(PRGDIR)/datagen.c datagencli.c
- $(CC) $(FLAGS) $^ -o $@$(EXT)
+ $(CC) $(FLAGS) $^ -o $@
test-adapt-correctness: datagen adapt
@./test-correctness.sh
@@ -45,3 +46,31 @@
@$(RM) -f tests/*.zst
@$(RM) -f tests/tmp*
@echo "finished cleaning"
+
+#-----------------------------------------------------------------------------
+# make install is validated only for Linux, OSX, BSD, Hurd and Solaris targets
+#-----------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
+
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+PREFIX ?= /usr/local
+DESTDIR ?=
+BINDIR ?= $(PREFIX)/bin
+
+INSTALL_PROGRAM ?= $(INSTALL) -m 755
+
+install: adapt
+ @echo Installing binaries
+ @$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/
+ @$(INSTALL_PROGRAM) adapt $(DESTDIR)$(BINDIR)/zstd-adaptive
+ @echo zstd-adaptive installation completed
+
+uninstall:
+ @$(RM) $(DESTDIR)$(BINDIR)/zstd-adaptive
+ @echo zstd-adaptive programs successfully uninstalled
+endif
diff --git a/contrib/adaptive-compression/adapt.c b/contrib/adaptive-compression/adapt.c
index 5cf3b97..40ebb07 100644
--- a/contrib/adaptive-compression/adapt.c
+++ b/contrib/adaptive-compression/adapt.c
@@ -42,6 +42,8 @@
static unsigned g_useProgressBar = 1;
static UTIL_freq_t g_ticksPerSecond;
static unsigned g_forceCompressionLevel = 0;
+static unsigned g_minCLevel = 1;
+static unsigned g_maxCLevel;
typedef struct {
void* start;
@@ -331,7 +333,7 @@
pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex);
pthread_mutex_lock(&ctx->jobCompressed_mutex.pMutex);
- pthread_cond_signal(&ctx->jobCompressed_cond.pCond);
+ pthread_cond_broadcast(&ctx->jobCompressed_cond.pCond);
pthread_mutex_unlock(&ctx->jobReady_mutex.pMutex);
pthread_mutex_lock(&ctx->jobWrite_mutex.pMutex);
@@ -412,6 +414,8 @@
pthread_mutex_unlock(&ctx->createCompletion_mutex.pMutex);
DEBUG(2, "convergence counter: %u\n", ctx->convergenceCounter);
+ assert(g_minCLevel <= ctx->compressionLevel && g_maxCLevel >= ctx->compressionLevel);
+
/* adaptation logic */
if (ctx->cooldown) ctx->cooldown--;
@@ -420,7 +424,7 @@
/* use whichever one waited less because it was slower */
double const completion = MAX(createWaitCompressionCompletion, writeWaitCompressionCompletion);
unsigned const change = convertCompletionToChange(completion);
- unsigned const boundChange = MIN(change, ctx->compressionLevel - 1);
+ unsigned const boundChange = MIN(change, ctx->compressionLevel - g_minCLevel);
if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) {
/* reset convergence counter, might have been a spike */
ctx->convergenceCounter = 0;
@@ -438,7 +442,7 @@
/* compress waiting on write */
double const completion = MIN(compressWaitWriteCompletion, compressWaitCreateCompletion);
unsigned const change = convertCompletionToChange(completion);
- unsigned const boundChange = MIN(change, ZSTD_maxCLevel() - ctx->compressionLevel);
+ unsigned const boundChange = MIN(change, g_maxCLevel - ctx->compressionLevel);
if (ctx->convergenceCounter >= CONVERGENCE_LOWER_BOUND && boundChange != 0) {
/* reset convergence counter, might have been a spike */
ctx->convergenceCounter = 0;
@@ -619,18 +623,20 @@
static void displayProgress(unsigned cLevel, unsigned last)
{
- if (!g_useProgressBar) return;
UTIL_time_t currTime;
UTIL_getTime(&currTime);
- double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
- double const sizeMB = (double)g_streamedSize / (1 << 20);
- double const avgCompRate = sizeMB * 1000 / timeElapsed;
- fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
- if (last) {
- fprintf(stderr, "\n");
- }
- else {
- fflush(stderr);
+ if (!g_useProgressBar) return;
+ {
+ double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0);
+ double const sizeMB = (double)g_streamedSize / (1 << 20);
+ double const avgCompRate = sizeMB * 1000 / timeElapsed;
+ fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate);
+ if (last) {
+ fprintf(stderr, "\n");
+ }
+ else {
+ fflush(stderr);
+ }
}
}
@@ -928,9 +934,9 @@
static int compressFilename(const char* const srcFilename, const char* const dstFilenameOrNull)
{
int ret = 0;
+ fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
UTIL_getTime(&g_startTime);
g_streamedSize = 0;
- fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull);
ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg);
ret |= freeFileCompressionResources(&fcr);
return ret;
@@ -973,19 +979,21 @@
return result;
}
-static void help()
+static void help(void)
{
PRINT("Usage:\n");
PRINT(" ./multi [options] [file(s)]\n");
PRINT("\n");
PRINT("Options:\n");
PRINT(" -oFILE : specify the output file name\n");
- PRINT(" -i# : provide initial compression level\n");
+ PRINT(" -i# : provide initial compression level -- default %d, must be in the range [L, U] where L and U are bound values (see below for defaults)\n", DEFAULT_COMPRESSION_LEVEL);
PRINT(" -h : display help/information\n");
PRINT(" -f : force the compression level to stay constant\n");
PRINT(" -c : force write to stdout\n");
PRINT(" -p : hide progress bar\n");
PRINT(" -q : quiet mode -- do not show progress bar or other information\n");
+ PRINT(" -l# : provide lower bound for compression level -- default 1\n");
+ PRINT(" -u# : provide upper bound for compression level -- default %u\n", ZSTD_maxCLevel());
}
/* return 0 if successful, else return error */
int main(int argCount, const char* argv[])
@@ -993,10 +1001,12 @@
const char* outFilename = NULL;
const char** filenameTable = (const char**)malloc(argCount*sizeof(const char*));
unsigned filenameIdx = 0;
- filenameTable[0] = stdinmark;
unsigned forceStdout = 0;
+ unsigned providedInitialCLevel = 0;
int ret = 0;
int argNum;
+ filenameTable[0] = stdinmark;
+ g_maxCLevel = ZSTD_maxCLevel();
UTIL_initTimer(&g_ticksPerSecond);
@@ -1018,6 +1028,7 @@
case 'i':
argument += 2;
g_compressionLevel = readU32FromChar(&argument);
+ providedInitialCLevel = 1;
break;
case 'h':
help();
@@ -1036,6 +1047,14 @@
g_useProgressBar = 0;
g_displayLevel = 0;
break;
+ case 'l':
+ argument += 2;
+ g_minCLevel = readU32FromChar(&argument);
+ break;
+ case 'u':
+ argument += 2;
+ g_maxCLevel = readU32FromChar(&argument);
+ break;
default:
DISPLAY("Error: invalid argument provided\n");
ret = 1;
@@ -1048,6 +1067,20 @@
filenameTable[filenameIdx++] = argument;
}
+ /* check initial, max, and min compression levels */
+ {
+ unsigned const minMaxInconsistent = g_minCLevel > g_maxCLevel;
+ unsigned const initialNotInRange = g_minCLevel > g_compressionLevel || g_maxCLevel < g_compressionLevel;
+ if (minMaxInconsistent || (initialNotInRange && providedInitialCLevel)) {
+ DISPLAY("Error: provided compression level parameters are invalid\n");
+ ret = 1;
+ goto _main_exit;
+ }
+ else if (initialNotInRange) {
+ g_compressionLevel = g_minCLevel;
+ }
+ }
+
/* error checking with number of files */
if (filenameIdx > 1 && (outFilename != NULL && strcmp(outFilename, stdoutmark))) {
DISPLAY("Error: multiple input files provided, cannot use specified output file\n");
diff --git a/contrib/adaptive-compression/test-correctness.sh b/contrib/adaptive-compression/test-correctness.sh
index 8ae6604..3bea867 100755
--- a/contrib/adaptive-compression/test-correctness.sh
+++ b/contrib/adaptive-compression/test-correctness.sh
@@ -242,4 +242,11 @@
./datagen -s39 -g1GB | pv -L 25m -q | ./adapt -i1 | pv -q > tmp.zst
zstd -d tmp.zst
rm tmp*
+
+echo -e "\ncorrectness tests -- testing bounds"
+./datagen -s40 -g1GB | pv -L 25m -q | ./adapt -i1 -u4 | pv -q > tmp.zst
+rm tmp*
+
+./datagen -s41 -g1GB | ./adapt -i14 -l4 > tmp.zst
+rm tmp*
make clean
diff --git a/lib/common/pool.c b/lib/common/pool.c
index aeaca7e..e140f1e 100644
--- a/lib/common/pool.c
+++ b/lib/common/pool.c
@@ -39,6 +39,12 @@
size_t queueHead;
size_t queueTail;
size_t queueSize;
+
+ /* The number of threads working on jobs */
+ size_t numThreadsBusy;
+ /* Indicates if the queue is empty */
+ int queueEmpty;
+
/* The mutex protects the queue */
pthread_mutex_t queueMutex;
/* Condition variable for pushers to wait on when the queue is full */
@@ -60,21 +66,37 @@
for (;;) {
/* Lock the mutex and wait for a non-empty queue or until shutdown */
pthread_mutex_lock(&ctx->queueMutex);
- while (ctx->queueHead == ctx->queueTail && !ctx->shutdown) {
+
+ while (ctx->queueEmpty && !ctx->shutdown) {
pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
}
/* empty => shutting down: so stop */
- if (ctx->queueHead == ctx->queueTail) {
+ if (ctx->queueEmpty) {
pthread_mutex_unlock(&ctx->queueMutex);
return opaque;
}
/* Pop a job off the queue */
- { POOL_job const job = ctx->queue[ctx->queueHead];
+ {
+ POOL_job const job = ctx->queue[ctx->queueHead];
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
+ ctx->numThreadsBusy++;
+ ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
/* Unlock the mutex, signal a pusher, and run the job */
pthread_mutex_unlock(&ctx->queueMutex);
- pthread_cond_signal(&ctx->queuePushCond);
+
+ if (ctx->queueSize > 1) {
+ pthread_cond_signal(&ctx->queuePushCond);
+ }
+
job.function(job.opaque);
+
+ /* If the intended queue size was 0, signal after finishing job */
+ if (ctx->queueSize == 1) {
+ pthread_mutex_lock(&ctx->queueMutex);
+ ctx->numThreadsBusy--;
+ pthread_mutex_unlock(&ctx->queueMutex);
+ pthread_cond_signal(&ctx->queuePushCond);
+ }
}
}
/* Unreachable */
@@ -83,7 +105,7 @@
POOL_ctx *POOL_create(size_t numThreads, size_t queueSize) {
POOL_ctx *ctx;
/* Check the parameters */
- if (!numThreads || !queueSize) { return NULL; }
+ if (!numThreads) { return NULL; }
/* Allocate the context and zero initialize */
ctx = (POOL_ctx *)calloc(1, sizeof(POOL_ctx));
if (!ctx) { return NULL; }
@@ -95,6 +117,8 @@
ctx->queue = (POOL_job*) malloc(ctx->queueSize * sizeof(POOL_job));
ctx->queueHead = 0;
ctx->queueTail = 0;
+ ctx->numThreadsBusy = 0;
+ ctx->queueEmpty = 1;
(void)pthread_mutex_init(&ctx->queueMutex, NULL);
(void)pthread_cond_init(&ctx->queuePushCond, NULL);
(void)pthread_cond_init(&ctx->queuePopCond, NULL);
@@ -153,22 +177,37 @@
+ ctx->numThreads * sizeof(pthread_t);
}
+/**
+ * Returns 1 if the queue is full and 0 otherwise.
+ *
+ * If the queueSize is 1 (the pool was created with an intended queueSize of 0),
+ * then a queue is empty if there is a thread free and no job is waiting.
+ */
+static int isQueueFull(POOL_ctx const* ctx) {
+ if (ctx->queueSize > 1) {
+ return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
+ } else {
+ return ctx->numThreadsBusy == ctx->numThreads ||
+ !ctx->queueEmpty;
+ }
+}
+
void POOL_add(void* ctxVoid, POOL_function function, void *opaque) {
POOL_ctx* const ctx = (POOL_ctx*)ctxVoid;
if (!ctx) { return; }
pthread_mutex_lock(&ctx->queueMutex);
{ POOL_job const job = {function, opaque};
+
/* Wait until there is space in the queue for the new job */
- size_t newTail = (ctx->queueTail + 1) % ctx->queueSize;
- while (ctx->queueHead == newTail && !ctx->shutdown) {
+ while (isQueueFull(ctx) && !ctx->shutdown) {
pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
- newTail = (ctx->queueTail + 1) % ctx->queueSize;
}
/* The queue is still going => there is space */
if (!ctx->shutdown) {
+ ctx->queueEmpty = 0;
ctx->queue[ctx->queueTail] = job;
- ctx->queueTail = newTail;
+ ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
}
}
pthread_mutex_unlock(&ctx->queueMutex);
diff --git a/lib/common/pool.h b/lib/common/pool.h
index 957100f..ed27119 100644
--- a/lib/common/pool.h
+++ b/lib/common/pool.h
@@ -22,7 +22,6 @@
* Create a thread pool with at most `numThreads` threads.
* `numThreads` must be at least 1.
* The maximum number of queued jobs before blocking is `queueSize`.
- * `queueSize` must be at least 1.
* @return : POOL_ctx pointer on success, else NULL.
*/
POOL_ctx *POOL_create(size_t numThreads, size_t queueSize);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index a73763d..a70a666 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -942,7 +942,7 @@
else { entropy->hufCTable_repeatMode = HUF_repeat_check; } /* now have a table to reuse */
}
- if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) {
+ if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
entropy->hufCTable_repeatMode = HUF_repeat_none;
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
}
@@ -1156,11 +1156,10 @@
}
}
-MEM_STATIC size_t ZSTD_compressSequences (seqStore_t* seqStorePtr,
+MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_entropyCTables_t* entropy,
ZSTD_compressionParameters const* cParams,
- void* dst, size_t dstCapacity,
- size_t srcSize)
+ void* dst, size_t dstCapacity)
{
const int longOffsets = cParams->windowLog > STREAM_ACCUMULATOR_MIN;
U32 count[MaxSeq+1];
@@ -1195,7 +1194,7 @@
if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq;
else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
- if (nbSeq==0) goto _check_compressibility;
+ if (nbSeq==0) return op - ostart;
/* seqHead : flags for FSE encoding type */
seqHead = op++;
@@ -1244,23 +1243,40 @@
op += streamSize;
}
+ return op - ostart;
+}
- /* check compressibility */
-_check_compressibility:
- { size_t const minGain = ZSTD_minGain(srcSize);
- size_t const maxCSize = srcSize - minGain;
- if ((size_t)(op-ostart) >= maxCSize) {
- entropy->hufCTable_repeatMode = HUF_repeat_none;
- entropy->offcode_repeatMode = FSE_repeat_none;
- entropy->matchlength_repeatMode = FSE_repeat_none;
- entropy->litlength_repeatMode = FSE_repeat_none;
- return 0;
- } }
+MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr,
+ ZSTD_entropyCTables_t* entropy,
+ ZSTD_compressionParameters const* cParams,
+ void* dst, size_t dstCapacity,
+ size_t srcSize)
+{
+ size_t const cSize = ZSTD_compressSequences_internal(seqStorePtr, entropy, cParams,
+ dst, dstCapacity);
+ size_t const minGain = ZSTD_minGain(srcSize);
+ size_t const maxCSize = srcSize - minGain;
+ /* If the srcSize <= dstCapacity, then there is enough space to write a
+ * raw uncompressed block. Since we ran out of space, the block must not
+ * be compressible, so fall back to a raw uncompressed block.
+ */
+ int const uncompressibleError = cSize == ERROR(dstSize_tooSmall) && srcSize <= dstCapacity;
+
+ if (ZSTD_isError(cSize) && !uncompressibleError)
+ return cSize;
+ /* Check compressibility */
+ if (cSize >= maxCSize || uncompressibleError) {
+ entropy->hufCTable_repeatMode = HUF_repeat_none;
+ entropy->offcode_repeatMode = FSE_repeat_none;
+ entropy->matchlength_repeatMode = FSE_repeat_none;
+ entropy->litlength_repeatMode = FSE_repeat_none;
+ return 0;
+ }
+ assert(!ZSTD_isError(cSize));
/* confirm repcodes */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) seqStorePtr->rep[i] = seqStorePtr->repToConfirm[i]; }
-
- return op - ostart;
+ return cSize;
}
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 2a1b70e..0a47a3d 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -917,11 +917,11 @@
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
-* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
+* Assumption : 0 < cSrcSize, dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
{
/* decoder timing evaluation */
- U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
+ U32 const Q = cSrcSize >= dstSize ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
U32 const D256 = (U32)(dstSize >> 8);
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
@@ -977,7 +977,7 @@
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
- if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
+ if (cSrcSize == 0) return ERROR(corruption_detected);
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X4_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 92e80c1..159b7b1 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1731,7 +1731,7 @@
return 0;
}
dctx->expected = 0; /* not necessary to copy more */
-
+ /* fall-through */
case ZSTDds_decodeFrameHeader:
assert(src != NULL);
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
@@ -2391,7 +2391,7 @@
zds->outBuffSize = neededOutSize;
} }
zds->streamStage = zdss_read;
- /* pass-through */
+ /* fall-through */
case zdss_read:
DEBUGLOG(5, "stage zdss_read");
@@ -2416,8 +2416,7 @@
} }
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zds->streamStage = zdss_load;
- /* pass-through */
-
+ /* fall-through */
case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
@@ -2439,8 +2438,7 @@
zds->outEnd = zds->outStart + decodedSize;
} }
zds->streamStage = zdss_flush;
- /* pass-through */
-
+ /* fall-through */
case zdss_flush:
{ size_t const toFlushSize = zds->outEnd - zds->outStart;
size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize);
diff --git a/tests/files/huffman-compressed-larger b/tests/files/huffman-compressed-larger
new file mode 100644
index 0000000..f594f1a
--- /dev/null
+++ b/tests/files/huffman-compressed-larger
Binary files differ
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 77853b1..bc8584e 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -386,6 +386,13 @@
./datagen | $ZSTD -c | $ZSTD -t
+
+$ECHO "\n**** golden files tests **** "
+
+$ZSTD -t -r files
+$ZSTD -c -r files | $ZSTD -t
+
+
$ECHO "\n**** benchmark mode tests **** "
$ECHO "bench one file"
diff --git a/tests/poolTests.c b/tests/poolTests.c
index adc5947..9e11281 100644
--- a/tests/poolTests.c
+++ b/tests/poolTests.c
@@ -1,5 +1,6 @@
#include "pool.h"
#include "threading.h"
+#include "util.h"
#include <stddef.h>
#include <stdio.h>
@@ -50,21 +51,45 @@
return 0;
}
+void waitFn(void *opaque) {
+ (void)opaque;
+ UTIL_sleepMilli(1);
+}
+
+/* Tests for deadlock */
+int testWait(size_t numThreads, size_t queueSize) {
+ struct data data;
+ POOL_ctx *ctx = POOL_create(numThreads, queueSize);
+ ASSERT_TRUE(ctx);
+ {
+ size_t i;
+ for (i = 0; i < 16; ++i) {
+ POOL_add(ctx, &waitFn, &data);
+ }
+ }
+ POOL_free(ctx);
+ return 0;
+}
+
int main(int argc, const char **argv) {
size_t numThreads;
for (numThreads = 1; numThreads <= 4; ++numThreads) {
size_t queueSize;
- for (queueSize = 1; queueSize <= 2; ++queueSize) {
+ for (queueSize = 0; queueSize <= 2; ++queueSize) {
if (testOrder(numThreads, queueSize)) {
printf("FAIL: testOrder\n");
return 1;
}
+ if (testWait(numThreads, queueSize)) {
+ printf("FAIL: testWait\n");
+ return 1;
+ }
}
}
printf("PASS: testOrder\n");
(void)argc;
(void)argv;
- return (POOL_create(0, 1) || POOL_create(1, 0)) ? printf("FAIL: testInvalid\n"), 1
- : printf("PASS: testInvalid\n"), 0;
+ return (POOL_create(0, 1)) ? printf("FAIL: testInvalid\n"), 1
+ : printf("PASS: testInvalid\n"), 0;
return 0;
}