commit dd54bbc184c7b260a7ed419fbbc248e03afd9942
author Yann Collet <yann.collet.73@gmail.com> Tue Mar 08 02:35:34 2016 +0100
committer Yann Collet <yann.collet.73@gmail.com> Tue Mar 08 02:35:34 2016 +0100
tree 2ccffee3fcd4a8b6c95dfb833f378af2c0898ed6
parent 370b08e84041845f8ae7845aaad65b90cb6541f7

Fixed large NbSeq > 32 K
Added a test in Fuzzer to check NbSeq > 32 K

lib/zstd_compress.c

diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c
index a5127b7..50979ce 100644
--- a/lib/zstd_compress.c
+++ b/lib/zstd_compress.c
@@ -165,8 +165,8 @@
         U32 srcLog = ZSTD_highbit((U32)(params->srcSize)-1) + 1;
         if (params->windowLog > srcLog) params->windowLog = srcLog;
     }
-    if (params->windowLog   < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
-    if (params->contentLog  > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus;   /* <= ZSTD_CONTENTLOG_MAX */
+    if (params->windowLog  < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* required for frame header */
+    if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus;   /* <= ZSTD_CONTENTLOG_MAX */
 }
 
 
@@ -176,9 +176,12 @@
     const size_t blockSize = MIN(BLOCKSIZE, (size_t)1 << params.windowLog);
     /* reserve table memory */
     const U32    contentLog = (params.strategy == ZSTD_fast) ? 1 : params.contentLog;
+    const U32    divider = (params.searchLength==3) ? 3 : 4;
+    const size_t maxNbSeq = blockSize / divider;
+    const size_t tokenSpace = blockSize + 8*maxNbSeq;
     const size_t tableSpace = ((1 << contentLog) + (1 << params.hashLog) + (1 << params.hashLog3)) * sizeof(U32);
     const size_t optSpace   = ((1<<MLbits) + (1<<LLbits) + (1<<Offbits) + (1<<Litbits))*sizeof(U32) + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
-    const size_t neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + (3*blockSize)
+    const size_t neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
                            + ((params.strategy == ZSTD_btopt) ? optSpace : 0);
 
     if (zc->workSpaceSize < neededSpace) {
@@ -206,13 +209,13 @@
     zc->blockSize = blockSize;
 
     zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer);
-    zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (blockSize>>2));
-    zc->seqStore.litStart = zc->seqStore.offCodeStart + (blockSize>>2);
+    zc->seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + maxNbSeq);
+    zc->seqStore.litStart = zc->seqStore.offCodeStart + maxNbSeq;
     zc->seqStore.litLengthStart =  zc->seqStore.litStart + blockSize;
-    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2);
-    zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2);
+    zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + maxNbSeq;
+    zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + maxNbSeq;
     if (params.strategy == ZSTD_btopt) {
-        zc->seqStore.litFreq = (U32*)((void*)(zc->seqStore.dumpsStart + (blockSize>>2)));
+        zc->seqStore.litFreq = (U32*)((void*)(zc->seqStore.dumpsStart + maxNbSeq));
         zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
         zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<<LLbits);
         zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<<MLbits);

programs/fuzzer.c

diff --git a/programs/fuzzer.c b/programs/fuzzer.c
index 6d57080..697695b 100644
--- a/programs/fuzzer.c
+++ b/programs/fuzzer.c
@@ -344,7 +344,6 @@
     DISPLAYLEVEL(4, "OK \n");
 
     /* nbSeq limit test */
-    if (0)
     {
         #define _3BYTESTESTLENGTH 131000
         #define NB3BYTESSEQLOG   9
@@ -360,7 +359,7 @@
             _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&r) & 255);
         }
 
-        for (i=0; i < _3BYTESTESTLENGTH; ){
+        for (i=0; i < _3BYTESTESTLENGTH; ) {
             U32 id = FUZ_rand(&r) & NB3BYTESSEQMASK;
             ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
             ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];