Merge pull request #2397 from dscheg/fixtrainfrombuffer
Pass dictBufferCapacity to COVER_selectDict()
diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index bd9dd86..c78af13 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -955,7 +955,7 @@
free(selection.dictContent);
}
-COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
+COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize) {
@@ -963,8 +963,8 @@
size_t largestCompressed = 0;
BYTE* customDictContentEnd = customDictContent + dictContentSize;
- BYTE * largestDictbuffer = (BYTE *)malloc(dictContentSize);
- BYTE * candidateDictBuffer = (BYTE *)malloc(dictContentSize);
+ BYTE * largestDictbuffer = (BYTE *)malloc(dictBufferCapacity);
+ BYTE * candidateDictBuffer = (BYTE *)malloc(dictBufferCapacity);
double regressionTolerance = ((double)params.shrinkDictMaxRegression / 100.0) + 1.00;
if (!largestDictbuffer || !candidateDictBuffer) {
@@ -976,7 +976,7 @@
/* Initial dictionary size and compressed size */
memcpy(largestDictbuffer, customDictContent, dictContentSize);
dictContentSize = ZDICT_finalizeDictionary(
- largestDictbuffer, dictContentSize, customDictContent, dictContentSize,
+ largestDictbuffer, dictBufferCapacity, customDictContent, dictContentSize,
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
if (ZDICT_isError(dictContentSize)) {
@@ -1010,7 +1010,7 @@
while (dictContentSize < largestDict) {
memcpy(candidateDictBuffer, largestDictbuffer, largestDict);
dictContentSize = ZDICT_finalizeDictionary(
- candidateDictBuffer, dictContentSize, customDictContentEnd - dictContentSize, dictContentSize,
+ candidateDictBuffer, dictBufferCapacity, customDictContentEnd - dictContentSize, dictContentSize,
samplesBuffer, samplesSizes, nbFinalizeSamples, params.zParams);
if (ZDICT_isError(dictContentSize)) {
@@ -1088,7 +1088,7 @@
{
const size_t tail = COVER_buildDictionary(ctx, freqs, &activeDmers, dict,
dictBufferCapacity, parameters);
- selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
ctx->samples, ctx->samplesSizes, (unsigned)ctx->nbTrainSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
totalCompressedSize);
diff --git a/lib/dictBuilder/cover.h b/lib/dictBuilder/cover.h
index f2aa0e3..9f1cb5f 100644
--- a/lib/dictBuilder/cover.h
+++ b/lib/dictBuilder/cover.h
@@ -152,6 +152,6 @@
* smallest dictionary within a specified regression of the compressed size
* from the largest dictionary.
*/
- COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent,
+ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBufferCapacity,
size_t dictContentSize, const BYTE* samplesBuffer, const size_t* samplesSizes, unsigned nbFinalizeSamples,
size_t nbCheckSamples, size_t nbSamples, ZDICT_cover_params_t params, size_t* offsets, size_t totalCompressedSize);
diff --git a/lib/dictBuilder/fastcover.c b/lib/dictBuilder/fastcover.c
index 8265e2c..5e60f24 100644
--- a/lib/dictBuilder/fastcover.c
+++ b/lib/dictBuilder/fastcover.c
@@ -487,7 +487,7 @@
parameters, segmentFreqs);
const unsigned nbFinalizeSamples = (unsigned)(ctx->nbTrainSamples * ctx->accelParams.finalize / 100);
- selection = COVER_selectDict(dict + tail, dictBufferCapacity - tail,
+ selection = COVER_selectDict(dict + tail, dictBufferCapacity, dictBufferCapacity - tail,
ctx->samples, ctx->samplesSizes, nbFinalizeSamples, ctx->nbTrainSamples, ctx->nbSamples, parameters, ctx->offsets,
totalCompressedSize);