Initial release
diff --git a/programs/fullbench.c b/programs/fullbench.c
new file mode 100644
index 0000000..b13ea60
--- /dev/null
+++ b/programs/fullbench.c
@@ -0,0 +1,682 @@
+/*
+ fullbench.c - Detailed bench program for zstd
+ Copyright (C) Yann Collet 2014-2015
+
+ GPL v2 License
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ You can contact the author at :
+ - zstd source repository : https://github.com/Cyan4973/zstd
+ - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/**************************************
+* Compiler Options
+**************************************/
+/* Disable some Visual warning messages */
+#define _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
+
+/* Unix Large Files support (>4GB) */
+#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */
+# define _LARGEFILE_SOURCE
+# define _FILE_OFFSET_BITS 64
+#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */
+# define _LARGEFILE64_SOURCE
+#endif
+
+/* S_ISREG & gettimeofday() are not supported by MSVC */
+#if defined(_MSC_VER) || defined(_WIN32)
+# define BMK_LEGACY_TIMER 1
+#endif
+
+
+/**************************************
+* Includes
+**************************************/
+#include <stdlib.h> /* malloc */
+#include <stdio.h> /* fprintf, fopen, ftello64 */
+#include <sys/types.h> /* stat64 */
+#include <sys/stat.h> /* stat64 */
+#include <string.h> /* strcmp */
+
+/* Use ftime() if gettimeofday() is not available on your target */
+#if defined(BMK_LEGACY_TIMER)
+# include <sys/timeb.h> /* timeb, ftime */
+#else
+# include <sys/time.h> /* gettimeofday */
+#endif
+
+#include "zstd.h"
+#include "fse_static.h"
+
+
+/**************************************
+* Compiler Options
+**************************************/
+/* S_ISREG & gettimeofday() are not supported by MSVC */
+#if !defined(S_ISREG)
+# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
+#endif
+
+
+/**************************************
+* Basic Types
+**************************************/
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
+# include <stdint.h>
+ typedef uint8_t BYTE;
+ typedef uint16_t U16;
+ typedef uint32_t U32;
+ typedef int32_t S32;
+ typedef uint64_t U64;
+#else
+ typedef unsigned char BYTE;
+ typedef unsigned short U16;
+ typedef unsigned int U32;
+ typedef signed int S32;
+ typedef unsigned long long U64;
+#endif
+
+
+/**************************************
+* Constants
+**************************************/
+#define PROGRAM_DESCRIPTION "zStandard speed analyzer"
+#ifndef ZSTD_VERSION
+# define ZSTD_VERSION ""
+#endif
+#define AUTHOR "Yann Collet"
+#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION, (int)(sizeof(void*)*8), AUTHOR, __DATE__
+
+
+#define KB *(1<<10)
+#define MB *(1<<20)
+
+#define NBLOOPS 6
+#define TIMELOOP 2500
+
+#define KNUTH 2654435761U
+#define MAX_MEM (1984 MB)
+#define DEFAULT_CHUNKSIZE (4<<20)
+
+static double g_compressibilityDefault = 0.50;
+static const U32 prime1 = 2654435761U;
+static const U32 prime2 = 2246822519U;
+static const size_t sampleSize = 10000000;
+
+
+/**************************************
+* Macros
+**************************************/
+#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+
+
+/**************************************
+* Benchmark Parameters
+**************************************/
+static int nbIterations = NBLOOPS;
+
+void BMK_SetNbIterations(int nbLoops)
+{
+ nbIterations = nbLoops;
+ DISPLAY("- %i iterations -\n", nbIterations);
+}
+
+
+/*********************************************************
+* Private functions
+*********************************************************/
+
+#if defined(BMK_LEGACY_TIMER)
+
+static int BMK_GetMilliStart(void)
+{
+ /* Based on Legacy ftime()
+ * Rolls over every ~ 12.1 days (0x100000/24/60/60)
+ * Use GetMilliSpan to correct for rollover */
+ struct timeb tb;
+ int nCount;
+ ftime( &tb );
+ nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000);
+ return nCount;
+}
+
+#else
+
+static int BMK_GetMilliStart(void)
+{
+ /* Based on newer gettimeofday()
+ * Use GetMilliSpan to correct for rollover */
+ struct timeval tv;
+ int nCount;
+ gettimeofday(&tv, NULL);
+ nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000);
+ return nCount;
+}
+
+#endif
+
+
+static int BMK_GetMilliSpan( int nTimeStart )
+{
+ int nSpan = BMK_GetMilliStart() - nTimeStart;
+ if ( nSpan < 0 )
+ nSpan += 0x100000 * 1000;
+ return nSpan;
+}
+
+
+static size_t BMK_findMaxMem(U64 requiredMem)
+{
+ size_t step = 64 MB;
+ BYTE* testmem=NULL;
+
+ requiredMem = (((requiredMem >> 26) + 1) << 26);
+ if (requiredMem > MAX_MEM) requiredMem = MAX_MEM;
+
+ requiredMem += 2*step;
+ while (!testmem)
+ {
+ requiredMem -= step;
+ testmem = (BYTE*) malloc ((size_t)requiredMem);
+ }
+
+ free (testmem);
+ return (size_t) (requiredMem - step);
+}
+
+
+static U64 BMK_GetFileSize(char* infilename)
+{
+ int r;
+#if defined(_MSC_VER)
+ struct _stat64 statbuf;
+ r = _stat64(infilename, &statbuf);
+#else
+ struct stat statbuf;
+ r = stat(infilename, &statbuf);
+#endif
+ if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
+ return (U64)statbuf.st_size;
+}
+
+
+static U32 BMK_rotl32(unsigned val32, unsigned nbBits) { return((val32 << nbBits) | (val32 >> (32 - nbBits))); }
+
+static U32 BMK_rand(U32* src)
+{
+ U32 rand32 = *src;
+ rand32 *= prime1;
+ rand32 += prime2;
+ rand32 = BMK_rotl32(rand32, 13);
+ *src = rand32;
+ return rand32 >> 9;
+}
+
+#define BMK_RAND15BITS ( BMK_rand(&seed) & 0x7FFF)
+#define BMK_RANDLENGTH ((BMK_rand(&seed) & 3) ? (BMK_rand(&seed) % 15) : (BMK_rand(&seed) % 510) + 15)
+#define BMK_RANDCHAR (BYTE)((BMK_rand(&seed) & 63) + '0')
+static void BMK_datagen(void* buffer, size_t bufferSize, double proba, U32 seed)
+{
+ BYTE* BBuffer = (BYTE*)buffer;
+ unsigned pos = 0;
+ U32 P32 = (U32)(32768 * proba);
+
+ /* First Byte */
+ BBuffer[pos++] = BMK_RANDCHAR;
+
+ while (pos < bufferSize)
+ {
+ /* Select : Literal (noise) or copy (within 64K) */
+ if (BMK_RAND15BITS < P32)
+ {
+ /* Match */
+ size_t match, end;
+ unsigned length = BMK_RANDLENGTH + 4;
+ unsigned offset = BMK_RAND15BITS + 1;
+ if (offset > pos) offset = pos;
+ match = pos - offset;
+ end = pos + length;
+ if (end > bufferSize) end = bufferSize;
+ while (pos < end) BBuffer[pos++] = BBuffer[match++];
+ }
+ else
+ {
+ /* Literal */
+ size_t end;
+ unsigned length = BMK_RANDLENGTH;
+ end = pos + length;
+ if (end > bufferSize) end = bufferSize;
+ while (pos < end) BBuffer[pos++] = BMK_RANDCHAR;
+ }
+ }
+}
+
+
+/*********************************************************
+* Benchmark wrappers
+*********************************************************/
+typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
+typedef struct
+{
+ blockType_t blockType;
+ U32 unusedBits;
+ U32 origSize;
+} blockProperties_t;
+
+static size_t g_cSize = 0;
+
+extern size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr);
+extern size_t ZSTD_decodeLiteralsBlock(void* ctx, void* dst, size_t maxDstSize, const BYTE** litPtr, const void* src, size_t srcSize);
+extern size_t ZSTD_decodeSeqHeaders(size_t* lastLLPtr, const BYTE** dumpsPtr, void* DTableLL, void* DTableML, void* DTableOffb, const void* src, size_t srcSize);
+
+
+size_t local_ZSTD_compress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+{
+ (void)buff2;
+ return ZSTD_compress(dst, dstSize, src, srcSize);
+}
+
+size_t local_ZSTD_decompress(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+{
+ (void)src; (void)srcSize;
+ return ZSTD_decompress(dst, dstSize, buff2, g_cSize);
+}
+
+size_t local_ZSTD_decodeLiteralsBlock(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+{
+ U32 ctx[1<<12];
+ const BYTE* ll;
+ (void)src; (void)srcSize;
+ ZSTD_decodeLiteralsBlock(ctx, dst, dstSize, &ll, buff2, g_cSize);
+ return (const BYTE*)dst + dstSize - ll;
+}
+
+size_t local_ZSTD_decodeSeqHeaders(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+{
+ U32 DTableML[1<<11], DTableLL[1<<10], DTableOffb[1<<9];
+ const BYTE* dumps;
+ size_t lastllSize;
+ (void)src; (void)srcSize; (void)dst; (void)dstSize;
+ return ZSTD_decodeSeqHeaders(&lastllSize, &dumps, DTableLL, DTableML, DTableOffb, buff2, g_cSize);
+}
+
+size_t local_conditionalNull(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+{
+ U32 i;
+ size_t total = 0;
+ BYTE* data = buff2;
+
+ (void)dst; (void)dstSize; (void)src;
+ for (i=0; i < srcSize; i++)
+ {
+ U32 b = data[i];
+ total += b;
+ if (b==0) total = 0; // 825
+ //if (!b) total = 0; // 825
+ //total = b ? total : 0; // 622
+ //total *= !!b; // 465
+ //total &= -!b; // 622
+ }
+ return total;
+}
+
+size_t local_decodeLiteralsForward(void* dst, size_t dstSize, void* buff2, const void* src, size_t srcSize)
+{
+ (void)src; (void)srcSize;
+ return FSE_decompress(dst, dstSize, buff2, g_cSize);
+}
+
+
+
+/*********************************************************
+* Bench functions
+*********************************************************/
+size_t benchMem(void* src, size_t srcSize, U32 benchNb)
+{
+ BYTE* dstBuff;
+ size_t dstBuffSize;
+ BYTE* buff2;
+ int loopNb;
+ const char* benchName;
+ size_t (*benchFunction)(void* dst, size_t dstSize, void* verifBuff, const void* src, size_t srcSize);
+ double bestTime = 100000000.;
+ size_t errorCode = 0;
+
+ // Declaration
+ switch(benchNb)
+ {
+ case 1:
+ benchFunction = local_ZSTD_compress; benchName = "ZSTD_compress";
+ break;
+ case 11:
+ benchFunction = local_ZSTD_decompress; benchName = "ZSTD_decompress";
+ break;
+ case 31:
+ benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "ZSTD_decodeLiteralsBlock";
+ break;
+ case 32:
+ benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "ZSTD_decodeSeqHeaders";
+ break;
+ case 101:
+ benchFunction = local_conditionalNull; benchName = "conditionalNull";
+ break;
+ case 102:
+ benchFunction = local_decodeLiteralsForward; benchName = "ZSTD_decodeLiteralsForward";
+ break;
+ default :
+ return 0;
+ }
+
+ /* Allocation */
+ dstBuffSize = srcSize + 512;
+ dstBuff = malloc(dstBuffSize);
+ buff2 = malloc(dstBuffSize);
+ if ((!dstBuff) || (!buff2))
+ {
+ DISPLAY("\nError: not enough memory!\n");
+ free(dstBuff); free(buff2);
+ return 12;
+ }
+
+ /* Preparation */
+ switch(benchNb)
+ {
+ case 11:
+ g_cSize = ZSTD_compress(buff2, dstBuffSize, src, srcSize);
+ break;
+ case 31: // ZSTD_decodeLiteralsBlock
+ {
+ blockProperties_t bp;
+ ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
+ g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
+ memcpy(buff2, dstBuff+7, g_cSize);
+ //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
+ srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
+ break;
+ }
+ case 32: // ZSTD_decodeSeqHeaders
+ {
+ blockProperties_t bp;
+ const BYTE* ip = dstBuff;
+ const BYTE* iend;
+ size_t blockSize;
+ ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
+ ip += 4; // Jump magic Number
+ blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
+ iend = ip + 3 + blockSize; // Get end of first block
+ ip += 3; // jump first block header
+ ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
+ g_cSize = iend-ip;
+ memcpy(buff2, ip, g_cSize); // copy rest of block (starting with SeqHeader)
+ srcSize = srcSize > 128 KB ? 128 KB : srcSize; // speed relative to block
+ break;
+ }
+
+ /* test functions */
+
+ case 101: // conditionalNull
+ {
+ size_t i;
+ U32 seed = (U32)srcSize;
+ for (i=0; i<srcSize; i++)
+ buff2[i] = (BYTE)(BMK_rand(&seed) & 15);
+ break;
+ }
+ case 102: //
+ {
+ blockProperties_t bp;
+ ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
+ g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp);
+ memcpy(buff2, dstBuff+10, g_cSize);
+ //srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
+ srcSize = srcSize > 128 KB ? 128 KB : srcSize; // relative to block
+ break;
+ }
+ default : ;
+ }
+
+ for (loopNb = 1; loopNb <= nbIterations; loopNb++)
+ {
+ double averageTime;
+ int milliTime;
+ U32 nbRounds=0;
+
+ DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
+ { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
+
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliStart() == milliTime);
+ milliTime = BMK_GetMilliStart();
+ while(BMK_GetMilliSpan(milliTime) < TIMELOOP)
+ {
+ errorCode = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize);
+ if (ZSTD_isError(errorCode)) { DISPLAY("ERROR ! %s() => %s !! \n", benchName, ZSTD_getErrorName(errorCode)); exit(1); }
+ nbRounds++;
+ }
+ milliTime = BMK_GetMilliSpan(milliTime);
+
+ averageTime = (double)milliTime / nbRounds;
+ if (averageTime < bestTime) bestTime = averageTime;
+ DISPLAY("%2i- %-30.30s : %7.1f MB/s (%9u)\r", loopNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode);
+ }
+
+ DISPLAY("%2u- %-30.30s : %7.1f MB/s (%9u)\n", benchNb, benchName, (double)srcSize / bestTime / 1000., (U32)errorCode);
+
+ free(dstBuff);
+ free(buff2);
+ return 0;
+}
+
+
+int benchSample(U32 benchNb)
+{
+ char* origBuff;
+ size_t benchedSize = sampleSize;
+ const char* name = "Sample50";
+
+ /* Allocation */
+ origBuff = (char*) malloc((size_t)benchedSize);
+ if(!origBuff)
+ {
+ DISPLAY("\nError: not enough memory!\n");
+ return 12;
+ }
+
+ /* Fill buffer */
+ BMK_datagen(origBuff, benchedSize, g_compressibilityDefault, 0);
+
+ /* bench */
+ DISPLAY("\r%79s\r", "");
+ DISPLAY(" %s : \n", name);
+ if (benchNb)
+ benchMem(origBuff, benchedSize, benchNb);
+ else
+ for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
+
+ free(origBuff);
+ return 0;
+}
+
+
+int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
+{
+ int fileIdx=0;
+
+ /* Loop for each file */
+ while (fileIdx<nbFiles)
+ {
+ FILE* inFile;
+ char* inFileName;
+ U64 inFileSize;
+ size_t benchedSize;
+ size_t readSize;
+ char* origBuff;
+
+ /* Check file existence */
+ inFileName = fileNamesTable[fileIdx++];
+ inFile = fopen( inFileName, "rb" );
+ if (inFile==NULL)
+ {
+ DISPLAY( "Pb opening %s\n", inFileName);
+ return 11;
+ }
+
+ // Memory allocation & restrictions
+ inFileSize = BMK_GetFileSize(inFileName);
+ benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3;
+ if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
+ if (benchedSize < inFileSize)
+ {
+ DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
+ }
+
+ // Alloc
+ origBuff = (char*) malloc((size_t)benchedSize);
+ if(!origBuff)
+ {
+ DISPLAY("\nError: not enough memory!\n");
+ fclose(inFile);
+ return 12;
+ }
+
+ // Fill input buffer
+ DISPLAY("Loading %s... \r", inFileName);
+ readSize = fread(origBuff, 1, benchedSize, inFile);
+ fclose(inFile);
+
+ if(readSize != benchedSize)
+ {
+ DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
+ free(origBuff);
+ return 13;
+ }
+
+ // bench
+ DISPLAY("\r%79s\r", "");
+ DISPLAY(" %s : \n", inFileName);
+ if (benchNb)
+ benchMem(origBuff, benchedSize, benchNb);
+ else
+ for (benchNb=0; benchNb<100; benchNb++) benchMem(origBuff, benchedSize, benchNb);
+ }
+
+ return 0;
+}
+
+
+int usage(char* exename)
+{
+ DISPLAY( "Usage :\n");
+ DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename);
+ DISPLAY( "Arguments :\n");
+ DISPLAY( " -H/-h : Help (this text + advanced options)\n");
+ return 0;
+}
+
+int usage_advanced(void)
+{
+ DISPLAY( "\nAdvanced options :\n");
+ DISPLAY( " -b# : test only function # \n");
+ DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS);
+ return 0;
+}
+
+int badusage(char* exename)
+{
+ DISPLAY("Wrong parameters\n");
+ usage(exename);
+ return 0;
+}
+
+int main(int argc, char** argv)
+{
+ int i,
+ filenamesStart=0,
+ result;
+ char* exename=argv[0];
+ char* input_filename=0;
+ U32 benchNb = 0, main_pause = 0;
+
+ // Welcome message
+ DISPLAY(WELCOME_MESSAGE);
+
+ if (argc<1) { badusage(exename); return 1; }
+
+ for(i=1; i<argc; i++)
+ {
+ char* argument = argv[i];
+
+ if(!argument) continue; // Protection if argument empty
+
+ // Decode command (note : aggregated commands are allowed)
+ if (argument[0]=='-')
+ {
+ while (argument[1]!=0)
+ {
+ argument ++;
+
+ switch(argument[0])
+ {
+ // Display help on usage
+ case 'h' :
+ case 'H': usage(exename); usage_advanced(); return 0;
+
+ // Pause at the end (hidden option)
+ case 'p': main_pause = 1; break;
+
+ // Select specific bench algorithm only
+ case 'b':
+ benchNb = 0;
+ while ((argument[1]>= '0') && (argument[1]<= '9'))
+ {
+ benchNb *= 10;
+ benchNb += argument[1] - '0';
+ argument++;
+ }
+ break;
+
+ // Modify Nb Iterations
+ case 'i':
+ if ((argument[1] >='1') && (argument[1] <='9'))
+ {
+ int iters = argument[1] - '0';
+ BMK_SetNbIterations(iters);
+ argument++;
+ }
+ break;
+
+ // Unknown command
+ default : badusage(exename); return 1;
+ }
+ }
+ continue;
+ }
+
+ // first provided filename is input
+ if (!input_filename) { input_filename=argument; filenamesStart=i; continue; }
+ }
+
+ if (filenamesStart==0)
+ result = benchSample(benchNb);
+ else result = benchFiles(argv+filenamesStart, argc-filenamesStart, benchNb);
+
+ if (main_pause) { printf("press enter...\n"); getchar(); }
+
+ return result;
+}
+