Merge remote-tracking branch 'refs/remotes/facebook/dev' into zlibWrapper
diff --git a/.travis.yml b/.travis.yml
index 1165196..ff6ab0f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -67,12 +67,18 @@
- os: linux
dist: trusty
sudo: required
- env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest"
+ install:
+ - export CXX="g++-4.8" CC="gcc-4.8"
+ env: PLATFORM="Ubuntu 14.04" CMD="make gpptest && make clean && make gnu90test && make clean && make c99test && make clean && make gnu99test && make clean && make clangtest && make clean && make -C contrib/pzstd pzstd32 && make -C contrib/pzstd googletest32 && make -C contrib/pzstd test32 && make -C contrib/pzstd clean"
addons:
apt:
packages:
- libc6-dev-i386
- - g++-multilib
+ - g++-multilib
+ - gcc-4.8
+ - gcc-4.8-multilib
+ - g++-4.8
+ - g++-4.8-multilib
- os: linux
dist: trusty
sudo: required
diff --git a/appveyor.yml b/appveyor.yml
index 8f4e450..6345c7b 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -52,6 +52,8 @@
ECHO *** &&
ECHO make -C contrib\pzstd pzstd &&
make -C contrib\pzstd pzstd &&
+ make -C contrib\pzstd googletest-mingw64 &&
+ make -C contrib\pzstd test &&
make -C contrib\pzstd clean
)
- if [%COMPILER%]==[gcc] (
diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj
index ff45e39..68c3578 100644
--- a/build/VS2005/zstd/zstd.vcproj
+++ b/build/VS2005/zstd/zstd.vcproj
@@ -44,7 +44,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -121,7 +121,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -196,7 +196,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -274,7 +274,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj
index 2313c87..7ea3d9b 100644
--- a/build/VS2005/zstdlib/zstdlib.vcproj
+++ b/build/VS2005/zstdlib/zstdlib.vcproj
@@ -44,7 +44,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -120,7 +120,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -194,7 +194,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -271,7 +271,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -379,6 +379,34 @@
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v01.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v02.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v03.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v04.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v05.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v06.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v07.c"
+ >
+ </File>
</Filter>
<Filter
Name="Header Files"
@@ -481,6 +509,14 @@
RelativePath="..\..\..\lib\legacy\zstd_v05.h"
>
</File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v06.h"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v07.h"
+ >
+ </File>
</Filter>
<Filter
Name="Resource Files"
diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index 9d71f6e..b272ffd 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -45,7 +45,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -122,7 +122,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -197,7 +197,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -275,7 +275,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj
index db596b4..fa4cd26 100644
--- a/build/VS2008/zstdlib/zstdlib.vcproj
+++ b/build/VS2008/zstdlib/zstdlib.vcproj
@@ -45,7 +45,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -121,7 +121,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
Name="VCCLCompilerTool"
Optimization="0"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
MinimalRebuild="true"
BasicRuntimeChecks="3"
RuntimeLibrary="3"
@@ -272,7 +272,7 @@
EnableIntrinsicFunctions="true"
OmitFramePointers="true"
AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
- PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE"
+ PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
RuntimeLibrary="0"
EnableFunctionLevelLinking="true"
UsePrecompiledHeader="0"
@@ -380,6 +380,34 @@
RelativePath="..\..\..\lib\decompress\zstd_decompress.c"
>
</File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v01.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v02.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v03.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v04.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v05.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v06.c"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v07.c"
+ >
+ </File>
</Filter>
<Filter
Name="Header Files"
@@ -482,6 +510,14 @@
RelativePath="..\..\..\lib\legacy\zstd_v05.h"
>
</File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v06.h"
+ >
+ </File>
+ <File
+ RelativePath="..\..\..\lib\legacy\zstd_v07.h"
+ >
+ </File>
</Filter>
<Filter
Name="Resource Files"
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index 0f4e06a..eb7e7b5 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -149,7 +149,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@@ -165,7 +165,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
</ClCompile>
@@ -183,7 +183,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<TreatWarningAsError>false</TreatWarningAsError>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
@@ -204,7 +204,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
diff --git a/build/VS2010/zstdlib/zstdlib.vcxproj b/build/VS2010/zstdlib/zstdlib.vcxproj
index 232fdf4..b97808d 100644
--- a/build/VS2010/zstdlib/zstdlib.vcxproj
+++ b/build/VS2010/zstdlib/zstdlib.vcxproj
@@ -32,6 +32,13 @@
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\zdict.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v01.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v02.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v03.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v04.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v05.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v06.c" />
+ <ClCompile Include="..\..\..\lib\legacy\zstd_v07.c" />
<ClInclude Include="..\..\..\lib\common\bitstream.h" />
<ClInclude Include="..\..\..\lib\common\error_private.h" />
<ClInclude Include="..\..\..\lib\common\error_public.h" />
@@ -40,6 +47,14 @@
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zbuff.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v03.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v04.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v05.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v06.h" />
+ <ClInclude Include="..\..\..\lib\legacy\zstd_v07.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_opt.h" />
@@ -126,7 +141,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<MinimalRebuild>true</MinimalRebuild>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -146,7 +161,7 @@
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
- <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>true</TreatWarningAsError>
<BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
<RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
@@ -166,7 +181,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
<DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
@@ -188,7 +203,7 @@
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
- <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_HEAPMODE=0;ZSTD_LEGACY_SUPPORT=0;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <PreprocessorDefinitions>ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<TreatWarningAsError>false</TreatWarningAsError>
<EnablePREfast>false</EnablePREfast>
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile
index d71cf5b..e30be0b 100644
--- a/contrib/pzstd/Makefile
+++ b/contrib/pzstd/Makefile
@@ -30,7 +30,7 @@
EXT =
endif
-.PHONY: default all test clean
+.PHONY: default all test clean test32 googletest googletest32
default: pzstd
@@ -41,7 +41,6 @@
$(MAKE) -C $(ZSTDDIR) libzstd
@cp $(ZSTDDIR)/libzstd.a .
-
Pzstd.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
$(CXX) $(FLAGS) -c Pzstd.cpp -o $@
@@ -54,23 +53,66 @@
main.o: main.cpp *.h utils/*.h
$(CXX) $(FLAGS) -c main.cpp -o $@
-pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a
+pzstd: Pzstd.o SkippableFrame.o Options.o main.o libzstd.a
$(CXX) $(FLAGS) $^ -o $@$(EXT) -lpthread
+libzstd32.a: $(ZSTD_FILES)
+ $(MAKE) -C $(ZSTDDIR) libzstd MOREFLAGS="-m32"
+ @cp $(ZSTDDIR)/libzstd.a libzstd32.a
+
+Pzstd32.o: Pzstd.h Pzstd.cpp ErrorHolder.h utils/*.h
+ $(CXX) -m32 $(FLAGS) -c Pzstd.cpp -o $@
+
+SkippableFrame32.o: SkippableFrame.h SkippableFrame.cpp utils/*.h
+ $(CXX) -m32 $(FLAGS) -c SkippableFrame.cpp -o $@
+
+Options32.o: Options.h Options.cpp
+ $(CXX) -m32 $(FLAGS) -c Options.cpp -o $@
+
+main32.o: main.cpp *.h utils/*.h
+ $(CXX) -m32 $(FLAGS) -c main.cpp -o $@
+
+pzstd32: Pzstd32.o SkippableFrame32.o Options32.o main32.o libzstd32.a
+ $(CXX) -m32 $(FLAGS) $^ -o $@$(EXT) -lpthread
+
googletest:
+ @$(RM) -rf googletest
@git clone https://github.com/google/googletest
@mkdir -p googletest/build
@cd googletest/build && cmake .. && make
-test: libzstd.a Pzstd.o Options.o SkippableFrame.o
+googletest32:
+ @$(RM) -rf googletest
+ @git clone https://github.com/google/googletest
+ @mkdir -p googletest/build
+ @cd googletest/build && cmake .. -DCMAKE_CXX_FLAGS=-m32 && make
+
+googletest-mingw64:
+ $(RM) -rf googletest
+ git clone https://github.com/google/googletest
+ mkdir -p googletest/build
+ cd googletest/build && cmake -G "MSYS Makefiles" .. && $(MAKE)
+
+test:
+ $(MAKE) libzstd.a
+ $(MAKE) pzstd MOREFLAGS="-Wall -Wextra -pedantic -Werror"
$(MAKE) -C utils/test clean
- $(MAKE) -C utils/test test
+ $(MAKE) -C utils/test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
$(MAKE) -C test clean
- $(MAKE) -C test test
+ $(MAKE) -C test test MOREFLAGS="-Wall -Wextra -pedantic -Werror"
+
+test32:
+ $(MAKE) libzstd.a MOREFLAGS="-m32"
+ $(MAKE) pzstd MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+ $(MAKE) -C utils/test clean
+ $(MAKE) -C utils/test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+ $(MAKE) -C test clean
+ $(MAKE) -C test test MOREFLAGS="-m32 -Wall -Wextra -pedantic -Werror"
+
clean:
$(MAKE) -C $(ZSTDDIR) clean
$(MAKE) -C utils/test clean
$(MAKE) -C test clean
- @$(RM) -rf libzstd.a *.o pzstd$(EXT)
+ @$(RM) -rf libzstd.a *.o pzstd$(EXT) pzstd32$(EXT)
@echo Cleaning completed
diff --git a/contrib/pzstd/Options.cpp b/contrib/pzstd/Options.cpp
index 122f4fb..5562ee1 100644
--- a/contrib/pzstd/Options.cpp
+++ b/contrib/pzstd/Options.cpp
@@ -7,182 +7,419 @@
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include "Options.h"
+#include "utils/ScopeGuard.h"
+#include <algorithm>
+#include <cassert>
#include <cstdio>
#include <cstring>
#include <thread>
+#include <util.h>
+#include <vector>
+
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || \
+ defined(__CYGWIN__)
+#include <io.h> /* _isatty */
+#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
+#else
+#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || \
+ defined(_POSIX_SOURCE) || \
+ (defined(__APPLE__) && \
+ defined( \
+ __MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \
+ */
+#include <unistd.h> /* isatty */
+#define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
+#else
+#define IS_CONSOLE(stdStream) 0
+#endif
+#endif
namespace pzstd {
namespace {
-unsigned parseUnsigned(const char* arg) {
+unsigned defaultNumThreads() {
+#ifdef PZSTD_NUM_THREADS
+ return PZSTD_NUM_THREADS;
+#else
+ return std::thread::hardware_concurrency();
+#endif
+}
+
+unsigned parseUnsigned(const char **arg) {
unsigned result = 0;
- while (*arg >= '0' && *arg <= '9') {
+ while (**arg >= '0' && **arg <= '9') {
result *= 10;
- result += *arg - '0';
- ++arg;
+ result += **arg - '0';
+ ++(*arg);
}
return result;
}
-const std::string zstdExtension = ".zst";
-constexpr unsigned defaultCompressionLevel = 3;
-constexpr unsigned maxNonUltraCompressionLevel = 19;
+const char *getArgument(const char *options, const char **argv, int &i,
+ int argc) {
+ if (options[1] != 0) {
+ return options + 1;
+ }
+ ++i;
+ if (i == argc) {
+ std::fprintf(stderr, "Option -%c requires an argument, but none provided\n",
+ *options);
+ return nullptr;
+ }
+ return argv[i];
+}
+
+const std::string kZstdExtension = ".zst";
+constexpr char kStdIn[] = "-";
+constexpr char kStdOut[] = "-";
+constexpr unsigned kDefaultCompressionLevel = 3;
+constexpr unsigned kMaxNonUltraCompressionLevel = 19;
+
+#ifdef _WIN32
+const char nullOutput[] = "nul";
+#else
+const char nullOutput[] = "/dev/null";
+#endif
+
+void notSupported(const char *option) {
+ std::fprintf(stderr, "Operation not supported: %s\n", option);
+}
void usage() {
std::fprintf(stderr, "Usage:\n");
- std::fprintf(stderr, "\tpzstd [args] FILE\n");
+ std::fprintf(stderr, " pzstd [args] [FILE(s)]\n");
std::fprintf(stderr, "Parallel ZSTD options:\n");
- std::fprintf(stderr, "\t-n/--num-threads #: Number of threads to spawn\n");
- std::fprintf(stderr, "\t-p/--pzstd-headers: Write pzstd headers to enable parallel decompression\n");
+ std::fprintf(stderr, " -p, --processes # : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads());
std::fprintf(stderr, "ZSTD options:\n");
- std::fprintf(stderr, "\t-u/--ultra : enable levels beyond %i, up to %i (requires more memory)\n", maxNonUltraCompressionLevel, ZSTD_maxCLevel());
- std::fprintf(stderr, "\t-h/--help : display help and exit\n");
- std::fprintf(stderr, "\t-V/--version : display version number and exit\n");
- std::fprintf(stderr, "\t-d/--decompress : decompression\n");
- std::fprintf(stderr, "\t-f/--force : overwrite output\n");
- std::fprintf(stderr, "\t-o/--output file : result stored into `file`\n");
- std::fprintf(stderr, "\t-c/--stdout : write output to standard output\n");
- std::fprintf(stderr, "\t-# : # compression level (1-%d, default:%d)\n", maxNonUltraCompressionLevel, defaultCompressionLevel);
+ std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
+ std::fprintf(stderr, " -d, --decompress : decompression\n");
+ std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n");
+ std::fprintf(stderr, " -f, --force : overwrite output without prompting\n");
+ std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n");
+ std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n");
+ std::fprintf(stderr, " -h, --help : display help and exit\n");
+ std::fprintf(stderr, " -V, --version : display version number and exit\n");
+ std::fprintf(stderr, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n");
+ std::fprintf(stderr, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n");
+ std::fprintf(stderr, " -c, --stdout : force wrtie to standard output, even if it is the console\n");
+#ifdef UTIL_HAS_CREATEFILELIST
+ std::fprintf(stderr, " -r : operate recursively on directories\n");
+#endif
+ std::fprintf(stderr, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel());
+ std::fprintf(stderr, " -C, --check : integrity check (default)\n");
+ std::fprintf(stderr, " --no-check : no integrity check\n");
+ std::fprintf(stderr, " -t, --test : test compressed file integrity\n");
+ std::fprintf(stderr, " -- : all arguments after \"--\" are treated as files\n");
}
} // anonymous namespace
Options::Options()
- : numThreads(0),
- maxWindowLog(23),
- compressionLevel(defaultCompressionLevel),
- decompress(false),
- overwrite(false),
- pzstdHeaders(false) {}
+ : numThreads(defaultNumThreads()), maxWindowLog(23),
+ compressionLevel(kDefaultCompressionLevel), decompress(false),
+ overwrite(false), keepSource(true), writeMode(WriteMode::Auto),
+ checksum(true), verbosity(2) {}
-bool Options::parse(int argc, const char** argv) {
+Options::Status Options::parse(int argc, const char **argv) {
+ bool test = false;
+ bool recursive = false;
bool ultra = false;
+ bool forceStdout = false;
+ // Local copy of input files, which are pointers into argv.
+ std::vector<const char *> localInputFiles;
for (int i = 1; i < argc; ++i) {
- const char* arg = argv[i];
- // Arguments with a short option
- char option = 0;
- if (!std::strcmp(arg, "--num-threads")) {
- option = 'n';
- } else if (!std::strcmp(arg, "--pzstd-headers")) {
- option = 'p';
- } else if (!std::strcmp(arg, "--ultra")) {
- option = 'u';
- } else if (!std::strcmp(arg, "--version")) {
- option = 'V';
- } else if (!std::strcmp(arg, "--help")) {
- option = 'h';
- } else if (!std::strcmp(arg, "--decompress")) {
- option = 'd';
- } else if (!std::strcmp(arg, "--force")) {
- option = 'f';
- } else if (!std::strcmp(arg, "--output")) {
- option = 'o';
- } else if (!std::strcmp(arg, "--stdout")) {
- option = 'c';
- }else if (arg[0] == '-' && arg[1] != 0) {
- // Parse the compression level or short option
- if (arg[1] >= '0' && arg[1] <= '9') {
- compressionLevel = parseUnsigned(arg + 1);
- continue;
- }
- option = arg[1];
- } else if (inputFile.empty()) {
- inputFile = arg;
+ const char *arg = argv[i];
+ // Protect against empty arguments
+ if (arg[0] == 0) {
continue;
- } else {
- std::fprintf(stderr, "Invalid argument: %s.\n", arg);
- return false;
}
-
- switch (option) {
- case 'n':
- if (++i == argc) {
- std::fprintf(stderr, "Invalid argument: -n requires an argument.\n");
- return false;
- }
- numThreads = parseUnsigned(argv[i]);
- if (numThreads == 0) {
- std::fprintf(stderr, "Invalid argument: # of threads must be > 0.\n");
- return false;
- }
- break;
- case 'p':
- pzstdHeaders = true;
- break;
- case 'u':
+ // Everything after "--" is an input file
+ if (!std::strcmp(arg, "--")) {
+ ++i;
+ std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles));
+ break;
+ }
+ // Long arguments that don't have a short option
+ {
+ bool isLongOption = true;
+ if (!std::strcmp(arg, "--rm")) {
+ keepSource = false;
+ } else if (!std::strcmp(arg, "--ultra")) {
ultra = true;
maxWindowLog = 0;
- break;
- case 'V':
- std::fprintf(stderr, "ZSTD version: %s.\n", ZSTD_VERSION_STRING);
- return false;
+ } else if (!std::strcmp(arg, "--no-check")) {
+ checksum = false;
+ } else if (!std::strcmp(arg, "--sparse")) {
+ writeMode = WriteMode::Sparse;
+ notSupported("Sparse mode");
+ return Status::Failure;
+ } else if (!std::strcmp(arg, "--no-sparse")) {
+ writeMode = WriteMode::Regular;
+ notSupported("Sparse mode");
+ return Status::Failure;
+ } else if (!std::strcmp(arg, "--dictID")) {
+ notSupported(arg);
+ return Status::Failure;
+ } else if (!std::strcmp(arg, "--no-dictID")) {
+ notSupported(arg);
+ return Status::Failure;
+ } else {
+ isLongOption = false;
+ }
+ if (isLongOption) {
+ continue;
+ }
+ }
+ // Arguments with a short option simply set their short option.
+ const char *options = nullptr;
+ if (!std::strcmp(arg, "--processes")) {
+ options = "p";
+ } else if (!std::strcmp(arg, "--version")) {
+ options = "V";
+ } else if (!std::strcmp(arg, "--help")) {
+ options = "h";
+ } else if (!std::strcmp(arg, "--decompress")) {
+ options = "d";
+ } else if (!std::strcmp(arg, "--force")) {
+ options = "f";
+ } else if (!std::strcmp(arg, "--stdout")) {
+ options = "c";
+ } else if (!std::strcmp(arg, "--keep")) {
+ options = "k";
+ } else if (!std::strcmp(arg, "--verbose")) {
+ options = "v";
+ } else if (!std::strcmp(arg, "--quiet")) {
+ options = "q";
+ } else if (!std::strcmp(arg, "--check")) {
+ options = "C";
+ } else if (!std::strcmp(arg, "--test")) {
+ options = "t";
+ } else if (arg[0] == '-' && arg[1] != 0) {
+ options = arg + 1;
+ } else {
+ localInputFiles.emplace_back(arg);
+ continue;
+ }
+ assert(options != nullptr);
+
+ bool finished = false;
+ while (!finished && *options != 0) {
+ // Parse the compression level
+ if (*options >= '0' && *options <= '9') {
+ compressionLevel = parseUnsigned(&options);
+ continue;
+ }
+
+ switch (*options) {
case 'h':
+ case 'H':
usage();
- return false;
+ return Status::Message;
+ case 'V':
+ std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING);
+ return Status::Message;
+ case 'p': {
+ finished = true;
+ const char *optionArgument = getArgument(options, argv, i, argc);
+ if (optionArgument == nullptr) {
+ return Status::Failure;
+ }
+ if (*optionArgument < '0' || *optionArgument > '9') {
+ std::fprintf(stderr, "Option -p expects a number, but %s provided\n",
+ optionArgument);
+ return Status::Failure;
+ }
+ numThreads = parseUnsigned(&optionArgument);
+ if (*optionArgument != 0) {
+ std::fprintf(stderr,
+ "Option -p expects a number, but %u%s provided\n",
+ numThreads, optionArgument);
+ return Status::Failure;
+ }
+ break;
+ }
+ case 'o': {
+ finished = true;
+ const char *optionArgument = getArgument(options, argv, i, argc);
+ if (optionArgument == nullptr) {
+ return Status::Failure;
+ }
+ outputFile = optionArgument;
+ break;
+ }
+ case 'C':
+ checksum = true;
+ break;
+ case 'k':
+ keepSource = true;
+ break;
case 'd':
decompress = true;
break;
case 'f':
overwrite = true;
+ forceStdout = true;
break;
- case 'o':
- if (++i == argc) {
- std::fprintf(stderr, "Invalid argument: -o requires an argument.\n");
- return false;
- }
- outputFile = argv[i];
+ case 't':
+ test = true;
+ decompress = true;
break;
+#ifdef UTIL_HAS_CREATEFILELIST
+ case 'r':
+ recursive = true;
+ break;
+#endif
case 'c':
- outputFile = '-';
+ outputFile = kStdOut;
+ forceStdout = true;
break;
+ case 'v':
+ ++verbosity;
+ break;
+ case 'q':
+ --verbosity;
+ // Ignore them for now
+ break;
+ // Unsupported options from Zstd
+ case 'D':
+ case 's':
+ notSupported("Zstd dictionaries.");
+ return Status::Failure;
+ case 'b':
+ case 'e':
+ case 'i':
+ case 'B':
+ notSupported("Zstd benchmarking options.");
+ return Status::Failure;
default:
- std::fprintf(stderr, "Invalid argument: %s.\n", arg);
- return false;
- }
- }
- // Determine input file if not specified
- if (inputFile.empty()) {
- inputFile = "-";
- }
- // Determine output file if not specified
- if (outputFile.empty()) {
- if (inputFile == "-") {
- outputFile = "-";
- } else {
- // Attempt to add/remove zstd extension from the input file
- if (decompress) {
- int stemSize = inputFile.size() - zstdExtension.size();
- if (stemSize > 0 && inputFile.substr(stemSize) == zstdExtension) {
- outputFile = inputFile.substr(0, stemSize);
- } else {
- std::fprintf(
- stderr, "Invalid argument: Unable to determine output file.\n");
- return false;
- }
- } else {
- outputFile = inputFile + zstdExtension;
+ std::fprintf(stderr, "Invalid argument: %s\n", arg);
+ return Status::Failure;
}
+ if (!finished) {
+ ++options;
+ }
+ } // while (*options != 0);
+ } // for (int i = 1; i < argc; ++i);
+
+ // Input file defaults to standard input if not provided.
+ if (localInputFiles.empty()) {
+ localInputFiles.emplace_back(kStdIn);
+ }
+
+ // Check validity of input files
+ if (localInputFiles.size() > 1) {
+ const auto it = std::find(localInputFiles.begin(), localInputFiles.end(),
+ std::string{kStdIn});
+ if (it != localInputFiles.end()) {
+ std::fprintf(
+ stderr,
+ "Cannot specify standard input when handling multiple files\n");
+ return Status::Failure;
}
}
+ if (localInputFiles.size() > 1 || recursive) {
+ if (!outputFile.empty() && outputFile != nullOutput) {
+ std::fprintf(
+ stderr,
+ "Cannot specify an output file when handling multiple inputs\n");
+ return Status::Failure;
+ }
+ }
+
+ // Translate input files/directories into files to (de)compress
+ if (recursive) {
+ char *scratchBuffer = nullptr;
+ unsigned numFiles = 0;
+ const char **files =
+ UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
+ &scratchBuffer, &numFiles);
+ if (files == nullptr) {
+ std::fprintf(stderr, "Error traversing directories\n");
+ return Status::Failure;
+ }
+ auto guard =
+ makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); });
+ if (numFiles == 0) {
+ std::fprintf(stderr, "No files found\n");
+ return Status::Failure;
+ }
+ inputFiles.resize(numFiles);
+ std::copy(files, files + numFiles, inputFiles.begin());
+ } else {
+ inputFiles.resize(localInputFiles.size());
+ std::copy(localInputFiles.begin(), localInputFiles.end(),
+ inputFiles.begin());
+ }
+ localInputFiles.clear();
+ assert(!inputFiles.empty());
+
+ // If reading from standard input, default to standard output
+ if (inputFiles[0] == kStdIn && outputFile.empty()) {
+ assert(inputFiles.size() == 1);
+ outputFile = "-";
+ }
+
+ if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) {
+ assert(inputFiles.size() == 1);
+ std::fprintf(stderr, "Cannot read input from interactive console\n");
+ return Status::Failure;
+ }
+ if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) {
+ std::fprintf(stderr, "Will not write to console stdout unless -c or -f is "
+ "specified and decompressing\n");
+ return Status::Failure;
+ }
+
// Check compression level
{
- unsigned maxCLevel = ultra ? ZSTD_maxCLevel() : maxNonUltraCompressionLevel;
- if (compressionLevel > maxCLevel) {
- std::fprintf(
- stderr, "Invalid compression level %u.\n", compressionLevel);
- return false;
+ unsigned maxCLevel =
+ ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel;
+ if (compressionLevel > maxCLevel || compressionLevel == 0) {
+ std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel);
+ return Status::Failure;
}
}
+
// Check that numThreads is set
if (numThreads == 0) {
- numThreads = std::thread::hardware_concurrency();
- if (numThreads == 0) {
- std::fprintf(stderr, "Invalid arguments: # of threads not specified "
- "and unable to determine hardware concurrency.\n");
- return false;
- }
+ std::fprintf(stderr, "Invalid arguments: # of threads not specified "
+ "and unable to determine hardware concurrency.\n");
+ return Status::Failure;
}
- return true;
+
+ // Modify verbosity
+ // If we are piping input and output, turn off interaction
+ if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) {
+ verbosity = 1;
+ }
+ // If we are in multi-file mode, turn off interaction
+ if (inputFiles.size() > 1 && verbosity == 2) {
+ verbosity = 1;
+ }
+
+ // Set options for test mode
+ if (test) {
+ outputFile = nullOutput;
+ keepSource = true;
+ }
+ return Status::Success;
+}
+
+std::string Options::getOutputFile(const std::string &inputFile) const {
+ if (!outputFile.empty()) {
+ return outputFile;
+ }
+ // Attempt to add/remove zstd extension from the input file
+ if (decompress) {
+ int stemSize = inputFile.size() - kZstdExtension.size();
+ if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) {
+ return inputFile.substr(0, stemSize);
+ } else {
+ return "";
+ }
+ } else {
+ return inputFile + kZstdExtension;
+ }
}
}
diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h
index 47c5f78..97c3885 100644
--- a/contrib/pzstd/Options.h
+++ b/contrib/pzstd/Options.h
@@ -14,47 +14,55 @@
#include <cstdint>
#include <string>
+#include <vector>
namespace pzstd {
struct Options {
+ enum class WriteMode { Regular, Auto, Sparse };
+
unsigned numThreads;
unsigned maxWindowLog;
unsigned compressionLevel;
bool decompress;
- std::string inputFile;
+ std::vector<std::string> inputFiles;
std::string outputFile;
bool overwrite;
- bool pzstdHeaders;
+ bool keepSource;
+ WriteMode writeMode;
+ bool checksum;
+ int verbosity;
+
+ enum class Status {
+ Success, // Successfully parsed options
+ Failure, // Failure to parse options
+ Message // Options specified to print a message (e.g. "-h")
+ };
Options();
- Options(
- unsigned numThreads,
- unsigned maxWindowLog,
- unsigned compressionLevel,
- bool decompress,
- const std::string& inputFile,
- const std::string& outputFile,
- bool overwrite,
- bool pzstdHeaders)
- : numThreads(numThreads),
- maxWindowLog(maxWindowLog),
- compressionLevel(compressionLevel),
- decompress(decompress),
- inputFile(inputFile),
- outputFile(outputFile),
- overwrite(overwrite),
- pzstdHeaders(pzstdHeaders) {}
+ Options(unsigned numThreads, unsigned maxWindowLog, unsigned compressionLevel,
+ bool decompress, std::vector<std::string> inputFiles,
+ std::string outputFile, bool overwrite, bool keepSource,
+ WriteMode writeMode, bool checksum, int verbosity)
+ : numThreads(numThreads), maxWindowLog(maxWindowLog),
+ compressionLevel(compressionLevel), decompress(decompress),
+ inputFiles(std::move(inputFiles)), outputFile(std::move(outputFile)),
+ overwrite(overwrite), keepSource(keepSource), writeMode(writeMode),
+ checksum(checksum), verbosity(verbosity) {}
- bool parse(int argc, const char** argv);
+ Status parse(int argc, const char **argv);
ZSTD_parameters determineParameters() const {
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
+ params.fParams.contentSizeFlag = 1;
+ params.fParams.checksumFlag = checksum;
if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) {
params.cParams.windowLog = maxWindowLog;
params.cParams = ZSTD_adjustCParams(params.cParams, 0, 0);
}
return params;
}
+
+ std::string getOutputFile(const std::string &inputFile) const;
};
}
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index 87c4c20..ccd4f62 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -19,6 +19,15 @@
#include <memory>
#include <string>
+#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
+# include <fcntl.h> /* _O_BINARY */
+# include <io.h> /* _setmode, _isatty */
+# define SET_BINARY_MODE(file) { if (_setmode(_fileno(file), _O_BINARY) == -1) perror("Cannot set _O_BINARY"); }
+#else
+# include <unistd.h> /* isatty */
+# define SET_BINARY_MODE(file)
+#endif
+
namespace pzstd {
namespace {
@@ -31,43 +40,27 @@
using std::size_t;
-size_t pzstdMain(const Options& options, ErrorHolder& errorHolder) {
- // Open the input file and attempt to determine its size
- FILE* inputFd = stdin;
- std::uintmax_t inputSize = 0;
- if (options.inputFile != "-") {
- inputFd = std::fopen(options.inputFile.c_str(), "rb");
- if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
- return 0;
- }
- std::error_code ec;
- inputSize = file_size(options.inputFile, ec);
- if (ec) {
- inputSize = 0;
- }
+static std::uintmax_t fileSizeOrZero(const std::string &file) {
+ if (file == "-") {
+ return 0;
}
- auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
-
- // Check if the output file exists and then open it
- FILE* outputFd = stdout;
- if (options.outputFile != "-") {
- if (!options.overwrite && options.outputFile != nullOutput) {
- outputFd = std::fopen(options.outputFile.c_str(), "rb");
- if (!errorHolder.check(outputFd == nullptr, "Output file exists")) {
- return 0;
- }
- }
- outputFd = std::fopen(options.outputFile.c_str(), "wb");
- if (!errorHolder.check(
- outputFd != nullptr, "Failed to open output file")) {
- return 0;
- }
+ std::error_code ec;
+ auto size = file_size(file, ec);
+ if (ec) {
+ size = 0;
}
- auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
+ return size;
+}
+static size_t handleOneInput(const Options &options,
+ const std::string &inputFile,
+ FILE* inputFd,
+ FILE* outputFd,
+ ErrorHolder &errorHolder) {
+ auto inputSize = fileSizeOrZero(inputFile);
// WorkQueue outlives ThreadPool so in the case of error we are certain
// we don't accidently try to call push() on it after it is destroyed.
- WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{2 * options.numThreads};
+ WorkQueue<std::shared_ptr<BufferWorkQueue>> outs{options.numThreads + 1};
size_t bytesWritten;
{
// Initialize the thread pool with numThreads + 1
@@ -89,21 +82,136 @@
options.determineParameters());
});
// Start writing
- bytesWritten =
- writeFile(errorHolder, outs, outputFd, options.pzstdHeaders);
+ bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress);
} else {
// Add a job that reads the input and starts all the decompression jobs
executor.add([&errorHolder, &outs, &executor, inputFd] {
asyncDecompressFrames(errorHolder, outs, executor, inputFd);
});
// Start writing
- bytesWritten = writeFile(
- errorHolder, outs, outputFd, /* writeSkippableFrames */ false);
+ bytesWritten = writeFile(errorHolder, outs, outputFd, options.decompress);
}
}
return bytesWritten;
}
+static FILE *openInputFile(const std::string &inputFile,
+ ErrorHolder &errorHolder) {
+ if (inputFile == "-") {
+ SET_BINARY_MODE(stdin);
+ return stdin;
+ }
+ // Check if input file is a directory
+ {
+ std::error_code ec;
+ if (is_directory(inputFile, ec)) {
+ errorHolder.setError("Output file is a directory -- ignored");
+ return nullptr;
+ }
+ }
+ auto inputFd = std::fopen(inputFile.c_str(), "rb");
+ if (!errorHolder.check(inputFd != nullptr, "Failed to open input file")) {
+ return nullptr;
+ }
+ return inputFd;
+}
+
+static FILE *openOutputFile(const Options &options,
+ const std::string &outputFile,
+ ErrorHolder &errorHolder) {
+ if (outputFile == "-") {
+ SET_BINARY_MODE(stdout);
+ return stdout;
+ }
+ // Check if the output file exists and then open it
+ if (!options.overwrite && outputFile != nullOutput) {
+ auto outputFd = std::fopen(outputFile.c_str(), "rb");
+ if (outputFd != nullptr) {
+ std::fclose(outputFd);
+ if (options.verbosity <= 1) {
+ errorHolder.setError("Output file exists");
+ return nullptr;
+ }
+ std::fprintf(
+ stderr,
+ "pzstd: %s already exists; do you wish to overwrite (y/n) ? ",
+ outputFile.c_str());
+ int c = getchar();
+ if (c != 'y' && c != 'Y') {
+ errorHolder.setError("Not overwritten");
+ return nullptr;
+ }
+ }
+ }
+ auto outputFd = std::fopen(outputFile.c_str(), "wb");
+ if (!errorHolder.check(
+ outputFd != nullptr, "Failed to open output file")) {
+ return 0;
+ }
+ return outputFd;
+}
+
+int pzstdMain(const Options &options) {
+ int returnCode = 0;
+ for (const auto& input : options.inputFiles) {
+ // Setup the error holder
+ ErrorHolder errorHolder;
+ auto printErrorGuard = makeScopeGuard([&] {
+ if (errorHolder.hasError()) {
+ returnCode = 1;
+ if (options.verbosity > 0) {
+ std::fprintf(stderr, "pzstd: %s: %s.\n", input.c_str(),
+ errorHolder.getError().c_str());
+ }
+ } else {
+
+ }
+ });
+ // Open the input file
+ auto inputFd = openInputFile(input, errorHolder);
+ if (inputFd == nullptr) {
+ continue;
+ }
+ auto closeInputGuard = makeScopeGuard([&] { std::fclose(inputFd); });
+ // Open the output file
+ auto outputFile = options.getOutputFile(input);
+ if (!errorHolder.check(outputFile != "",
+ "Input file does not have extension .zst")) {
+ continue;
+ }
+ auto outputFd = openOutputFile(options, outputFile, errorHolder);
+ if (outputFd == nullptr) {
+ continue;
+ }
+ auto closeOutputGuard = makeScopeGuard([&] { std::fclose(outputFd); });
+ // (de)compress the file
+ handleOneInput(options, input, inputFd, outputFd, errorHolder);
+ if (errorHolder.hasError()) {
+ continue;
+ }
+ // Delete the input file if necessary
+ if (!options.keepSource) {
+ // Be sure that we are done and have written everything before we delete
+ if (!errorHolder.check(std::fclose(inputFd) == 0,
+ "Failed to close input file")) {
+ continue;
+ }
+ closeInputGuard.dismiss();
+ if (!errorHolder.check(std::fclose(outputFd) == 0,
+ "Failed to close output file")) {
+ continue;
+ }
+ closeOutputGuard.dismiss();
+ if (std::remove(input.c_str()) != 0) {
+ errorHolder.setError("Failed to remove input file");
+ continue;
+ }
+ }
+ }
+ // Returns 1 if any of the files failed to (de)compress.
+ return returnCode;
+}
+
/// Construct a `ZSTD_inBuffer` that points to the data in `buffer`.
static ZSTD_inBuffer makeZstdInBuffer(const Buffer& buffer) {
return ZSTD_inBuffer{buffer.data(), buffer.size(), 0};
@@ -224,10 +332,9 @@
size_t step = size_t{1} << (params.cParams.windowLog + 2);
// If file size is known, see if a smaller step will spread work more evenly
if (size != 0) {
- const std::uintmax_t newStep = size / std::uintmax_t{numThreads};
- if (newStep != 0 &&
- newStep <= std::uintmax_t{std::numeric_limits<size_t>::max()}) {
- step = std::min(step, size_t{newStep});
+ const std::uintmax_t newStep = size / numThreads;
+ if (newStep != 0 && newStep <= std::numeric_limits<size_t>::max()) {
+ step = std::min(step, static_cast<size_t>(newStep));
}
}
return step;
@@ -451,12 +558,12 @@
ErrorHolder& errorHolder,
WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
FILE* outputFd,
- bool writeSkippableFrames) {
+ bool decompress) {
size_t bytesWritten = 0;
std::shared_ptr<BufferWorkQueue> out;
// Grab the output queue for each decompression job (in order).
while (outs.pop(out) && !errorHolder.hasError()) {
- if (writeSkippableFrames) {
+ if (!decompress) {
// If we are compressing and want to write skippable frames we can't
// start writing before compression is done because we need to know the
// compressed size.
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index 51d1584..0c21d13 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -28,11 +28,9 @@
* An error occurred if `errorHandler.hasError()`.
*
* @param options The pzstd options to use for (de)compression
- * @param errorHolder Used to report errors and coordinate early shutdown
- * if an error occured
- * @returns The number of bytes written.
+ * @returns 0 upon success and non-zero on failure.
*/
-std::size_t pzstdMain(const Options& options, ErrorHolder& errorHolder);
+int pzstdMain(const Options& options);
/**
* Streams input from `fd`, breaks input up into chunks, and compresses each
@@ -79,16 +77,16 @@
* Streams input in from each queue in `outs` in order, and writes the data to
* `outputFd`.
*
- * @param errorHolder Used to report errors and coordinate early exit
- * @param outs A queue of output queues, one for each
- * (de)compression job.
- * @param outputFd The file descriptor to write to
- * @param writeSkippableFrames Should we write pzstd headers?
- * @returns The number of bytes written
+ * @param errorHolder Used to report errors and coordinate early exit
+ * @param outs A queue of output queues, one for each
+ * (de)compression job.
+ * @param outputFd The file descriptor to write to
+ * @param decompress Are we decompressing?
+ * @returns The number of bytes written
*/
std::size_t writeFile(
ErrorHolder& errorHolder,
WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
FILE* outputFd,
- bool writeSkippableFrames);
+ bool decompress);
}
diff --git a/contrib/pzstd/main.cpp b/contrib/pzstd/main.cpp
index 7ff2cef..279cbfb 100644
--- a/contrib/pzstd/main.cpp
+++ b/contrib/pzstd/main.cpp
@@ -19,16 +19,14 @@
int main(int argc, const char** argv) {
Options options;
- if (!options.parse(argc, argv)) {
+ switch (options.parse(argc, argv)) {
+ case Options::Status::Failure:
return 1;
+ case Options::Status::Message:
+ return 0;
+ default:
+ break;
}
- ErrorHolder errorHolder;
- pzstdMain(options, errorHolder);
-
- if (errorHolder.hasError()) {
- std::fprintf(stderr, "Error: %s.\n", errorHolder.getError().c_str());
- return 1;
- }
- return 0;
+ return pzstdMain(options);
}
diff --git a/contrib/pzstd/test/Makefile b/contrib/pzstd/test/Makefile
index 5fd167d..4f6ba99 100644
--- a/contrib/pzstd/test/Makefile
+++ b/contrib/pzstd/test/Makefile
@@ -21,19 +21,19 @@
# Set GTEST_INC and GTEST_LIB to work with your install of gtest
GTEST_INC ?= -isystem $(PZSTDDIR)/googletest/googletest/include
GTEST_LIB ?= -L $(PZSTDDIR)/googletest/build/googlemock/gtest
-
-CPPFLAGS = -I$(PZSTDDIR) $(GTEST_INC) $(GTEST_LIB) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
+GTEST_FLAGS = $(GTEST_INC) $(GTEST_LIB)
+CPPFLAGS = -I$(PZSTDDIR) -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(PROGDIR) -I.
CXXFLAGS ?= -O3
-CXXFLAGS += -std=c++11
+CXXFLAGS += -std=c++11 -Wno-deprecated-declarations
CXXFLAGS += $(MOREFLAGS)
FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(LDFLAGS)
datagen.o: $(PROGDIR)/datagen.*
- $(CXX) $(FLAGS) $(PROGDIR)/datagen.c -c -o $@
+ $(CC) $(CPPFLAGS) -O3 $(MOREFLAGS) $(LDFLAGS) -Wno-long-long -Wno-variadic-macros $(PROGDIR)/datagen.c -c -o $@
%: %.cpp *.h datagen.o
- $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) -lgtest -lgtest_main -lpthread
+ $(CXX) $(FLAGS) $@.cpp datagen.o $(PZSTDDIR)/Pzstd.o $(PZSTDDIR)/SkippableFrame.o $(PZSTDDIR)/Options.o $(PZSTDDIR)/libzstd.a -o $@$(EXT) $(GTEST_FLAGS) -lgtest -lgtest_main -lpthread
.PHONY: test clean
diff --git a/contrib/pzstd/test/OptionsTest.cpp b/contrib/pzstd/test/OptionsTest.cpp
index b87358c..e7d4b2b 100644
--- a/contrib/pzstd/test/OptionsTest.cpp
+++ b/contrib/pzstd/test/OptionsTest.cpp
@@ -8,172 +8,535 @@
*/
#include "Options.h"
-#include <gtest/gtest.h>
#include <array>
+#include <gtest/gtest.h>
using namespace pzstd;
namespace pzstd {
-bool operator==(const Options& lhs, const Options& rhs) {
+bool operator==(const Options &lhs, const Options &rhs) {
return lhs.numThreads == rhs.numThreads &&
- lhs.maxWindowLog == rhs.maxWindowLog &&
- lhs.compressionLevel == rhs.compressionLevel &&
- lhs.decompress == rhs.decompress && lhs.inputFile == rhs.inputFile &&
- lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite &&
- lhs.pzstdHeaders == rhs.pzstdHeaders;
+ lhs.maxWindowLog == rhs.maxWindowLog &&
+ lhs.compressionLevel == rhs.compressionLevel &&
+ lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles &&
+ lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite &&
+ lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode &&
+ lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity;
}
+
+std::ostream &operator<<(std::ostream &out, const Options &opt) {
+ out << "{";
+ {
+ out << "\n\t"
+ << "numThreads: " << opt.numThreads;
+ out << ",\n\t"
+ << "maxWindowLog: " << opt.maxWindowLog;
+ out << ",\n\t"
+ << "compressionLevel: " << opt.compressionLevel;
+ out << ",\n\t"
+ << "decompress: " << opt.decompress;
+ out << ",\n\t"
+ << "inputFiles: {";
+ {
+ bool first = true;
+ for (const auto &file : opt.inputFiles) {
+ if (!first) {
+ out << ",";
+ }
+ first = false;
+ out << "\n\t\t" << file;
+ }
+ }
+ out << "\n\t}";
+ out << ",\n\t"
+ << "outputFile: " << opt.outputFile;
+ out << ",\n\t"
+ << "overwrite: " << opt.overwrite;
+ out << ",\n\t"
+ << "keepSource: " << opt.keepSource;
+ out << ",\n\t"
+ << "writeMode: " << static_cast<int>(opt.writeMode);
+ out << ",\n\t"
+ << "checksum: " << opt.checksum;
+ out << ",\n\t"
+ << "verbosity: " << opt.verbosity;
+ }
+ out << "\n}";
+ return out;
+}
+}
+
+namespace {
+#ifdef _WIN32
+const char nullOutput[] = "nul";
+#else
+const char nullOutput[] = "/dev/null";
+#endif
+
+constexpr auto autoMode = Options::WriteMode::Auto;
+} // anonymous namespace
+
+#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__)
+#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__)
+#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__)
+
+template <typename... Args>
+std::array<const char *, sizeof...(Args) + 1> makeArray(Args... args) {
+ return {{nullptr, args...}};
}
TEST(Options, ValidInputs) {
{
Options options;
- std::array<const char*, 6> args = {
- {nullptr, "--num-threads", "5", "-o", "-", "-f"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {5, 23, 3, false, "-", "-", true, false};
+ auto args = makeArray("--processes", "5", "-o", "x", "y", "-f");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {5, 23, 3, false, {"y"}, "x",
+ true, true, autoMode, true, 2};
EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 6> args = {
- {nullptr, "-n", "1", "input", "-19", "-p"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {1, 23, 19, false, "input", "input.zst", false, true};
+ auto args = makeArray("-p", "1", "input", "-19");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {1, 23, 19, false, {"input"}, "",
+ false, true, autoMode, true, 2};
EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 10> args = {{nullptr,
- "--ultra",
- "-22",
- "-n",
- "1",
- "--output",
- "x",
- "-d",
- "x.zst",
- "-f"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {1, 0, 22, true, "x.zst", "x", true, false};
+ auto args =
+ makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {1, 0, 22, true, {"x.zst"}, "x",
+ true, true, autoMode, true, 2};
EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 6> args = {{nullptr,
- "--num-threads",
- "100",
- "hello.zst",
- "--decompress",
- "--force"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {100, 23, 3, true, "hello.zst", "hello", true, false};
+ auto args = makeArray("--processes", "100", "hello.zst", "--decompress",
+ "--force");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {100, 23, 3, true, {"hello.zst"}, "", true,
+ true, autoMode, true, 2};
EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 5> args = {{nullptr, "-", "-n", "1", "-c"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {1, 23, 3, false, "-", "-", false, false};
+ auto args = makeArray("x", "-dp", "1", "-c");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {1, 23, 3, true, {"x"}, "-",
+ false, true, autoMode, true, 2};
EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 5> args = {{nullptr, "-", "-n", "1", "--stdout"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {1, 23, 3, false, "-", "-", false, false};
+ auto args = makeArray("x", "-dp", "1", "--stdout");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {1, 23, 3, true, {"x"}, "-",
+ false, true, autoMode, true, 2};
EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 10> args = {{nullptr,
- "-n",
- "1",
- "-",
- "-5",
- "-o",
- "-",
- "-u",
- "-d",
- "--pzstd-headers"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {1, 0, 5, true, "-", "-", false, true};
+ auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {1, 0, 5, true, {"x"}, "-",
+ true, true, autoMode, true, 2};
+ EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 6> args = {
- {nullptr, "silesia.tar", "-o", "silesia.tar.pzstd", "-n", "2"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
- Options expected = {
- 2, 23, 3, false, "silesia.tar", "silesia.tar.pzstd", false, false};
+ auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {2,
+ 23,
+ 3,
+ false,
+ {"silesia.tar"},
+ "silesia.tar.pzstd",
+ false,
+ true,
+ autoMode,
+ true,
+ 2};
+ EXPECT_EQ(expected, options);
}
{
Options options;
- std::array<const char*, 3> args = {{nullptr, "-n", "1"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "-p", "1");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
}
{
Options options;
- std::array<const char*, 4> args = {{nullptr, "-", "-n", "1"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "-p", "1");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ }
+}
+
+TEST(Options, GetOutputFile) {
+ {
+ Options options;
+ auto args = makeArray("x");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0]));
+ }
+ {
+ Options options;
+ auto args = makeArray("-o-");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ EXPECT_EQ("-", options.getOutputFile(options.inputFiles[0]));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "y", "-o", nullOutput);
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0]));
+ }
+ {
+ Options options;
+ auto args = makeArray("x.zst", "-do", nullOutput);
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0]));
+ }
+ {
+ Options options;
+ auto args = makeArray("x.zst", "-d");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0]));
+ }
+ {
+ Options options;
+ auto args = makeArray("xzst", "-d");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ("", options.getOutputFile(options.inputFiles[0]));
+ }
+ {
+ Options options;
+ auto args = makeArray("xzst", "-doxx");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0]));
+ }
+}
+
+TEST(Options, MultipleFiles) {
+ {
+ Options options;
+ auto args = makeArray("x", "y", "z");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected;
+ expected.inputFiles = {"x", "y", "z"};
+ expected.verbosity = 1;
+ EXPECT_EQ(expected, options);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "y", "z", "-o", nullOutput);
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected;
+ expected.inputFiles = {"x", "y", "z"};
+ expected.outputFile = nullOutput;
+ expected.verbosity = 1;
+ EXPECT_EQ(expected, options);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "y", "-o-");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "y", "-o", "file");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ Options expected = {4, 23, 12, true, {"x", "--rm", "-c"},
+ "", true, true, autoMode, true,
+ 0};
+ EXPECT_EQ(expected, options);
}
}
TEST(Options, NumThreads) {
{
Options options;
- std::array<const char*, 3> args = {{nullptr, "-o", "-"}};
- EXPECT_TRUE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "-dfo", "-");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
}
{
Options options;
- std::array<const char*, 5> args = {{nullptr, "-n", "0", "-o", "-"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "-p", "0", "-fo", "-");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
}
{
Options options;
- std::array<const char*, 4> args = {{nullptr, "-n", "-o", "-"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("-f", "-p", "-o", "-");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
}
}
TEST(Options, BadCompressionLevel) {
{
Options options;
- std::array<const char*, 3> args = {{nullptr, "x", "-20"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "-20");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
}
{
Options options;
- std::array<const char*, 4> args = {{nullptr, "x", "-u", "-23"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "--ultra", "-23");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "--1"); // negative 1?
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
}
}
TEST(Options, InvalidOption) {
{
Options options;
- std::array<const char*, 3> args = {{nullptr, "x", "-x"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("x", "-x");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
}
}
TEST(Options, BadOutputFile) {
{
Options options;
- std::array<const char*, 5> args = {{nullptr, "notzst", "-d", "-n", "1"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("notzst", "-d", "-p", "1");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ("", options.getOutputFile(options.inputFiles.front()));
+ }
+}
+
+TEST(Options, BadOptionsWithArguments) {
+ {
+ Options options;
+ auto args = makeArray("x", "-pf");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "-p", "10f");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "-p");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "-o");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "-o");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+}
+
+TEST(Options, KeepSource) {
+ {
+ Options options;
+ auto args = makeArray("x", "--rm", "-k");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.keepSource);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "--rm", "--keep");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.keepSource);
+ }
+ {
+ Options options;
+ auto args = makeArray("x");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.keepSource);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "--rm");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(false, options.keepSource);
+ }
+}
+
+TEST(Options, Verbosity) {
+ {
+ Options options;
+ auto args = makeArray("x");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(2, options.verbosity);
+ }
+ {
+ Options options;
+ auto args = makeArray("--quiet", "-qq", "x");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(-1, options.verbosity);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "y");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(1, options.verbosity);
+ }
+ {
+ Options options;
+ auto args = makeArray("--", "x", "y");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(1, options.verbosity);
+ }
+ {
+ Options options;
+ auto args = makeArray("-qv", "x", "y");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(1, options.verbosity);
+ }
+ {
+ Options options;
+ auto args = makeArray("-v", "x", "y");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(3, options.verbosity);
+ }
+ {
+ Options options;
+ auto args = makeArray("-v", "x");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(3, options.verbosity);
+ }
+}
+
+TEST(Options, TestMode) {
+ {
+ Options options;
+ auto args = makeArray("x", "-t");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.keepSource);
+ EXPECT_EQ(true, options.decompress);
+ EXPECT_EQ(nullOutput, options.outputFile);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "--test", "--rm", "-ohello");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.keepSource);
+ EXPECT_EQ(true, options.decompress);
+ EXPECT_EQ(nullOutput, options.outputFile);
+ }
+}
+
+TEST(Options, Checksum) {
+ {
+ Options options;
+ auto args = makeArray("x.zst", "--no-check", "-Cd");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.checksum);
+ }
+ {
+ Options options;
+ auto args = makeArray("x");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.checksum);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "--no-check", "--check");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(true, options.checksum);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "--no-check");
+ EXPECT_SUCCESS(options.parse(args.size(), args.data()));
+ EXPECT_EQ(false, options.checksum);
+ }
+}
+
+TEST(Options, InputFiles) {
+ {
+ Options options;
+ auto args = makeArray("-cd");
+ options.parse(args.size(), args.data());
+ EXPECT_EQ(1, options.inputFiles.size());
+ EXPECT_EQ("-", options.inputFiles[0]);
+ EXPECT_EQ("-", options.outputFile);
+ }
+ {
+ Options options;
+ auto args = makeArray();
+ options.parse(args.size(), args.data());
+ EXPECT_EQ(1, options.inputFiles.size());
+ EXPECT_EQ("-", options.inputFiles[0]);
+ EXPECT_EQ("-", options.outputFile);
+ }
+ {
+ Options options;
+ auto args = makeArray("-d");
+ options.parse(args.size(), args.data());
+ EXPECT_EQ(1, options.inputFiles.size());
+ EXPECT_EQ("-", options.inputFiles[0]);
+ EXPECT_EQ("-", options.outputFile);
+ }
+ {
+ Options options;
+ auto args = makeArray("x", "-");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+}
+
+TEST(Options, InvalidOptions) {
+ {
+ Options options;
+ auto args = makeArray("-ibasdf");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("- ");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("-n15");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("-0", "x");
+ EXPECT_FAILURE(options.parse(args.size(), args.data()));
}
}
TEST(Options, Extras) {
{
Options options;
- std::array<const char*, 2> args = {{nullptr, "-h"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("-h");
+ EXPECT_MESSAGE(options.parse(args.size(), args.data()));
}
{
Options options;
- std::array<const char*, 2> args = {{nullptr, "-V"}};
- EXPECT_FALSE(options.parse(args.size(), args.data()));
+ auto args = makeArray("-H");
+ EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("-V");
+ EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("--help");
+ EXPECT_MESSAGE(options.parse(args.size(), args.data()));
+ }
+ {
+ Options options;
+ auto args = makeArray("--version");
+ EXPECT_MESSAGE(options.parse(args.size(), args.data()));
}
}
diff --git a/contrib/pzstd/test/PzstdTest.cpp b/contrib/pzstd/test/PzstdTest.cpp
index 9d1256f..64bcf9c 100644
--- a/contrib/pzstd/test/PzstdTest.cpp
+++ b/contrib/pzstd/test/PzstdTest.cpp
@@ -6,14 +6,16 @@
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
-#include "datagen.h"
#include "Pzstd.h"
+extern "C" {
+#include "datagen.h"
+}
#include "test/RoundTrip.h"
#include "utils/ScopeGuard.h"
-#include <gtest/gtest.h>
#include <cstddef>
#include <cstdio>
+#include <gtest/gtest.h>
#include <memory>
#include <random>
@@ -25,11 +27,14 @@
std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed);
std::mt19937 gen(seed);
- for (unsigned len = 1; len < 1028; ++len) {
+ for (unsigned len = 1; len < 256; ++len) {
+ if (len % 16 == 0) {
+ std::fprintf(stderr, "%u / 16\n", len / 16);
+ }
std::string inputFile = std::tmpnam(nullptr);
auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
{
- static uint8_t buf[1028];
+ static uint8_t buf[256];
RDG_genBuffer(buf, len, 0.5, 0.0, gen());
auto fd = std::fopen(inputFile.c_str(), "wb");
auto written = std::fwrite(buf, 1, len, fd);
@@ -37,19 +42,16 @@
ASSERT_EQ(written, len);
}
for (unsigned headers = 0; headers <= 1; ++headers) {
- for (unsigned numThreads = 1; numThreads <= 4; numThreads *= 2) {
- for (unsigned level = 1; level <= 8; level *= 8) {
+ for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) {
+ for (unsigned level = 1; level <= 4; level *= 4) {
auto errorGuard = makeScopeGuard([&] {
- guard.dismiss();
- std::fprintf(stderr, "file: %s\n", inputFile.c_str());
std::fprintf(stderr, "pzstd headers: %u\n", headers);
std::fprintf(stderr, "# threads: %u\n", numThreads);
std::fprintf(stderr, "compression level: %u\n", level);
});
Options options;
- options.pzstdHeaders = headers;
options.overwrite = true;
- options.inputFile = inputFile;
+ options.inputFiles = {inputFile};
options.numThreads = numThreads;
options.compressionLevel = level;
ASSERT_TRUE(roundTrip(options));
@@ -80,17 +82,14 @@
for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) {
for (unsigned level = 1; level <= 4; level *= 2) {
auto errorGuard = makeScopeGuard([&] {
- guard.dismiss();
- std::fprintf(stderr, "file: %s\n", inputFile.c_str());
std::fprintf(stderr, "pzstd headers: %u\n", headers);
std::fprintf(stderr, "# threads: %u\n", numThreads);
std::fprintf(stderr, "compression level: %u\n", level);
});
Options options;
- options.pzstdHeaders = headers;
options.overwrite = true;
- options.inputFile = inputFile;
- options.numThreads = numThreads;
+ options.inputFiles = {inputFile};
+ options.numThreads = std::min(numThreads, options.numThreads);
options.compressionLevel = level;
ASSERT_TRUE(roundTrip(options));
errorGuard.dismiss();
@@ -100,6 +99,40 @@
}
}
+TEST(Pzstd, ExtremelyLargeSize) {
+ unsigned seed = std::random_device{}();
+ std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed);
+ std::mt19937 gen(seed);
+
+ std::string inputFile = std::tmpnam(nullptr);
+ auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
+
+ {
+ // Write 4GB + 64 MB
+ constexpr size_t kLength = 1 << 26;
+ std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]);
+ auto fd = std::fopen(inputFile.c_str(), "wb");
+ auto closeGuard = makeScopeGuard([&] { std::fclose(fd); });
+ for (size_t i = 0; i < (1 << 6) + 1; ++i) {
+ RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen());
+ auto written = std::fwrite(buf.get(), 1, kLength, fd);
+ if (written != kLength) {
+ std::fprintf(stderr, "Failed to write file, skipping test\n");
+ return;
+ }
+ }
+ }
+
+ Options options;
+ options.overwrite = true;
+ options.inputFiles = {inputFile};
+ options.compressionLevel = 1;
+ if (options.numThreads == 0) {
+ options.numThreads = 1;
+ }
+ ASSERT_TRUE(roundTrip(options));
+}
+
TEST(Pzstd, ExtremelyCompressible) {
std::string inputFile = std::tmpnam(nullptr);
auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); });
@@ -112,9 +145,8 @@
ASSERT_EQ(written, 10000);
}
Options options;
- options.pzstdHeaders = false;
options.overwrite = true;
- options.inputFile = inputFile;
+ options.inputFiles = {inputFile};
options.numThreads = 1;
options.compressionLevel = 1;
ASSERT_TRUE(roundTrip(options));
diff --git a/contrib/pzstd/test/RoundTrip.h b/contrib/pzstd/test/RoundTrip.h
index 829c95c..8b90884 100644
--- a/contrib/pzstd/test/RoundTrip.h
+++ b/contrib/pzstd/test/RoundTrip.h
@@ -55,7 +55,10 @@
}
inline bool roundTrip(Options& options) {
- std::string source = options.inputFile;
+ if (options.inputFiles.size() != 1) {
+ return false;
+ }
+ std::string source = options.inputFiles.front();
std::string compressedFile = std::tmpnam(nullptr);
std::string decompressedFile = std::tmpnam(nullptr);
auto guard = makeScopeGuard([&] {
@@ -66,21 +69,15 @@
{
options.outputFile = compressedFile;
options.decompress = false;
- ErrorHolder errorHolder;
- pzstdMain(options, errorHolder);
- if (errorHolder.hasError()) {
- errorHolder.getError();
+ if (pzstdMain(options) != 0) {
return false;
}
}
{
options.decompress = true;
- options.inputFile = compressedFile;
+ options.inputFiles.front() = compressedFile;
options.outputFile = decompressedFile;
- ErrorHolder errorHolder;
- pzstdMain(options, errorHolder);
- if (errorHolder.hasError()) {
- errorHolder.getError();
+ if (pzstdMain(options) != 0) {
return false;
}
}
diff --git a/contrib/pzstd/test/RoundTripTest.cpp b/contrib/pzstd/test/RoundTripTest.cpp
index 01c1c81..ed2ea77 100644
--- a/contrib/pzstd/test/RoundTripTest.cpp
+++ b/contrib/pzstd/test/RoundTripTest.cpp
@@ -6,7 +6,9 @@
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
+extern "C" {
#include "datagen.h"
+}
#include "Options.h"
#include "test/RoundTrip.h"
#include "utils/ScopeGuard.h"
@@ -46,14 +48,12 @@
template <typename Generator>
Options generateOptions(Generator& gen, const string& inputFile) {
Options options;
- options.inputFile = inputFile;
+ options.inputFiles = {inputFile};
options.overwrite = true;
- std::bernoulli_distribution pzstdHeaders{0.75};
std::uniform_int_distribution<unsigned> numThreads{1, 32};
std::uniform_int_distribution<unsigned> compressionLevel{1, 10};
- options.pzstdHeaders = pzstdHeaders(gen);
options.numThreads = numThreads(gen);
options.compressionLevel = compressionLevel(gen);
@@ -61,7 +61,7 @@
}
}
-int main(int argc, char** argv) {
+int main() {
std::mt19937 gen(std::random_device{}());
auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); });
@@ -77,8 +77,6 @@
std::fprintf(stderr, "numThreads: %u\n", options.numThreads);
std::fprintf(stderr, "level: %u\n", options.compressionLevel);
std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress);
- std::fprintf(
- stderr, "pzstd headers? %u\n", (unsigned)options.pzstdHeaders);
std::fprintf(stderr, "file: %s\n", inputFile.c_str());
return 1;
}
diff --git a/contrib/pzstd/utils/FileSystem.h b/contrib/pzstd/utils/FileSystem.h
index 979c82b..7d59704 100644
--- a/contrib/pzstd/utils/FileSystem.h
+++ b/contrib/pzstd/utils/FileSystem.h
@@ -21,10 +21,11 @@
namespace pzstd {
+// using file_status = ... causes gcc to emit a false positive warning
#if defined(_MSC_VER)
-using file_status = struct ::_stat64;
+typedef struct ::_stat64 file_status;
#else
-using file_status = struct ::stat;
+typedef struct ::stat file_status;
#endif
/// http://en.cppreference.com/w/cpp/filesystem/status
@@ -59,6 +60,22 @@
return is_regular_file(status(path, ec));
}
+/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+inline bool is_directory(file_status status) noexcept {
+#if defined(S_ISDIR)
+ return S_ISDIR(status.st_mode);
+#elif !defined(S_ISDIR) && defined(S_IFMT) && defined(S_IFDIR)
+ return (status.st_mode & S_IFMT) == S_IFDIR;
+#else
+ static_assert(false, "NO POSIX stat() support.");
+#endif
+}
+
+/// http://en.cppreference.com/w/cpp/filesystem/is_directory
+inline bool is_directory(StringPiece path, std::error_code& ec) noexcept {
+ return is_directory(status(path, ec));
+}
+
/// http://en.cppreference.com/w/cpp/filesystem/file_size
inline std::uintmax_t file_size(
StringPiece path,
diff --git a/contrib/pzstd/utils/test/ThreadPoolTest.cpp b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
index 9b9868c..1d857aa 100644
--- a/contrib/pzstd/utils/test/ThreadPoolTest.cpp
+++ b/contrib/pzstd/utils/test/ThreadPoolTest.cpp
@@ -20,12 +20,12 @@
{
ThreadPool executor(1);
- for (int i = 0; i < 100; ++i) {
+ for (int i = 0; i < 10; ++i) {
executor.add([ &results, i ] { results.push_back(i); });
}
}
- for (int i = 0; i < 100; ++i) {
+ for (int i = 0; i < 10; ++i) {
EXPECT_EQ(i, results[i]);
}
}
@@ -35,7 +35,7 @@
std::atomic<bool> start{false};
{
ThreadPool executor(5);
- for (int i = 0; i < 1000; ++i) {
+ for (int i = 0; i < 10; ++i) {
executor.add([ &numFinished, &start ] {
while (!start.load()) {
// spin
@@ -45,7 +45,7 @@
}
start.store(true);
}
- EXPECT_EQ(1000, numFinished.load());
+ EXPECT_EQ(10, numFinished.load());
}
TEST(ThreadPool, AddJobWhileJoining) {
diff --git a/contrib/pzstd/utils/test/WorkQueueTest.cpp b/contrib/pzstd/utils/test/WorkQueueTest.cpp
index 84d8573..ebf375a 100644
--- a/contrib/pzstd/utils/test/WorkQueueTest.cpp
+++ b/contrib/pzstd/utils/test/WorkQueueTest.cpp
@@ -89,14 +89,14 @@
TEST(WorkQueue, SPMC) {
WorkQueue<int> queue;
- std::vector<int> results(10000, -1);
+ std::vector<int> results(50, -1);
std::mutex mutex;
std::vector<std::thread> threads;
- for (int i = 0; i < 100; ++i) {
+ for (int i = 0; i < 5; ++i) {
threads.emplace_back(Popper{&queue, results.data(), &mutex});
}
- for (int i = 0; i < 10000; ++i) {
+ for (int i = 0; i < 50; ++i) {
queue.push(i);
}
queue.finish();
@@ -105,24 +105,24 @@
thread.join();
}
- for (int i = 0; i < 10000; ++i) {
+ for (int i = 0; i < 50; ++i) {
EXPECT_EQ(i, results[i]);
}
}
TEST(WorkQueue, MPMC) {
WorkQueue<int> queue;
- std::vector<int> results(10000, -1);
+ std::vector<int> results(100, -1);
std::mutex mutex;
std::vector<std::thread> popperThreads;
- for (int i = 0; i < 100; ++i) {
+ for (int i = 0; i < 4; ++i) {
popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
}
std::vector<std::thread> pusherThreads;
- for (int i = 0; i < 10; ++i) {
- auto min = i * 1000;
- auto max = (i + 1) * 1000;
+ for (int i = 0; i < 2; ++i) {
+ auto min = i * 50;
+ auto max = (i + 1) * 50;
pusherThreads.emplace_back(
[ &queue, min, max ] {
for (int i = min; i < max; ++i) {
@@ -140,7 +140,7 @@
thread.join();
}
- for (int i = 0; i < 10000; ++i) {
+ for (int i = 0; i < 100; ++i) {
EXPECT_EQ(i, results[i]);
}
}
@@ -197,16 +197,16 @@
}
TEST(WorkQueue, BoundedSizeMPMC) {
- WorkQueue<int> queue(100);
- std::vector<int> results(10000, -1);
+ WorkQueue<int> queue(10);
+ std::vector<int> results(200, -1);
std::mutex mutex;
std::vector<std::thread> popperThreads;
- for (int i = 0; i < 10; ++i) {
+ for (int i = 0; i < 4; ++i) {
popperThreads.emplace_back(Popper{&queue, results.data(), &mutex});
}
std::vector<std::thread> pusherThreads;
- for (int i = 0; i < 100; ++i) {
+ for (int i = 0; i < 2; ++i) {
auto min = i * 100;
auto max = (i + 1) * 100;
pusherThreads.emplace_back(
@@ -226,7 +226,7 @@
thread.join();
}
- for (int i = 0; i < 10000; ++i) {
+ for (int i = 0; i < 200; ++i) {
EXPECT_EQ(i, results[i]);
}
}
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index 5a9bc40..d9e89f8 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -326,7 +326,7 @@
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTDv06_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
-ZSTDLIB_API size_t ZSTDv06_decompress_usingPreparedDCtx(
+ZSTDLIBv06_API size_t ZSTDv06_decompress_usingPreparedDCtx(
ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
@@ -337,7 +337,7 @@
static const size_t ZSTDv06_frameHeaderSize_min = 5;
static const size_t ZSTDv06_frameHeaderSize_max = ZSTDv06_FRAMEHEADERSIZE_MAX;
-ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
+ZSTDLIBv06_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
/*
Streaming decompression, direct mode (bufferless)
@@ -396,7 +396,7 @@
*/
#define ZSTDv06_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */
-ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index bcc6efb..14040ab 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -14,23 +14,19 @@
extern "C" {
#endif
-/*-*************************************
-* Dependencies
-***************************************/
+/*====== Dependency ======*/
#include <stddef.h> /* size_t */
-/*-***************************************************************
-* Export parameters
-*****************************************************************/
+/*====== Export for Windows ======*/
/*!
* ZSTDv06_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+# define ZSTDLIBv06_API __declspec(dllexport)
#else
-# define ZSTDLIB_API
+# define ZSTDLIBv06_API
#endif
@@ -42,18 +38,18 @@
`dstCapacity` must be large enough, equal or larger than originalSize.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */
-ZSTDLIB_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
- const void* src, size_t compressedSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
/* *************************************
* Helper functions
***************************************/
-ZSTDLIB_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */
+ZSTDLIBv06_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */
/* Error Management */
-ZSTDLIB_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */
+ZSTDLIBv06_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+ZSTDLIBv06_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */
/* *************************************
@@ -61,12 +57,12 @@
***************************************/
/** Decompression context */
typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx;
-ZSTDLIB_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);
-ZSTDLIB_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */
+ZSTDLIBv06_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void);
+ZSTDLIBv06_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv06_decompressDCtx() :
* Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */
-ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-***********************
@@ -76,10 +72,10 @@
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted.
* Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */
-ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize);
+ZSTDLIBv06_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
/*-************************
@@ -88,12 +84,12 @@
struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; };
typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams;
-ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
-ZSTDLIB_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx);
+ZSTDLIBv06_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+ZSTDLIBv06_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv06_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx);
-ZSTDLIB_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx);
-ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv06_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx);
+ZSTDLIBv06_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -102,15 +98,15 @@
***************************************/
typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx;
-ZSTDLIB_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void);
-ZSTDLIB_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx);
+ZSTDLIBv06_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void);
+ZSTDLIBv06_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv06_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx);
+ZSTDLIBv06_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
- void* dst, size_t* dstCapacityPtr,
- const void* src, size_t* srcSizePtr);
+ZSTDLIBv06_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx,
+ void* dst, size_t* dstCapacityPtr,
+ const void* src, size_t* srcSizePtr);
/*-***************************************************************************
* Streaming decompression howto
@@ -140,13 +136,13 @@
/* *************************************
* Tool functions
***************************************/
-ZSTDLIB_API unsigned ZBUFFv06_isError(size_t errorCode);
-ZSTDLIB_API const char* ZBUFFv06_getErrorName(size_t errorCode);
+ZSTDLIBv06_API unsigned ZBUFFv06_isError(size_t errorCode);
+ZSTDLIBv06_API const char* ZBUFFv06_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
-ZSTDLIB_API size_t ZBUFFv06_recommendedDInSize(void);
-ZSTDLIB_API size_t ZBUFFv06_recommendedDOutSize(void);
+ZSTDLIBv06_API size_t ZBUFFv06_recommendedDInSize(void);
+ZSTDLIBv06_API size_t ZBUFFv06_recommendedDOutSize(void);
/*-*************************************
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index dac71ae..f4c8073 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -68,27 +68,27 @@
/*! ZSTDv07_estimateDCtxSize() :
* Gives the potential amount of memory allocated to create a ZSTDv07_DCtx */
-ZSTDLIB_API size_t ZSTDv07_estimateDCtxSize(void);
+ZSTDLIBv07_API size_t ZSTDv07_estimateDCtxSize(void);
/*! ZSTDv07_createDCtx_advanced() :
* Create a ZSTD decompression context using external alloc and free functions */
-ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem);
+ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx_advanced(ZSTDv07_customMem customMem);
/*! ZSTDv07_sizeofDCtx() :
* Gives the amount of memory used by a given ZSTDv07_DCtx */
-ZSTDLIB_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZSTDv07_sizeofDCtx(const ZSTDv07_DCtx* dctx);
/* ******************************************************************
* Buffer-less streaming functions (synchronous mode)
********************************************************************/
-ZSTDLIB_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx);
-ZSTDLIB_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx);
+ZSTDLIBv07_API size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZSTDv07_decompressBegin_usingDict(ZSTDv07_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv07_API void ZSTDv07_copyDCtx(ZSTDv07_DCtx* dctx, const ZSTDv07_DCtx* preparedDCtx);
-ZSTDLIB_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx);
-ZSTDLIB_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv07_API size_t ZSTDv07_nextSrcSizeToDecompress(ZSTDv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZSTDv07_decompressContinue(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*
Buffer-less streaming decompression (synchronous mode)
@@ -169,8 +169,8 @@
*/
#define ZSTDv07_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
-ZSTDLIB_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
+ZSTDLIBv07_API size_t ZSTDv07_decompressBlock(ZSTDv07_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */
#endif /* ZSTDv07_STATIC_LINKING_ONLY */
@@ -650,8 +650,8 @@
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
MEM_STATIC BITv07_DStream_status BITv07_reloadDStream(BITv07_DStream_t* bitD)
{
- if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
- return BITv07_DStream_overflow;
+ if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
+ return BITv07_DStream_overflow;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
bitD->ptr -= bitD->bitsConsumed >> 3;
@@ -3831,7 +3831,7 @@
/** ZSTDv07_insertBlock() :
insert `src` block into `dctx` history. Useful to track uncompressed blocks. */
-ZSTDLIB_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize)
+ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockStart, size_t blockSize)
{
ZSTDv07_checkContinuity(dctx, blockStart);
dctx->previousDstEnd = (const char*)blockStart + blockSize;
@@ -4233,7 +4233,7 @@
/*! ZSTDv07_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */
-ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
+ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTDv07_DDict* ddict)
@@ -4320,7 +4320,7 @@
ZSTDv07_customMem customMem;
}; /* typedef'd to ZBUFFv07_DCtx within "zstd_buffered.h" */
-ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem);
+ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx_advanced(ZSTDv07_customMem customMem);
ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void)
{
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index d1fbc08..30725dc 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -24,13 +24,12 @@
* Enable exporting of functions when building a Windows DLL
*/
#if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1)
-# define ZSTDLIB_API __declspec(dllexport)
+# define ZSTDLIBv07_API __declspec(dllexport)
#else
-# define ZSTDLIB_API
+# define ZSTDLIBv07_API
#endif
-
/* *************************************
* Simple API
***************************************/
@@ -46,12 +45,12 @@
`dstCapacity` must be equal or larger than originalSize.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */
-ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
- const void* src, size_t compressedSize);
+ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity,
+ const void* src, size_t compressedSize);
/*====== Helper functions ======*/
-ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
-ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */
+ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
+ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */
/*-*************************************
@@ -59,12 +58,12 @@
***************************************/
/** Decompression context */
typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx;
-ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void);
-ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */
+ZSTDLIBv07_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void);
+ZSTDLIBv07_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */
/** ZSTDv07_decompressDCtx() :
* Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */
-ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTDLIBv07_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-************************
@@ -74,10 +73,10 @@
* Decompression using a pre-defined Dictionary content (see dictBuilder).
* Dictionary must be identical to the one used during compression.
* Note : This function load the dictionary, resulting in a significant startup time */
-ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const void* dict,size_t dictSize);
+ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const void* dict,size_t dictSize);
/*-**************************
@@ -87,16 +86,16 @@
* Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */
typedef struct ZSTDv07_DDict_s ZSTDv07_DDict;
-ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict);
+ZSTDLIBv07_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize);
+ZSTDLIBv07_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict);
/*! ZSTDv07_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
* Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */
-ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
- void* dst, size_t dstCapacity,
- const void* src, size_t srcSize,
- const ZSTDv07_DDict* ddict);
+ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx,
+ void* dst, size_t dstCapacity,
+ const void* src, size_t srcSize,
+ const ZSTDv07_DDict* ddict);
typedef struct {
unsigned long long frameContentSize;
@@ -105,7 +104,7 @@
unsigned checksumFlag;
} ZSTDv07_frameParams;
-ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
+ZSTDLIBv07_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */
@@ -114,13 +113,13 @@
* Streaming functions
***************************************/
typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx;
-ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void);
-ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx);
+ZSTDLIBv07_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void);
+ZSTDLIBv07_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx);
-ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize);
+ZSTDLIBv07_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx);
+ZSTDLIBv07_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize);
-ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx,
+ZSTDLIBv07_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
@@ -152,13 +151,13 @@
/* *************************************
* Tool functions
***************************************/
-ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode);
-ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode);
+ZSTDLIBv07_API unsigned ZBUFFv07_isError(size_t errorCode);
+ZSTDLIBv07_API const char* ZBUFFv07_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
-ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void);
-ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void);
+ZSTDLIBv07_API size_t ZBUFFv07_recommendedDInSize(void);
+ZSTDLIBv07_API size_t ZBUFFv07_recommendedDOutSize(void);
/*-*************************************
diff --git a/programs/datagen.h b/programs/datagen.h
index 55f9d82..094056b 100644
--- a/programs/datagen.h
+++ b/programs/datagen.h
@@ -6,7 +6,8 @@
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
-
+#ifndef DATAGEN_H
+#define DATAGEN_H
#include <stddef.h> /* size_t */
@@ -22,3 +23,5 @@
RDG_genStdout
Same as RDG_genBuffer, but generates data into stdout
*/
+
+#endif
diff --git a/programs/util.h b/programs/util.h
index 9d28c82..aabebe9 100644
--- a/programs/util.h
+++ b/programs/util.h
@@ -31,12 +31,12 @@
/* Unix Large Files support (>4GB) */
-#if !defined(__LP64__) /* No point defining Large file for 64 bit */
-# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
-# if defined(__sun__) /* Sun Solaris 32-bits requires specific definitions */
-# define _LARGEFILE_SOURCE /* fseeko, ftello */
-# else
-# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */
+#if !defined(__LP64__) /* No point defining Large file for 64 bit */
+# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
+# if defined(__sun__) && !defined(_LARGEFILE_SOURCE) /* Sun Solaris 32-bits requires specific definitions */
+# define _LARGEFILE_SOURCE /* fseeko, ftello */
+# elif !defined(_LARGEFILE64_SOURCE)
+# define _LARGEFILE64_SOURCE /* off64_t, fseeko64, ftello64 */
# endif
#endif