fileio : decoding malformed lzma frame does no longer exit()
makes it possible to continue decoding file list
diff --git a/programs/fileio.c b/programs/fileio.c
index 7794371..780c29b 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -107,6 +107,17 @@
/*-*************************************
* Errors
***************************************/
+/*-*************************************
+* Debug
+***************************************/
+#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
+# include <assert.h>
+#else
+# ifndef assert
+# define assert(condition) ((void)0)
+# endif
+#endif
+
#ifndef ZSTD_DEBUG
# define ZSTD_DEBUG 0
#endif
@@ -1347,10 +1358,10 @@
/** FIO_decompressFrame() :
- @return : size of decoded frame, or an error code
+ * @return : size of decoded zstd frame, or an error code
*/
-#define FIO_ERROR_ZSTD_DECODING ((unsigned long long)(-2))
-unsigned long long FIO_decompressFrame(dRess_t* ress,
+#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
+unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
FILE* finput,
const char* srcFileName,
U64 alreadyDecoded)
@@ -1375,7 +1386,7 @@
if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint));
- return FIO_ERROR_ZSTD_DECODING;
+ return FIO_ERROR_FRAME_DECODING;
}
/* Write block */
@@ -1393,7 +1404,7 @@
if (inBuff.size != inBuff.pos) {
DISPLAYLEVEL(1, "%s : Decoding error (37) : should consume entire input \n",
srcFileName);
- return FIO_ERROR_ZSTD_DECODING;
+ return FIO_ERROR_FRAME_DECODING;
}
/* Fill input buffer */
@@ -1404,7 +1415,7 @@
if (ress->srcBufferLoaded < toRead) {
DISPLAYLEVEL(1, "%s : Read error (39) : premature end \n",
srcFileName);
- return FIO_ERROR_ZSTD_DECODING;
+ return FIO_ERROR_FRAME_DECODING;
} } }
FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
@@ -1444,7 +1455,7 @@
strm.avail_in = (uInt)ress->srcBufferLoaded;
}
ret = inflate(&strm, flush);
- if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+ if (ret == Z_BUF_ERROR) EXM_THROW(89, "zstd: %s: premature gz end", srcFileName);
if (ret != Z_OK && ret != Z_STREAM_END) {
DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret);
return 0;
@@ -1476,19 +1487,23 @@
unsigned long long outFileSize = 0;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_action action = LZMA_RUN;
- lzma_ret ret;
+ lzma_ret initRet;
+ int decodingError = 0;
strm.next_in = 0;
strm.avail_in = 0;
if (plain_lzma) {
- ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
+ initRet = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
} else {
- ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
+ initRet = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
}
- if (ret != LZMA_OK)
- EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d",
- srcFileName, ret);
+ if (initRet != LZMA_OK) {
+ DISPLAYLEVEL(1, "zstd: %s: %s error %d \n",
+ plain_lzma ? "lzma_alone_decoder" : "lzma_stream_decoder",
+ srcFileName, initRet);
+ return FIO_ERROR_FRAME_DECODING;
+ }
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
@@ -1496,6 +1511,7 @@
strm.avail_in = ress->srcBufferLoaded;
for ( ; ; ) {
+ lzma_ret ret;
if (strm.avail_in == 0) {
ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
if (ress->srcBufferLoaded == 0) action = LZMA_FINISH;
@@ -1504,16 +1520,21 @@
}
ret = lzma_code(&strm, action);
- if (ret == LZMA_BUF_ERROR)
- EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+ if (ret == LZMA_BUF_ERROR) {
+ DISPLAYLEVEL(1, "zstd: %s: premature lzma end \n", srcFileName);
+ decodingError = 1; break;
+ }
if (ret != LZMA_OK && ret != LZMA_STREAM_END) {
- DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret);
- return 0;
+ DISPLAYLEVEL(1, "zstd: %s: lzma_code decoding error %d \n",
+ srcFileName, ret);
+ decodingError = 1; break;
}
{ size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
- if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes)
- EXM_THROW(31, "Write error : cannot write to output file");
+ if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) {
+ DISPLAYLEVEL(1, "zstd: %s \n", strerror(errno));
+ decodingError = 1; break;
+ }
outFileSize += decompBytes;
strm.next_out = (BYTE*)ress->dstBuffer;
strm.avail_out = ress->dstBufferSize;
@@ -1524,7 +1545,7 @@
if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
ress->srcBufferLoaded = strm.avail_in;
lzma_end(&strm);
- return outFileSize;
+ return decodingError ? FIO_ERROR_FRAME_DECODING : outFileSize;
}
#endif
@@ -1596,6 +1617,87 @@
+/** FIO_decompressFrames() :
+ * Find and decode frames inside srcFile
+ * srcFile presumed opened and valid
+ * @return : 0 : OK
+ * 1 : error
+ */
+static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
+ const char* dstFileName, const char* srcFileName)
+{
+ unsigned readSomething = 0;
+ unsigned long long filesize = 0;
+ assert(srcFile != NULL);
+
+ /* for each frame */
+ for ( ; ; ) {
+ /* check magic number -> version */
+ size_t const toRead = 4;
+ const BYTE* const buf = (const BYTE*)ress.srcBuffer;
+ if (ress.srcBufferLoaded < toRead)
+ ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded,
+ (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
+ if (ress.srcBufferLoaded==0) {
+ if (readSomething==0) {
+ DISPLAYLEVEL(1, "zstd: %s: unexpected end of file \n", srcFileName);
+ return 1;
+ } /* else, srcFileName is just empty */
+ break; /* no more input */
+ }
+ readSomething = 1; /* there is at least 1 byte in srcFile */
+ if (ress.srcBufferLoaded < toRead) {
+ DISPLAYLEVEL(1, "zstd: %s: unknown header \n", srcFileName);
+ return 1;
+ }
+ if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
+#ifdef ZSTD_GZDECOMPRESS
+ unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
+ if (result == 0) return 1;
+ filesize += result;
+#else
+ DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without HAVE_ZLIB) -- ignored \n", srcFileName);
+ return 1;
+#endif
+ } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
+ || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
+#ifdef ZSTD_LZMADECOMPRESS
+ unsigned long long const frameSize = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+ if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
+ filesize += frameSize;
+#else
+ DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without HAVE_LZMA) -- ignored \n", srcFileName);
+ return 1;
+#endif
+ } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
+#ifdef ZSTD_LZ4DECOMPRESS
+ unsigned long long const result = FIO_decompressLz4Frame(&ress, srcFile, srcFileName);
+ if (result == 0) return 1;
+ filesize += result;
+#else
+ DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
+ return 1;
+#endif
+ } else if (ZSTD_isFrame(buf, toRead)) {
+ unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize);
+ if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
+ filesize += frameSize;
+ } else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
+ return FIO_passThrough(ress.dstFile, srcFile,
+ ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
+ } else {
+ DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
+ return 1;
+ } } /* for each frame */
+
+ /* Final Status */
+ DISPLAYLEVEL(2, "\r%79s\r", "");
+ DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
+
+ return 0;
+}
+
+
/** FIO_decompressSrcFile() :
Decompression `srcFileName` into `ress.dstFile`
@return : 0 : OK
@@ -1604,8 +1706,7 @@
static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName)
{
FILE* srcFile;
- unsigned readSomething = 0;
- unsigned long long filesize = 0;
+ int result;
if (UTIL_isDirectory(srcFileName)) {
DISPLAYLEVEL(1, "zstd: %s is a directory -- ignored \n", srcFileName);
@@ -1615,88 +1716,18 @@
srcFile = FIO_openSrcFile(srcFileName);
if (srcFile==NULL) return 1;
- /* for each frame */
- for ( ; ; ) {
- /* check magic number -> version */
- size_t const toRead = 4;
- const BYTE* buf = (const BYTE*)ress.srcBuffer;
- if (ress.srcBufferLoaded < toRead)
- ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded,
- (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
- if (ress.srcBufferLoaded==0) {
- if (readSomething==0) {
- DISPLAY("zstd: %s: unexpected end of file \n", srcFileName);
- fclose(srcFile);
- return 1;
- } /* srcFileName is empty */
- break; /* no more input */
- }
- readSomething = 1; /* there is at least >= 4 bytes in srcFile */
- if (ress.srcBufferLoaded < toRead) {
- DISPLAY("zstd: %s: unknown header \n", srcFileName);
- fclose(srcFile);
- return 1;
- }
- if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
-#ifdef ZSTD_GZDECOMPRESS
- unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
- if (result == 0) return 1;
- filesize += result;
-#else
- DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
- return 1;
-#endif
- } else if ((buf[0] == 0xFD && buf[1] == 0x37) /* xz magic number */
- || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
-#ifdef ZSTD_LZMADECOMPRESS
- unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
- if (result == 0) return 1;
- filesize += result;
-#else
- DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName);
- return 1;
-#endif
- } else if (MEM_readLE32(buf) == LZ4_MAGICNUMBER) {
-#ifdef ZSTD_LZ4DECOMPRESS
- unsigned long long const result = FIO_decompressLz4Frame(&ress, srcFile, srcFileName);
- if (result == 0) return 1;
- filesize += result;
-#else
- DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without ZSTD_LZ4DECOMPRESS) -- ignored \n", srcFileName);
- return 1;
-#endif
- } else {
- if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
- if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
- unsigned const result = FIO_passThrough(ress.dstFile, srcFile,
- ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
- if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
- return result;
- } else {
- DISPLAYLEVEL(1, "zstd: %s: not in zstd format \n", srcFileName);
- fclose(srcFile);
- return 1;
- } }
- { unsigned long long const frameSize = FIO_decompressFrame(&ress, srcFile, srcFileName, filesize);
- if (frameSize == FIO_ERROR_ZSTD_DECODING) {
- fclose(srcFile);
- return 1;
- }
- filesize += frameSize;
- } }
- } /* for each frame */
-
- /* Final Status */
- DISPLAYLEVEL(2, "\r%79s\r", "");
- DISPLAYLEVEL(2, "%-20s: %llu bytes \n", srcFileName, filesize);
+ result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName);
/* Close file */
- if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName); /* error should never happen */
+ if (fclose(srcFile)) {
+ EXM_THROW(33, "zstd: %s: %s", srcFileName, strerror(errno)); /* error should never happen */
+ return 1;
+ }
if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) {
if (remove(srcFileName))
EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno));
}
- return 0;
+ return result;
}