Removed duplication between ZipArchiveOutputStream and StreamCompressor
Reatained all compatibility in ZipArchiveOutputStream
Made StreamCompressor and all instantiation of this package private.
git-svn-id: https://svn.apache.org/repos/asf/commons/proper/compress/trunk@1648585 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java
index 9a01a96..9c5c7ab 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java
@@ -21,9 +21,7 @@
import org.apache.commons.compress.utils.BoundedInputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Queue;
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java b/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java
index b5e53a2..26ee7aa 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java
@@ -17,6 +17,7 @@
*/
package org.apache.commons.compress.archivers.zip;
+import java.io.Closeable;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InputStream;
@@ -30,7 +31,7 @@
*
* @since 1.10
*/
-public abstract class StreamCompressor {
+abstract class StreamCompressor implements Closeable {
/*
* Apparently Deflater.setInput gets slowed down a lot on Sun JVMs
@@ -45,27 +46,26 @@
private final CRC32 crc = new CRC32();
- int writtenToOutputStream = 0;
- int sourcePayloadLength = 0;
- long actualCrc;
+ private long writtenToOutputStream = 0;
+ private long sourcePayloadLength = 0;
+ private long totalWrittenToOutputStream = 0;
private final int bufferSize = 4096;
private final byte[] outputBuffer = new byte[bufferSize];
private final byte[] readerBuf = new byte[bufferSize];
- protected StreamCompressor(Deflater deflater) {
+ StreamCompressor(Deflater deflater) {
this.def = deflater;
}
/**
* Create a stream compressor with the given compression level.
*
- * @param compressionLevel The #Deflater compression level
- * @param os The #OutputStream stream to receive output
+ * @param os The #OutputStream stream to receive output
+ * @param deflater The deflater to use
* @return A stream compressor
*/
- public static StreamCompressor create(int compressionLevel, OutputStream os) {
- final Deflater deflater = new Deflater(compressionLevel, true);
+ static StreamCompressor create(OutputStream os, Deflater deflater) {
return new OutputStreamCompressor(deflater, os);
}
@@ -75,19 +75,18 @@
* @param os The #OutputStream stream to receive output
* @return A stream compressor
*/
- public static StreamCompressor create( OutputStream os) {
- return create(Deflater.DEFAULT_COMPRESSION, os);
+ static StreamCompressor create(OutputStream os) {
+ return create(os, new Deflater(Deflater.DEFAULT_COMPRESSION, true));
}
/**
* Create a stream compressor with the given compression level.
*
- * @param compressionLevel The #Deflater compression level
- * @param os The #DataOutput to receive output
+ * @param os The #DataOutput to receive output
+ * @param deflater The deflater to use for the compressor
* @return A stream compressor
*/
- public static StreamCompressor create(int compressionLevel, DataOutput os) {
- final Deflater deflater = new Deflater(compressionLevel, true);
+ static StreamCompressor create(DataOutput os, Deflater deflater) {
return new DataOutputCompressor(deflater, os);
}
@@ -95,7 +94,7 @@
* Create a stream compressor with the given compression level.
*
* @param compressionLevel The #Deflater compression level
- * @param bs The #ScatterGatherBackingStore to receive output
+ * @param bs The #ScatterGatherBackingStore to receive output
* @return A stream compressor
*/
public static StreamCompressor create(int compressionLevel, ScatterGatherBackingStore bs) {
@@ -109,37 +108,51 @@
* @param bs The #ScatterGatherBackingStore to receive output
* @return A stream compressor
*/
- public static StreamCompressor create( ScatterGatherBackingStore bs) {
+ public static StreamCompressor create(ScatterGatherBackingStore bs) {
return create(Deflater.DEFAULT_COMPRESSION, bs);
}
/**
* The crc32 of the last deflated file
+ *
* @return the crc32
*/
public long getCrc32() {
- return actualCrc;
+ return crc.getValue();
}
/**
* Return the number of bytes read from the source stream
+ *
* @return The number of bytes read, never negative
*/
- public int getBytesRead() {
+ public long getBytesRead() {
return sourcePayloadLength;
}
/**
* The number of bytes written to the output
+ *
* @return The number of bytes, never negative
*/
- public int getBytesWritten() {
+ public long getBytesWritten() {
return writtenToOutputStream;
}
/**
+ * The total number of bytes written to the output for all files
+ *
+ * @return The number of bytes, never negative
+ */
+ public long getTotalBytesWritten() {
+ return totalWrittenToOutputStream;
+ }
+
+
+ /**
* Deflate the given source using the supplied compression method
+ *
* @param source The source to compress
* @param method The #ZipArchiveEntry compression method
* @throws IOException When failures happen
@@ -149,39 +162,56 @@
reset();
int length;
- while(( length = source.read(readerBuf, 0, readerBuf.length)) >= 0){
- crc.update(readerBuf, 0, length);
- if (method == ZipArchiveEntry.DEFLATED) {
- writeDeflated(readerBuf, 0, length);
- } else {
- writeOut(readerBuf, 0, length);
- writtenToOutputStream += length;
- }
- sourcePayloadLength += length;
+ while ((length = source.read(readerBuf, 0, readerBuf.length)) >= 0) {
+ write(readerBuf, 0, length, method);
}
if (method == ZipArchiveEntry.DEFLATED) {
flushDeflater();
}
- actualCrc = crc.getValue();
-
-
}
- private void reset(){
+ /**
+ * Writes bytes to ZIP entry.
+ *
+ * @param b the byte array to write
+ * @param offset the start position to write from
+ * @param length the number of bytes to write
+ * @param method the comrpession method to use
+ * @return the number of bytes written to the stream this time
+ * @throws IOException on error
+ */
+ long write(byte[] b, int offset, int length, int method) throws IOException {
+ long current = writtenToOutputStream;
+ crc.update(b, offset, length);
+ if (method == ZipArchiveEntry.DEFLATED) {
+ writeDeflated(b, offset, length);
+ } else {
+ writeCounted(b, offset, length);
+ }
+ sourcePayloadLength += length;
+ return writtenToOutputStream - current;
+ }
+
+
+ void reset() {
crc.reset();
def.reset();
sourcePayloadLength = 0;
writtenToOutputStream = 0;
}
- private void flushDeflater() throws IOException {
+ public void close() throws IOException {
+ def.end();
+ }
+
+ void flushDeflater() throws IOException {
def.finish();
while (!def.finished()) {
deflate();
}
}
- private void writeDeflated(byte[]b, int offset, int length)
+ private void writeDeflated(byte[] b, int offset, int length)
throws IOException {
if (length > 0 && !def.finished()) {
if (length <= DEFLATER_BLOCK_SIZE) {
@@ -212,12 +242,21 @@
private void deflate() throws IOException {
int len = def.deflate(outputBuffer, 0, outputBuffer.length);
if (len > 0) {
- writeOut(outputBuffer, 0, len);
- writtenToOutputStream += len;
+ writeCounted(outputBuffer, 0, len);
}
}
- protected abstract void writeOut(byte[] data, int offset, int length) throws IOException ;
+ public void writeCounted(byte[] data) throws IOException {
+ writeCounted(data, 0, data.length);
+ }
+
+ public void writeCounted(byte[] data, int offset, int length) throws IOException {
+ writeOut(data, offset, length);
+ writtenToOutputStream += length;
+ totalWrittenToOutputStream += length;
+ }
+
+ protected abstract void writeOut(byte[] data, int offset, int length) throws IOException;
private static final class ScatterGatherBackingStoreCompressor extends StreamCompressor {
private final ScatterGatherBackingStore bs;
@@ -229,7 +268,7 @@
protected final void writeOut(byte[] data, int offset, int length)
throws IOException {
- bs.writeOut(data, offset, length);
+ bs.writeOut(data, offset, length);
}
}
@@ -243,12 +282,13 @@
protected final void writeOut(byte[] data, int offset, int length)
throws IOException {
- os.write(data, offset, length);
+ os.write(data, offset, length);
}
}
private static final class DataOutputCompressor extends StreamCompressor {
private final DataOutput raf;
+
public DataOutputCompressor(Deflater deflater, DataOutput raf) {
super(deflater);
this.raf = raf;
diff --git a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
index fe2e519..86f91ea 100644
--- a/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
+++ b/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java
@@ -28,7 +28,6 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.zip.CRC32;
import java.util.zip.Deflater;
import java.util.zip.ZipException;
@@ -181,15 +180,7 @@
private final List<ZipArchiveEntry> entries =
new LinkedList<ZipArchiveEntry>();
- /**
- * CRC instance to avoid parsing DEFLATED data twice.
- */
- private final CRC32 crc = new CRC32();
-
- /**
- * Count the bytes written to out.
- */
- private long written = 0;
+ private final StreamCompressor streamCompressor;
/**
* Start of central directory.
@@ -235,18 +226,12 @@
private ZipEncoding zipEncoding =
ZipEncodingHelper.getZipEncoding(DEFAULT_ENCODING);
+
/**
* This Deflater object is used for output.
*
*/
- protected final Deflater def = new Deflater(level, true);
-
- /**
- * This buffer serves as a Deflater.
- *
- */
- private final byte[] buf = new byte[BUFFER_SIZE];
-
+ protected final Deflater def;
/**
* Optional random access output.
*/
@@ -286,6 +271,8 @@
public ZipArchiveOutputStream(OutputStream out) {
this.out = out;
this.raf = null;
+ def = new Deflater(level, true);
+ streamCompressor = StreamCompressor.create(out, def);
}
/**
@@ -305,6 +292,8 @@
_raf = null;
o = new FileOutputStream(file);
}
+ def = new Deflater(level, true);
+ streamCompressor = StreamCompressor.create(_raf, def);
out = o;
raf = _raf;
}
@@ -441,16 +430,16 @@
throw new IOException("This archive contains unclosed entries.");
}
- cdOffset = written;
+ cdOffset = streamCompressor.getTotalBytesWritten();
for (ZipArchiveEntry ze : entries) {
writeCentralFileHeader(ze);
}
- cdLength = written - cdOffset;
+ cdLength = streamCompressor.getTotalBytesWritten() - cdOffset;
writeZip64CentralDirectory();
writeCentralDirectoryEnd();
offsets.clear();
entries.clear();
- def.end();
+ streamCompressor.close();
finished = true;
}
@@ -467,11 +456,11 @@
flushDeflater();
- long bytesWritten = written - entry.dataStart;
- long realCrc = crc.getValue();
- crc.reset();
-
+ long bytesWritten = streamCompressor.getTotalBytesWritten() - entry.dataStart;
+ long realCrc = streamCompressor.getCrc32();
+ entry.bytesRead = streamCompressor.getBytesRead();
doCloseEntry(realCrc, bytesWritten);
+ streamCompressor.reset();
}
/**
@@ -484,16 +473,24 @@
*/
private void closeCopiedEntry() throws IOException {
preClose();
- long realCrc = entry.entry.getCrc();
entry.bytesRead = entry.entry.getSize();
- doCloseEntry(realCrc, entry.entry.getCompressedSize());
+ doCloseCopiedEntry();
}
private void doCloseEntry(long realCrc, long bytesWritten) throws IOException {
final Zip64Mode effectiveMode = getEffectiveZip64Mode(entry.entry);
- final boolean actuallyNeedsZip64 =
- handleSizesAndCrc(bytesWritten, realCrc, effectiveMode);
+ final boolean actuallyNeedsZip64 = handleSizesAndCrc(bytesWritten, realCrc, effectiveMode);
+ closeEntry(actuallyNeedsZip64);
+ }
+
+ private void doCloseCopiedEntry() throws IOException {
+ Zip64Mode effectiveMode = getEffectiveZip64Mode(entry.entry);
+ boolean actuallyNeedsZip64 = checkIfNeedsZip64(effectiveMode);
+ closeEntry(actuallyNeedsZip64);
+ }
+
+ private void closeEntry(boolean actuallyNeedsZip64) throws IOException {
if (raf != null) {
rewriteSizesAndCrc(actuallyNeedsZip64);
}
@@ -538,10 +535,7 @@
*/
private void flushDeflater() throws IOException {
if (entry.entry.getMethod() == DEFLATED) {
- def.finish();
- while (!def.finished()) {
- deflate();
- }
+ streamCompressor.flushDeflater();
}
}
@@ -563,7 +557,6 @@
entry.entry.setCompressedSize(bytesWritten);
entry.entry.setCrc(crc);
- def.reset();
} else if (raf == null) {
if (entry.entry.getCrc() != crc) {
throw new ZipException("bad CRC checksum for entry "
@@ -586,16 +579,25 @@
entry.entry.setCrc(crc);
}
+ return checkIfNeedsZip64(effectiveMode);
+ }
+
+ /**
+ * Ensures the current entry's size and CRC information is set to
+ * the values just written, verifies it isn't too big in the
+ * Zip64Mode.Never case and returns whether the entry would
+ * require a Zip64 extra field.
+ */
+ private boolean checkIfNeedsZip64(Zip64Mode effectiveMode)
+ throws ZipException {
final boolean actuallyNeedsZip64 = effectiveMode == Zip64Mode.Always
- || entry.entry.getSize() >= ZIP64_MAGIC
- || entry.entry.getCompressedSize() >= ZIP64_MAGIC;
+ || entry.entry.getSize() >= ZIP64_MAGIC
+ || entry.entry.getCompressedSize() >= ZIP64_MAGIC;
if (actuallyNeedsZip64 && effectiveMode == Zip64Mode.Never) {
- throw new Zip64RequiredException(Zip64RequiredException
- .getEntryTooBigMessage(entry.entry));
+ throw new Zip64RequiredException(Zip64RequiredException.getEntryTooBigMessage(entry.entry));
}
return actuallyNeedsZip64;
}
-
/**
* When using random access output, write the local file header
* and potentiall the ZIP64 extra containing the correct CRC and
@@ -692,7 +694,7 @@
def.setLevel(level);
hasCompressionLevelChanged = false;
}
- writeLocalFileHeader((ZipArchiveEntry)archiveEntry);
+ writeLocalFileHeader((ZipArchiveEntry) archiveEntry);
}
/**
@@ -828,17 +830,23 @@
throw new IllegalStateException("No current entry");
}
ZipUtil.checkRequestedFeatures(entry.entry);
- entry.hasWritten = true;
- if (entry.entry.getMethod() == DEFLATED) {
- writeDeflated(b, offset, length);
- } else {
- writeOut(b, offset, length);
- written += length;
- }
- crc.update(b, offset, length);
- count(length);
+ long writtenThisTime = streamCompressor.write(b, offset, length, entry.entry.getMethod());
+ count(writtenThisTime);
}
+ /**
+ * Write bytes to output or random access file.
+ * @param data the byte array to write
+ * @throws IOException on error
+ */
+ private void writeCounted(byte[] data) throws IOException {
+ streamCompressor.writeCounted(data);
+ }
+
+
+
+
+
private void copyFromZipInputStream(InputStream src) throws IOException {
if (entry == null) {
throw new IllegalStateException("No current entry");
@@ -846,41 +854,11 @@
ZipUtil.checkRequestedFeatures(entry.entry);
entry.hasWritten = true;
byte[] tmpBuf = new byte[4096];
- int length = src.read( tmpBuf );
- while ( length >= 0 )
+ int length;
+ while ((length = src.read( tmpBuf )) >= 0 )
{
- writeOut( tmpBuf, 0, length );
- written += length;
- crc.update( tmpBuf, 0, length );
-
+ streamCompressor.writeCounted(tmpBuf, 0, length);
count( length );
- length = src.read( tmpBuf );
- }
- }
-
- /**
- * write implementation for DEFLATED entries.
- */
- private void writeDeflated(byte[]b, int offset, int length)
- throws IOException {
- if (length > 0 && !def.finished()) {
- entry.bytesRead += length;
- if (length <= DEFLATER_BLOCK_SIZE) {
- def.setInput(b, offset, length);
- deflateUntilInputIsNeeded();
- } else {
- final int fullblocks = length / DEFLATER_BLOCK_SIZE;
- for (int i = 0; i < fullblocks; i++) {
- def.setInput(b, offset + i * DEFLATER_BLOCK_SIZE,
- DEFLATER_BLOCK_SIZE);
- deflateUntilInputIsNeeded();
- }
- final int done = fullblocks * DEFLATER_BLOCK_SIZE;
- if (done < length) {
- def.setInput(b, offset + done, length - done);
- deflateUntilInputIsNeeded();
- }
- }
}
}
@@ -943,18 +921,6 @@
static final byte[] ZIP64_EOCD_LOC_SIG = ZipLong.getBytes(0X07064B50L);
/**
- * Writes next block of compressed data to the output stream.
- * @throws IOException on error
- */
- protected final void deflate() throws IOException {
- int len = def.deflate(buf, 0, buf.length);
- if (len > 0) {
- writeOut(buf, 0, len);
- written += len;
- }
- }
-
- /**
* Writes the local file header entry
* @param ze the entry to write
* @throws IOException on error
@@ -968,11 +934,11 @@
}
final byte[] localHeader = createLocalFileHeader(ze, name, encodable);
- offsets.put(ze, written);
- entry.localDataStart = written + 14; // Right before crc
- writeOut( localHeader);
- written += localHeader.length;
- entry.dataStart = written;
+ long localHeaderStart = streamCompressor.getTotalBytesWritten();
+ offsets.put(ze, localHeaderStart);
+ entry.localDataStart = localHeaderStart + LFH_CRC_OFFSET; // At crc offset
+ writeCounted(localHeader);
+ entry.dataStart = streamCompressor.getTotalBytesWritten();
}
@@ -1084,18 +1050,15 @@
if (ze.getMethod() != DEFLATED || raf != null) {
return;
}
- writeOut(DD_SIG);
- writeOut(ZipLong.getBytes(ze.getCrc()));
- int sizeFieldSize = WORD;
+ writeCounted(DD_SIG);
+ writeCounted(ZipLong.getBytes(ze.getCrc()));
if (!hasZip64Extra(ze)) {
- writeOut(ZipLong.getBytes(ze.getCompressedSize()));
- writeOut(ZipLong.getBytes(ze.getSize()));
+ writeCounted(ZipLong.getBytes(ze.getCompressedSize()));
+ writeCounted(ZipLong.getBytes(ze.getSize()));
} else {
- sizeFieldSize = DWORD;
- writeOut(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
- writeOut(ZipEightByteInteger.getBytes(ze.getSize()));
+ writeCounted(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
+ writeCounted(ZipEightByteInteger.getBytes(ze.getSize()));
}
- written += 2 * WORD + 2 * sizeFieldSize;
}
/**
@@ -1125,10 +1088,8 @@
handleZip64Extra(ze, lfhOffset, needsZip64Extra);
- byte[] centralFileHeader = createCentralFileHeader(ze, getName(ze), lfhOffset,
- needsZip64Extra);
- writeOut(centralFileHeader);
- written += centralFileHeader.length;
+ byte[] centralFileHeader = createCentralFileHeader(ze, getName(ze), lfhOffset, needsZip64Extra);
+ writeCounted(centralFileHeader);
}
/**
@@ -1249,11 +1210,11 @@
* and {@link Zip64Mode #setUseZip64} is {@link Zip64Mode#Never}.
*/
protected void writeCentralDirectoryEnd() throws IOException {
- writeOut(EOCD_SIG);
+ writeCounted(EOCD_SIG);
// disk numbers
- writeOut(ZERO);
- writeOut(ZERO);
+ writeCounted(ZERO);
+ writeCounted(ZERO);
// number of entries
int numberOfEntries = entries.size();
@@ -1269,18 +1230,17 @@
byte[] num = ZipShort.getBytes(Math.min(numberOfEntries,
ZIP64_MAGIC_SHORT));
- writeOut(num);
- writeOut(num);
+ writeCounted(num);
+ writeCounted(num);
// length and location of CD
- writeOut(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC)));
- writeOut(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC)));
+ writeCounted(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC)));
+ writeCounted(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC)));
// ZIP file comment
ByteBuffer data = this.zipEncoding.encode(comment);
- writeOut(ZipShort.getBytes(data.limit()));
- writeOut(data.array(), data.arrayOffset(),
- data.limit() - data.position());
+ writeCounted(ZipShort.getBytes(data.limit()));
+ streamCompressor.writeCounted(data.array(), data.arrayOffset(), data.limit() - data.position());
}
private static final byte[] ONE = ZipLong.getBytes(1L);
@@ -1307,7 +1267,7 @@
return;
}
- long offset = written;
+ long offset = streamCompressor.getBytesWritten();
writeOut(ZIP64_EOCD_SIG);
// size, we don't have any variable length as we don't support
@@ -1359,9 +1319,10 @@
* @throws IOException on error
*/
protected final void writeOut(byte[] data) throws IOException {
- writeOut(data, 0, data.length);
+ streamCompressor.writeOut(data, 0, data.length);
}
+
/**
* Write bytes to output or random access file.
* @param data the byte array to write
@@ -1370,19 +1331,10 @@
* @throws IOException on error
*/
protected final void writeOut(byte[] data, int offset, int length)
- throws IOException {
- if (raf != null) {
- raf.write(data, offset, length);
- } else {
- out.write(data, offset, length);
- }
+ throws IOException {
+ streamCompressor.writeOut(data, offset, length);
}
- private void deflateUntilInputIsNeeded() throws IOException {
- while (!def.needsInput()) {
- deflate();
- }
- }
private GeneralPurposeBit getGeneralPurposeBits(final int zipMethod, final boolean utfFallback) {
GeneralPurposeBit b = new GeneralPurposeBit();