Merge "Fix the OOME in ScannerParseLargeFileBenchmarkTest"
diff --git a/expectations/knownfailures.txt b/expectations/knownfailures.txt
index 698ebc5..01991b0 100644
--- a/expectations/knownfailures.txt
+++ b/expectations/knownfailures.txt
@@ -1374,6 +1374,11 @@
]
},
{
+ description: "ScannerParseLargeFileBenchmark can cause a failure due to a timeout",
+ bug: 14865710,
+ name: "org.apache.harmony.tests.java.util.ScannerParseLargeFileBenchmarkTest"
+},
+{
description: "Known failure in GregorianCalendarTest",
bug: 12778197,
name: "org.apache.harmony.tests.java.util.GregorianCalendarTest#test_computeTime"
diff --git a/harmony-tests/src/test/java/org/apache/harmony/tests/java/util/ScannerParseLargeFileBenchmarkTest.java b/harmony-tests/src/test/java/org/apache/harmony/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
index 4b0d1ea..c0f9e58 100644
--- a/harmony-tests/src/test/java/org/apache/harmony/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
+++ b/harmony-tests/src/test/java/org/apache/harmony/tests/java/util/ScannerParseLargeFileBenchmarkTest.java
@@ -24,11 +24,11 @@
public class ScannerParseLargeFileBenchmarkTest extends TestCase {
/**
- * This test will check when parse a large file like more than 200M bytes if
- * the Scanner will exhaust all heap memory
+ * Check whether the Scanner will exhaust all heap memory when parsing a
+ * large file.
*/
public void testParseLargeFile() throws Exception {
- MyReader reader = new MyReader();
+ FakeLargeFile reader = new FakeLargeFile();
String delimiter = "\r?\n";
Scanner scanner = new Scanner(reader).useDelimiter(delimiter);
@@ -39,14 +39,9 @@
reader.close();
}
- private static class MyReader extends Reader {
- static final char[] CONTENT = "large file!\n".toCharArray();
-
- static long fileLength = (8 << 21) * 12;
-
- static boolean first = true;
-
- static int position = 0;
+ private static class FakeLargeFile extends Reader {
+ private static final char[] CONTENT = "large file!\n".toCharArray();
+ private static final int FILE_LENGTH = 192 * 1024 * 1024; // 192 MB
private int count = 0;
@@ -55,22 +50,24 @@
}
@Override
- public int read(char[] buf, int offset, int length) {
- if (count >= fileLength) {
+ public int read(char[] buffer, int offset, int length) {
+ if (count >= FILE_LENGTH) {
return -1;
}
- if (first == true) {
- position = 0;
- first = false;
- }
- for (int i = offset; i < length; i++) {
- buf[i] = CONTENT[(i + position) % CONTENT.length];
- count++;
- }
- position = (length + position) % CONTENT.length;
-
- return length - offset;
+ final int charsToRead = Math.min(FILE_LENGTH - count, length);
+ int bufferIndex = offset;
+ int contentIndex = count % CONTENT.length;
+ int charsRead = 0;
+ while (charsRead < charsToRead) {
+ buffer[bufferIndex++] = CONTENT[contentIndex++];
+ if (contentIndex == CONTENT.length) {
+ contentIndex = 0;
+ }
+ charsRead++;
+ }
+ count += charsRead;
+ return charsToRead;
}
}
}
diff --git a/luni/src/main/java/java/util/Scanner.java b/luni/src/main/java/java/util/Scanner.java
index 7d504b7..7d0e795 100644
--- a/luni/src/main/java/java/util/Scanner.java
+++ b/luni/src/main/java/java/util/Scanner.java
@@ -159,12 +159,15 @@
if (charsetName == null) {
throw new IllegalArgumentException("charsetName == null");
}
+
+ InputStreamReader streamReader;
try {
- setInput(new InputStreamReader(fis, charsetName));
+ streamReader = new InputStreamReader(fis, charsetName);
} catch (UnsupportedEncodingException e) {
IoUtils.closeQuietly(fis);
throw new IllegalArgumentException(e.getMessage());
}
+ initialize(streamReader);
}
/**
@@ -174,7 +177,7 @@
* the string to be scanned.
*/
public Scanner(String src) {
- setInput(new StringReader(src));
+ initialize(new StringReader(src));
}
/**
@@ -203,11 +206,14 @@
if (src == null) {
throw new NullPointerException("src == null");
}
+
+ InputStreamReader streamReader;
try {
- setInput(new InputStreamReader(src, charsetName));
+ streamReader = new InputStreamReader(src, charsetName);
} catch (UnsupportedEncodingException e) {
throw new IllegalArgumentException(e.getMessage());
}
+ initialize(streamReader);
}
/**
@@ -220,7 +226,7 @@
if (src == null) {
throw new NullPointerException("src == null");
}
- setInput(src);
+ initialize(src);
}
/**
@@ -252,13 +258,14 @@
if (charsetName == null) {
throw new IllegalArgumentException("charsetName == null");
}
- setInput(Channels.newReader(src, charsetName));
+ initialize(Channels.newReader(src, charsetName));
}
- private void setInput(Readable input) {
+ private void initialize(Readable input) {
this.input = input;
- buffer.limit(0);
- matcher = delimiter.matcher(buffer);
+ matcher = delimiter.matcher("");
+ matcher.useTransparentBounds(true);
+ matcher.useAnchoringBounds(false);
}
/**
@@ -535,7 +542,7 @@
checkOpen();
checkNotNull(pattern);
matchSuccessful = false;
- saveCurrentStatus();
+ prepareForScan();
// if the next token exists, set the match region, otherwise return
// false
if (!setTokenRegion()) {
@@ -790,7 +797,7 @@
* @throws IllegalStateException if this {@code Scanner} is closed.
*/
public boolean hasNextLine() {
- saveCurrentStatus();
+ prepareForScan();
String result = findWithinHorizon(LINE_PATTERN, 0);
recoverPreviousStatus();
return result != null;
@@ -954,7 +961,7 @@
checkOpen();
checkNotNull(pattern);
matchSuccessful = false;
- saveCurrentStatus();
+ prepareForScan();
if (!setTokenRegion()) {
recoverPreviousStatus();
// if setting match region fails
@@ -1204,7 +1211,7 @@
Pattern floatPattern = getFloatPattern();
String floatString = next(floatPattern);
floatString = removeLocaleInfoFromFloat(floatString);
- double doubleValue = 0;
+ double doubleValue;
try {
doubleValue = Double.parseDouble(floatString);
} catch (NumberFormatException e) {
@@ -1248,7 +1255,7 @@
Pattern floatPattern = getFloatPattern();
String floatString = next(floatPattern);
floatString = removeLocaleInfoFromFloat(floatString);
- float floatValue = 0;
+ float floatValue;
try {
floatValue = Float.parseFloat(floatString);
} catch (NumberFormatException e) {
@@ -1310,7 +1317,7 @@
Pattern integerPattern = getIntegerPattern(radix);
String intString = next(integerPattern);
intString = removeLocaleInfo(intString, int.class);
- int intValue = 0;
+ int intValue;
try {
intValue = Integer.parseInt(intString, radix);
} catch (NumberFormatException e) {
@@ -1340,7 +1347,7 @@
matcher.usePattern(LINE_PATTERN);
matcher.region(findStartIndex, bufferLength);
- String result = null;
+ String result;
while (true) {
if (matcher.find()) {
if (inputExhausted || matcher.end() != bufferLength
@@ -1422,7 +1429,7 @@
Pattern integerPattern = getIntegerPattern(radix);
String intString = next(integerPattern);
intString = removeLocaleInfo(intString, int.class);
- long longValue = 0;
+ long longValue;
try {
longValue = Long.parseLong(intString, radix);
} catch (NumberFormatException e) {
@@ -1484,7 +1491,7 @@
Pattern integerPattern = getIntegerPattern(radix);
String intString = next(integerPattern);
intString = removeLocaleInfo(intString, int.class);
- short shortValue = 0;
+ short shortValue;
try {
shortValue = Short.parseShort(intString, radix);
} catch (NumberFormatException e) {
@@ -1662,23 +1669,46 @@
}
/*
- * Change the matcher's string after reading input
+ * Change the matcher's input after modifying the contents of the buffer.
+ * The current implementation of Matcher causes a copy of the buffer to be taken.
*/
private void resetMatcher() {
- if (matcher == null) {
- matcher = delimiter.matcher(buffer);
- } else {
- matcher.reset(buffer);
- }
- matcher.useTransparentBounds(true);
- matcher.useAnchoringBounds(false);
+ matcher.reset(buffer);
matcher.region(findStartIndex, bufferLength);
}
/*
- * Save the matcher's last find position
+ * Recover buffer space for characters that are already processed and save the matcher's state
+ * in case parsing fails. See recoverPrevousState. This method must be called before
+ * any buffer offsets are calculated.
*/
- private void saveCurrentStatus() {
+ private void prepareForScan() {
+ // Compacting the buffer recovers space taken by already processed characters. This does not
+ // prevent the buffer growing in all situations but keeps the buffer small when delimiters
+ // exist regularly.
+ if (findStartIndex >= buffer.capacity() / 2) {
+ // When over half the buffer is filled with characters no longer being considered by the
+ // scanner we take the cost of compacting the buffer.
+
+ // Move all characters from [findStartIndex, findStartIndex + remaining()) to
+ // [0, remaining()).
+ int oldPosition = buffer.position();
+ buffer.position(findStartIndex);
+ buffer.compact();
+ buffer.position(oldPosition);
+
+ // Update Scanner state to reflect the new buffer state.
+ bufferLength -= findStartIndex;
+ findStartIndex = 0;
+ preStartIndex = -1;
+
+ // The matcher must also be informed that the buffer has changed because it operates on
+ // a String copy.
+ resetMatcher();
+ }
+
+ // Save the matcher's last find position so it can be returned to if the next token cannot
+ // be parsed.
preStartIndex = findStartIndex;
}
@@ -1822,7 +1852,7 @@
boolean negative = removeLocaleSign(tokenBuilder);
// Remove group separator
String groupSeparator = String.valueOf(dfs.getGroupingSeparator());
- int separatorIndex = -1;
+ int separatorIndex;
while ((separatorIndex = tokenBuilder.indexOf(groupSeparator)) != -1) {
tokenBuilder.delete(separatorIndex, separatorIndex + 1);
}
@@ -1909,9 +1939,9 @@
*/
private boolean setTokenRegion() {
// The position where token begins
- int tokenStartIndex = 0;
+ int tokenStartIndex;
// The position where token ends
- int tokenEndIndex = 0;
+ int tokenEndIndex;
// Use delimiter pattern
matcher.usePattern(delimiter);
matcher.region(findStartIndex, bufferLength);
@@ -1945,8 +1975,7 @@
if (matcher.find()) {
findComplete = true;
// If just delimiter remains
- if (matcher.start() == findStartIndex
- && matcher.end() == bufferLength) {
+ if (matcher.start() == findStartIndex && matcher.end() == bufferLength) {
// If more input resource exists
if (!inputExhausted) {
readMore();
@@ -1964,7 +1993,7 @@
}
}
tokenStartIndex = matcher.end();
- findStartIndex = matcher.end();
+ findStartIndex = tokenStartIndex;
return tokenStartIndex;
}
@@ -1984,7 +2013,7 @@
setSuccess = true;
}
// If the first delimiter of scanner is not at the find start position
- if (-1 != findIndex && preStartIndex != matcher.start()) {
+ if (findIndex != -1 && preStartIndex != matcher.start()) {
tokenStartIndex = preStartIndex;
tokenEndIndex = matcher.start();
findStartIndex = matcher.start();
@@ -1996,7 +2025,7 @@
}
private int findDelimiterAfter() {
- int tokenEndIndex = 0;
+ int tokenEndIndex;
boolean findComplete = false;
while (!findComplete) {
if (matcher.find()) {
@@ -2014,7 +2043,7 @@
}
}
tokenEndIndex = matcher.start();
- findStartIndex = matcher.start();
+ findStartIndex = tokenEndIndex;
return tokenEndIndex;
}
@@ -2032,7 +2061,7 @@
}
// Read input resource
- int readCount = 0;
+ int readCount;
try {
buffer.limit(buffer.capacity());
buffer.position(oldBufferLength);