Merge pull request #380 from rfoltyns/feature/filtering_parser_match_count
#208 FilteringParserDelegate match count support
diff --git a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java
index 3f2abae..b686ba9 100644
--- a/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java
+++ b/src/main/java/com/fasterxml/jackson/core/base/ParserMinimalBase.java
@@ -36,6 +36,7 @@
protected final static int INT_LCURLY = '{';
protected final static int INT_RCURLY = '}';
protected final static int INT_QUOTE = '"';
+ protected final static int INT_APOS = '\'';
protected final static int INT_BACKSLASH = '\\';
protected final static int INT_SLASH = '/';
protected final static int INT_COLON = ':';
diff --git a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java
index b810a21..56a0b20 100644
--- a/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java
+++ b/src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java
@@ -1982,7 +1982,7 @@
protected String _handleOddName(int ch) throws IOException
{
// First: may allow single quotes
- if (ch == '\'' && isEnabled(Feature.ALLOW_SINGLE_QUOTES)) {
+ if (ch == INT_APOS && isEnabled(Feature.ALLOW_SINGLE_QUOTES)) {
return _parseAposName();
}
// Allow unquoted names if feature enabled:
@@ -2059,7 +2059,7 @@
}
}
int ch = _inputBuffer[_inputPtr++] & 0xFF;
- if (ch == '\'') { // special case, ''
+ if (ch == INT_APOS) { // special case, ''
return "";
}
int[] quads = _quadBuffer;
@@ -2072,7 +2072,7 @@
final int[] codes = _icLatin1;
while (true) {
- if (ch == '\'') {
+ if (ch == INT_APOS) {
break;
}
// additional check to skip handling of double-quotes
@@ -2658,7 +2658,7 @@
}
while (_inputPtr < max) {
c = (int) inputBuffer[_inputPtr++] & 0xFF;
- if (c == '\'' || codes[c] != 0) {
+ if (c == INT_APOS || codes[c] != 0) {
break ascii_loop;
}
outBuf[outPtr++] = (char) c;
@@ -2666,7 +2666,7 @@
}
// Ok: end marker, escape or multi-byte?
- if (c == '\'') {
+ if (c == INT_APOS) {
break main_loop;
}
diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java
index 846463c..37b687f 100644
--- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java
+++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java
@@ -48,21 +48,6 @@
/*
/**********************************************************************
- /* Location tracking, additional
- /**********************************************************************
- */
-
- /**
- * Alternate row tracker, used to keep track of position by `\r` marker
- * (whereas <code>_currInputRow</code> tracks `\n`). Used to simplify
- * tracking of linefeeds, assuming that input typically uses various
- * linefeed combinations (`\r`, `\n` or `\r\n`) consistently, in which
- * case we can simply choose max of two row candidates.
- */
- protected int _currInputRowAlt = 1;
-
- /*
- /**********************************************************************
/* Life-cycle
/**********************************************************************
*/
@@ -106,6 +91,9 @@
// Time to update pointers first
_currInputProcessed += _origBufferLen;
+ // Also need to adjust row start, to work as if it extended into the past wrt new buffer
+ _currInputRowStart = start - (_inputEnd - _currInputRowStart);
+
// And then update buffer settings
_inputBuffer = buf;
_inputPtr = start;
@@ -143,7 +131,15 @@
}
return avail;
}
-
+
+ // Should never be called: can not be implemented quite as expected
+ // due to non-blocking behavior
+ @Override
+ protected char _decodeEscaped() throws IOException {
+ VersionUtil.throwInternal();
+ return ' ';
+ }
+
/*
/**********************************************************************
/* Main-level decoding
@@ -229,6 +225,10 @@
return _parseEscapedName(_quadLength, _pending32, _pendingBytes);
case MINOR_FIELD_NAME_ESCAPE:
return _finishFieldWithEscape();
+ case MINOR_FIELD_APOS_NAME:
+ return _finishAposName(_quadLength, _pending32, _pendingBytes);
+ case MINOR_FIELD_UNQUOTED_NAME:
+ return _finishUnquotedName(_quadLength, _pending32, _pendingBytes);
// Value states
@@ -248,10 +248,12 @@
case MINOR_VALUE_TOKEN_ERROR: // case of "almost token", just need tokenize for error
return _finishErrorToken();
- case MINOR_NUMBER_LEADING_MINUS:
- return _finishNumberLeadingMinus(_inputBuffer[_inputPtr++] & 0xFF);
- case MINOR_NUMBER_LEADING_ZERO:
+ case MINOR_NUMBER_MINUS:
+ return _finishNumberMinus(_inputBuffer[_inputPtr++] & 0xFF);
+ case MINOR_NUMBER_ZERO:
return _finishNumberLeadingZeroes();
+ case MINOR_NUMBER_MINUSZERO:
+ return _finishNumberLeadingNegZeroes();
case MINOR_NUMBER_INTEGER_DIGITS:
return _finishNumberIntegralPart(_textBuffer.getBufferWithoutReset(),
_textBuffer.getCurrentSegmentSize());
@@ -266,16 +268,25 @@
return _finishRegularString();
case MINOR_VALUE_STRING_UTF8_2:
_textBuffer.append((char) _decodeUTF8_2(_pending32, _inputBuffer[_inputPtr++]));
+ if (_minorStateAfterSplit == MINOR_VALUE_APOS_STRING) {
+ return _finishAposString();
+ }
return _finishRegularString();
case MINOR_VALUE_STRING_UTF8_3:
if (!_decodeSplitUTF8_3(_pending32, _pendingBytes, _inputBuffer[_inputPtr++])) {
return JsonToken.NOT_AVAILABLE;
}
+ if (_minorStateAfterSplit == MINOR_VALUE_APOS_STRING) {
+ return _finishAposString();
+ }
return _finishRegularString();
case MINOR_VALUE_STRING_UTF8_4:
if (!_decodeSplitUTF8_4(_pending32, _pendingBytes, _inputBuffer[_inputPtr++])) {
return JsonToken.NOT_AVAILABLE;
}
+ if (_minorStateAfterSplit == MINOR_VALUE_APOS_STRING) {
+ return _finishAposString();
+ }
return _finishRegularString();
case MINOR_VALUE_STRING_ESCAPE:
@@ -286,7 +297,13 @@
}
_textBuffer.append((char) c);
}
+ if (_minorStateAfterSplit == MINOR_VALUE_APOS_STRING) {
+ return _finishAposString();
+ }
return _finishRegularString();
+
+ case MINOR_VALUE_APOS_STRING:
+ return _finishAposString();
}
VersionUtil.throwInternal();
return null;
@@ -320,7 +337,10 @@
return _finishErrorTokenWithEOF();
// Number-parsing states; valid stopping points, more explicit errors
- case MINOR_NUMBER_LEADING_ZERO:
+ case MINOR_NUMBER_ZERO:
+ case MINOR_NUMBER_MINUSZERO:
+ // NOTE: does NOT retain possible leading minus-sign (can change if
+ // absolutely needs be)
return _valueCompleteInt(0, "0");
case MINOR_NUMBER_INTEGER_DIGITS:
// Fine: just need to ensure we have value fully defined
@@ -426,7 +446,7 @@
// it is not allowed per se, it may be erroneously used,
// and could be indicate by a more specific error message.
case '0':
- return _startLeadingZero();
+ return _startNumberLeadingZero();
case '1':
case '2':
case '3':
@@ -504,7 +524,7 @@
// it is not allowed per se, it may be erroneously used,
// and could be indicate by a more specific error message.
case '0':
- return _startLeadingZero();
+ return _startNumberLeadingZero();
case '1':
case '2': case '3':
@@ -581,7 +601,7 @@
// it is not allowed per se, it may be erroneously used,
// and could be indicate by a more specific error message.
case '0':
- return _startLeadingZero();
+ return _startNumberLeadingZero();
case '1':
case '2': case '3':
@@ -808,20 +828,18 @@
protected JsonToken _startPositiveNumber(int ch) throws IOException
{
_numberNegative = false;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ outBuf[0] = (char) ch;
// in unlikely event of not having more input, denote location
if (_inputPtr >= _inputEnd) {
_minorState = MINOR_NUMBER_INTEGER_DIGITS;
- _textBuffer.emptyAndGetCurrentSegment();
- _textBuffer.append((char) ch);
+ _textBuffer.setCurrentLength(1);
return (_currToken = JsonToken.NOT_AVAILABLE);
}
- char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
- outBuf[0] = (char) ch;
- ch = _inputBuffer[_inputPtr];
int outPtr = 1;
- ch &= 0xFF;
+ ch = _inputBuffer[_inputPtr] & 0xFF;
while (true) {
if (ch < INT_0) {
if (ch == INT_PERIOD) {
@@ -856,17 +874,18 @@
_textBuffer.setCurrentLength(outPtr);
return _valueComplete(JsonToken.VALUE_NUMBER_INT);
}
-
+
protected JsonToken _startNegativeNumber() throws IOException
{
+ _numberNegative = true;
if (_inputPtr >= _inputEnd) {
- _minorState = MINOR_NUMBER_LEADING_MINUS;
+ _minorState = MINOR_NUMBER_MINUS;
return (_currToken = JsonToken.NOT_AVAILABLE);
}
int ch = _inputBuffer[_inputPtr++] & 0xFF;
if (ch <= INT_0) {
if (ch == INT_0) {
- return _startLeadingZero();
+ return _finishNumberLeadingNegZeroes();
}
// One special case: if first char is 0, must not be followed by a digit
reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
@@ -875,7 +894,6 @@
reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
}
char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
- _numberNegative = true;
outBuf[0] = '-';
outBuf[1] = (char) ch;
if (_inputPtr >= _inputEnd) {
@@ -921,11 +939,11 @@
return _valueComplete(JsonToken.VALUE_NUMBER_INT);
}
- protected JsonToken _startLeadingZero() throws IOException
+ protected JsonToken _startNumberLeadingZero() throws IOException
{
int ptr = _inputPtr;
if (ptr >= _inputEnd) {
- _minorState = MINOR_NUMBER_LEADING_ZERO;
+ _minorState = MINOR_NUMBER_ZERO;
return (_currToken = JsonToken.NOT_AVAILABLE);
}
@@ -935,21 +953,13 @@
int ch = _inputBuffer[ptr++] & 0xFF;
// one early check: leading zeroes may or may not be allowed
- if (ch <= INT_0) {
+ if (ch < INT_0) {
if (ch == INT_PERIOD) {
_inputPtr = ptr;
_intLength = 1;
char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
outBuf[0] = '0';
- outBuf[1] = '.';
- return _startFloat(outBuf, 2, ch);
- }
- // although not guaranteed, seems likely valid separator (white space,
- // comma, end bracket/curly); next time token needed will verify
- if (ch == INT_0) {
- _inputPtr = ptr;
- _minorState = MINOR_NUMBER_LEADING_ZERO;
- return _finishNumberLeadingZeroes();
+ return _startFloat(outBuf, 1, ch);
}
} else if (ch > INT_9) {
if (ch == INT_e || ch == INT_E) {
@@ -957,8 +967,7 @@
_intLength = 1;
char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
outBuf[0] = '0';
- outBuf[1] = (char) ch;
- return _startFloat(outBuf, 2, ch);
+ return _startFloat(outBuf, 1, ch);
}
// Ok; unfortunately we have closing bracket/curly that are valid so need
// (colon not possible since this is within value, not after key)
@@ -967,43 +976,107 @@
reportUnexpectedNumberChar(ch,
"expected digit (0-9), decimal point (.) or exponent indicator (e/E) to follow '0'");
}
+ } else { // leading zero case (zero followed by a digit)
+ // leave inputPtr as is (i.e. "push back" digit)
+ return _finishNumberLeadingZeroes();
}
// leave _inputPtr as-is, to push back byte we checked
return _valueCompleteInt(0, "0");
}
+ protected JsonToken _finishNumberMinus(int ch) throws IOException
+ {
+ if (ch <= INT_0) {
+ if (ch == INT_0) {
+ return _finishNumberLeadingNegZeroes();
+ }
+ reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
+ } else if (ch > INT_9) {
+ // !!! TODO: -Infinite etc
+ reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
+ }
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ outBuf[0] = '-';
+ outBuf[1] = (char) ch;
+ _intLength = 1;
+ return _finishNumberIntegralPart(outBuf, 2);
+ }
+
protected JsonToken _finishNumberLeadingZeroes() throws IOException
{
// In general, skip further zeroes (if allowed), look for legal follow-up
// numeric characters; likely legal separators, or, known illegal (letters).
-
while (true) {
if (_inputPtr >= _inputEnd) {
+ _minorState = MINOR_NUMBER_ZERO;
return (_currToken = JsonToken.NOT_AVAILABLE);
}
int ch = _inputBuffer[_inputPtr++] & 0xFF;
- if (ch <= INT_0) {
+ if (ch < INT_0) {
if (ch == INT_PERIOD) {
- _intLength = 1;
char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
outBuf[0] = '0';
- outBuf[1] = '.';
- return _startFloat(outBuf, 2, ch);
- }
- // although not guaranteed, seems likely valid separator (white space,
- // comma, end bracket/curly); next time token needed will verify
- if (ch == INT_0) {
- if (!isEnabled(Feature.ALLOW_NUMERIC_LEADING_ZEROS)) {
- reportInvalidNumber("Leading zeroes not allowed");
- }
- continue;
+ _intLength = 1;
+ return _startFloat(outBuf, 1, ch);
}
} else if (ch > INT_9) {
if (ch == INT_e || ch == INT_E) {
- _intLength = 1;
char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
outBuf[0] = '0';
- outBuf[1] = (char) ch;
+ _intLength = 1;
+ return _startFloat(outBuf, 1, ch);
+ }
+ // Ok; unfortunately we have closing bracket/curly that are valid so need
+ // (colon not possible since this is within value, not after key)
+ //
+ if ((ch != INT_RBRACKET) && (ch != INT_RCURLY)) {
+ reportUnexpectedNumberChar(ch,
+ "expected digit (0-9), decimal point (.) or exponent indicator (e/E) to follow '0'");
+ }
+ } else { // Number between 0 and 9
+ // although not guaranteed, seems likely valid separator (white space,
+ // comma, end bracket/curly); next time token needed will verify
+ if (!isEnabled(Feature.ALLOW_NUMERIC_LEADING_ZEROS)) {
+ reportInvalidNumber("Leading zeroes not allowed");
+ }
+ if (ch == INT_0) { // coalesce multiple leading zeroes into just one
+ continue;
+ }
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ // trim out leading zero
+ outBuf[0] = (char) ch;
+ _intLength = 1;
+ return _finishNumberIntegralPart(outBuf, 1);
+ }
+ --_inputPtr;
+ return _valueCompleteInt(0, "0");
+ }
+ }
+
+ protected JsonToken _finishNumberLeadingNegZeroes() throws IOException
+ {
+ // In general, skip further zeroes (if allowed), look for legal follow-up
+ // numeric characters; likely legal separators, or, known illegal (letters).
+ while (true) {
+ if (_inputPtr >= _inputEnd) {
+ _minorState = MINOR_NUMBER_MINUSZERO;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+ int ch = _inputBuffer[_inputPtr++] & 0xFF;
+ if (ch < INT_0) {
+ if (ch == INT_PERIOD) {
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ outBuf[0] = '-';
+ outBuf[1] = '0';
+ _intLength = 1;
+ return _startFloat(outBuf, 2, ch);
+ }
+ } else if (ch > INT_9) {
+ if (ch == INT_e || ch == INT_E) {
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ outBuf[0] = '-';
+ outBuf[1] = '0';
+ _intLength = 1;
return _startFloat(outBuf, 2, ch);
}
// Ok; unfortunately we have closing bracket/curly that are valid so need
@@ -1014,42 +1087,33 @@
"expected digit (0-9), decimal point (.) or exponent indicator (e/E) to follow '0'");
}
} else { // Number between 1 and 9; go integral
- --_inputPtr;
- _minorState = MINOR_NUMBER_INTEGER_DIGITS;
- return _finishNumberIntegralPart(_textBuffer.emptyAndGetCurrentSegment(), 0);
+ // although not guaranteed, seems likely valid separator (white space,
+ // comma, end bracket/curly); next time token needed will verify
+ if (!isEnabled(Feature.ALLOW_NUMERIC_LEADING_ZEROS)) {
+ reportInvalidNumber("Leading zeroes not allowed");
+ }
+ if (ch == INT_0) { // coalesce multiple leading zeroes into just one
+ continue;
+ }
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ // trim out leading zero
+ outBuf[0] = '-';
+ outBuf[1] = (char) ch;
+ _intLength = 1;
+ return _finishNumberIntegralPart(outBuf, 2);
}
--_inputPtr;
return _valueCompleteInt(0, "0");
}
}
- protected JsonToken _finishNumberLeadingMinus(int ch) throws IOException
- {
- if (ch <= INT_0) {
- if (ch == INT_0) {
- return _startLeadingZero();
- }
- // One special case: if first char is 0, must not be followed by a digit
- reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
- } else if (ch > INT_9) {
- // !!! TODO: -Infinite etc
- reportUnexpectedNumberChar(ch, "expected digit (0-9) to follow minus sign, for valid numeric value");
- }
- char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
- _numberNegative = true;
- outBuf[0] = '-';
- outBuf[1] = (char) ch;
- _intLength = 1;
- _minorState = MINOR_NUMBER_INTEGER_DIGITS;
- return _finishNumberIntegralPart(outBuf, 2);
- }
-
protected JsonToken _finishNumberIntegralPart(char[] outBuf, int outPtr) throws IOException
{
int negMod = _numberNegative ? -1 : 0;
while (true) {
if (_inputPtr >= _inputEnd) {
+ _minorState = MINOR_NUMBER_INTEGER_DIGITS;
_textBuffer.setCurrentLength(outPtr);
return (_currToken = JsonToken.NOT_AVAILABLE);
}
@@ -1087,11 +1151,11 @@
{
int fractLen = 0;
if (ch == INT_PERIOD) {
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.expandCurrentSegment();
+ }
+ outBuf[outPtr++] = '.';
while (true) {
- if (outPtr >= outBuf.length) {
- outBuf = _textBuffer.expandCurrentSegment();
- }
- outBuf[outPtr++] = (char) ch;
if (_inputPtr >= _inputEnd) {
_textBuffer.setCurrentLength(outPtr);
_minorState = MINOR_NUMBER_FRACTION_DIGITS;
@@ -1107,6 +1171,10 @@
}
break;
}
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.expandCurrentSegment();
+ }
+ outBuf[outPtr++] = (char) ch;
++fractLen;
}
}
@@ -1135,7 +1203,7 @@
_expLength = 0;
return (_currToken = JsonToken.NOT_AVAILABLE);
}
- ch = _inputBuffer[_inputPtr];
+ ch = _inputBuffer[_inputPtr++];
}
while (ch >= INT_0 && ch <= INT_9) {
++expLen;
@@ -1225,7 +1293,7 @@
_expLength = 0;
return JsonToken.NOT_AVAILABLE;
}
- ch = _inputBuffer[_inputPtr];
+ ch = _inputBuffer[_inputPtr++];
}
}
@@ -1503,7 +1571,7 @@
* and hence is offlined to a separate method.
*/
private final JsonToken _parseEscapedName(int qlen, int currQuad, int currQuadBytes)
- throws IOException
+ throws IOException
{
// This may seem weird, but here we do not want to worry about
// UTF-8 decoding yet. Rather, we'll assume that part is ok (if not it will get
@@ -1548,6 +1616,7 @@
ch = _decodeCharEscape();
if (ch < 0) { // method has set up state about escape sequence
_minorState = MINOR_FIELD_NAME_ESCAPE;
+ _minorStateAfterSplit = MINOR_FIELD_NAME;
_quadLength = qlen;
_pending32 = currQuad;
_pendingBytes = currQuadBytes;
@@ -1623,7 +1692,7 @@
// First: may allow single quotes
if (ch == '\'') {
if (isEnabled(Feature.ALLOW_SINGLE_QUOTES)) {
- return _parseAposName();
+ return _finishAposName(0, 0, 0);
}
} else if (ch == ']') { // for better error reporting...
return _closeArrayScope();
@@ -1643,14 +1712,35 @@
_reportUnexpectedChar(ch, "was expecting either valid name character (for unquoted name) or double-quote (for quoted) to start field name");
}
+ return _finishUnquotedName(0, ch, 1);
+ }
+
+ /**
+ * Parsing of optionally supported non-standard "unquoted" names: names without
+ * either double-quotes or apostrophes surrounding them.
+ * Unlike other
+ */
+ private JsonToken _finishUnquotedName(int qlen, int currQuad, int currQuadBytes)
+ throws IOException
+ {
+ int[] quads = _quadBuffer;
+ final int[] codes = CharTypes.getInputCodeUtf8JsNames();
+
// Ok, now; instead of ultra-optimizing parsing here (as with regular JSON names),
// let's just use the generic "slow" variant. Can measure its impact later on if need be.
- int[] quads = _quadBuffer;
- int qlen = 0;
- int currQuad = 0;
- int currQuadBytes = 0;
-
while (true) {
+ if (_inputPtr >= _inputEnd) {
+ _quadLength = qlen;
+ _pending32 = currQuad;
+ _pendingBytes = currQuadBytes;
+ _minorState = MINOR_FIELD_UNQUOTED_NAME;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+ int ch = _inputBuffer[_inputPtr] & 0xFF;
+ if (codes[ch] != 0) {
+ break;
+ }
+ ++_inputPtr;
// Ok, we have one more byte to add at any rate:
if (currQuadBytes < 4) {
++currQuadBytes;
@@ -1663,16 +1753,6 @@
currQuad = ch;
currQuadBytes = 1;
}
- if (_inputPtr >= _inputEnd) {
- if (!_loadMore()) {
- _reportInvalidEOF(" in field name", JsonToken.FIELD_NAME);
- }
- }
- ch = _inputBuffer[_inputPtr] & 0xFF;
- if (codes[ch] != 0) {
- break;
- }
- ++_inputPtr;
}
if (currQuadBytes > 0) {
@@ -1688,33 +1768,22 @@
return _fieldComplete(name);
}
- /* Parsing to support [JACKSON-173]. Plenty of duplicated code;
- * main reason being to try to avoid slowing down fast path
- * for valid JSON -- more alternatives, more code, generally
- * bit slower execution.
- */
- private JsonToken _parseAposName() throws IOException
+ private JsonToken _finishAposName(int qlen, int currQuad, int currQuadBytes)
+ throws IOException
{
- if (_inputPtr >= _inputEnd) {
- if (!_loadMore()) {
- _reportInvalidEOF(": was expecting closing '\'' for field name", JsonToken.FIELD_NAME);
- }
- }
- int ch = _inputBuffer[_inputPtr++] & 0xFF;
- if (ch == '\'') { // special case, ''
- return _fieldComplete("");
- }
int[] quads = _quadBuffer;
- int qlen = 0;
- int currQuad = 0;
- int currQuadBytes = 0;
-
- // Copied from parseEscapedFieldName, with minor mods:
-
final int[] codes = _icLatin1;
while (true) {
- if (ch == '\'') {
+ if (_inputPtr >= _inputEnd) {
+ _quadLength = qlen;
+ _pending32 = currQuad;
+ _pendingBytes = currQuadBytes;
+ _minorState = MINOR_FIELD_APOS_NAME;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+ int ch = _inputBuffer[_inputPtr++] & 0xFF;
+ if (ch == INT_APOS) {
break;
}
// additional check to skip handling of double-quotes
@@ -1725,6 +1794,14 @@
} else {
// Nope, escape sequence
ch = _decodeCharEscape();
+ if (ch < 0) { // method has set up state about escape sequence
+ _minorState = MINOR_FIELD_NAME_ESCAPE;
+ _minorStateAfterSplit = MINOR_FIELD_APOS_NAME;
+ _quadLength = qlen;
+ _pending32 = currQuad;
+ _pendingBytes = currQuadBytes;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
}
if (ch > 127) {
// Ok, we'll need room for first byte right away
@@ -1771,12 +1848,6 @@
currQuad = ch;
currQuadBytes = 1;
}
- if (_inputPtr >= _inputEnd) {
- if (!_loadMore()) {
- _reportInvalidEOF(" in field name", JsonToken.FIELD_NAME);
- }
- }
- ch = _inputBuffer[_inputPtr++] & 0xFF;
}
if (currQuadBytes > 0) {
@@ -1784,6 +1855,8 @@
_quadBuffer = quads = growArrayBy(quads, quads.length);
}
quads[qlen++] = _padLastQuad(currQuad, currQuadBytes);
+ } else if (qlen == 0) { // rare case but possible
+ return _fieldComplete("");
}
String name = _symbols.findName(quads, qlen);
if (name == null) {
@@ -1792,12 +1865,6 @@
return _fieldComplete(name);
}
- @Override
- protected char _decodeEscaped() throws IOException {
- VersionUtil.throwInternal();
- return ' ';
- }
-
protected final JsonToken _finishFieldWithEscape() throws IOException
{
// First: try finishing what wasn't yet:
@@ -1824,9 +1891,8 @@
// Second byte gets output below:
} else { // 3 bytes; no need to worry about surrogates here
currQuad = (currQuad << 8) | (0xe0 | (ch >> 12));
- ++currQuadBytes;
// need room for middle byte?
- if (currQuadBytes >= 4) {
+ if (++currQuadBytes >= 4) {
_quadBuffer[_quadLength++] = currQuad;
currQuad = 0;
currQuadBytes = 0;
@@ -1845,6 +1911,9 @@
currQuad = ch;
currQuadBytes = 1;
}
+ if (_minorStateAfterSplit == MINOR_FIELD_APOS_NAME) {
+ return _finishAposName(_quadLength, currQuad, currQuadBytes);
+ }
return _parseEscapedName(_quadLength, currQuad, currQuadBytes);
}
@@ -1920,12 +1989,6 @@
/**********************************************************************
*/
- protected JsonToken _startAposString() throws IOException
- {
- VersionUtil.throwInternal();
- return null;
- }
-
protected JsonToken _startString() throws IOException
{
int ptr = _inputPtr;
@@ -2001,6 +2064,7 @@
_inputPtr = ptr;
_textBuffer.setCurrentLength(outPtr);
if (!_decodeSplitMultiByte(c, codes[c], ptr < _inputEnd)) {
+ _minorStateAfterSplit = MINOR_VALUE_STRING;
return (_currToken = JsonToken.NOT_AVAILABLE);
}
outBuf = _textBuffer.getBufferWithoutReset();
@@ -2052,6 +2116,131 @@
}
}
+ protected JsonToken _startAposString() throws IOException
+ {
+ int ptr = _inputPtr;
+ int outPtr = 0;
+ char[] outBuf = _textBuffer.emptyAndGetCurrentSegment();
+ final int[] codes = _icUTF8;
+
+ final int max = Math.min(_inputEnd, (ptr + outBuf.length));
+ final byte[] inputBuffer = _inputBuffer;
+ while (ptr < max) {
+ int c = (int) inputBuffer[ptr] & 0xFF;
+ if (c == INT_APOS) {
+ _inputPtr = ptr+1;
+ _textBuffer.setCurrentLength(outPtr);
+ return _valueComplete(JsonToken.VALUE_STRING);
+ }
+
+ if (codes[c] != 0) {
+ break;
+ }
+ ++ptr;
+ outBuf[outPtr++] = (char) c;
+ }
+ _textBuffer.setCurrentLength(outPtr);
+ _inputPtr = ptr;
+ return _finishAposString();
+ }
+
+ private final JsonToken _finishAposString() throws IOException
+ {
+ int c;
+ final int[] codes = _icUTF8;
+ final byte[] inputBuffer = _inputBuffer;
+
+ char[] outBuf = _textBuffer.getBufferWithoutReset();
+ int outPtr = _textBuffer.getCurrentSegmentSize();
+ int ptr = _inputPtr;
+ final int safeEnd = _inputEnd - 5; // longest escape is 6 chars
+
+ main_loop:
+ while (true) {
+ ascii_loop:
+ while (true) {
+ if (ptr >= _inputEnd) {
+ _inputPtr = ptr;
+ _minorState = MINOR_VALUE_APOS_STRING;
+ _textBuffer.setCurrentLength(outPtr);
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ final int max = Math.min(_inputEnd, (ptr + (outBuf.length - outPtr)));
+ while (ptr < max) {
+ c = inputBuffer[ptr++] & 0xFF;
+ if ((codes[c] != 0) && (c != INT_QUOTE)) {
+ break ascii_loop;
+ }
+ if (c == INT_APOS) {
+ _inputPtr = ptr;
+ _textBuffer.setCurrentLength(outPtr);
+ return _valueComplete(JsonToken.VALUE_STRING);
+ }
+ outBuf[outPtr++] = (char) c;
+ }
+ }
+ // Escape or multi-byte?
+ // If possibly split, use off-lined longer version
+ if (ptr >= safeEnd) {
+ _inputPtr = ptr;
+ _textBuffer.setCurrentLength(outPtr);
+ if (!_decodeSplitMultiByte(c, codes[c], ptr < _inputEnd)) {
+ _minorStateAfterSplit = MINOR_VALUE_APOS_STRING;
+ return (_currToken = JsonToken.NOT_AVAILABLE);
+ }
+ outBuf = _textBuffer.getBufferWithoutReset();
+ outPtr = _textBuffer.getCurrentSegmentSize();
+ ptr = _inputPtr;
+ continue main_loop;
+ }
+ // otherwise use inlined
+ switch (codes[c]) {
+ case 1: // backslash
+ _inputPtr = ptr;
+ c = _decodeFastCharEscape(); // since we know it's not split
+ ptr = _inputPtr;
+ break;
+ case 2: // 2-byte UTF
+ c = _decodeUTF8_2(c, _inputBuffer[ptr++]);
+ break;
+ case 3: // 3-byte UTF
+ c = _decodeUTF8_3(c, _inputBuffer[ptr++], _inputBuffer[ptr++]);
+ break;
+ case 4: // 4-byte UTF
+ c = _decodeUTF8_4(c, _inputBuffer[ptr++], _inputBuffer[ptr++],
+ _inputBuffer[ptr++]);
+ // Let's add first part right away:
+ outBuf[outPtr++] = (char) (0xD800 | (c >> 10));
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ c = 0xDC00 | (c & 0x3FF);
+ // And let the other char output down below
+ break;
+ default:
+ if (c < INT_SPACE) {
+ // Note: call can now actually return (to allow unquoted linefeeds)
+ _throwUnquotedSpace(c, "string value");
+ } else {
+ // Is this good enough error message?
+ _reportInvalidChar(c);
+ }
+ }
+ // Need more room?
+ if (outPtr >= outBuf.length) {
+ outBuf = _textBuffer.finishCurrentSegment();
+ outPtr = 0;
+ }
+ // Ok, let's add char to output:
+ outBuf[outPtr++] = (char) c;
+ }
+ }
+
private final boolean _decodeSplitMultiByte(int c, int type, boolean gotNext)
throws IOException
{
@@ -2296,10 +2485,4 @@
/* Internal methods, other
/**********************************************************************
*/
- // !!! TODO: only temporarily here
-
- private final boolean _loadMore() {
- VersionUtil.throwInternal();
- return false;
- }
}
diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java
index 29626b3..35d0643 100644
--- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java
+++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java
@@ -79,6 +79,9 @@
// encountered either just backslash, or backslash and 'u' and 0 - 3 hex digits,
protected final static int MINOR_FIELD_NAME_ESCAPE = 11;
+ protected final static int MINOR_FIELD_APOS_NAME = 12;
+ protected final static int MINOR_FIELD_UNQUOTED_NAME = 13;
+
protected final static int MINOR_VALUE_LEADING_WS = 14;
protected final static int MINOR_VALUE_LEADING_COMMA = 15;
protected final static int MINOR_VALUE_LEADING_COLON = 16;
@@ -87,9 +90,10 @@
protected final static int MINOR_VALUE_TOKEN_TRUE = 18;
protected final static int MINOR_VALUE_TOKEN_FALSE = 19;
- protected final static int MINOR_NUMBER_LEADING_MINUS = 20;
- protected final static int MINOR_NUMBER_LEADING_ZERO = 21;
- protected final static int MINOR_NUMBER_INTEGER_DIGITS = 22;
+ protected final static int MINOR_NUMBER_MINUS = 20;
+ protected final static int MINOR_NUMBER_ZERO = 21; // zero as first, possibly trimming multiple
+ protected final static int MINOR_NUMBER_MINUSZERO = 22; // "-0" (and possibly more zeroes) receive
+ protected final static int MINOR_NUMBER_INTEGER_DIGITS = 23;
protected final static int MINOR_NUMBER_FRACTION_DIGITS = 24;
protected final static int MINOR_NUMBER_EXPONENT_MARKER = 25;
@@ -100,6 +104,7 @@
protected final static int MINOR_VALUE_STRING_UTF8_2 = 32;
protected final static int MINOR_VALUE_STRING_UTF8_3 = 33;
protected final static int MINOR_VALUE_STRING_UTF8_4 = 34;
+ protected final static int MINOR_VALUE_APOS_STRING = 35;
/**
* Special state at which point decoding of a non-quoted token has encountered
@@ -145,21 +150,27 @@
*/
/**
- * Current main decoding state
+ * Current main decoding state within logical tree
*/
protected int _majorState;
/**
- * Addition indicator within state; contextually relevant for just that state
- */
- protected int _minorState;
-
- /**
* Value of {@link #_majorState} after completing a scalar value
*/
protected int _majorStateAfterValue;
/**
+ * Additional indicator within state; contextually relevant for just that state
+ */
+ protected int _minorState;
+
+ /**
+ * Secondary minor state indicator used during decoding of escapes and/or
+ * multi-byte Unicode characters
+ */
+ protected int _minorStateAfterSplit;
+
+ /**
* Flag that is sent when calling application indicates that there will
* be no more input to parse.
*/
@@ -167,6 +178,21 @@
/*
/**********************************************************************
+ /* Location tracking, additional
+ /**********************************************************************
+ */
+
+ /**
+ * Alternate row tracker, used to keep track of position by `\r` marker
+ * (whereas <code>_currInputRow</code> tracks `\n`). Used to simplify
+ * tracking of linefeeds, assuming that input typically uses various
+ * linefeed combinations (`\r`, `\n` or `\r\n`) consistently, in which
+ * case we can simply choose max of two row candidates.
+ */
+ protected int _currInputRowAlt = 1;
+
+ /*
+ /**********************************************************************
/* Life-cycle
/**********************************************************************
*/
@@ -236,7 +262,6 @@
// 30-May-2017, tatu: Seems like this is the most certain way to prevent
// further decoding... not the optimal place, but due to inheritance
// hierarchy most convenient.
- _inputPtr = 0;
_inputEnd = 0;
}
@@ -261,6 +286,17 @@
return false;
}
+ @Override
+ public JsonLocation getCurrentLocation()
+ {
+ int col = _inputPtr - _currInputRowStart + 1; // 1-based
+ // Since we track CR and LF separately, max should gives us right answer
+ int row = Math.max(_currInputRow, _currInputRowAlt);
+ return new JsonLocation(_getSourceReference(),
+ _currInputProcessed + _inputPtr, -1L, // bytes, chars
+ row, col);
+ }
+
/*
/**********************************************************************
/* Public API, access to token information, text
@@ -751,7 +787,7 @@
protected final void _updateLocation()
{
- _tokenInputRow = _currInputRow;
+ _tokenInputRow = Math.max(_currInputRow, _currInputRowAlt);
final int ptr = _inputPtr;
_tokenInputTotal = _currInputProcessed + ptr;
_tokenInputCol = ptr - _currInputRowStart;
diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/package-info.java b/src/main/java/com/fasterxml/jackson/core/json/async/package-info.java
new file mode 100644
index 0000000..00db974
--- /dev/null
+++ b/src/main/java/com/fasterxml/jackson/core/json/async/package-info.java
@@ -0,0 +1,6 @@
+/**
+ * Non-blocking ("async") JSON parser implementation.
+ *
+ * @since 2.9
+ */
+package com.fasterxml.jackson.core.json.async;
diff --git a/src/main/java/com/fasterxml/jackson/core/sym/ByteQuadsCanonicalizer.java b/src/main/java/com/fasterxml/jackson/core/sym/ByteQuadsCanonicalizer.java
index b758a43..2f9b925 100644
--- a/src/main/java/com/fasterxml/jackson/core/sym/ByteQuadsCanonicalizer.java
+++ b/src/main/java/com/fasterxml/jackson/core/sym/ByteQuadsCanonicalizer.java
@@ -557,13 +557,16 @@
* that contains actual quads.
*/
if (qlen < 4) { // another sanity check
- if (qlen == 3) {
+ switch (qlen) {
+ case 3:
return findName(q[0], q[1], q[2]);
- }
- if (qlen == 2) {
+ case 2:
return findName(q[0], q[1]);
+ case 1:
+ return findName(q[0]);
+ default: // if 0 ever passed
+ return "";
}
- return findName(q[0]);
}
final int hash = calcHash(q, qlen);
int offset = _calcOffset(hash);
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncCharEscapingTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncCharEscapingTest.java
index e886f60..9bb61a9 100644
--- a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncCharEscapingTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncCharEscapingTest.java
@@ -101,4 +101,3 @@
r.close();
}
}
-
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncFieldNamesTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncFieldNamesTest.java
index 641bc88..e3796bb 100644
--- a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncFieldNamesTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncFieldNamesTest.java
@@ -10,6 +10,41 @@
{
private final JsonFactory JSON_F = new JsonFactory();
+ private final JsonFactory JSON_APOS_F = new JsonFactory();
+ {
+ JSON_APOS_F.enable(JsonParser.Feature.ALLOW_SINGLE_QUOTES);
+ }
+
+ // Mainly to test "fast" parse for shortish names
+ public void testSimpleFieldNames() throws IOException
+ {
+ for (String name : new String[] { "", "a", "ab", "abc", "abcd",
+ "abcd1", "abcd12", "abcd123", "abcd1234",
+ "abcd1234a", "abcd1234ab", "abcd1234abc", "abcd1234abcd",
+ "abcd1234abcd1"
+ }) {
+ _testSimpleFieldName(name);
+ }
+ }
+
+ private void _testSimpleFieldName(String fieldName) throws IOException
+ {
+ // use long buffer to ensure fast decoding may be used
+ AsyncReaderWrapper r = asyncForBytes(JSON_F, 99,
+ _jsonDoc(String.format("{\"%s\":true} \r", fieldName)),
+ 0);
+ assertNull(r.currentToken());
+ assertToken(JsonToken.START_OBJECT, r.nextToken());
+ assertToken(JsonToken.FIELD_NAME, r.nextToken());
+ assertEquals(fieldName, r.currentName());
+ assertToken(JsonToken.VALUE_TRUE, r.nextToken());
+ assertToken(JsonToken.END_OBJECT, r.nextToken());
+ assertNull(r.nextToken());
+ JsonLocation loc = r.parser().getCurrentLocation();
+ assertEquals(2, loc.getLineNr());
+ assertEquals(1, loc.getColumnNr());
+ }
+
public void testEscapedFieldNames() throws IOException
{
_testEscapedFieldNames("\\'foo\\'", "'foo'");
@@ -23,13 +58,17 @@
private void _testEscapedFieldNames(String nameEncoded, String nameExp) throws IOException
{
+ byte[] doc;
+ StringWriter w;
+
nameEncoded = aposToQuotes(nameEncoded);
nameExp = aposToQuotes(nameExp);
- StringWriter w = new StringWriter();
+
+ w = new StringWriter();
w.append("{\"");
w.append(nameEncoded);
w.append("\":true}");
- byte[] doc = w.toString().getBytes("UTF-8");
+ doc = w.toString().getBytes("UTF-8");
_testEscapedFieldNames(doc, nameExp, 0, 99);
_testEscapedFieldNames(doc, nameExp, 0, 5);
@@ -40,6 +79,22 @@
_testEscapedFieldNames(doc, nameExp, 1, 99);
_testEscapedFieldNames(doc, nameExp, 1, 3);
_testEscapedFieldNames(doc, nameExp, 1, 1);
+
+ w = new StringWriter();
+ w.append("{'");
+ w.append(nameEncoded);
+ w.append("':true}");
+ doc = w.toString().getBytes("UTF-8");
+
+ _testEscapedAposFieldNames(doc, nameExp, 0, 99);
+ _testEscapedAposFieldNames(doc, nameExp, 0, 5);
+ _testEscapedAposFieldNames(doc, nameExp, 0, 3);
+ _testEscapedAposFieldNames(doc, nameExp, 0, 2);
+ _testEscapedAposFieldNames(doc, nameExp, 0, 1);
+
+ _testEscapedAposFieldNames(doc, nameExp, 1, 99);
+ _testEscapedAposFieldNames(doc, nameExp, 1, 3);
+ _testEscapedAposFieldNames(doc, nameExp, 1, 1);
}
private void _testEscapedFieldNames(byte[] doc, String expName,
@@ -55,4 +110,18 @@
r.close();
assertNull(r.nextToken());
}
+
+ private void _testEscapedAposFieldNames(byte[] doc, String expName,
+ int offset, int readSize) throws IOException
+ {
+ AsyncReaderWrapper r = asyncForBytes(JSON_APOS_F, readSize, doc, offset);
+ assertNull(r.currentToken());
+ assertToken(JsonToken.START_OBJECT, r.nextToken());
+ assertToken(JsonToken.FIELD_NAME, r.nextToken());
+ assertEquals(expName, r.currentName());
+ assertToken(JsonToken.VALUE_TRUE, r.nextToken());
+
+ r.close();
+ assertNull(r.nextToken());
+ }
}
diff --git a/src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdParsingTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNonStdParsingTest.java
similarity index 65%
rename from src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdParsingTest.java
rename to src/test/java/com/fasterxml/jackson/core/json/async/AsyncNonStdParsingTest.java
index 492429b..c49be65 100644
--- a/src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdParsingTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNonStdParsingTest.java
@@ -1,4 +1,4 @@
-package com.fasterxml.jackson.failing.async;
+package com.fasterxml.jackson.core.json.async;
import java.io.IOException;
@@ -8,11 +8,13 @@
public class AsyncNonStdParsingTest extends AsyncTestBase
{
- public void testLargeUnquoted() throws Exception
+ public void testLargeUnquotedNames() throws Exception
{
+ JsonFactory f = new JsonFactory();
+ f.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+
StringBuilder sb = new StringBuilder(5000);
sb.append("[\n");
- //final int REPS = 2000;
final int REPS = 1050;
for (int i = 0; i < REPS; ++i) {
if (i > 0) {
@@ -27,12 +29,24 @@
sb.append("}\n");
}
sb.append("]");
- String JSON = sb.toString();
- JsonFactory f = new JsonFactory();
- f.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
- AsyncReaderWrapper p = createParser(f, JSON);
+ String doc = sb.toString();
+
+ _testLargeUnquoted(f, REPS, doc, 0, 99);
+ _testLargeUnquoted(f, REPS, doc, 0, 5);
+ _testLargeUnquoted(f, REPS, doc, 0, 3);
+ _testLargeUnquoted(f, REPS, doc, 0, 2);
+ _testLargeUnquoted(f, REPS, doc, 0, 1);
+
+ _testLargeUnquoted(f, REPS, doc, 1, 99);
+ _testLargeUnquoted(f, REPS, doc, 1, 1);
+ }
+
+ private void _testLargeUnquoted(JsonFactory f, int reps, String doc,
+ int offset, int readSize) throws Exception
+ {
+ AsyncReaderWrapper p = createParser(f, doc, offset, readSize);
assertToken(JsonToken.START_ARRAY, p.nextToken());
- for (int i = 0; i < REPS; ++i) {
+ for (int i = 0; i < reps; ++i) {
assertToken(JsonToken.START_OBJECT, p.nextToken());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
assertEquals("abc"+(i&127), p.currentName());
@@ -43,13 +57,27 @@
p.close();
}
- public void testSimpleUnquoted() throws Exception
+ public void testSimpleUnquotedNames() throws Exception
{
final JsonFactory f = new JsonFactory();
f.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
- String JSON = "{ a : 1, _foo:true, $:\"money!\", \" \":null }";
- AsyncReaderWrapper p = createParser(f, JSON);
+ _testSimpleUnquoted(f, 0, 99);
+ _testSimpleUnquoted(f, 0, 5);
+ _testSimpleUnquoted(f, 0, 3);
+ _testSimpleUnquoted(f, 0, 2);
+ _testSimpleUnquoted(f, 0, 1);
+
+ _testSimpleUnquoted(f, 1, 99);
+ _testSimpleUnquoted(f, 1, 3);
+ _testSimpleUnquoted(f, 1, 1);
+ }
+
+ private void _testSimpleUnquoted(JsonFactory f,
+ int offset, int readSize) throws Exception
+ {
+ String doc = "{ a : 1, _foo:true, $:\"money!\", \" \":null }";
+ AsyncReaderWrapper p = createParser(f, doc, offset, readSize);
assertToken(JsonToken.START_OBJECT, p.nextToken());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
@@ -72,10 +100,9 @@
assertToken(JsonToken.END_OBJECT, p.nextToken());
p.close();
- // Another thing, as per [Issue#102]: numbers
+ // Another thing, as per [jackson-cre#102]: numbers
- JSON = "{ 123:true,4:false }";
- p = createParser(f, JSON);
+ p = createParser(f, "{ 123:true,4:false }", offset, readSize);
assertToken(JsonToken.START_OBJECT, p.nextToken());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
@@ -95,12 +122,24 @@
* accept single-quotes for String values (field names,
* textual values)
*/
- public void testSingleQuotesDefault() throws Exception
+ public void testAposQuotingDisabled() throws Exception
{
JsonFactory f = new JsonFactory();
+ _testSingleQuotesDefault(f, 0, 99);
+ _testSingleQuotesDefault(f, 0, 5);
+ _testSingleQuotesDefault(f, 0, 3);
+ _testSingleQuotesDefault(f, 0, 1);
+
+ _testSingleQuotesDefault(f, 1, 99);
+ _testSingleQuotesDefault(f, 1, 1);
+ }
+
+ private void _testSingleQuotesDefault(JsonFactory f,
+ int offset, int readSize) throws Exception
+ {
// First, let's see that by default they are not allowed
String JSON = "[ 'text' ]";
- AsyncReaderWrapper p = createParser(f, JSON);
+ AsyncReaderWrapper p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.START_ARRAY, p.nextToken());
try {
p.nextToken();
@@ -112,7 +151,7 @@
}
JSON = "{ 'a':1 }";
- p = createParser(f, JSON);
+ p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.START_OBJECT, p.nextToken());
try {
p.nextToken();
@@ -129,14 +168,31 @@
* single quotes, to allow handling invalid (but, alas, common)
* JSON.
*/
- public void testSingleQuotesEnabled() throws Exception
+ public void testAposQuotingEnabled() throws Exception
{
JsonFactory f = new JsonFactory();
f.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
- String JSON = "{ 'a' : 1, \"foobar\": 'b', '_abcde1234':'d', '\"' : '\"\"', '':'' }";
- AsyncReaderWrapper p = createParser(f, JSON);
+ _testAposQuotingEnabled(f, 0, 99);
+ _testAposQuotingEnabled(f, 0, 5);
+ _testAposQuotingEnabled(f, 0, 3);
+ _testAposQuotingEnabled(f, 0, 2);
+ _testAposQuotingEnabled(f, 0, 1);
+ _testAposQuotingEnabled(f, 1, 99);
+ _testAposQuotingEnabled(f, 2, 1);
+ _testAposQuotingEnabled(f, 1, 1);
+ }
+
+ private void _testAposQuotingEnabled(JsonFactory f,
+ int offset, int readSize) throws Exception
+ {
+ String UNINAME = String.format("Uni%c-key-%c", UNICODE_2BYTES, UNICODE_3BYTES);
+ String UNIVALUE = String.format("Uni%c-value-%c", UNICODE_3BYTES, UNICODE_2BYTES);
+ String JSON = String.format(
+ "{ 'a' : 1, \"foobar\": 'b', '_abcde1234':'d', '\"' : '\"\"', '':'', '%s':'%s'}",
+ UNINAME, UNIVALUE);
+ AsyncReaderWrapper p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.START_OBJECT, p.nextToken());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
@@ -154,19 +210,23 @@
assertToken(JsonToken.FIELD_NAME, p.nextToken());
assertEquals("\"", p.currentText());
assertToken(JsonToken.VALUE_STRING, p.nextToken());
- //assertEquals("\"\"", p.currentText());
+ assertEquals("\"\"", p.currentText());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
assertEquals("", p.currentText());
assertToken(JsonToken.VALUE_STRING, p.nextToken());
assertEquals("", p.currentText());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals(UNINAME, p.currentText());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(UNIVALUE, p.currentText());
+
assertToken(JsonToken.END_OBJECT, p.nextToken());
p.close();
-
- JSON = "{'b':1,'array':[{'b':3}],'ob':{'b':4,'x':0,'y':3,'a':false }}";
- p = createParser(f, JSON);
+ JSON = "{'b':1,'array':[{'b':3}],'ob':{'b':4,'x':0,'y':'"+UNICODE_SEGMENT+"','a':false }}";
+ p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.START_OBJECT, p.nextToken());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
assertEquals("b", p.currentName());
@@ -194,8 +254,8 @@
assertEquals(0, p.getIntValue());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
assertEquals("y", p.currentName());
- assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
- assertEquals(3, p.getIntValue());
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals(UNICODE_SEGMENT, p.currentText());
assertToken(JsonToken.FIELD_NAME, p.nextToken());
assertEquals("a", p.currentName());
assertToken(JsonToken.VALUE_FALSE, p.nextToken());
@@ -211,8 +271,20 @@
JsonFactory f = new JsonFactory();
f.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true);
+ _testSingleQuotesEscaped(f, 0, 99);
+ _testSingleQuotesEscaped(f, 0, 5);
+ _testSingleQuotesEscaped(f, 0, 3);
+ _testSingleQuotesEscaped(f, 0, 1);
+
+ _testSingleQuotesEscaped(f, 1, 99);
+ _testSingleQuotesEscaped(f, 1, 1);
+ }
+
+ private void _testSingleQuotesEscaped(JsonFactory f,
+ int offset, int readSize) throws Exception
+ {
String JSON = "[ '16\\'' ]";
- AsyncReaderWrapper p = createParser(f, JSON);
+ AsyncReaderWrapper p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.START_ARRAY, p.nextToken());
assertToken(JsonToken.VALUE_STRING, p.nextToken());
@@ -220,15 +292,28 @@
assertToken(JsonToken.END_ARRAY, p.nextToken());
p.close();
}
-
+
public void testNonStandardNameChars() throws Exception
{
JsonFactory f = new JsonFactory();
f.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+
+ _testNonStandardNameChars(f, 0, 99);
+ _testNonStandardNameChars(f, 0, 6);
+ _testNonStandardNameChars(f, 0, 3);
+ _testNonStandardNameChars(f, 0, 1);
+
+ _testNonStandardNameChars(f, 1, 99);
+ _testNonStandardNameChars(f, 2, 1);
+ }
+
+ private void _testNonStandardNameChars(JsonFactory f,
+ int offset, int readSize) throws Exception
+ {
String JSON = "{ @type : \"mytype\", #color : 123, *error* : true, "
+" hyphen-ated : \"yes\", me+my : null"
+"}";
- AsyncReaderWrapper p = createParser(f, JSON);
+ AsyncReaderWrapper p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.START_OBJECT, p.nextToken());
@@ -259,13 +344,25 @@
p.close();
}
- public void testNonStandarBackslashQuoting(int mode) throws Exception
+ public void testNonStandarBackslashQuotingForValues(int mode) throws Exception
+ {
+ _testNonStandarBackslashQuoting(0, 99);
+ _testNonStandarBackslashQuoting(0, 6);
+ _testNonStandarBackslashQuoting(0, 3);
+ _testNonStandarBackslashQuoting(0, 1);
+
+ _testNonStandarBackslashQuoting(2, 99);
+ _testNonStandarBackslashQuoting(1, 1);
+ }
+
+ private void _testNonStandarBackslashQuoting(
+ int offset, int readSize) throws Exception
{
// first: verify that we get an exception
JsonFactory f = new JsonFactory();
assertFalse(f.isEnabled(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER));
final String JSON = quote("\\'");
- AsyncReaderWrapper p = createParser(f, JSON);
+ AsyncReaderWrapper p = createParser(f, JSON, offset, readSize);
try {
p.nextToken();
p.currentText();
@@ -278,14 +375,15 @@
// and then verify it's ok...
f.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true);
assertTrue(f.isEnabled(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER));
- p = createParser(f, JSON);
+ p = createParser(f, JSON, offset, readSize);
assertToken(JsonToken.VALUE_STRING, p.nextToken());
assertEquals("'", p.currentText());
p.close();
}
- private AsyncReaderWrapper createParser(JsonFactory f, String doc) throws IOException
+ private AsyncReaderWrapper createParser(JsonFactory f, String doc,
+ int offset, int readSize) throws IOException
{
- return asyncForBytes(f, 1, _jsonDoc(doc), 1);
+ return asyncForBytes(f, readSize, _jsonDoc(doc), offset);
}
}
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberCoercionTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberCoercionTest.java
index cc92576..6fe2693 100644
--- a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberCoercionTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberCoercionTest.java
@@ -46,6 +46,12 @@
assertEquals(2, p.getIntValue());
p.close();
+ p = createParser("0.1");
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(0.1, p.getDoubleValue());
+ assertEquals(0, p.getIntValue());
+ p.close();
+
// BigDecimal->int
p = createParser("10");
assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberLeadingZeroesTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberLeadingZeroesTest.java
new file mode 100644
index 0000000..768430e
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncNumberLeadingZeroesTest.java
@@ -0,0 +1,103 @@
+package com.fasterxml.jackson.core.json.async;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
+
+public class AsyncNumberLeadingZeroesTest extends AsyncTestBase
+{
+ public void testLeadingZeroesInt() throws Exception
+ {
+ _testLeadingZeroesInt("00003", 3);
+ _testLeadingZeroesInt("00003 ", 3);
+ _testLeadingZeroesInt(" 00003", 3);
+
+ _testLeadingZeroesInt("-00007", -7);
+ _testLeadingZeroesInt("-00007 ", -7);
+ _testLeadingZeroesInt(" -00007", -7);
+
+ _testLeadingZeroesInt("056", 56);
+ _testLeadingZeroesInt("056 ", 56);
+ _testLeadingZeroesInt(" 056", 56);
+
+ _testLeadingZeroesInt("-04", -4);
+ _testLeadingZeroesInt("-04 ", -4);
+ _testLeadingZeroesInt(" -04", -4);
+
+ _testLeadingZeroesInt("0"+Integer.MAX_VALUE, Integer.MAX_VALUE);
+ _testLeadingZeroesInt(" 0"+Integer.MAX_VALUE, Integer.MAX_VALUE);
+ _testLeadingZeroesInt("0"+Integer.MAX_VALUE+" ", Integer.MAX_VALUE);
+ }
+
+ public void _testLeadingZeroesInt(String valueStr, int value) throws Exception
+ {
+ // first: verify that we get an exception
+ JsonFactory f = new JsonFactory();
+ assertFalse(f.isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS));
+ String JSON = valueStr;
+ AsyncReaderWrapper p = createParser(f, JSON);
+ try {
+ p.nextToken();
+ p.currentText();
+ fail("Should have thrown an exception for doc <"+JSON+">");
+ } catch (JsonParseException e) {
+ verifyException(e, "invalid numeric value");
+ } finally {
+ p.close();
+ }
+
+ // and then verify it's ok when enabled
+ f.configure(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS, true);
+ assertTrue(f.isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS));
+ p = createParser(f, JSON);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(value, p.getIntValue());
+ assertEquals(String.valueOf(value), p.currentText());
+ p.close();
+ }
+
+ public void testLeadingZeroesFloat() throws Exception
+ {
+ _testLeadingZeroesFloat("00.25", 0.25);
+ _testLeadingZeroesFloat(" 00.25", 0.25);
+ _testLeadingZeroesFloat("00.25 ", 0.25);
+
+ _testLeadingZeroesFloat("-000.5", -0.5);
+ _testLeadingZeroesFloat(" -000.5", -0.5);
+ _testLeadingZeroesFloat("-000.5 ", -0.5);
+ }
+
+ public void _testLeadingZeroesFloat(String valueStr, double value) throws Exception
+ {
+ // first: verify that we get an exception
+ JsonFactory f = new JsonFactory();
+ assertFalse(f.isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS));
+ String JSON = valueStr;
+ AsyncReaderWrapper p = createParser(f, JSON);
+ try {
+ p.nextToken();
+ p.currentText();
+ fail("Should have thrown an exception for doc <"+JSON+">");
+ } catch (JsonParseException e) {
+ verifyException(e, "invalid numeric value");
+ } finally {
+ p.close();
+ }
+
+ // and then verify it's ok when enabled
+ f.configure(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS, true);
+ assertTrue(f.isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS));
+ p = createParser(f, JSON);
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, p.nextToken());
+ assertEquals(String.valueOf(value), p.currentText());
+ assertEquals(value, p.getDoubleValue());
+ p.close();
+ }
+
+ private AsyncReaderWrapper createParser(JsonFactory f, String doc) throws IOException
+ {
+ return asyncForBytes(f, 1, _jsonDoc(doc), 1);
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootNumbersTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootNumbersTest.java
new file mode 100644
index 0000000..6cd43f2
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootNumbersTest.java
@@ -0,0 +1,123 @@
+package com.fasterxml.jackson.core.json.async;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
+
+public class AsyncRootNumbersTest extends AsyncTestBase
+{
+ private final JsonFactory JSON_F = new JsonFactory();
+
+ public void testRootInts() throws Exception {
+ _testRootInts("10", 10);
+ _testRootInts(" 10", 10);
+ _testRootInts("10 ", 10);
+
+ _testRootInts("0", 0);
+ _testRootInts(" 0", 0);
+ _testRootInts("0 ", 0);
+
+ _testRootInts("-1234", -1234);
+ _testRootInts(" -1234", -1234);
+ _testRootInts(" -1234 ", -1234);
+ }
+
+ private void _testRootInts(String doc, int value) throws Exception
+ {
+ byte[] input = _jsonDoc(doc);
+ JsonFactory f = JSON_F;
+ _testRootInts(value, f, input, 0, 90);
+ _testRootInts(value, f, input, 0, 3);
+ _testRootInts(value, f, input, 0, 2);
+ _testRootInts(value, f, input, 0, 1);
+
+ _testRootInts(value, f, input, 1, 90);
+ _testRootInts(value, f, input, 1, 3);
+ _testRootInts(value, f, input, 1, 1);
+ }
+
+ private void _testRootInts(int value, JsonFactory f,
+ byte[] data, int offset, int readSize) throws IOException
+ {
+ AsyncReaderWrapper r = asyncForBytes(f, readSize, data, offset);
+ assertNull(r.currentToken());
+
+ assertToken(JsonToken.VALUE_NUMBER_INT, r.nextToken());
+ assertEquals(value, r.getIntValue());
+ assertNull(r.nextToken());
+ assertTrue(r.isClosed());
+ }
+
+ public void testRootDoublesSimple() throws Exception {
+ _testRootDoubles("10.0", 10.0);
+ _testRootDoubles(" 10.0", 10.0);
+ _testRootDoubles("10.0 ", 10.0);
+
+ _testRootDoubles("-1234.25", -1234.25);
+ _testRootDoubles(" -1234.25", -1234.25);
+ _testRootDoubles(" -1234.25 ", -1234.25);
+
+ _testRootDoubles("0.25", 0.25);
+ _testRootDoubles(" 0.25", 0.25);
+ _testRootDoubles("0.25 ", 0.25);
+ }
+
+ public void testRootDoublesScientific() throws Exception
+ {
+ _testRootDoubles("9e3", 9e3);
+ _testRootDoubles(" 9e3", 9e3);
+ _testRootDoubles("9e3 ", 9e3);
+
+ _testRootDoubles("9e-2", 9e-2);
+ _testRootDoubles(" 9e-2", 9e-2);
+ _testRootDoubles("9e-2 ", 9e-2);
+
+ _testRootDoubles("-12.5e3", -12.5e3);
+ _testRootDoubles(" -12.5e3", -12.5e3);
+ _testRootDoubles(" -12.5e3 ", -12.5e3);
+
+ _testRootDoubles("-12.5E3", -12.5e3);
+ _testRootDoubles(" -12.5E3", -12.5e3);
+ _testRootDoubles("-12.5E3 ", -12.5e3);
+
+ _testRootDoubles("-12.5E-2", -12.5e-2);
+ _testRootDoubles(" -12.5E-2", -12.5e-2);
+ _testRootDoubles(" -12.5E-2 ", -12.5e-2);
+
+ _testRootDoubles("0e-05", 0e-5);
+ _testRootDoubles("0e-5 ", 0e-5);
+ _testRootDoubles(" 0e-5", 0e-5);
+
+ _testRootDoubles("0e1", 0e1);
+ _testRootDoubles("0e1 ", 0e1);
+ _testRootDoubles(" 0e1", 0e1);
+ }
+
+ private void _testRootDoubles(String doc, double value) throws Exception
+ {
+ byte[] input = _jsonDoc(doc);
+ JsonFactory f = JSON_F;
+ _testRootDoubles(value, f, input, 0, 90);
+ _testRootDoubles(value, f, input, 0, 3);
+ _testRootDoubles(value, f, input, 0, 2);
+ _testRootDoubles(value, f, input, 0, 1);
+
+ _testRootDoubles(value, f, input, 1, 90);
+ _testRootDoubles(value, f, input, 1, 3);
+ _testRootDoubles(value, f, input, 1, 1);
+ }
+
+ private void _testRootDoubles(double value, JsonFactory f,
+ byte[] data, int offset, int readSize) throws IOException
+ {
+ AsyncReaderWrapper r = asyncForBytes(f, readSize, data, offset);
+ assertNull(r.currentToken());
+
+ assertToken(JsonToken.VALUE_NUMBER_FLOAT, r.nextToken());
+ assertEquals(value, r.getDoubleValue());
+ assertNull(r.nextToken());
+ assertTrue(r.isClosed());
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootValuesTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootValuesTest.java
index eae4a44..16b24c1 100644
--- a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootValuesTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncRootValuesTest.java
@@ -52,86 +52,6 @@
assertTrue(r.isClosed());
}
- public void testRootInts() throws Exception {
- _testRootInts("10", 10);
- _testRootInts(" 10", 10);
- _testRootInts("10 ", 10);
-
- _testRootInts("0", 0);
- _testRootInts(" 0", 0);
- _testRootInts("0 ", 0);
-
- _testRootInts("-1234", -1234);
- _testRootInts(" -1234", -1234);
- _testRootInts(" -1234 ", -1234);
- }
-
- private void _testRootInts(String doc, int value) throws Exception
- {
- byte[] input = _jsonDoc(doc);
- JsonFactory f = JSON_F;
- _testRootInts(value, f, input, 0, 90);
- _testRootInts(value, f, input, 0, 3);
- _testRootInts(value, f, input, 0, 2);
- _testRootInts(value, f, input, 0, 1);
-
- _testRootInts(value, f, input, 1, 90);
- _testRootInts(value, f, input, 1, 3);
- _testRootInts(value, f, input, 1, 1);
- }
-
- private void _testRootInts(int value, JsonFactory f,
- byte[] data, int offset, int readSize) throws IOException
- {
- AsyncReaderWrapper r = asyncForBytes(f, readSize, data, offset);
- assertNull(r.currentToken());
-
- assertToken(JsonToken.VALUE_NUMBER_INT, r.nextToken());
- assertEquals(value, r.getIntValue());
- assertNull(r.nextToken());
- assertTrue(r.isClosed());
- }
-
- public void testRootFloats() throws Exception {
- _testRootFloats("10.0", 10.0);
- _testRootFloats(" 10.0", 10.0);
- _testRootFloats("10.0 ", 10.0);
-
- _testRootFloats("-1234.25", -1234.25);
- _testRootFloats(" -1234.25", -1234.25);
- _testRootFloats(" -1234.25 ", -1234.25);
-
- _testRootFloats("-12.5e3", -12500.);
- _testRootFloats(" -12.5e3", -12500.);
- _testRootFloats(" -12.5e3 ", -12500.);
- }
-
- private void _testRootFloats(String doc, double value) throws Exception
- {
- byte[] input = _jsonDoc(doc);
- JsonFactory f = JSON_F;
- _testRootFloats(value, f, input, 0, 90);
- _testRootFloats(value, f, input, 0, 3);
- _testRootFloats(value, f, input, 0, 2);
- _testRootFloats(value, f, input, 0, 1);
-
- _testRootFloats(value, f, input, 1, 90);
- _testRootFloats(value, f, input, 1, 3);
- _testRootFloats(value, f, input, 1, 1);
- }
-
- private void _testRootFloats(double value, JsonFactory f,
- byte[] data, int offset, int readSize) throws IOException
- {
- AsyncReaderWrapper r = asyncForBytes(f, readSize, data, offset);
- assertNull(r.currentToken());
-
- assertToken(JsonToken.VALUE_NUMBER_FLOAT, r.nextToken());
- assertEquals(value, r.getDoubleValue());
- assertNull(r.nextToken());
- assertTrue(r.isClosed());
- }
-
/*
/**********************************************************************
/* Root-level sequences
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScalarArrayTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScalarArrayTest.java
index 8541d8f..3920dbf 100644
--- a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScalarArrayTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScalarArrayTest.java
@@ -1,6 +1,8 @@
package com.fasterxml.jackson.core.json.async;
import java.io.*;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.JsonParser.NumberType;
@@ -97,6 +99,13 @@
assertToken(JsonToken.VALUE_NUMBER_INT, r.nextToken());
assertEquals(values[i], r.getIntValue());
assertEquals(NumberType.INT, r.getNumberType());
+
+ // and then couple of special modes, just to get better test coverage
+ String asStr = String.valueOf(values[i]);
+ assertEquals(asStr, r.currentText());
+ StringWriter sw = new StringWriter();
+ assertEquals(asStr.length(), r.parser().getText(sw));
+ assertEquals(asStr, sw.toString());
}
assertToken(JsonToken.END_ARRAY, r.nextToken());
@@ -108,7 +117,7 @@
public void testLong() throws IOException
{
final long[] input = new long[] {
- // SmileGenerator will try to minimize so....
+ // JsonParser will determine minimum size needed, so can't do these
// 1, -1, 16, -17, 131, -155, 1000, -3000, 0xFFFF, -99999,
-1L + Integer.MIN_VALUE, 1L + Integer.MAX_VALUE,
19L * Integer.MIN_VALUE, 27L * Integer.MAX_VALUE,
@@ -157,7 +166,7 @@
/**********************************************************************
*/
-/* public void testFloats() throws IOException
+ public void testFloats() throws IOException
{
final float[] input = new float[] { 0.0f, 0.25f, -0.5f, 10000.125f, - 99999.075f };
ByteArrayOutputStream bytes = new ByteArrayOutputStream(100);
@@ -189,7 +198,8 @@
for (int i = 0; i < values.length; ++i) {
assertToken(JsonToken.VALUE_NUMBER_FLOAT, r.nextToken());
assertEquals(values[i], r.getFloatValue());
- assertEquals(NumberType.FLOAT, r.getNumberType());
+ // json can't distinguish floats from doubles so
+ assertEquals(NumberType.DOUBLE, r.getNumberType());
}
assertToken(JsonToken.END_ARRAY, r.nextToken());
// and end up with "no token" as well
@@ -199,7 +209,8 @@
public void testDoubles() throws IOException
{
- final double[] input = new double[] { 0.0, 0.25, -0.5, 10000.125, -99999.075 };
+ final double[] input = new double[] { 0.0, 0.25, -0.5, 10000.125,
+ -99999.075 };
ByteArrayOutputStream bytes = new ByteArrayOutputStream(100);
JsonFactory f = JSON_F;
JsonGenerator g = f.createGenerator(bytes);
@@ -210,11 +221,11 @@
g.writeEndArray();
g.close();
byte[] data = bytes.toByteArray();
- _testDoubles(f, input, data, 0, 100);
+ _testDoubles(f, input, data, 0, 99);
_testDoubles(f, input, data, 0, 3);
_testDoubles(f, input, data, 0, 1);
- _testDoubles(f, input, data, 1, 100);
+ _testDoubles(f, input, data, 1, 99);
_testDoubles(f, input, data, 1, 3);
_testDoubles(f, input, data, 1, 1);
}
@@ -228,7 +239,9 @@
assertToken(JsonToken.START_ARRAY, r.nextToken());
for (int i = 0; i < values.length; ++i) {
assertToken(JsonToken.VALUE_NUMBER_FLOAT, r.nextToken());
- assertEquals(values[i], r.getDoubleValue());
+ assertEquals(String.format("Entry #%d: %s (textual '%s')",
+ i, values[i], r.currentText()),
+ values[i], r.getDoubleValue());
assertEquals(NumberType.DOUBLE, r.getNumberType());
}
assertToken(JsonToken.END_ARRAY, r.nextToken());
@@ -237,26 +250,30 @@
assertNull(r.nextToken());
assertTrue(r.isClosed());
}
-*/
+
/*
/**********************************************************************
/* BigInteger, BigDecimal
/**********************************************************************
*/
-/*
+
public void testBigIntegers() throws IOException
{
- BigInteger bigBase = BigInteger.valueOf(1234567890344656736L);
+ BigInteger bigBase = BigInteger.valueOf(Long.MAX_VALUE);
final BigInteger[] input = new BigInteger[] {
+ // Since JSON doesn't denote "real" type, just deduces from magnitude,
+ // let's not test any values within int/long range
+ /*
BigInteger.ZERO,
BigInteger.ONE,
BigInteger.TEN,
BigInteger.valueOf(-999L),
bigBase,
+ */
bigBase.shiftLeft(100).add(BigInteger.valueOf(123456789L)),
bigBase.add(bigBase),
bigBase.multiply(BigInteger.valueOf(17)),
- bigBase.negate()
+ bigBase.negate().subtract(BigInteger.TEN)
};
ByteArrayOutputStream bytes = new ByteArrayOutputStream(100);
JsonFactory f = JSON_F;
@@ -299,9 +316,10 @@
{
BigDecimal bigBase = new BigDecimal("1234567890344656736.125");
final BigDecimal[] input = new BigDecimal[] {
- BigDecimal.ZERO,
- BigDecimal.ONE,
- BigDecimal.TEN,
+ // 04-Jun-2017, tatu: these look like integral numbers in JSON so can't use:
+// BigDecimal.ZERO,
+// BigDecimal.ONE,
+// BigDecimal.TEN,
BigDecimal.valueOf(-999.25),
bigBase,
bigBase.divide(new BigDecimal("5")),
@@ -330,21 +348,20 @@
}
private void _testBigDecimals(JsonFactory f, BigDecimal[] values,
- byte[] data, int offset, int readSize) throws IOException
+ byte[] doc, int offset, int readSize) throws IOException
{
- AsyncReaderWrapper r = asyncForBytes(f, readSize, data, offset);
+ AsyncReaderWrapper r = asyncForBytes(f, readSize, doc, offset);
// start with "no token"
assertNull(r.currentToken());
assertToken(JsonToken.START_ARRAY, r.nextToken());
for (int i = 0; i < values.length; ++i) {
BigDecimal expValue = values[i];
assertToken(JsonToken.VALUE_NUMBER_FLOAT, r.nextToken());
- assertEquals(expValue, r.getBigDecimalValue());
+ assertEquals(expValue, r.getDecimalValue());
assertEquals(NumberType.BIG_DECIMAL, r.getNumberType());
}
assertToken(JsonToken.END_ARRAY, r.nextToken());
assertNull(r.nextToken());
assertTrue(r.isClosed());
}
-*/
}
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScopeMatchingTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScopeMatchingTest.java
new file mode 100644
index 0000000..f4b1d42
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncScopeMatchingTest.java
@@ -0,0 +1,115 @@
+package com.fasterxml.jackson.core.json.async;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
+
+/**
+ * Set of basic unit tests for verifying that Array/Object scopes
+ * are properly matched.
+ */
+public class AsyncScopeMatchingTest extends AsyncTestBase
+{
+ private final JsonFactory JSON_F = new JsonFactory();
+
+ public void testUnclosedArray(int mode) throws Exception
+ {
+ AsyncReaderWrapper p = asyncForBytes(JSON_F, 3, _jsonDoc("[ 1, 2 "), 0);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(2, p.getIntValue());
+
+ try {
+ p.nextToken();
+ fail("Expected an exception for unclosed ARRAY (mode: "+mode+")");
+ } catch (JsonParseException pe) {
+ verifyException(pe, "expected close marker for ARRAY");
+ }
+ }
+
+ public void testUnclosedObject(int mode) throws Exception
+ {
+ AsyncReaderWrapper p = asyncForBytes(JSON_F, 3, _jsonDoc("{ \"key\" : 3 "), 0);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+
+ try {
+ p.nextToken();
+ fail("Expected an exception for unclosed OBJECT (mode: "+mode+")");
+ } catch (JsonParseException pe) {
+ verifyException(pe, "expected close marker for OBJECT");
+ }
+ }
+
+ public void testEOFInName(int mode) throws Exception
+ {
+ final String JSON = "{ \"abcd";
+ AsyncReaderWrapper p = asyncForBytes(JSON_F, 3, _jsonDoc(JSON), 0);
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ try {
+ p.nextToken();
+ fail("Expected an exception for EOF");
+ } catch (JsonParseException pe) {
+ verifyException(pe, "Unexpected end-of-input");
+ } catch (IOException ie) {
+ // DataInput behaves bit differently
+ if (mode == MODE_DATA_INPUT) {
+ verifyException(ie, "end-of-input");
+ return;
+ }
+ }
+ }
+
+ public void testMismatchArrayToObject() throws Exception
+ {
+ final String JSON = "[ 1, 2 }";
+ AsyncReaderWrapper p = asyncForBytes(JSON_F, 3, _jsonDoc(JSON), 0);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ try {
+ p.nextToken();
+ fail("Expected an exception for incorrectly closed ARRAY");
+ } catch (JsonParseException pe) {
+ verifyException(pe, "Unexpected close marker '}': expected ']'");
+ }
+ p.close();
+ }
+
+ public void testMismatchObjectToArray() throws Exception
+ {
+ final String JSON = "{ ]";
+ AsyncReaderWrapper p = asyncForBytes(JSON_F, 3, _jsonDoc(JSON), 0);
+
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+
+ try {
+ p.nextToken();
+ fail("Expected an exception for incorrectly closed OBJECT");
+ } catch (JsonParseException pe) {
+ verifyException(pe, "Unexpected close marker ']': expected '}'");
+ }
+ p.close();
+ }
+
+ public void testMisssingColon(int mode) throws Exception
+ {
+ final String JSON = "{ \"a\" \"b\" }";
+ AsyncReaderWrapper p = asyncForBytes(JSON_F, 3, _jsonDoc(JSON), 0);
+
+ assertToken(JsonToken.START_OBJECT, p.nextToken());
+ try {
+ // can be either here, or with next one...
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ p.nextToken();
+ fail("Expected an exception for missing semicolon");
+ } catch (JsonParseException pe) {
+ verifyException(pe, "was expecting a colon");
+ }
+ p.close();
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncUnicodeHandlingTest.java b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncUnicodeHandlingTest.java
index 4213389..05915fe 100644
--- a/src/test/java/com/fasterxml/jackson/core/json/async/AsyncUnicodeHandlingTest.java
+++ b/src/test/java/com/fasterxml/jackson/core/json/async/AsyncUnicodeHandlingTest.java
@@ -13,11 +13,10 @@
public void testShortUnicodeWithSurrogates() throws IOException
{
JsonFactory f = JSON_F;
+
// first, no buffer boundaries
- /*
_testUnicodeWithSurrogates(f, 28, 99);
_testUnicodeWithSurrogates(f, 53, 99);
- */
// then small chunks
_testUnicodeWithSurrogates(f, 28, 3);
@@ -32,11 +31,9 @@
{
JsonFactory f = JSON_F;
- /*
_testUnicodeWithSurrogates(f, 230, Integer.MAX_VALUE);
_testUnicodeWithSurrogates(f, 700, Integer.MAX_VALUE);
_testUnicodeWithSurrogates(f, 9600, Integer.MAX_VALUE);
- */
_testUnicodeWithSurrogates(f, 230, 3);
_testUnicodeWithSurrogates(f, 700, 3);
diff --git a/src/test/java/com/fasterxml/jackson/failing/async/AsyncCommentParsingTest.java b/src/test/java/com/fasterxml/jackson/failing/async/AsyncCommentParsingTest.java
new file mode 100644
index 0000000..512aebe
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/failing/async/AsyncCommentParsingTest.java
@@ -0,0 +1,236 @@
+package com.fasterxml.jackson.failing.async;
+
+import java.io.*;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
+
+/**
+ * Unit tests for verifying that support for (non-standard) comments
+ * works as expected.
+ */
+public class AsyncCommentParsingTest extends AsyncTestBase
+{
+ final static String DOC_WITH_SLASHSTAR_COMMENT =
+ "[ /* comment:\n ends here */ 1 /* one more ok to have \"unquoted\" */ ]"
+ ;
+
+ final static String DOC_WITH_SLASHSLASH_COMMENT =
+ "[ // comment...\n 1 \r // one more, not array: [] \n ]"
+ ;
+
+ /*
+ /**********************************************************
+ /* Test method wrappers
+ /**********************************************************
+ */
+
+ public void testCommentsDisabled() throws Exception
+ {
+ _testDisabled(DOC_WITH_SLASHSTAR_COMMENT);
+ _testDisabled(DOC_WITH_SLASHSLASH_COMMENT);
+ }
+
+ public void testCommentsEnabled() throws Exception
+ {
+ _testEnabled(DOC_WITH_SLASHSTAR_COMMENT);
+ _testEnabled(DOC_WITH_SLASHSLASH_COMMENT);
+ }
+
+ public void testCommentsWithUTF8() throws Exception
+ {
+ final String JSON = "/* \u00a9 2099 Yoyodyne Inc. */\n [ \"bar? \u00a9\" ]\n";
+ _testWithUTF8Chars(JSON);
+ }
+
+ public void testYAMLComments() throws Exception
+ {
+ JsonFactory f = new JsonFactory();
+ f.configure(JsonParser.Feature.ALLOW_YAML_COMMENTS, true);
+
+ _testYAMLComments(f);
+ _testCommentsBeforePropValue(f, "# foo\n");
+
+ _testCommentsBetweenArrayValues(f, "# foo\n");
+ }
+
+ public void testCCommentsBytes() throws Exception {
+ JsonFactory f = new JsonFactory();
+ f.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+ final String COMMENT = "/* foo */\n";
+ _testCommentsBeforePropValue(f, COMMENT);
+ }
+
+ public void testCppCommentsBytes() throws Exception {
+ JsonFactory f = new JsonFactory();
+ f.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+ final String COMMENT = "// foo\n";
+ _testCommentsBeforePropValue(f, COMMENT);
+ }
+
+ private void _testCommentsBeforePropValue(JsonFactory f,
+ String comment) throws Exception
+ {
+ for (String arg : new String[] {
+ ":%s123",
+ " :%s123",
+ "\t:%s123",
+ ": %s123",
+ ":\t%s123",
+ }) {
+ String commented = String.format(arg, comment);
+
+ final String DOC = "{\"abc\"" + commented + "}";
+ AsyncReaderWrapper p = _createParser(f, DOC, 3);
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+ JsonToken t = null;
+ try {
+ t = p.nextToken();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed on '"+DOC+"' due to "+e, e);
+ }
+ assertEquals(JsonToken.FIELD_NAME, t);
+
+ try {
+ t = p.nextToken();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed on '"+DOC+"' due to "+e, e);
+ }
+ assertEquals(JsonToken.VALUE_NUMBER_INT, t);
+ assertEquals(123, p.getIntValue());
+ assertEquals(JsonToken.END_OBJECT, p.nextToken());
+ p.close();
+ }
+
+ }
+
+ private void _testCommentsBetweenArrayValues(JsonFactory f,
+ String comment) throws Exception
+ {
+ for (String tmpl : new String[] {
+ "%s,",
+ " %s,",
+ "\t%s,",
+ "%s ,",
+ "%s\t,",
+ " %s ,",
+ "\t%s\t,",
+ "\n%s,",
+ "%s\n,",
+ }) {
+ String commented = String.format(tmpl, comment);
+
+ final String DOC = "[1"+commented+"2]";
+ AsyncReaderWrapper p = _createParser(f, DOC, 3);
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+ JsonToken t = null;
+ try {
+ t = p.nextToken();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed on '"+DOC+"' due to "+e, e);
+ }
+ assertEquals(JsonToken.VALUE_NUMBER_INT, t);
+ assertEquals(1, p.getIntValue());
+
+ try {
+ t = p.nextToken();
+ } catch (Exception e) {
+ throw new RuntimeException("Failed on '"+DOC+"' due to "+e, e);
+ }
+ assertEquals(JsonToken.VALUE_NUMBER_INT, t);
+ assertEquals(2, p.getIntValue());
+ assertEquals(JsonToken.END_ARRAY, p.nextToken());
+ p.close();
+ }
+
+ }
+
+ private void _testYAMLComments(JsonFactory f) throws Exception
+ {
+ final String DOC = "# foo\n"
+ +" {\"a\" # xyz\n"
+ +" : # foo\n"
+ +" 1, # more\n"
+ +"\"b\": [ \n"
+ +" #all!\n"
+ +" 3 #yay!\n"
+ +"] # foobar\n"
+ +"} # x"
+ ;
+ AsyncReaderWrapper p = _createParser(f, DOC, 3);
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.currentName());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(1, p.getIntValue());
+ assertEquals(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.currentName());
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+ assertEquals(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(3, p.getIntValue());
+ assertEquals(JsonToken.END_ARRAY, p.nextToken());
+ assertEquals(JsonToken.END_OBJECT, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ /*
+ /**********************************************************
+ /* Helper methods
+ /**********************************************************
+ */
+
+ private void _testWithUTF8Chars(String doc) throws IOException
+ {
+ // should basically just stream through
+ AsyncReaderWrapper p = _createParser(doc, false, 3);
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertNull(p.nextToken());
+ p.close();
+ }
+
+ private void _testDisabled(String doc) throws IOException
+ {
+ AsyncReaderWrapper p = _createParser(doc, false, 3);
+ try {
+ p.nextToken();
+ fail("Expected exception for unrecognized comment");
+ } catch (JsonParseException je) {
+ // Should have something denoting that user may want to enable 'ALLOW_COMMENTS'
+ verifyException(je, "ALLOW_COMMENTS");
+ }
+ p.close();
+ }
+
+ private void _testEnabled(String doc) throws IOException
+ {
+ AsyncReaderWrapper p = _createParser(doc, false, 3);
+ assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
+ assertEquals(1, p.getIntValue());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ p.close();
+ }
+
+ private AsyncReaderWrapper _createParser(String doc, boolean enabled,
+ int bytesPerRead)
+ throws IOException
+ {
+ JsonFactory f = new JsonFactory();
+
+ f.configure(JsonParser.Feature.ALLOW_COMMENTS, enabled);
+ AsyncReaderWrapper p = asyncForBytes(f, bytesPerRead, _jsonDoc(doc), 0);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ return p;
+ }
+
+ private AsyncReaderWrapper _createParser(JsonFactory f, String doc, int bytesPerRead)
+ throws IOException
+ {
+ AsyncReaderWrapper p = asyncForBytes(f, bytesPerRead, _jsonDoc(doc), 0);
+ assertToken(JsonToken.START_ARRAY, p.nextToken());
+ return p;
+ }
+}
diff --git a/src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdNumbersTest.java b/src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdNumbersTest.java
index f88bdac..946b5bf 100644
--- a/src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdNumbersTest.java
+++ b/src/test/java/com/fasterxml/jackson/failing/async/AsyncNonStdNumbersTest.java
@@ -8,54 +8,6 @@
public class AsyncNonStdNumbersTest extends AsyncTestBase
{
- public void testLeadingZeroes() throws Exception {
- _testLeadingZeroes(false);
- _testLeadingZeroes(true);
- }
-
- public void _testLeadingZeroes(boolean appendSpace) throws Exception
- {
- // first: verify that we get an exception
- JsonFactory f = new JsonFactory();
- assertFalse(f.isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS));
- String JSON = "00003";
- if (appendSpace) {
- JSON += " ";
- }
- AsyncReaderWrapper p = createParser(f, JSON);
- try {
- p.nextToken();
- p.currentText();
- fail("Should have thrown an exception for doc <"+JSON+">");
- } catch (JsonParseException e) {
- verifyException(e, "invalid numeric value");
- } finally {
- p.close();
- }
-
- // and then verify it's ok when enabled
- f.configure(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS, true);
- assertTrue(f.isEnabled(JsonParser.Feature.ALLOW_NUMERIC_LEADING_ZEROS));
- p = createParser(f, JSON);
- assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
- assertEquals("3", p.currentText());
- assertEquals(3, p.getIntValue());
- p.close();
-
- // Plus, also: verify that leading zero magnitude is ok:
- JSON = "0"+Integer.MAX_VALUE;
- if (appendSpace) {
- JSON += " ";
- }
- p = createParser(f, JSON);
- assertToken(JsonToken.VALUE_NUMBER_INT, p.nextToken());
- assertEquals(String.valueOf(Integer.MAX_VALUE), p.currentText());
- assertEquals(Integer.MAX_VALUE, p.getIntValue());
- Number nr = p.getNumberValue();
- assertSame(Integer.class, nr.getClass());
- p.close();
- }
-
public void testAllowNaN() throws Exception
{
final String JSON = "[ NaN]";
diff --git a/src/test/java/com/fasterxml/jackson/failing/async/AsyncTrailingCommasTest.java b/src/test/java/com/fasterxml/jackson/failing/async/AsyncTrailingCommasTest.java
new file mode 100644
index 0000000..cbec888
--- /dev/null
+++ b/src/test/java/com/fasterxml/jackson/failing/async/AsyncTrailingCommasTest.java
@@ -0,0 +1,316 @@
+package com.fasterxml.jackson.failing.async;
+
+import java.io.IOException;
+import java.util.*;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.core.async.AsyncTestBase;
+import com.fasterxml.jackson.core.testsupport.AsyncReaderWrapper;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class AsyncTrailingCommasTest extends AsyncTestBase
+{
+ private final JsonFactory factory;
+ private final HashSet<JsonParser.Feature> features;
+
+ public AsyncTrailingCommasTest(List<Feature> features) {
+ this.factory = new JsonFactory();
+ this.features = new HashSet<JsonParser.Feature>(features);
+
+ for (JsonParser.Feature feature : features) {
+ factory.enable(feature);
+ }
+ }
+
+ @Parameterized.Parameters(name = "Features {0}")
+ public static Collection<Object[]> getTestCases()
+ {
+ ArrayList<Object[]> cases = new ArrayList<Object[]>();
+ cases.add(new Object[]{Collections.emptyList()});
+ cases.add(new Object[]{Arrays.asList(Feature.ALLOW_MISSING_VALUES)});
+ cases.add(new Object[]{Arrays.asList(Feature.ALLOW_TRAILING_COMMA)});
+ cases.add(new Object[]{Arrays.asList(Feature.ALLOW_MISSING_VALUES, Feature.ALLOW_TRAILING_COMMA)});
+ return cases;
+ }
+
+ @Test
+ public void testArrayBasic() throws Exception {
+ String json = "[\"a\", \"b\"]";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("a", p.currentText());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("b", p.currentText());
+
+ assertEquals(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ }
+
+ @Test
+ public void testArrayInnerComma() throws Exception {
+ String json = "[\"a\",, \"b\"]";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("a", p.currentText());
+
+ if (!features.contains(Feature.ALLOW_MISSING_VALUES)) {
+ assertUnexpected(p, ',');
+ return;
+ }
+
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("b", p.currentText());
+
+ assertEquals(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ }
+
+ @Test
+ public void testArrayLeadingComma() throws Exception {
+ String json = "[,\"a\", \"b\"]";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+
+ if (!features.contains(Feature.ALLOW_MISSING_VALUES)) {
+ assertUnexpected(p, ',');
+ return;
+ }
+
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("a", p.currentText());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("b", p.currentText());
+
+ assertEquals(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ p.close();
+ }
+
+ @Test
+ public void testArrayTrailingComma() throws Exception {
+ String json = "[\"a\", \"b\",]";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("a", p.currentText());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("b", p.currentText());
+
+ // ALLOW_TRAILING_COMMA takes priority over ALLOW_MISSING_VALUES
+ if (features.contains(Feature.ALLOW_TRAILING_COMMA)) {
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ } else if (features.contains(Feature.ALLOW_MISSING_VALUES)) {
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ } else {
+ assertUnexpected(p, ']');
+ }
+ p.close();
+ }
+
+ @Test
+ public void testArrayTrailingCommas() throws Exception {
+ String json = "[\"a\", \"b\",,]";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("a", p.currentText());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("b", p.currentText());
+
+ // ALLOW_TRAILING_COMMA takes priority over ALLOW_MISSING_VALUES
+ if (features.contains(Feature.ALLOW_MISSING_VALUES) &&
+ features.contains(Feature.ALLOW_TRAILING_COMMA)) {
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ } else if (features.contains(Feature.ALLOW_MISSING_VALUES)) {
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ } else {
+ assertUnexpected(p, ',');
+ }
+ p.close();
+ }
+
+ @Test
+ public void testArrayTrailingCommasTriple() throws Exception {
+ String json = "[\"a\", \"b\",,,]";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_ARRAY, p.nextToken());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("a", p.currentText());
+
+ assertToken(JsonToken.VALUE_STRING, p.nextToken());
+ assertEquals("b", p.currentText());
+
+ // ALLOW_TRAILING_COMMA takes priority over ALLOW_MISSING_VALUES
+ if (features.contains(Feature.ALLOW_MISSING_VALUES) &&
+ features.contains(Feature.ALLOW_TRAILING_COMMA)) {
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ } else if (features.contains(Feature.ALLOW_MISSING_VALUES)) {
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.VALUE_NULL, p.nextToken());
+ assertToken(JsonToken.END_ARRAY, p.nextToken());
+ assertEnd(p);
+ } else {
+ assertUnexpected(p, ',');
+ }
+ p.close();
+ }
+
+ @Test
+ public void testObjectBasic() throws Exception {
+ String json = "{\"a\": true, \"b\": false}";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.currentText());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.currentText());
+ assertToken(JsonToken.VALUE_FALSE, p.nextToken());
+
+ assertEquals(JsonToken.END_OBJECT, p.nextToken());
+ assertEnd(p);
+ p.close();
+ }
+
+ @Test
+ public void testObjectInnerComma() throws Exception {
+ String json = "{\"a\": true,, \"b\": false}";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.currentText());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+
+ assertUnexpected(p, ',');
+ p.close();
+ }
+
+ @Test
+ public void testObjectLeadingComma() throws Exception {
+ String json = "{,\"a\": true, \"b\": false}";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+
+ assertUnexpected(p, ',');
+ p.close();
+ }
+
+ @Test
+ public void testObjectTrailingComma() throws Exception {
+ String json = "{\"a\": true, \"b\": false,}";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.currentText());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.currentText());
+ assertToken(JsonToken.VALUE_FALSE, p.nextToken());
+
+ if (features.contains(Feature.ALLOW_TRAILING_COMMA)) {
+ assertToken(JsonToken.END_OBJECT, p.nextToken());
+ assertEnd(p);
+ } else {
+ assertUnexpected(p, '}');
+ }
+ p.close();
+ }
+
+ @Test
+ public void testObjectTrailingCommas() throws Exception {
+ String json = "{\"a\": true, \"b\": false,,}";
+
+ AsyncReaderWrapper p = createParser(factory, json);
+
+ assertEquals(JsonToken.START_OBJECT, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("a", p.currentText());
+ assertToken(JsonToken.VALUE_TRUE, p.nextToken());
+
+ assertToken(JsonToken.FIELD_NAME, p.nextToken());
+ assertEquals("b", p.currentText());
+ assertToken(JsonToken.VALUE_FALSE, p.nextToken());
+
+ assertUnexpected(p, ',');
+ p.close();
+ }
+
+ private void assertEnd(AsyncReaderWrapper p) throws IOException {
+ JsonToken next = p.nextToken();
+ assertNull("expected end of stream but found " + next, next);
+ }
+
+ private void assertUnexpected(AsyncReaderWrapper p, char c) throws IOException {
+ try {
+ p.nextToken();
+ fail("No exception thrown");
+ } catch (JsonParseException e) {
+ verifyException(e, String.format("Unexpected character ('%s' (code %d))", c, (int) c));
+ }
+ }
+
+ private AsyncReaderWrapper createParser(JsonFactory f, String doc) throws IOException
+ {
+ int bytesPerRead = 3; // should vary but...
+ AsyncReaderWrapper p = asyncForBytes(f, bytesPerRead, _jsonDoc(doc), 0);
+ return p;
+ }
+}