Some skeletal work for non-blocking parsing
diff --git a/src/main/java/com/fasterxml/jackson/core/json/ByteSourceJsonBootstrapper.java b/src/main/java/com/fasterxml/jackson/core/json/ByteSourceJsonBootstrapper.java
index 6ff84e9..bc50907 100644
--- a/src/main/java/com/fasterxml/jackson/core/json/ByteSourceJsonBootstrapper.java
+++ b/src/main/java/com/fasterxml/jackson/core/json/ByteSourceJsonBootstrapper.java
@@ -18,9 +18,9 @@
*/
public final class ByteSourceJsonBootstrapper
{
- final static byte UTF8_BOM_1 = (byte) 0xEF;
- final static byte UTF8_BOM_2 = (byte) 0xBB;
- final static byte UTF8_BOM_3 = (byte) 0xBF;
+ public final static byte UTF8_BOM_1 = (byte) 0xEF;
+ public final static byte UTF8_BOM_2 = (byte) 0xBB;
+ public final static byte UTF8_BOM_3 = (byte) 0xBF;
/*
/**********************************************************
diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java
index 6c67e26..d53bdd9 100644
--- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java
+++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java
@@ -7,6 +7,7 @@
import com.fasterxml.jackson.core.async.ByteArrayFeeder;
import com.fasterxml.jackson.core.async.NonBlockingInputFeeder;
import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.json.ByteSourceJsonBootstrapper;
import com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer;
import com.fasterxml.jackson.core.util.VersionUtil;
@@ -39,6 +40,29 @@
/*
/**********************************************************************
+ /* Location tracking, additional
+ /**********************************************************************
+ */
+
+ /**
+ * Alternate row tracker, used to keep track of position by `\r` marker
+ * (whereas <code>_currInputRow</code> tracks `\n`). Used to simplify
+ * tracking of linefeeds, assuming that input typically uses various
+ * linefeed combinations (`\r`, `\n` or `\r\n`) consistently, in which
+ * case we can simply choose max of two row candidates.
+ */
+ protected int _currInputRowAlt = 1;
+
+ /*
+ /**********************************************************************
+ /* Other state
+ /**********************************************************************
+ */
+
+ protected int _currentQuote;
+
+ /*
+ /**********************************************************************
/* Life-cycle
/**********************************************************************
*/
@@ -148,14 +172,14 @@
// No: fresh new token; may or may not have existing one
_numTypesValid = NR_UNKNOWN;
-// _tokenInputTotal = _currInputProcessed + _inputPtr;
+ _tokenInputTotal = _currInputProcessed + _inputPtr;
// also: clear any data retained so far
_binaryValue = null;
int ch = _inputBuffer[_inputPtr++];
switch (_majorState) {
case MAJOR_INITIAL:
- // TODO: Bootstrapping? BOM?
+ return _startDocument(ch);
case MAJOR_ROOT:
return _startValue(ch);
@@ -193,6 +217,50 @@
/**********************************************************************
*/
+ private final JsonToken _startDocument(int ch) throws IOException
+ {
+ ch &= 0xFF;
+
+ // Very first byte: could be BOM
+ if (ch == ByteSourceJsonBootstrapper.UTF8_BOM_1) {
+ // !!! TODO
+ }
+
+ // If not BOM (or we got past it), could be whitespace or comment to skip
+ while (ch <= 0x020) {
+ if (ch != INT_SPACE) {
+ if (ch == INT_LF) {
+ ++_currInputRow;
+ _currInputRowStart = _inputPtr;
+ } else if (ch == INT_CR) {
+ ++_currInputRowAlt;
+ _currInputRowStart = _inputPtr;
+ } else if (ch != INT_TAB) {
+ _throwInvalidSpace(ch);
+ }
+ }
+ if (_inputPtr >= _inputEnd) {
+ _minorState = MINOR_FIELD_ROOT_GOT_SEPARATOR;
+ if (_closed) {
+ return null;
+ }
+ // note: if so, do not even bother changing state
+ if (_endOfInput) { // except for this special case
+ return _eofAsNextToken();
+ }
+ return JsonToken.NOT_AVAILABLE;
+ }
+ ch = _inputBuffer[_inputPtr++] & 0xFF;
+ }
+ return _startValue(ch);
+ }
+
+ /*
+ /**********************************************************************
+ /* Second-level decoding, value parsing
+ /**********************************************************************
+ */
+
/**
* Helper method called to detect type of a value token (at any level), and possibly
* decode it if contained in input buffer.
@@ -200,6 +268,121 @@
*/
private final JsonToken _startValue(int ch) throws IOException
{
+ if (ch == INT_QUOTE) {
+ return _startString(ch);
+ }
+ switch (ch) {
+ case '-':
+ return _startNegativeNumber();
+
+ // Should we have separate handling for plus? Although
+ // it is not allowed per se, it may be erroneously used,
+ // and could be indicate by a more specific error message.
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9':
+ return _startPositiveNumber(ch);
+ case 'f':
+ return _startFalseToken();
+ case 'n':
+ return _startNullToken();
+ case 't':
+ return _startTrueToken();
+ case '[':
+ return _startArrayScope();
+ case ']':
+ return _closeArrayScope();
+ case '{':
+ return _startObjectScope();
+ case '}':
+ return _closeObjectScope();
+ default:
+ }
+ return _startUnexpectedValue(ch);
+ }
+
+ protected JsonToken _startUnexpectedValue(int ch) throws IOException
+ {
+ // TODO: Maybe support non-standard tokens that streaming parser does:
+ //
+ // * NaN
+ // * Infinity
+ // * Plus-prefix for numbers
+ // * Apostrophe for Strings
+
+ switch (ch) {
+ case '\'':
+ return _startString(ch);
+
+ case ',':
+ // If Feature.ALLOW_MISSING_VALUES is enabled we may allow "missing values",
+ // that is, encountering a trailing comma or closing marker where value would be expected
+ if (!_parsingContext.inObject() && isEnabled(Feature.ALLOW_MISSING_VALUES)) {
+ // Important to "push back" separator, to be consumed before next value;
+ // does not lead to infinite loop
+ --_inputPtr;
+ return _valueComplete(JsonToken.VALUE_NULL);
+ }
+ break;
+ }
+ // !!! TODO: maybe try to collect more information for better diagnostics
+ _reportUnexpectedChar(ch, "expected a valid value (number, String, array, object, 'true', 'false' or 'null')");
+ return null;
+ }
+
+ /*
+ /**********************************************************************
+ /* Second-level decoding, simple tokens
+ /**********************************************************************
+ */
+
+ protected JsonToken _startFalseToken() throws IOException
+ {
+ return null;
+ }
+
+ protected JsonToken _startTrueToken() throws IOException
+ {
+ return null;
+ }
+
+ protected JsonToken _startNullToken() throws IOException
+ {
+ return null;
+ }
+
+ /*
+ /**********************************************************************
+ /* Second-level decoding, String decoding
+ /**********************************************************************
+ */
+
+ protected JsonToken _startString(int q) throws IOException
+ {
+ _currentQuote = q;
+ return null;
+ }
+
+ /*
+ /**********************************************************************
+ /* Second-level decoding, String decoding
+ /**********************************************************************
+ */
+
+ protected JsonToken _startPositiveNumber(int ch) throws IOException
+ {
+ return null;
+ }
+
+ protected JsonToken _startNegativeNumber() throws IOException
+ {
return null;
}
diff --git a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java
index 4955417..2482c07 100644
--- a/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java
+++ b/src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParserBase.java
@@ -6,6 +6,7 @@
import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.base.ParserBase;
import com.fasterxml.jackson.core.io.IOContext;
+import com.fasterxml.jackson.core.json.JsonReadContext;
import com.fasterxml.jackson.core.sym.ByteQuadsCanonicalizer;
/**
@@ -42,12 +43,29 @@
* is forthcoming AND we have exhausted all the input
*/
protected final static int MAJOR_CLOSED = 5;
+
+ /*
+ /**********************************************************************
+ /* Minor state constants
+ /**********************************************************************
+ */
+
+ /**
+ * State between root-level value, waiting for at least one white-space
+ * character as separator
+ */
+ protected final static int MINOR_FIELD_ROOT_NEED_SEPARATOR = 1;
+
+ /**
+ * State between root-level value, having processed at least one white-space
+ * character, and expecting either more, start of a value, or end of input
+ * stream.
+ */
+ protected final static int MINOR_FIELD_ROOT_GOT_SEPARATOR = 2;
- // // // "Sub-states"
+ protected final static int MINOR_FIELD_NAME = 10;
- protected final static int MINOR_FIELD_NAME = 1;
-
- protected final static int MINOR_VALUE_NUMBER = 6;
+ protected final static int MINOR_VALUE_NUMBER = 11;
protected final static int MINOR_VALUE_STRING = 15;
@@ -148,6 +166,7 @@
_currToken = null;
_majorState = MAJOR_INITIAL;
+ _majorStateAfterValue = MAJOR_ROOT;
}
@Override
@@ -352,6 +371,68 @@
/*
/**********************************************************************
+ /* Handling of nested scope, state
+ /**********************************************************************
+ */
+
+ protected final JsonToken _startArrayScope() throws IOException
+ {
+ _parsingContext = _parsingContext.createChildArrayContext(-1, -1);
+ _majorState = MAJOR_ARRAY_ELEMENT;
+ _majorStateAfterValue = MAJOR_ARRAY_ELEMENT;
+ return (_currToken = JsonToken.START_ARRAY);
+ }
+
+ protected final JsonToken _startObjectScope() throws IOException
+ {
+ _parsingContext = _parsingContext.createChildObjectContext(-1, -1);
+ _majorState = MAJOR_OBJECT_FIELD;
+ _majorStateAfterValue = MAJOR_OBJECT_FIELD;
+ return (_currToken = JsonToken.START_OBJECT);
+ }
+
+ protected final JsonToken _closeArrayScope() throws IOException
+ {
+ if (!_parsingContext.inArray()) {
+ _reportMismatchedEndMarker(']', '}');
+ }
+ JsonReadContext ctxt = _parsingContext.getParent();
+ _parsingContext = ctxt;
+ int st;
+ if (ctxt.inObject()) {
+ st = MAJOR_OBJECT_FIELD;
+ } else if (ctxt.inArray()) {
+ st = MAJOR_ARRAY_ELEMENT;
+ } else {
+ st = MAJOR_ROOT;
+ }
+ _majorState = st;
+ _majorStateAfterValue = st;
+ return (_currToken = JsonToken.END_ARRAY);
+ }
+
+ protected final JsonToken _closeObjectScope() throws IOException
+ {
+ if (!_parsingContext.inObject()) {
+ _reportMismatchedEndMarker('}', ']');
+ }
+ JsonReadContext ctxt = _parsingContext.getParent();
+ _parsingContext = ctxt;
+ int st;
+ if (ctxt.inObject()) {
+ st = MAJOR_OBJECT_FIELD;
+ } else if (ctxt.inArray()) {
+ st = MAJOR_ARRAY_ELEMENT;
+ } else {
+ st = MAJOR_ROOT;
+ }
+ _majorState = st;
+ _majorStateAfterValue = st;
+ return (_currToken = JsonToken.END_OBJECT);
+ }
+
+ /*
+ /**********************************************************************
/* Internal methods, field name parsing
/**********************************************************************
*/