Fix #616

Author: cowtowncoder

release-notes/CREDITS-2.x

@@ -216,3 +216,8 @@ Volkan Yazıcı (vy@github)
   * Reported #609: (partial fix) `FilteringGeneratorDelegate` does not handle
     `writeString(Reader, int)`
   (2.10.4 [partial], 2.11.0 [full fix])
+
+Justin Liu (jusliu@github)
+  * Reported #616: Parsing JSON with `ALLOW_MISSING_VALUE` enabled results in endless stream
+   of `VALUE_NULL` tokens
+  (2.10.5)

release-notes/VERSION-2.x

@@ -14,6 +14,12 @@ JSON library.
 === Releases ===
 ------------------------------------------------------------------------
 
+2.10.5 (not yet released)
+
+#616: Parsing JSON with `ALLOW_MISSING_VALUE` enabled results in endless stream
+  of `VALUE_NULL` tokens
+ (reported by Justin L)
+
 2.10.4 (03-May-2020)
 
 #605: Handle case when system property access is restricted

src/main/java/com/fasterxml/jackson/core/json/ReaderBasedJsonParser.java

@@ -1132,17 +1132,20 @@ private final JsonToken _nextTokenNotInObject(int i) throws IOException
             return (_currToken = _parsePosNumber(i));
         /*
          * This check proceeds only if the Feature.ALLOW_MISSING_VALUES is enabled
-         * The Check is for missing values. Incase of missing values in an array, the next token will be either ',' or ']'.
+         * The Check is for missing values. In case of missing values in an array, the next token will be either ',' or ']'.
          * This case, decrements the already incremented _inputPtr in the buffer in case of comma(,) 
          * so that the existing flow goes back to checking the next token which will be comma again and
          * it continues the parsing.
          * Also the case returns NULL as current token in case of ',' or ']'.    
          */
+// case ']':  // 11-May-2020, tatu: related to [core#616], this should never be reached
         case ',':
-        case ']':
-            if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
-                --_inputPtr;
-                return (_currToken = JsonToken.VALUE_NULL);  
+            // 11-May-2020, tatu: [core#616] No commas in root level
+            if (!_parsingContext.inRoot()) {
+                if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
+                    --_inputPtr;
+                    return (_currToken = JsonToken.VALUE_NULL);
+                }
             }
         }
         return (_currToken = _handleOddValue(i));
@@ -1881,9 +1884,12 @@ protected JsonToken _handleOddValue(int i) throws IOException
             }
             // fall through
         case ',':
-            if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
-                --_inputPtr;
-                return JsonToken.VALUE_NULL;
+            // 11-May-2020, tatu: [core#616] No commas in root level
+            if (!_parsingContext.inRoot()) {
+                if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
+                    --_inputPtr;
+                    return JsonToken.VALUE_NULL;
+                }
             }
             break;
         case 'N':

src/main/java/com/fasterxml/jackson/core/json/UTF8DataInputJsonParser.java

@@ -2027,10 +2027,13 @@ protected JsonToken _handleUnexpectedValue(int c)
             /* !!! TODO: 08-May-2016, tatu: To support `Feature.ALLOW_MISSING_VALUES` would
              *    need handling here...
              */
-            if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
+            // 11-May-2020, tatu: [core#616] No commas in root level
+            if (!_parsingContext.inRoot()) {
+                if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
 //               _inputPtr--;
-                _nextByte = c;
-               return JsonToken.VALUE_NULL;
+                    _nextByte = c;
+                   return JsonToken.VALUE_NULL;
+                }
             }
             // fall through
         case '}':

src/main/java/com/fasterxml/jackson/core/json/UTF8StreamJsonParser.java

@@ -2611,9 +2611,12 @@ protected JsonToken _handleUnexpectedValue(int c) throws IOException
             // 28-Mar-2016: [core#116]: If Feature.ALLOW_MISSING_VALUES is enabled
             //   we may allow "missing values", that is, encountering a trailing
             //   comma or closing marker where value would be expected
-            if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
-                --_inputPtr;
-                return JsonToken.VALUE_NULL;
+            // 11-May-2020, tatu: [core#616] No commas in root level
+            if (!_parsingContext.inRoot()) {
+                if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
+                    --_inputPtr;
+                    return JsonToken.VALUE_NULL;
+                }
             }
             // fall through
         case '}':

src/main/java/com/fasterxml/jackson/core/json/async/NonBlockingJsonParser.java

@@ -904,9 +904,12 @@ protected JsonToken _startUnexpectedValue(boolean leadingComma, int ch) throws I
             // 28-Mar-2016: [core#116]: If Feature.ALLOW_MISSING_VALUES is enabled
             //   we may allow "missing values", that is, encountering a trailing
             //   comma or closing marker where value would be expected
-            if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
-                --_inputPtr;
-                return _valueComplete(JsonToken.VALUE_NULL);
+            // 11-May-2020, tatu: [core#616] No commas in root level
+            if (!_parsingContext.inRoot()) {
+                if ((_features & FEAT_MASK_ALLOW_MISSING) != 0) {
+                    --_inputPtr;
+                    return _valueComplete(JsonToken.VALUE_NULL);
+                }
             }
             // fall through
         case INT_RCURLY:

src/test/java/com/fasterxml/jackson/core/read/TrailingCommas616Test.java

@@ -0,0 +1,40 @@
+package com.fasterxml.jackson.core.read;
+
+import com.fasterxml.jackson.core.*;
+import com.fasterxml.jackson.core.json.JsonReadFeature;
+
+public class TrailingCommas616Test extends BaseTest
+{
+    final JsonFactory f = JsonFactory.builder()
+            .enable(JsonReadFeature.ALLOW_MISSING_VALUES)
+            .build();
+
+    // [core#616]
+    public void testRootLevelComma616() throws Exception
+    {
+        _testRootLevel616(f, MODE_READER);
+    }
+
+    public void testRootLevelComma616Bytes() throws Exception
+    {
+        _testRootLevel616(f, MODE_INPUT_STREAM);
+        _testRootLevel616(f, MODE_INPUT_STREAM_THROTTLED);
+    }
+
+    public void testRootLevelComma616DataInput() throws Exception
+    {
+        _testRootLevel616(f, MODE_DATA_INPUT);
+    }
+
+    private void _testRootLevel616(JsonFactory f, int mode) throws Exception
+    {
+        JsonParser p = createParser(f, mode, ",");
+        try {
+            p.nextToken();
+            fail("Should not pass");
+        } catch (JsonParseException e) {
+            verifyException(e, "Unexpected character (','");
+        }
+        p.close();
+    }
+}