From 0aef5f8f6da5c5276b97c2acc7165db407243c96 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Mon, 7 Apr 2025 17:47:34 -0700 Subject: [PATCH 1/8] Add annotation tests --- annotations/README.md | 104 ++++++ annotations/assertion.schema.json | 21 ++ annotations/test-case.schema.json | 38 ++ annotations/test-suite.schema.json | 15 + annotations/test.schema.json | 16 + annotations/tests/applicators.json | 459 +++++++++++++++++++++++ annotations/tests/content.json | 64 ++++ annotations/tests/core.json | 38 ++ annotations/tests/format.json | 24 ++ annotations/tests/meta-data.json | 136 +++++++ annotations/tests/unevaluated.json | 579 +++++++++++++++++++++++++++++ annotations/tests/unknown.json | 25 ++ annotations/tests/validation.json | 341 +++++++++++++++++ 13 files changed, 1860 insertions(+) create mode 100644 annotations/README.md create mode 100644 annotations/assertion.schema.json create mode 100644 annotations/test-case.schema.json create mode 100644 annotations/test-suite.schema.json create mode 100644 annotations/test.schema.json create mode 100644 annotations/tests/applicators.json create mode 100644 annotations/tests/content.json create mode 100644 annotations/tests/core.json create mode 100644 annotations/tests/format.json create mode 100644 annotations/tests/meta-data.json create mode 100644 annotations/tests/unevaluated.json create mode 100644 annotations/tests/unknown.json create mode 100644 annotations/tests/validation.json diff --git a/annotations/README.md b/annotations/README.md new file mode 100644 index 00000000..826f98cc --- /dev/null +++ b/annotations/README.md @@ -0,0 +1,104 @@ +# Annotation Tests + +The annotations Test Suite tests which annotations should appear (or not appear) +on which values of an instance. These tests are agnostic of any output format. + +## Supported Dialects + +Although the concept of annotations didn't appear in the spec until 2019-09, the +concept is compatible with every version of JSON Schema. Test Cases in this Test +Suite are designed to be compatible with as many releases of JSON Schema as +possible. They do not include `$schema` or `$id`/`id` keywords so that +implementations can run the same Test Suite for each dialect they support. + +Since this Test Suite can be used for a variety of dialects, there are a couple +of options that can be used by Test Runners to filter out Test Cases that don't +apply to the dialect under test. + +## Test Case Components + +### description + +A short description of what behavior the Test Case is covering. + +### compatibility + +The `compatibility` option allows you to set which dialects the Test Case is +compatible with. Test Runners can use this value to filter out Test Cases that +don't apply the to dialect currently under test. Dialects are indicated by the +number corresponding to their release. Date-based releases use just the year. + +If this option isn't present, it means the Test Case is compatible with any +dialect. + +If this option is present with a number, the number indicates the minimum +release the Test Case is compatible with. This example indicates that the Test +Case is compatible with draft-07 and up. + +**Example**: `"compatibility": "7"` + +You can use a `<=` operator to indicate that the Test Case is compatible with +releases less then or equal to the given release. This example indicates that +the Test Case is compatible with 2019-09 and under. + +**Example**: `"compatibility": "<=2019"` + +You can use comma-separated values to indicate multiple constraints if needed. +This example indicates that the Test Case is compatible with releases between +draft-06 and 2019-09. + +**Example**: `"compatibility": "6,<=2019"` + +For convenience, you can use the `=` operator to indicate a Test Case is only +compatible with a single release. This example indicates that the Test Case is +compatible only with 2020-12. + +**Example**: `"compatibility": "=2020"` + +### schema + +The schema that will serve as the subject for the tests. Whenever possible, this +schema shouldn't include `$schema` or `id`/`$id` because Test Cases should be +designed to work with as many releases as possible. + +### externalSchemas + +`externalSchemas` allows you to define additional schemas that `schema` makes +references to. The value is an object where the keys are retrieval URIs and +values are schemas. Most external schemas aren't self identifying (using +`id`/`$id`) and rely on the retrieval URI for identification. This is done to +increase the number of dialects that the test is compatible with. Because `id` +changed to `$id` in draft-06, if you use `$id`, the test becomes incompatible +with draft-03/4 and in most cases, that's not necessary. + +### tests + +A collection of Tests to run to verify the Test Case. + +## Test Components + +### instance + +The JSON instance to be annotated. + +### assertions + +`assertions` are a collection of assertions that must be true for the test to pass. + +## Assertions Components + +### location + +The instance location. + +### keyword + +The annotating keyword. + +### expected + +An array of annotations on the `keyword` - instance `location` pair. `expected` +is an array because there's always a chance that an annotation is applied +multiple times to any given instance location. The `expected` array should be +sorted such that the most recently encountered value for an annotation during +evaluation comes before previously encountered values. diff --git a/annotations/assertion.schema.json b/annotations/assertion.schema.json new file mode 100644 index 00000000..2272f304 --- /dev/null +++ b/annotations/assertion.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "properties": { + "location": { + "markdownDescription": "The instance location.", + "type": "string", + "format": "json-pointer" + }, + "keyword": { + "markdownDescription": "The annotation keyword.", + "type": "string" + }, + "expected": { + "markdownDescription": "An array of annotations on the `keyword` - instance `location` pair.", + "type": "array" + } + }, + "required": ["location", "keyword", "expected"] +} diff --git a/annotations/test-case.schema.json b/annotations/test-case.schema.json new file mode 100644 index 00000000..6df5f109 --- /dev/null +++ b/annotations/test-case.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "properties": { + "description": { + "markdownDescription": "A short description of what behavior the Test Case is covering.", + "type": "string" + }, + "compatibility": { + "markdownDescription": "Set which dialects the Test Case is compatible with. Examples:\n- `\"7\"` -- draft-07 and above\n- `\"<=2019\"` -- 2019-09 and previous\n- `\"6,<=2019\"` -- Between draft-06 and 2019-09\n- `\"=2020\"` -- 2020-12 only", + "type": "string", + "pattern": "^(<=|=)?([123467]|2019|2020)(,(<=|=)?([123467]|2019|2020))*$" + }, + "schema": { + "markdownDescription": "This schema shouldn't include `$schema` or `id`/`$id` unless necesary for the test because Test Cases should be designed to work with as many releases as possible.", + "type": ["boolean", "object"] + }, + "externalSchemas": { + "markdownDescription": "The keys are retrieval URIs and values are schemas.", + "type": "object", + "patternProperties": { + "": { + "type": ["boolean", "object"] + } + }, + "propertyNames": { + "format": "uri" + } + }, + "tests": { + "markdownDescription": "A collection of Tests to run to verify the Test Case.", + "type": "array", + "items": { "$ref": "./test.schema.json" } + } + }, + "required": ["description", "schema", "tests"] +} diff --git a/annotations/test-suite.schema.json b/annotations/test-suite.schema.json new file mode 100644 index 00000000..c8b17f0d --- /dev/null +++ b/annotations/test-suite.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "suite": { + "type": "array", + "items": { "$ref": "./test-case.schema.json" } + } + }, + "required": ["description", "suite"] +} diff --git a/annotations/test.schema.json b/annotations/test.schema.json new file mode 100644 index 00000000..5ab48933 --- /dev/null +++ b/annotations/test.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "properties": { + "instance": { + "markdownDescription": "The JSON instance to be annotated." + }, + "assertions": { + "markdownDescription": "An array of annotations on the `keyword` - instance `location` pair.", + "type": "array", + "items": { "$ref": "./assertion.schema.json" } + } + }, + "required": ["instance", "assertions"] +} diff --git a/annotations/tests/applicators.json b/annotations/tests/applicators.json new file mode 100644 index 00000000..3d3977d9 --- /dev/null +++ b/annotations/tests/applicators.json @@ -0,0 +1,459 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The applicator vocabulary", + "suite": [ + { + "description": "`properties`, `patternProperties`, and `additionalProperties`", + "compatibility": "3", + "schema": { + "properties": { + "foo": { + "title": "Foo" + } + }, + "patternProperties": { + "^a": { + "title": "Bar" + } + }, + "additionalProperties": { + "title": "Baz" + } + }, + "tests": [ + { + "instance": {}, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": [] + }, + { + "location": "/apple", + "keyword": "title", + "expected": [] + }, + { + "location": "/bar", + "keyword": "title", + "expected": [] + } + ] + }, + { + "instance": { + "foo": {}, + "apple": {}, + "baz": {} + }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": [ + "Foo" + ] + }, + { + "location": "/apple", + "keyword": "title", + "expected": [ + "Bar" + ] + }, + { + "location": "/baz", + "keyword": "title", + "expected": [ + "Baz" + ] + } + ] + } + ] + }, + { + "description": "`propertyNames`", + "compatibility": "6", + "schema": { + "propertyNames": { + "const": "foo", + "title": "Foo" + } + }, + "tests": [ + { + "instance": { + "foo": 42 + }, + "assertions": [ + { + "location": "", + "keyword": "propertyNames", + "expected": [] + }, + { + "location": "/foo", + "keyword": "title", + "expected": [] + } + ] + } + ] + }, + { + "description": "`prefixItems` and `items`", + "compatibility": "2020", + "schema": { + "prefixItems": [ + { + "title": "Foo" + } + ], + "items": { + "title": "Bar" + } + }, + "tests": [ + { + "instance": [ + "foo", + "bar" + ], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": [ + "Foo" + ] + }, + { + "location": "/1", + "keyword": "title", + "expected": [ + "Bar" + ] + }, + { + "location": "/2", + "keyword": "title", + "expected": [] + } + ] + } + ] + }, + { + "description": "`contains`", + "compatibility": "6", + "schema": { + "contains": { + "type": "number", + "title": "Foo" + } + }, + "tests": [ + { + "instance": [ + "foo", + 42, + true + ], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": [] + }, + { + "location": "/1", + "keyword": "title", + "expected": [ + "Foo" + ] + }, + { + "location": "/2", + "keyword": "title", + "expected": [] + }, + { + "location": "/3", + "keyword": "title", + "expected": [] + } + ] + } + ] + }, + { + "description": "`allOf`", + "compatibility": "4", + "schema": { + "allOf": [ + { + "title": "Foo" + }, + { + "title": "Bar" + } + ] + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "allOf", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": [ + "Bar", + "Foo" + ] + } + ] + } + ] + }, + { + "description": "`anyOf`", + "compatibility": "4", + "schema": { + "anyOf": [ + { + "type": "integer", + "title": "Foo" + }, + { + "type": "number", + "title": "Bar" + } + ] + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "anyOf", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": [ + "Bar", + "Foo" + ] + } + ] + }, + { + "instance": 4.2, + "assertions": [ + { + "location": "", + "keyword": "title", + "expected": [ + "Bar" + ] + } + ] + } + ] + }, + { + "description": "`oneOf`", + "compatibility": "4", + "schema": { + "oneOf": [ + { + "type": "string", + "title": "Foo" + }, + { + "type": "number", + "title": "Bar" + } + ] + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "oneOf", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": [ + "Foo" + ] + } + ] + }, + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "title", + "expected": [ + "Bar" + ] + } + ] + } + ] + }, + { + "description": "`not`", + "compatibility": "4", + "schema": { + "title": "Foo", + "not": { + "not": { + "title": "Bar" + } + } + }, + "tests": [ + { + "instance": {}, + "assertions": [ + { + "location": "", + "keyword": "not", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": [ + "Foo" + ] + } + ] + } + ] + }, + { + "description": "`dependentSchemas`", + "compatibility": "2019", + "schema": { + "dependentSchemas": { + "foo": { + "title": "Foo" + } + } + }, + "tests": [ + { + "instance": { + "foo": 42 + }, + "assertions": [ + { + "keyword": "dependentSchemas", + "location": "", + "expected": [] + }, + { + "keyword": "title", + "location": "", + "expected": [ + "Foo" + ] + } + ] + }, + { + "instance": { + "foo": 42 + }, + "assertions": [ + { + "keyword": "title", + "location": "/foo", + "expected": [] + } + ] + } + ] + }, + { + "description": "`if`, `then`, and `else`", + "compatibility": "7", + "schema": { + "if": { + "title": "If", + "type": "string" + }, + "then": { + "title": "Then" + }, + "else": { + "title": "Else" + } + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "if", + "expected": [] + }, + { + "location": "", + "keyword": "then", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": [ + "Then", + "If" + ] + } + ] + }, + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "if", + "expected": [] + }, + { + "location": "", + "keyword": "else", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": [ + "Else" + ] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/content.json b/annotations/tests/content.json new file mode 100644 index 00000000..fe16e98b --- /dev/null +++ b/annotations/tests/content.json @@ -0,0 +1,64 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The content vocabulary", + "suite": [ + { + "description": "`contentMediaType` is an annotation", + "compatibility": "7", + "schema": { + "contentMediaType": "application/json" + }, + "tests": [ + { + "instance": "{ \"foo\": \"bar\" }", + "assertions": [ + { + "location": "", + "keyword": "contentMediaType", + "expected": ["application/json"] + } + ] + } + ] + }, + { + "description": "`contentEncoding` is an annotation", + "compatibility": "7", + "schema": { + "contentEncoding": "base64" + }, + "tests": [ + { + "instance": "SGVsbG8gZnJvbSBKU09OIFNjaGVtYQ==", + "assertions": [ + { + "location": "", + "keyword": "contentEncoding", + "expected": ["base64"] + } + ] + } + ] + }, + { + "description": "`contentSchema` is an annotation", + "compatibility": "2019", + "schema": { + "$id": "https://annotations.json-schema.org/test/contentSchema-is-an-annotation", + "contentSchema": { "type": "number" } + }, + "tests": [ + { + "instance": "42", + "assertions": [ + { + "location": "", + "keyword": "contentSchema", + "expected": ["https://annotations.json-schema.org/test/contentSchema-is-an-annotation#/contentSchema"] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/core.json b/annotations/tests/core.json new file mode 100644 index 00000000..0c6f3cea --- /dev/null +++ b/annotations/tests/core.json @@ -0,0 +1,38 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The core vocabulary", + "suite": [ + { + "description": "`$ref` and `$defs`", + "compatibility": "2019", + "schema": { + "$ref": "#/$defs/foo", + "$defs": { + "foo": { "title": "Foo" } + } + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "$ref", + "expected": [] + }, + { + "location": "", + "keyword": "$defs", + "expected": [] + }, + { + "location": "", + "keyword": "title", + "expected": ["Foo"] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/format.json b/annotations/tests/format.json new file mode 100644 index 00000000..a5ec6bbb --- /dev/null +++ b/annotations/tests/format.json @@ -0,0 +1,24 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The format vocabulary", + "suite": [ + { + "description": "`format` is an annotation", + "schema": { + "format": "email" + }, + "tests": [ + { + "instance": "foo@bar.com", + "assertions": [ + { + "location": "", + "keyword": "format", + "expected": ["email"] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/meta-data.json b/annotations/tests/meta-data.json new file mode 100644 index 00000000..6123d909 --- /dev/null +++ b/annotations/tests/meta-data.json @@ -0,0 +1,136 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The meta-data vocabulary", + "suite": [ + { + "description": "`title` is an annotation", + "schema": { + "title": "Foo" + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "title", + "expected": ["Foo"] + } + ] + } + ] + }, + { + "description": "`description` is an annotation", + "schema": { + "description": "Foo" + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "description", + "expected": ["Foo"] + } + ] + } + ] + }, + { + "description": "`default` is an annotation", + "schema": { + "default": "Foo" + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "default", + "expected": ["Foo"] + } + ] + } + ] + }, + { + "description": "`deprecated` is an annotation", + "compatibility": "2019", + "schema": { + "deprecated": true + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "deprecated", + "expected": [true] + } + ] + } + ] + }, + { + "description": "`readOnly` is an annotation", + "compatibility": "7", + "schema": { + "readOnly": true + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "readOnly", + "expected": [true] + } + ] + } + ] + }, + { + "description": "`writeOnly` is an annotation", + "compatibility": "7", + "schema": { + "writeOnly": true + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "writeOnly", + "expected": [true] + } + ] + } + ] + }, + { + "description": "`examples` is an annotation", + "compatibility": "6", + "schema": { + "examples": ["Foo", "Bar"] + }, + "tests": [ + { + "instance": "Foo", + "assertions": [ + { + "location": "", + "keyword": "examples", + "expected": [["Foo", "Bar"]] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/unevaluated.json b/annotations/tests/unevaluated.json new file mode 100644 index 00000000..b716c5c3 --- /dev/null +++ b/annotations/tests/unevaluated.json @@ -0,0 +1,579 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The unevaluated vocabulary", + "suite": [ + { + "description": "`unevaluatedProperties` alone", + "compatibility": "2019", + "schema": { + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Unevaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `properties`", + "compatibility": "2019", + "schema": { + "properties": { + "foo": { "title": "Evaluated" } + }, + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `patternProperties`", + "compatibility": "2019", + "schema": { + "patternProperties": { + "^a": { "title": "Evaluated" } + }, + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "apple": 42, "bar": 24 }, + "assertions": [ + { + "location": "/apple", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `additionalProperties`", + "compatibility": "2019", + "schema": { + "additionalProperties": { "title": "Evaluated" }, + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Evaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `dependentSchemas`", + "compatibility": "2019", + "schema": { + "dependentSchemas": { + "foo": { + "properties": { + "bar": { "title": "Evaluated" } + } + } + }, + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Unevaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Evaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `if`, `then`, and `else`", + "compatibility": "2019", + "schema": { + "if": { + "properties": { + "foo": { + "type": "string", + "title": "If" + } + } + }, + "then": { + "properties": { + "foo": { "title": "Then" } + } + }, + "else": { + "properties": { + "foo": { "title": "Else" } + } + }, + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": "", "bar": 42 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Then", "If"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + }, + { + "instance": { "foo": 42, "bar": "" }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Else"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `allOf`", + "compatibility": "2019", + "schema": { + "allOf": [ + { + "properties": { + "foo": { "title": "Evaluated" } + } + } + ], + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `anyOf`", + "compatibility": "2019", + "schema": { + "anyOf": [ + { + "properties": { + "foo": { "title": "Evaluated" } + } + } + ], + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `oneOf`", + "compatibility": "2019", + "schema": { + "oneOf": [ + { + "properties": { + "foo": { "title": "Evaluated" } + } + } + ], + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedProperties` with `not`", + "compatibility": "2019", + "schema": { + "not": { + "not": { + "properties": { + "foo": { "title": "Evaluated" } + } + } + }, + "unevaluatedProperties": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "/foo", + "keyword": "title", + "expected": ["Unevaluated"] + }, + { + "location": "/bar", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` alone", + "compatibility": "2019", + "schema": { + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": [42, 24], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Unevaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `prefixItems`", + "compatibility": "2020", + "schema": { + "prefixItems": [{ "title": "Evaluated" }], + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": [42, 24], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `contains`", + "compatibility": "2020", + "schema": { + "contains": { + "type": "string", + "title": "Evaluated" + }, + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": ["foo", 42], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `if`, `then`, and `else`", + "compatibility": "2020", + "schema": { + "if": { + "prefixItems": [ + { + "type": "string", + "title": "If" + } + ] + }, + "then": { + "prefixItems": [ + { "title": "Then" } + ] + }, + "else": { + "prefixItems": [ + { "title": "Else" } + ] + }, + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": ["", 42], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Then", "If"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + }, + { + "instance": [42, ""], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Else"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `allOf`", + "compatibility": "2020", + "schema": { + "allOf": [ + { + "prefixItems": [ + { "title": "Evaluated" } + ] + } + ], + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": [42, 24], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `anyOf`", + "compatibility": "2020", + "schema": { + "anyOf": [ + { + "prefixItems": [ + { "title": "Evaluated" } + ] + } + ], + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": [42, 24], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `oneOf`", + "compatibility": "2020", + "schema": { + "oneOf": [ + { + "prefixItems": [ + { "title": "Evaluated" } + ] + } + ], + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": [42, 24], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Evaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + }, + { + "description": "`unevaluatedItems` with `not`", + "compatibility": "2020", + "schema": { + "not": { + "not": { + "prefixItems": [ + { "title": "Evaluated" } + ] + } + }, + "unevaluatedItems": { "title": "Unevaluated" } + }, + "tests": [ + { + "instance": [42, 24], + "assertions": [ + { + "location": "/0", + "keyword": "title", + "expected": ["Unevaluated"] + }, + { + "location": "/1", + "keyword": "title", + "expected": ["Unevaluated"] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/unknown.json b/annotations/tests/unknown.json new file mode 100644 index 00000000..85a92e00 --- /dev/null +++ b/annotations/tests/unknown.json @@ -0,0 +1,25 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "Unknown keywords", + "suite": [ + { + "description": "`unknownKeyword` is an annotation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknownKeyword": "Foo" + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "unknownKeyword", + "expected": ["Foo"] + } + ] + } + ] + } + ] +} diff --git a/annotations/tests/validation.json b/annotations/tests/validation.json new file mode 100644 index 00000000..1faf46b0 --- /dev/null +++ b/annotations/tests/validation.json @@ -0,0 +1,341 @@ +{ + "$schema": "../test-suite.schema.json", + "description": "The validation library", + "suite": [ + { + "description": "`const` doesn't produce annotations", + "compatibility": "6", + "schema": { + "const": "foo" + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "const", + "expected": [] + } + ] + } + ] + }, + { + "description": "`dependentRequired` doesn't produce annotations", + "compatibility": "2019", + "schema": { + "dependentRequired": { + "foo": ["bar"] + } + }, + "tests": [ + { + "instance": { "foo": 1, "bar": 2 }, + "assertions": [ + { + "location": "", + "keyword": "dependentRequired", + "expected": [] + } + ] + } + ] + }, + { + "description": "`enum` doesn't produce annotations", + "schema": { + "enum": ["foo"] + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "enum", + "expected": [] + } + ] + } + ] + }, + { + "description": "`exclusiveMaximum` doesn't produce annotations", + "compatibility": "6", + "schema": { + "exclusiveMaximum": 50 + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "exclusiveMaximum", + "expected": [] + } + ] + } + ] + }, + { + "description": "`exclusiveMinimum` doesn't produce annotations", + "compatibility": "6", + "schema": { + "exclusiveMinimum": 5 + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "exclusiveMinimum", + "expected": [] + } + ] + } + ] + }, + { + "description": "`maxItems` doesn't produce annotations", + "schema": { + "maxItems": 42 + }, + "tests": [ + { + "instance": ["foo"], + "assertions": [ + { + "location": "", + "keyword": "maxItems", + "expected": [] + } + ] + } + ] + }, + { + "description": "`maxLength` doesn't produce annotations", + "schema": { + "maxLength": 42 + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "maxLength", + "expected": [] + } + ] + } + ] + }, + { + "description": "`maxProperties` doesn't produce annotations", + "compatibility": "4", + "schema": { + "maxProperties": 42 + }, + "tests": [ + { + "instance": { "foo": 42 }, + "assertions": [ + { + "location": "", + "keyword": "maxProperties", + "expected": [] + } + ] + } + ] + }, + { + "description": "`maximum` doesn't produce annotations", + "schema": { + "maximum": 50 + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "maximum", + "expected": [] + } + ] + } + ] + }, + { + "description": "`minItems` doesn't produce annotations", + "schema": { + "minItems": 2 + }, + "tests": [ + { + "instance": ["a", "b"], + "assertions": [ + { + "location": "", + "keyword": "minItems", + "expected": [] + } + ] + } + ] + }, + { + "description": "`minLength` doesn't produce annotations", + "schema": { + "minLength": 2 + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "minLength", + "expected": [] + } + ] + } + ] + }, + { + "description": "`minProperties` doesn't produce annotations", + "compatibility": "4", + "schema": { + "minProperties": 2 + }, + "tests": [ + { + "instance": { "foo": 42, "bar": 24 }, + "assertions": [ + { + "location": "", + "keyword": "minProperties", + "expected": [] + } + ] + } + ] + }, + { + "description": "`minimum` doesn't produce annotations", + "schema": { + "minimum": 42 + }, + "tests": [ + { + "instance": 50, + "assertions": [ + { + "location": "", + "keyword": "minimum", + "expected": [] + } + ] + } + ] + }, + { + "description": "`multipleOf` doesn't produce annotations", + "compatibility": "4", + "schema": { + "multipleOf": 2 + }, + "tests": [ + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "multipleOf", + "expected": [] + } + ] + } + ] + }, + { + "description": "`pattern` doesn't produce annotations", + "schema": { + "pattern": ".*" + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "pattern", + "expected": [] + } + ] + } + ] + }, + { + "description": "`required` doesn't produce annotations", + "compatibility": "3", + "schema": { + "required": ["foo"] + }, + "tests": [ + { + "instance": { "foo": 42 }, + "assertions": [ + { + "location": "", + "keyword": "required", + "expected": [] + } + ] + } + ] + }, + { + "description": "`type` doesn't produce annotations", + "schema": { + "type": "string" + }, + "tests": [ + { + "instance": "foo", + "assertions": [ + { + "location": "", + "keyword": "type", + "expected": [] + } + ] + } + ] + }, + { + "description": "`uniqueItems` doesn't produce annotations", + "compatibility": "2", + "schema": { + "uniqueItems": true + }, + "tests": [ + { + "instance": ["foo", "bar"], + "assertions": [ + { + "location": "", + "keyword": "uniqueItems", + "expected": [] + } + ] + } + ] + } + ] +} From 55b9fadba84353834ce366d72881644fda4cfc6e Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Mon, 7 Apr 2025 20:27:27 -0700 Subject: [PATCH 2/8] Add automation to check that annotation tests are valid --- .github/workflows/annotation-tests.yml | 21 +++++++++++++++++ bin/annotation-tests.ts | 31 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/workflows/annotation-tests.yml create mode 100755 bin/annotation-tests.ts diff --git a/.github/workflows/annotation-tests.yml b/.github/workflows/annotation-tests.yml new file mode 100644 index 00000000..16de7b16 --- /dev/null +++ b/.github/workflows/annotation-tests.yml @@ -0,0 +1,21 @@ +name: Validate annotation tests + +on: + pull_request: + paths: + - "annotations/**" + +jobs: + annotate: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Deno + uses: denoland/setup-deno@v2 + with: + deno-version: "2.x" + + - name: Validate annotation tests + run: deno --node-modules-dir=auto --allow-read --no-prompt bin/annotation-tests.ts diff --git a/bin/annotation-tests.ts b/bin/annotation-tests.ts new file mode 100755 index 00000000..2d3d1932 --- /dev/null +++ b/bin/annotation-tests.ts @@ -0,0 +1,31 @@ +#!/usr/bin/env deno +import { validate } from "npm:@hyperjump/json-schema/draft-07"; +import { BASIC } from "npm:@hyperjump/json-schema/experimental"; + +const validateTestSuite = await validate("./annotations/test-suite.schema.json"); + +console.log("Validating annotation tests ..."); + +let isValid = true; +for await (const entry of Deno.readDir("./annotations/tests")) { + if (entry.isFile) { + const json = await Deno.readTextFile(`./annotations/tests/${entry.name}`); + const suite = JSON.parse(json); + + const output = validateTestSuite(suite, BASIC); + + if (output.valid) { + console.log(`\x1b[32m✔\x1b[0m ${entry.name}`); + } else { + isValid = false; + console.log(`\x1b[31m✖\x1b[0m ${entry.name}`); + console.log(output); + } + } +} + +console.log("Done."); + +if (!isValid) { + Deno.exit(1); +} From 3426e76b2ce0719dad8e5b808917a571f58ac511 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Tue, 8 Apr 2025 16:13:24 -0700 Subject: [PATCH 3/8] Updates based on feedback from Karen --- annotations/README.md | 35 ++++++++++++++++++------------- annotations/assertion.schema.json | 2 +- annotations/test.schema.json | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/annotations/README.md b/annotations/README.md index 826f98cc..efa0686a 100644 --- a/annotations/README.md +++ b/annotations/README.md @@ -1,7 +1,10 @@ -# Annotation Tests +# Annotations Tests Suite -The annotations Test Suite tests which annotations should appear (or not appear) +The Annotations Test Suite tests which annotations should appear (or not appear) on which values of an instance. These tests are agnostic of any output format. +Similar to the validation tests, they only test the end result and don't include +details like mapping the annotation back to the schema location that contributed +it. This Test Suite leaves those details to be tested by output format tests. ## Supported Dialects @@ -63,13 +66,13 @@ designed to work with as many releases as possible. ### externalSchemas -`externalSchemas` allows you to define additional schemas that `schema` makes -references to. The value is an object where the keys are retrieval URIs and -values are schemas. Most external schemas aren't self identifying (using -`id`/`$id`) and rely on the retrieval URI for identification. This is done to -increase the number of dialects that the test is compatible with. Because `id` -changed to `$id` in draft-06, if you use `$id`, the test becomes incompatible -with draft-03/4 and in most cases, that's not necessary. +This allows you to define additional schemas that `schema` makes references to. +The value is an object where the keys are retrieval URIs and values are schemas. +Most external schemas aren't self identifying (using `id`/`$id`) and rely on the +retrieval URI for identification. This is done to increase the number of +dialects that the test is compatible with. Because `id` changed to `$id` in +draft-06, if you use `$id`, the test becomes incompatible with draft-03/4 and in +most cases, that's not necessary. ### tests @@ -83,7 +86,7 @@ The JSON instance to be annotated. ### assertions -`assertions` are a collection of assertions that must be true for the test to pass. +A collection of assertions that must be true for the test to pass. ## Assertions Components @@ -97,8 +100,10 @@ The annotating keyword. ### expected -An array of annotations on the `keyword` - instance `location` pair. `expected` -is an array because there's always a chance that an annotation is applied -multiple times to any given instance location. The `expected` array should be -sorted such that the most recently encountered value for an annotation during -evaluation comes before previously encountered values. +An array of `keyword` annotations expected on the instance at `location`. +`expected` is an array because there's always a chance that an annotation is +applied multiple times to any given instance location. Test runners can consider +this an unordered list, but as a convention for this Test Suite, the `expected` +array should be sorted such that the most recently encountered value for an +annotation given top-down evaluation of the schema comes before previously +encountered values. diff --git a/annotations/assertion.schema.json b/annotations/assertion.schema.json index 2272f304..e0b8a6a0 100644 --- a/annotations/assertion.schema.json +++ b/annotations/assertion.schema.json @@ -13,7 +13,7 @@ "type": "string" }, "expected": { - "markdownDescription": "An array of annotations on the `keyword` - instance `location` pair.", + "markdownDescription": "An array of `keyword` annotations expected on the instance at `location`.", "type": "array" } }, diff --git a/annotations/test.schema.json b/annotations/test.schema.json index 5ab48933..3581fbfc 100644 --- a/annotations/test.schema.json +++ b/annotations/test.schema.json @@ -7,7 +7,7 @@ "markdownDescription": "The JSON instance to be annotated." }, "assertions": { - "markdownDescription": "An array of annotations on the `keyword` - instance `location` pair.", + "markdownDescription": "A collection of assertions that must be true for the test to pass.", "type": "array", "items": { "$ref": "./assertion.schema.json" } } From 9f90e2d17c7e16478d27516b9363c44d4c1681e5 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Wed, 9 Apr 2025 13:07:22 -0700 Subject: [PATCH 4/8] Updates based on feedback from Juan --- annotations/README.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/annotations/README.md b/annotations/README.md index efa0686a..cef4a921 100644 --- a/annotations/README.md +++ b/annotations/README.md @@ -28,11 +28,15 @@ A short description of what behavior the Test Case is covering. The `compatibility` option allows you to set which dialects the Test Case is compatible with. Test Runners can use this value to filter out Test Cases that -don't apply the to dialect currently under test. Dialects are indicated by the -number corresponding to their release. Date-based releases use just the year. +don't apply the to dialect currently under test. The concept of annotations +didn't appear in the spec until 2019-09, but the concept is compatible with +older releases as well. When setting `compatibility`, test authors should take +into account dialects before 2019-09 for implementations that chose to support +annotations for older dialects. -If this option isn't present, it means the Test Case is compatible with any -dialect. +Dialects are indicated by the number corresponding to their release. Date-based +releases use just the year. If this option isn't present, it means the Test Case +is compatible with any dialect. If this option is present with a number, the number indicates the minimum release the Test Case is compatible with. This example indicates that the Test @@ -102,8 +106,11 @@ The annotating keyword. An array of `keyword` annotations expected on the instance at `location`. `expected` is an array because there's always a chance that an annotation is -applied multiple times to any given instance location. Test runners can consider -this an unordered list, but as a convention for this Test Suite, the `expected` -array should be sorted such that the most recently encountered value for an -annotation given top-down evaluation of the schema comes before previously -encountered values. +applied multiple times to any given instance location. An empty array is an +assertion that the annotation must not appear at the `location` for the +`keyword`. + +Test runners can consider this an unordered list, but as a convention for this +Test Suite, the `expected` array should be sorted such that the most recently +encountered value for an annotation given top-down evaluation of the schema +comes before previously encountered values. From 2779c991e855e25c4dd64e2165578efbf1b85310 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Thu, 10 Apr 2025 09:34:58 -0700 Subject: [PATCH 5/8] Make order of assertion properties consistent --- annotations/tests/applicators.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/annotations/tests/applicators.json b/annotations/tests/applicators.json index 3d3977d9..f0602288 100644 --- a/annotations/tests/applicators.json +++ b/annotations/tests/applicators.json @@ -365,13 +365,13 @@ }, "assertions": [ { - "keyword": "dependentSchemas", "location": "", + "keyword": "dependentSchemas", "expected": [] }, { - "keyword": "title", "location": "", + "keyword": "title", "expected": [ "Foo" ] @@ -384,8 +384,8 @@ }, "assertions": [ { - "keyword": "title", "location": "/foo", + "keyword": "title", "expected": [] } ] From 35ccff69f2b0a58d54b106bc17f515acd34f3ec1 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Fri, 11 Apr 2025 08:16:08 -0700 Subject: [PATCH 6/8] Remove tests that assert a keyword doesn't emit annotations --- annotations/tests/applicators.json | 50 ----- annotations/tests/core.json | 10 - annotations/tests/validation.json | 341 ----------------------------- 3 files changed, 401 deletions(-) delete mode 100644 annotations/tests/validation.json diff --git a/annotations/tests/applicators.json b/annotations/tests/applicators.json index f0602288..9f5b51c0 100644 --- a/annotations/tests/applicators.json +++ b/annotations/tests/applicators.json @@ -88,11 +88,6 @@ "foo": 42 }, "assertions": [ - { - "location": "", - "keyword": "propertyNames", - "expected": [] - }, { "location": "/foo", "keyword": "title", @@ -205,11 +200,6 @@ { "instance": "foo", "assertions": [ - { - "location": "", - "keyword": "allOf", - "expected": [] - }, { "location": "", "keyword": "title", @@ -241,11 +231,6 @@ { "instance": 42, "assertions": [ - { - "location": "", - "keyword": "anyOf", - "expected": [] - }, { "location": "", "keyword": "title", @@ -289,11 +274,6 @@ { "instance": "foo", "assertions": [ - { - "location": "", - "keyword": "oneOf", - "expected": [] - }, { "location": "", "keyword": "title", @@ -332,11 +312,6 @@ { "instance": {}, "assertions": [ - { - "location": "", - "keyword": "not", - "expected": [] - }, { "location": "", "keyword": "title", @@ -364,11 +339,6 @@ "foo": 42 }, "assertions": [ - { - "location": "", - "keyword": "dependentSchemas", - "expected": [] - }, { "location": "", "keyword": "title", @@ -411,16 +381,6 @@ { "instance": "foo", "assertions": [ - { - "location": "", - "keyword": "if", - "expected": [] - }, - { - "location": "", - "keyword": "then", - "expected": [] - }, { "location": "", "keyword": "title", @@ -434,16 +394,6 @@ { "instance": 42, "assertions": [ - { - "location": "", - "keyword": "if", - "expected": [] - }, - { - "location": "", - "keyword": "else", - "expected": [] - }, { "location": "", "keyword": "title", diff --git a/annotations/tests/core.json b/annotations/tests/core.json index 0c6f3cea..21a0ed7f 100644 --- a/annotations/tests/core.json +++ b/annotations/tests/core.json @@ -15,16 +15,6 @@ { "instance": "foo", "assertions": [ - { - "location": "", - "keyword": "$ref", - "expected": [] - }, - { - "location": "", - "keyword": "$defs", - "expected": [] - }, { "location": "", "keyword": "title", diff --git a/annotations/tests/validation.json b/annotations/tests/validation.json deleted file mode 100644 index 1faf46b0..00000000 --- a/annotations/tests/validation.json +++ /dev/null @@ -1,341 +0,0 @@ -{ - "$schema": "../test-suite.schema.json", - "description": "The validation library", - "suite": [ - { - "description": "`const` doesn't produce annotations", - "compatibility": "6", - "schema": { - "const": "foo" - }, - "tests": [ - { - "instance": "foo", - "assertions": [ - { - "location": "", - "keyword": "const", - "expected": [] - } - ] - } - ] - }, - { - "description": "`dependentRequired` doesn't produce annotations", - "compatibility": "2019", - "schema": { - "dependentRequired": { - "foo": ["bar"] - } - }, - "tests": [ - { - "instance": { "foo": 1, "bar": 2 }, - "assertions": [ - { - "location": "", - "keyword": "dependentRequired", - "expected": [] - } - ] - } - ] - }, - { - "description": "`enum` doesn't produce annotations", - "schema": { - "enum": ["foo"] - }, - "tests": [ - { - "instance": "foo", - "assertions": [ - { - "location": "", - "keyword": "enum", - "expected": [] - } - ] - } - ] - }, - { - "description": "`exclusiveMaximum` doesn't produce annotations", - "compatibility": "6", - "schema": { - "exclusiveMaximum": 50 - }, - "tests": [ - { - "instance": 42, - "assertions": [ - { - "location": "", - "keyword": "exclusiveMaximum", - "expected": [] - } - ] - } - ] - }, - { - "description": "`exclusiveMinimum` doesn't produce annotations", - "compatibility": "6", - "schema": { - "exclusiveMinimum": 5 - }, - "tests": [ - { - "instance": 42, - "assertions": [ - { - "location": "", - "keyword": "exclusiveMinimum", - "expected": [] - } - ] - } - ] - }, - { - "description": "`maxItems` doesn't produce annotations", - "schema": { - "maxItems": 42 - }, - "tests": [ - { - "instance": ["foo"], - "assertions": [ - { - "location": "", - "keyword": "maxItems", - "expected": [] - } - ] - } - ] - }, - { - "description": "`maxLength` doesn't produce annotations", - "schema": { - "maxLength": 42 - }, - "tests": [ - { - "instance": "foo", - "assertions": [ - { - "location": "", - "keyword": "maxLength", - "expected": [] - } - ] - } - ] - }, - { - "description": "`maxProperties` doesn't produce annotations", - "compatibility": "4", - "schema": { - "maxProperties": 42 - }, - "tests": [ - { - "instance": { "foo": 42 }, - "assertions": [ - { - "location": "", - "keyword": "maxProperties", - "expected": [] - } - ] - } - ] - }, - { - "description": "`maximum` doesn't produce annotations", - "schema": { - "maximum": 50 - }, - "tests": [ - { - "instance": 42, - "assertions": [ - { - "location": "", - "keyword": "maximum", - "expected": [] - } - ] - } - ] - }, - { - "description": "`minItems` doesn't produce annotations", - "schema": { - "minItems": 2 - }, - "tests": [ - { - "instance": ["a", "b"], - "assertions": [ - { - "location": "", - "keyword": "minItems", - "expected": [] - } - ] - } - ] - }, - { - "description": "`minLength` doesn't produce annotations", - "schema": { - "minLength": 2 - }, - "tests": [ - { - "instance": "foo", - "assertions": [ - { - "location": "", - "keyword": "minLength", - "expected": [] - } - ] - } - ] - }, - { - "description": "`minProperties` doesn't produce annotations", - "compatibility": "4", - "schema": { - "minProperties": 2 - }, - "tests": [ - { - "instance": { "foo": 42, "bar": 24 }, - "assertions": [ - { - "location": "", - "keyword": "minProperties", - "expected": [] - } - ] - } - ] - }, - { - "description": "`minimum` doesn't produce annotations", - "schema": { - "minimum": 42 - }, - "tests": [ - { - "instance": 50, - "assertions": [ - { - "location": "", - "keyword": "minimum", - "expected": [] - } - ] - } - ] - }, - { - "description": "`multipleOf` doesn't produce annotations", - "compatibility": "4", - "schema": { - "multipleOf": 2 - }, - "tests": [ - { - "instance": 42, - "assertions": [ - { - "location": "", - "keyword": "multipleOf", - "expected": [] - } - ] - } - ] - }, - { - "description": "`pattern` doesn't produce annotations", - "schema": { - "pattern": ".*" - }, - "tests": [ - { - "instance": "foo", - "assertions": [ - { - "location": "", - "keyword": "pattern", - "expected": [] - } - ] - } - ] - }, - { - "description": "`required` doesn't produce annotations", - "compatibility": "3", - "schema": { - "required": ["foo"] - }, - "tests": [ - { - "instance": { "foo": 42 }, - "assertions": [ - { - "location": "", - "keyword": "required", - "expected": [] - } - ] - } - ] - }, - { - "description": "`type` doesn't produce annotations", - "schema": { - "type": "string" - }, - "tests": [ - { - "instance": "foo", - "assertions": [ - { - "location": "", - "keyword": "type", - "expected": [] - } - ] - } - ] - }, - { - "description": "`uniqueItems` doesn't produce annotations", - "compatibility": "2", - "schema": { - "uniqueItems": true - }, - "tests": [ - { - "instance": ["foo", "bar"], - "assertions": [ - { - "location": "", - "keyword": "uniqueItems", - "expected": [] - } - ] - } - ] - } - ] -} From 265103e8a2dd485634aa39099286c71450dde4f1 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Fri, 18 Apr 2025 15:17:36 -0700 Subject: [PATCH 7/8] Update content tests to only apply to string instances --- annotations/tests/content.json | 59 +++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/annotations/tests/content.json b/annotations/tests/content.json index fe16e98b..659baa8d 100644 --- a/annotations/tests/content.json +++ b/annotations/tests/content.json @@ -3,7 +3,7 @@ "description": "The content vocabulary", "suite": [ { - "description": "`contentMediaType` is an annotation", + "description": "`contentMediaType` is an annotation for string instances", "compatibility": "7", "schema": { "contentMediaType": "application/json" @@ -18,11 +18,21 @@ "expected": ["application/json"] } ] + }, + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "contentMediaType", + "expected": [] + } + ] } ] }, { - "description": "`contentEncoding` is an annotation", + "description": "`contentEncoding` is an annotation for string instances", "compatibility": "7", "schema": { "contentEncoding": "base64" @@ -37,11 +47,52 @@ "expected": ["base64"] } ] + }, + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "contentEncoding", + "expected": [] + } + ] + } + ] + }, + { + "description": "`contentSchema` is an annotation for string instances", + "compatibility": "2019", + "schema": { + "$id": "https://annotations.json-schema.org/test/contentSchema-is-an-annotation", + "contentMediaType": "application/json", + "contentSchema": { "type": "number" } + }, + "tests": [ + { + "instance": "42", + "assertions": [ + { + "location": "", + "keyword": "contentSchema", + "expected": [{ "type": "number" }] + } + ] + }, + { + "instance": 42, + "assertions": [ + { + "location": "", + "keyword": "contentSchema", + "expected": [] + } + ] } ] }, { - "description": "`contentSchema` is an annotation", + "description": "`contentSchema` requires `contentMediaType`", "compatibility": "2019", "schema": { "$id": "https://annotations.json-schema.org/test/contentSchema-is-an-annotation", @@ -54,7 +105,7 @@ { "location": "", "keyword": "contentSchema", - "expected": ["https://annotations.json-schema.org/test/contentSchema-is-an-annotation#/contentSchema"] + "expected": [] } ] } From ec05504c8b253d7bbcda2e49237e4c26a496eff9 Mon Sep 17 00:00:00 2001 From: Jason Desrosiers Date: Tue, 22 Apr 2025 17:44:56 -0700 Subject: [PATCH 8/8] Change "expected" to an object with schema locations --- annotations/README.md | 34 +++--- annotations/assertion.schema.json | 7 +- annotations/tests/applicators.json | 114 ++++++++++---------- annotations/tests/content.json | 20 ++-- annotations/tests/core.json | 4 +- annotations/tests/format.json | 4 +- annotations/tests/meta-data.json | 28 +++-- annotations/tests/unevaluated.json | 162 ++++++++++++++++++++++------- annotations/tests/unknown.json | 8 +- 9 files changed, 246 insertions(+), 135 deletions(-) diff --git a/annotations/README.md b/annotations/README.md index cef4a921..69cd3dd7 100644 --- a/annotations/README.md +++ b/annotations/README.md @@ -2,16 +2,13 @@ The Annotations Test Suite tests which annotations should appear (or not appear) on which values of an instance. These tests are agnostic of any output format. -Similar to the validation tests, they only test the end result and don't include -details like mapping the annotation back to the schema location that contributed -it. This Test Suite leaves those details to be tested by output format tests. ## Supported Dialects -Although the concept of annotations didn't appear in the spec until 2019-09, the -concept is compatible with every version of JSON Schema. Test Cases in this Test -Suite are designed to be compatible with as many releases of JSON Schema as -possible. They do not include `$schema` or `$id`/`id` keywords so that +Although the annotation terminology of didn't appear in the spec until 2019-09, +the concept is compatible with every version of JSON Schema. Test Cases in this +Test Suite are designed to be compatible with as many releases of JSON Schema as +possible. They do not include `$schema` or `$id`/`id` keywords so implementations can run the same Test Suite for each dialect they support. Since this Test Suite can be used for a variety of dialects, there are a couple @@ -28,7 +25,7 @@ A short description of what behavior the Test Case is covering. The `compatibility` option allows you to set which dialects the Test Case is compatible with. Test Runners can use this value to filter out Test Cases that -don't apply the to dialect currently under test. The concept of annotations +don't apply the to dialect currently under test. The terminology for annotations didn't appear in the spec until 2019-09, but the concept is compatible with older releases as well. When setting `compatibility`, test authors should take into account dialects before 2019-09 for implementations that chose to support @@ -104,13 +101,16 @@ The annotating keyword. ### expected -An array of `keyword` annotations expected on the instance at `location`. -`expected` is an array because there's always a chance that an annotation is -applied multiple times to any given instance location. An empty array is an -assertion that the annotation must not appear at the `location` for the -`keyword`. +A collection of `keyword` annotations expected on the instance at `location`. +`expected` is an object where the keys are schema locations and the values are +the annotation that schema location contributed for the given `keyword`. -Test runners can consider this an unordered list, but as a convention for this -Test Suite, the `expected` array should be sorted such that the most recently -encountered value for an annotation given top-down evaluation of the schema -comes before previously encountered values. +There can be more than one expected annotation because multiple schema locations +could contribute annotations for a single keyword. + +An empty object is an assertion that the annotation must not appear at the +`location` for the `keyword`. + +As a convention for this Test Suite, the `expected` array should be sorted such +that the most recently encountered value for an annotation given top-down +evaluation of the schema comes before previously encountered values. diff --git a/annotations/assertion.schema.json b/annotations/assertion.schema.json index e0b8a6a0..88251788 100644 --- a/annotations/assertion.schema.json +++ b/annotations/assertion.schema.json @@ -13,8 +13,11 @@ "type": "string" }, "expected": { - "markdownDescription": "An array of `keyword` annotations expected on the instance at `location`.", - "type": "array" + "markdownDescription": "An object of schemaLocation/annotations pairs for `keyword` annotations expected on the instance at `location`.", + "type": "object", + "propertyNames": { + "format": "uri" + } } }, "required": ["location", "keyword", "expected"] diff --git a/annotations/tests/applicators.json b/annotations/tests/applicators.json index 9f5b51c0..919644b9 100644 --- a/annotations/tests/applicators.json +++ b/annotations/tests/applicators.json @@ -27,17 +27,17 @@ { "location": "/foo", "keyword": "title", - "expected": [] + "expected": {} }, { "location": "/apple", "keyword": "title", - "expected": [] + "expected": {} }, { "location": "/bar", "keyword": "title", - "expected": [] + "expected": {} } ] }, @@ -51,23 +51,23 @@ { "location": "/foo", "keyword": "title", - "expected": [ - "Foo" - ] + "expected": { + "#/properties/foo": "Foo" + } }, { "location": "/apple", "keyword": "title", - "expected": [ - "Bar" - ] + "expected": { + "#/patternProperties/%5Ea": "Bar" + } }, { "location": "/baz", "keyword": "title", - "expected": [ - "Baz" - ] + "expected": { + "#/additionalProperties": "Baz" + } } ] } @@ -91,7 +91,7 @@ { "location": "/foo", "keyword": "title", - "expected": [] + "expected": {} } ] } @@ -120,21 +120,21 @@ { "location": "/0", "keyword": "title", - "expected": [ - "Foo" - ] + "expected": { + "#/prefixItems/0": "Foo" + } }, { "location": "/1", "keyword": "title", - "expected": [ - "Bar" - ] + "expected": { + "#/items": "Bar" + } }, { "location": "/2", "keyword": "title", - "expected": [] + "expected": {} } ] } @@ -160,24 +160,24 @@ { "location": "/0", "keyword": "title", - "expected": [] + "expected": {} }, { "location": "/1", "keyword": "title", - "expected": [ - "Foo" - ] + "expected": { + "#/contains": "Foo" + } }, { "location": "/2", "keyword": "title", - "expected": [] + "expected": {} }, { "location": "/3", "keyword": "title", - "expected": [] + "expected": {} } ] } @@ -203,10 +203,10 @@ { "location": "", "keyword": "title", - "expected": [ - "Bar", - "Foo" - ] + "expected": { + "#/allOf/1": "Bar", + "#/allOf/0": "Foo" + } } ] } @@ -234,10 +234,10 @@ { "location": "", "keyword": "title", - "expected": [ - "Bar", - "Foo" - ] + "expected": { + "#/anyOf/1": "Bar", + "#/anyOf/0": "Foo" + } } ] }, @@ -247,9 +247,9 @@ { "location": "", "keyword": "title", - "expected": [ - "Bar" - ] + "expected": { + "#/anyOf/1": "Bar" + } } ] } @@ -277,9 +277,9 @@ { "location": "", "keyword": "title", - "expected": [ - "Foo" - ] + "expected": { + "#/oneOf/0": "Foo" + } } ] }, @@ -289,9 +289,9 @@ { "location": "", "keyword": "title", - "expected": [ - "Bar" - ] + "expected": { + "#/oneOf/1": "Bar" + } } ] } @@ -315,9 +315,9 @@ { "location": "", "keyword": "title", - "expected": [ - "Foo" - ] + "expected": { + "#": "Foo" + } } ] } @@ -342,9 +342,9 @@ { "location": "", "keyword": "title", - "expected": [ - "Foo" - ] + "expected": { + "#/dependentSchemas/foo": "Foo" + } } ] }, @@ -356,7 +356,7 @@ { "location": "/foo", "keyword": "title", - "expected": [] + "expected": {} } ] } @@ -384,10 +384,10 @@ { "location": "", "keyword": "title", - "expected": [ - "Then", - "If" - ] + "expected": { + "#/then": "Then", + "#/if": "If" + } } ] }, @@ -397,9 +397,9 @@ { "location": "", "keyword": "title", - "expected": [ - "Else" - ] + "expected": { + "#/else": "Else" + } } ] } diff --git a/annotations/tests/content.json b/annotations/tests/content.json index 659baa8d..07c17a69 100644 --- a/annotations/tests/content.json +++ b/annotations/tests/content.json @@ -15,7 +15,9 @@ { "location": "", "keyword": "contentMediaType", - "expected": ["application/json"] + "expected": { + "#": "application/json" + } } ] }, @@ -25,7 +27,7 @@ { "location": "", "keyword": "contentMediaType", - "expected": [] + "expected": {} } ] } @@ -44,7 +46,9 @@ { "location": "", "keyword": "contentEncoding", - "expected": ["base64"] + "expected": { + "#": "base64" + } } ] }, @@ -54,7 +58,7 @@ { "location": "", "keyword": "contentEncoding", - "expected": [] + "expected": {} } ] } @@ -75,7 +79,9 @@ { "location": "", "keyword": "contentSchema", - "expected": [{ "type": "number" }] + "expected": { + "#": { "type": "number" } + } } ] }, @@ -85,7 +91,7 @@ { "location": "", "keyword": "contentSchema", - "expected": [] + "expected": {} } ] } @@ -105,7 +111,7 @@ { "location": "", "keyword": "contentSchema", - "expected": [] + "expected": {} } ] } diff --git a/annotations/tests/core.json b/annotations/tests/core.json index 21a0ed7f..1d8dee55 100644 --- a/annotations/tests/core.json +++ b/annotations/tests/core.json @@ -18,7 +18,9 @@ { "location": "", "keyword": "title", - "expected": ["Foo"] + "expected": { + "#/$defs/foo": "Foo" + } } ] } diff --git a/annotations/tests/format.json b/annotations/tests/format.json index a5ec6bbb..d8cf9a7a 100644 --- a/annotations/tests/format.json +++ b/annotations/tests/format.json @@ -14,7 +14,9 @@ { "location": "", "keyword": "format", - "expected": ["email"] + "expected": { + "#": "email" + } } ] } diff --git a/annotations/tests/meta-data.json b/annotations/tests/meta-data.json index 6123d909..be99b652 100644 --- a/annotations/tests/meta-data.json +++ b/annotations/tests/meta-data.json @@ -14,7 +14,9 @@ { "location": "", "keyword": "title", - "expected": ["Foo"] + "expected": { + "#": "Foo" + } } ] } @@ -32,7 +34,9 @@ { "location": "", "keyword": "description", - "expected": ["Foo"] + "expected": { + "#": "Foo" + } } ] } @@ -50,7 +54,9 @@ { "location": "", "keyword": "default", - "expected": ["Foo"] + "expected": { + "#": "Foo" + } } ] } @@ -69,7 +75,9 @@ { "location": "", "keyword": "deprecated", - "expected": [true] + "expected": { + "#": true + } } ] } @@ -88,7 +96,9 @@ { "location": "", "keyword": "readOnly", - "expected": [true] + "expected": { + "#": true + } } ] } @@ -107,7 +117,9 @@ { "location": "", "keyword": "writeOnly", - "expected": [true] + "expected": { + "#": true + } } ] } @@ -126,7 +138,9 @@ { "location": "", "keyword": "examples", - "expected": [["Foo", "Bar"]] + "expected": { + "#": ["Foo", "Bar"] + } } ] } diff --git a/annotations/tests/unevaluated.json b/annotations/tests/unevaluated.json index b716c5c3..9f2db115 100644 --- a/annotations/tests/unevaluated.json +++ b/annotations/tests/unevaluated.json @@ -15,12 +15,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -42,12 +46,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/properties/foo": "Evaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -69,12 +77,16 @@ { "location": "/apple", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/patternProperties/%5Ea": "Evaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -94,12 +106,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/additionalProperties": "Evaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/additionalProperties": "Evaluated" + } } ] } @@ -125,12 +141,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/dependentSchemas/foo/properties/bar": "Evaluated" + } } ] } @@ -167,12 +187,17 @@ { "location": "/foo", "keyword": "title", - "expected": ["Then", "If"] + "expected": { + "#/then/properties/foo": "Then", + "#/if/properties/foo": "If" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] }, @@ -182,12 +207,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Else"] + "expected": { + "#/else/properties/foo": "Else" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -213,12 +242,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/allOf/0/properties/foo": "Evaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -244,12 +277,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/anyOf/0/properties/foo": "Evaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -275,12 +312,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/oneOf/0/properties/foo": "Evaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -306,12 +347,16 @@ { "location": "/foo", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } }, { "location": "/bar", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedProperties": "Unevaluated" + } } ] } @@ -330,12 +375,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -355,12 +404,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/prefixItems/0": "Evaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -383,12 +436,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/contains": "Evaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -425,12 +482,17 @@ { "location": "/0", "keyword": "title", - "expected": ["Then", "If"] + "expected": { + "#/then/prefixItems/0": "Then", + "#/if/prefixItems/0": "If" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] }, @@ -440,12 +502,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Else"] + "expected": { + "#/else/prefixItems/0": "Else" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -471,12 +537,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/allOf/0/prefixItems/0": "Evaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -502,12 +572,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/anyOf/0/prefixItems/0": "Evaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -533,12 +607,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Evaluated"] + "expected": { + "#/oneOf/0/prefixItems/0": "Evaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } @@ -564,12 +642,16 @@ { "location": "/0", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } }, { "location": "/1", "keyword": "title", - "expected": ["Unevaluated"] + "expected": { + "#/unevaluatedItems": "Unevaluated" + } } ] } diff --git a/annotations/tests/unknown.json b/annotations/tests/unknown.json index 85a92e00..b0c89003 100644 --- a/annotations/tests/unknown.json +++ b/annotations/tests/unknown.json @@ -6,7 +6,7 @@ "description": "`unknownKeyword` is an annotation", "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", - "unknownKeyword": "Foo" + "x-unknownKeyword": "Foo" }, "tests": [ { @@ -14,8 +14,10 @@ "assertions": [ { "location": "", - "keyword": "unknownKeyword", - "expected": ["Foo"] + "keyword": "x-unknownKeyword", + "expected": { + "#": "Foo" + } } ] }