diff --git a/.travis.yml b/.travis.yml index 6337236621..bfd530288e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,3 +20,13 @@ matrix: script: - python3 ./source/server-discovery-and-monitoring/tests/errors/generate-error-tests.py - cd source && make && git diff --exit-code + + - name: "Unifed Test Format schema checks" + dist: xenial + language: python + python: + - 3.7 + install: + - npm install -g ajv-cli + script: + - cd source/unified-test-format/tests && make diff --git a/source/unified-test-format/schema-1.0.json b/source/unified-test-format/schema-1.0.json new file mode 100644 index 0000000000..3ce636c952 --- /dev/null +++ b/source/unified-test-format/schema-1.0.json @@ -0,0 +1,286 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": ["description", "schemaVersion", "tests"], + "properties": { + "description": { "type": "string" }, + "schemaVersion": { "$ref": "#/definitions/version" }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/entity" } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/test" } + } + }, + + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { "$ref": "#/definitions/version" }, + "minServerVersion": { "$ref": "#/definitions/version" }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": ["single", "replicaset", "sharded", "sharded-replicaset"] + } + } + } + }, + + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": { "type": "string" }, + "uriOptions": { "type": "object" }, + "useMultipleMongoses": { "type": "boolean" }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": ["commandStartedEvent", "commandSucceededEvent", "commandFailedEvent"] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": ["id", "client", "databaseName"], + "properties": { + "id": { "type": "string" }, + "client": { "type": "string" }, + "databaseName": { "type": "string" }, + "databaseOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": ["id", "database", "collectionName"], + "properties": { + "id": { "type": "string" }, + "database": { "type": "string" }, + "collectionName": { "type": "string" }, + "collectionOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": ["id", "client"], + "properties": { + "id": { "type": "string" }, + "client": { "type": "string" }, + "sessionOptions": { "type": "object" } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": ["id", "database"], + "properties": { + "id": { "type": "string" }, + "database": { "type": "string" }, + "bucketOptions": { "type": "object" } + } + }, + "stream": { + "type": "object", + "additionalProperties": false, + "required": ["id", "hexBytes"], + "properties": { + "id": { "type": "string" }, + "hexBytes": { + "type": "string", + "pattern": "^([0-9A-Fa-f]{2})*$" + } + } + } + } + }, + + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": ["collectionName", "databaseName", "documents"], + "properties": { + "collectionName": { "type": "string" }, + "databaseName": { "type": "string" }, + "documents": { + "type": "array", + "items": { "type": "object" } + } + } + }, + + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": ["client", "events"], + "properties": { + "client": { "type": "string" }, + "events": { + "type": "array", + "items": { "$ref": "#/definitions/expectedEvent" } + } + } + }, + + "expectedEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { "type": "object" }, + "commandName": { "type": "string" }, + "databaseName": { "type": "string" } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { "type": "object" }, + "commandName": { "type": "string" } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { "type": "string" } + } + } + } + }, + + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { "type": "object" }, + "readPreference": { "type": "object" }, + "writeConcern": { "type": "object" } + } + }, + + "operation": { + "type": "object", + "additionalProperties": false, + "required": ["name", "object"], + "properties": { + "name": { "type": "string" }, + "object": { "type": "string" }, + "arguments": { "type": "object" }, + "expectError": { "$ref": "#/definitions/expectedError" }, + "expectResult": {}, + "saveResultAsEntity": { "type": "string" } + }, + "allOf": [ + { "not": { "required": ["expectError", "expectResult"] } }, + { "not": { "required": ["expectError", "saveResultAsEntity"] } } + ] + }, + + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { "type": "boolean" }, + "errorContains": { "type": "string" }, + "errorCode": { "type": "integer" }, + "errorCodeName": { "type": "string" }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "expectResult": {} + } + }, + + "test": { + "type": "object", + "additionalProperties": false, + "required": ["description", "operations"], + "properties": { + "description": { "type": "string" }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + "skipReason": { "type": "string" }, + "operations": { + "type": "array", + "items": { "$ref": "#/definitions/operation" } + }, + "expectEvents": { + "type": "array", + "items": { "$ref": "#/definitions/expectedEventsForClient" } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + } + } + } + } +} diff --git a/source/unified-test-format/schema-testing-example.json b/source/unified-test-format/schema-testing-example.json new file mode 100644 index 0000000000..eabb41674f --- /dev/null +++ b/source/unified-test-format/schema-testing-example.json @@ -0,0 +1,63 @@ +{ + "tests": [ + { + "description": "valid because error:true and result has only expected fields", + "operations": [ + { + "object": "collection", + "name": "runCommand", + "error": true, + "result": { + "errorContains": "string" + } + } + ] + }, + { + "description": "valid because error == false. result can be any type (non-object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "error": false, + "result": 1 + } + ] + }, + { + "description": "valid because error == false. result can be any type (object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "error": false, + "result": { + "foo": "bar" + } + } + ] + }, + { + "description": "valid because error != true. result can be any type (non-object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "result": 1 + } + ] + }, + { + "description": "valid because error != true. result can be any type (object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "result": { + "foo": "bar" + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile new file mode 100644 index 0000000000..2df2261eb5 --- /dev/null +++ b/source/unified-test-format/tests/Makefile @@ -0,0 +1,21 @@ +SCHEMA=../schema-1.0.json + +.PHONY: all invalid valid-fail valid-pass HAS_AJV + +all: invalid valid-fail valid-pass + +invalid: HAS_AJV + @# Redirect stdout to hide expected validation errors + @ajv test -s $(SCHEMA) -d "invalid/*.yml" --invalid > /dev/null && echo "invalid/*.yml passed test" + +valid-fail: HAS_AJV + @ajv test -s $(SCHEMA) -d "valid-fail/*.yml" --valid + +valid-pass: HAS_AJV + @ajv test -s $(SCHEMA) -d "valid-pass/*.yml" --valid + +HAS_AJV: + @if ! command -v ajv > /dev/null; then \ + echo 'Error: need "npm install -g ajv-cli"' 1>&2; \ + exit 1; \ + fi diff --git a/source/unified-test-format/tests/README.rst b/source/unified-test-format/tests/README.rst new file mode 100644 index 0000000000..dd422f7f02 --- /dev/null +++ b/source/unified-test-format/tests/README.rst @@ -0,0 +1,39 @@ +========================= +Unified Test Format Tests +========================= + +.. contents:: + +---- + +Introduction +============ + +This directory contains tests for the Unified Test Format's schema and test +runner implementation(s). Tests are organized in the following directories: + +- ``invalid``: These files do not validate against the schema and are used to + test the schema itself. + +- ``valid-pass``: These files validate against the schema and should pass when + executed with a test runner. + +- ``valid-fail``: These files validate against the schema but should produce + runtime errors or failures when executed with a test runner. Some do so by + violating the "SHOULD" and "SHOULD NOT" guidance in the spec (e.g. referencing + an undefined entity). + +Validating Test Files +===================== + +JSON and YAML test files can be validated using `Ajv `__ +and a schema from the parent directory (e.g. ``schema-1.0.json``). + +Test files can be validated individually like so:: + + ajv -s ../schema-1.0.json -d path/to/test.yml + +Ajv can also be used to assert the validity of test files:: + + ajv test -s ../schema-1.0.json -d "invalid/*.yml" --invalid + ajv test -s ../schema-1.0.json -d "valid-*/*.yml" --valid diff --git a/source/unified-test-format/tests/example-insertOne.json b/source/unified-test-format/tests/example-insertOne.json new file mode 100644 index 0000000000..be41f9eacb --- /dev/null +++ b/source/unified-test-format/tests/example-insertOne.json @@ -0,0 +1,100 @@ +{ + "description": "example-insertOne", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "2.6" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "test", + "documents": [ + { + "_id": 1 + } + ] + } + ], + "tests": [ + { + "description": "insertOne", + "operations": [ + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2 + } + }, + "expectResult": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "databaseName": "test", + "command": { + "insert": "coll", + "documents": [ + { + "_id": 2 + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "test", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/example-insertOne.yml b/source/unified-test-format/tests/example-insertOne.yml new file mode 100644 index 0000000000..22b27bb2f8 --- /dev/null +++ b/source/unified-test-format/tests/example-insertOne.yml @@ -0,0 +1,53 @@ +description: "example-insertOne" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "2.6" + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name test + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + +tests: + - description: "insertOne" + operations: + - + object: *collection0 + name: insertOne + arguments: + document: { _id: 2 } + expectResult: + insertedId: { $$unsetOrMatches: 2 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + databaseName: *database0Name + command: + insert: *collection0Name + documents: + - { _id: 2 } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } diff --git a/source/unified-test-format/tests/invalid/collectionData-additionalProperties.json b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.json new file mode 100644 index 0000000000..2d85093109 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.json @@ -0,0 +1,40 @@ +{ + "description": "collectionData-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo", + "documents": [], + "foo": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml new file mode 100644 index 0000000000..6a2256a639 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml @@ -0,0 +1,26 @@ +description: "collectionData-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-required.json b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.json new file mode 100644 index 0000000000..040dd86a1c --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.json @@ -0,0 +1,38 @@ +{ + "description": "collectionData-collectionName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "databaseName": "foo", + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml new file mode 100644 index 0000000000..2f20805581 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml @@ -0,0 +1,24 @@ +description: "collectionData-collectionName-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - databaseName: *database0Name + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-type.json b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.json new file mode 100644 index 0000000000..676d822e5e --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.json @@ -0,0 +1,39 @@ +{ + "description": "collectionData-collectionName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": 0, + "databaseName": "foo", + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml new file mode 100644 index 0000000000..a9da0e0129 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml @@ -0,0 +1,25 @@ +description: "collectionData-collectionName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: 0 + databaseName: *database0Name + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-required.json b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.json new file mode 100644 index 0000000000..7548f9d5be --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.json @@ -0,0 +1,38 @@ +{ + "description": "collectionData-databaseName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml new file mode 100644 index 0000000000..d8d6e4d571 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml @@ -0,0 +1,24 @@ +description: "collectionData-databaseName-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-type.json b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.json new file mode 100644 index 0000000000..ef719bbf6a --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.json @@ -0,0 +1,39 @@ +{ + "description": "collectionData-databaseName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": 0, + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml new file mode 100644 index 0000000000..27dfaafee1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml @@ -0,0 +1,25 @@ +description: "collectionData-databaseName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: 0 + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-items.json b/source/unified-test-format/tests/invalid/collectionData-documents-items.json new file mode 100644 index 0000000000..2916718d50 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-items.json @@ -0,0 +1,41 @@ +{ + "description": "collectionData-documents-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo", + "documents": [ + 0 + ] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-items.yml b/source/unified-test-format/tests/invalid/collectionData-documents-items.yml new file mode 100644 index 0000000000..6e860e896f --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-items.yml @@ -0,0 +1,25 @@ +description: "collectionData-documents-items" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-required.json b/source/unified-test-format/tests/invalid/collectionData-documents-required.json new file mode 100644 index 0000000000..7b8a7ead2a --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-required.json @@ -0,0 +1,38 @@ +{ + "description": "collectionData-documents-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-required.yml b/source/unified-test-format/tests/invalid/collectionData-documents-required.yml new file mode 100644 index 0000000000..0452842bc7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-required.yml @@ -0,0 +1,24 @@ +description: "collectionData-documents-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-type.json b/source/unified-test-format/tests/invalid/collectionData-documents-type.json new file mode 100644 index 0000000000..953cabae6e --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-type.json @@ -0,0 +1,39 @@ +{ + "description": "collectionData-documents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo", + "documents": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-type.yml b/source/unified-test-format/tests/invalid/collectionData-documents-type.yml new file mode 100644 index 0000000000..db6d8b417a --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-type.yml @@ -0,0 +1,25 @@ +description: "collectionData-documents-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json new file mode 100644 index 0000000000..beef260eed --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "foo": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml new file mode 100644 index 0000000000..e5b92562a2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml @@ -0,0 +1,17 @@ +description: "collectionOrDatabaseOptions-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json new file mode 100644 index 0000000000..1b9f4bcbea --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-readConcern-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "readConcern": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml new file mode 100644 index 0000000000..671a5b0241 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml @@ -0,0 +1,17 @@ +description: "collectionOrDatabaseOptions-readConcern-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + readConcern: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json new file mode 100644 index 0000000000..988b594d13 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-readPreference-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "readPreference": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml new file mode 100644 index 0000000000..84b328fc46 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml @@ -0,0 +1,17 @@ +description: "collectionOrDatabaseOptions-readPreference-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + readPreference: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json new file mode 100644 index 0000000000..bd2157c5cb --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-writeConcern-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "writeConcern": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml new file mode 100644 index 0000000000..5dc3f8b63c --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml @@ -0,0 +1,17 @@ +description: "collectionOrDatabaseOptions-writeConcern-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + writeConcern: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/createEntities-items.json b/source/unified-test-format/tests/invalid/createEntities-items.json new file mode 100644 index 0000000000..8e9d6ff702 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-items.json @@ -0,0 +1,13 @@ +{ + "description": "createEntities-items", + "schemaVersion": "1.0", + "createEntities": [ + 0 + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/createEntities-items.yml b/source/unified-test-format/tests/invalid/createEntities-items.yml new file mode 100644 index 0000000000..f0d00e33b2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-items.yml @@ -0,0 +1,9 @@ +description: "createEntities-items" + +schemaVersion: "1.0" + +createEntities: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/createEntities-minItems.json b/source/unified-test-format/tests/invalid/createEntities-minItems.json new file mode 100644 index 0000000000..3654923d28 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "createEntities-minItems", + "schemaVersion": "1.0", + "createEntities": [], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/createEntities-minItems.yml b/source/unified-test-format/tests/invalid/createEntities-minItems.yml new file mode 100644 index 0000000000..11641ea7d8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-minItems.yml @@ -0,0 +1,9 @@ +description: "createEntities-minItems" + +schemaVersion: "1.0" + +createEntities: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/createEntities-type.json b/source/unified-test-format/tests/invalid/createEntities-type.json new file mode 100644 index 0000000000..ce3c382c93 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-type.json @@ -0,0 +1,11 @@ +{ + "description": "createEntities-type", + "schemaVersion": "1.0", + "createEntities": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/createEntities-type.yml b/source/unified-test-format/tests/invalid/createEntities-type.yml new file mode 100644 index 0000000000..c181f31922 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-type.yml @@ -0,0 +1,9 @@ +description: "createEntities-type" + +schemaVersion: "1.0" + +createEntities: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/description-required.json b/source/unified-test-format/tests/invalid/description-required.json new file mode 100644 index 0000000000..e4e0d0efdf --- /dev/null +++ b/source/unified-test-format/tests/invalid/description-required.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/description-required.yml b/source/unified-test-format/tests/invalid/description-required.yml new file mode 100644 index 0000000000..5bc6ecff5e --- /dev/null +++ b/source/unified-test-format/tests/invalid/description-required.yml @@ -0,0 +1,7 @@ +# description: "description-required" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-additionalProperties.json new file mode 100644 index 0000000000..38b8898787 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-additionalProperties.json @@ -0,0 +1,15 @@ +{ + "description": "entity-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "foo": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-additionalProperties.yml new file mode 100644 index 0000000000..4978518c5e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-additionalProperties.yml @@ -0,0 +1,10 @@ +description: "entity-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json new file mode 100644 index 0000000000..46f9b4038e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "entity-bucket-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml new file mode 100644 index 0000000000..e864aab955 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml @@ -0,0 +1,19 @@ +description: "entity-bucket-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + database: *database0 + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json new file mode 100644 index 0000000000..c3d7423e65 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-bucket-bucketOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0", + "bucketOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml new file mode 100644 index 0000000000..2d80f5ca3f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml @@ -0,0 +1,19 @@ +description: "entity-bucket-bucketOptions-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + database: *database0 + bucketOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-required.json b/source/unified-test-format/tests/invalid/entity-bucket-database-required.json new file mode 100644 index 0000000000..1fde5a96c9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-required.json @@ -0,0 +1,29 @@ +{ + "description": "entity-bucket-database-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml b/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml new file mode 100644 index 0000000000..df7f8d9457 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml @@ -0,0 +1,17 @@ +description: "entity-bucket-database-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-type.json b/source/unified-test-format/tests/invalid/entity-bucket-database-type.json new file mode 100644 index 0000000000..798d273fb0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-type.json @@ -0,0 +1,30 @@ +{ + "description": "entity-bucket-database-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0", + "database": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml new file mode 100644 index 0000000000..e735d16e16 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml @@ -0,0 +1,18 @@ +description: "entity-bucket-database-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + database: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-required.json b/source/unified-test-format/tests/invalid/entity-bucket-id-required.json new file mode 100644 index 0000000000..c547d8ea3c --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-required.json @@ -0,0 +1,29 @@ +{ + "description": "entity-bucket-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "database": "database0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml b/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml new file mode 100644 index 0000000000..b47435870f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml @@ -0,0 +1,17 @@ +description: "entity-bucket-id-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + database: *database0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-type.json b/source/unified-test-format/tests/invalid/entity-bucket-id-type.json new file mode 100644 index 0000000000..f4e10ee630 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-type.json @@ -0,0 +1,30 @@ +{ + "description": "entity-bucket-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": 0, + "database": "database0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml new file mode 100644 index 0000000000..33522fea32 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml @@ -0,0 +1,18 @@ +description: "entity-bucket-id-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: 0 + database: *database0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.json new file mode 100644 index 0000000000..467e1d6ae1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml new file mode 100644 index 0000000000..b93e51d08a --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml @@ -0,0 +1,12 @@ +description: "entity-client-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-id-required.json b/source/unified-test-format/tests/invalid/entity-client-id-required.json new file mode 100644 index 0000000000..4be2fbf8e8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-required.json @@ -0,0 +1,15 @@ +{ + "description": "entity-client-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": {} + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-id-required.yml b/source/unified-test-format/tests/invalid/entity-client-id-required.yml new file mode 100644 index 0000000000..794a025f23 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-required.yml @@ -0,0 +1,10 @@ +description: "entity-client-id-required" + +schemaVersion: "1.0" + +createEntities: + - client: {} + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-id-type.json b/source/unified-test-format/tests/invalid/entity-client-id-type.json new file mode 100644 index 0000000000..cdc7cbc0e7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-type.json @@ -0,0 +1,17 @@ +{ + "description": "entity-client-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-id-type.yml b/source/unified-test-format/tests/invalid/entity-client-id-type.yml new file mode 100644 index 0000000000..fc872b2b22 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-type.yml @@ -0,0 +1,11 @@ +description: "entity-client-id-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json new file mode 100644 index 0000000000..1252ac82d7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json @@ -0,0 +1,20 @@ +{ + "description": "entity-client-ignoreCommandMonitoringEvents-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "ignoreCommandMonitoringEvents": [ + 0 + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml new file mode 100644 index 0000000000..8f63e4d7e6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml @@ -0,0 +1,12 @@ +description: "entity-client-ignoreCommandMonitoringEvents-items" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + ignoreCommandMonitoringEvents: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json new file mode 100644 index 0000000000..e78068a442 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-ignoreCommandMonitoringEvents-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "ignoreCommandMonitoringEvents": [] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml new file mode 100644 index 0000000000..128282c6f3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml @@ -0,0 +1,12 @@ +description: "entity-client-ignoreCommandMonitoringEvents-minItems" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + ignoreCommandMonitoringEvents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json new file mode 100644 index 0000000000..5ac2b340c5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-ignoreCommandMonitoringEvents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "ignoreCommandMonitoringEvents": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml new file mode 100644 index 0000000000..9fa8bfa323 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml @@ -0,0 +1,12 @@ +description: "entity-client-ignoreCommandMonitoringEvents-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + ignoreCommandMonitoringEvents: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json new file mode 100644 index 0000000000..c39c94eee2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json @@ -0,0 +1,20 @@ +{ + "description": "entity-client-observeEvents-enum", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "foo" + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml new file mode 100644 index 0000000000..5deaeb6212 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml @@ -0,0 +1,12 @@ +description: "entity-client-observeEvents-enum" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: ["foo"] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json new file mode 100644 index 0000000000..3aee11e3d5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json @@ -0,0 +1,20 @@ +{ + "description": "entity-client-observeEvents-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + 0 + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml new file mode 100644 index 0000000000..ad73f22699 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml @@ -0,0 +1,12 @@ +description: "entity-client-observeEvents-items" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json new file mode 100644 index 0000000000..e70d90c0a7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-observeEvents-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml new file mode 100644 index 0000000000..bd16e78c94 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml @@ -0,0 +1,12 @@ +description: "entity-client-observeEvents-minItems" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json new file mode 100644 index 0000000000..c144e32369 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-observeEvents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml new file mode 100644 index 0000000000..b75e97faea --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml @@ -0,0 +1,12 @@ +description: "entity-client-observeEvents-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json new file mode 100644 index 0000000000..4252480e98 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-uriOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "uriOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml new file mode 100644 index 0000000000..09bc9f25d3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml @@ -0,0 +1,12 @@ +description: "entity-client-uriOptions-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + uriOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json new file mode 100644 index 0000000000..e429cd71f8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-useMultipleMongoses-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml new file mode 100644 index 0000000000..ef5e4baae9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml @@ -0,0 +1,12 @@ +description: "entity-client-useMultipleMongoses-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + useMultipleMongoses: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json new file mode 100644 index 0000000000..90ee2b1ca0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json @@ -0,0 +1,32 @@ +{ + "description": "entity-collection-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml new file mode 100644 index 0000000000..2010de017a --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml @@ -0,0 +1,20 @@ +description: "entity-collection-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: "foo" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json new file mode 100644 index 0000000000..2446722e5e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json @@ -0,0 +1,30 @@ +{ + "description": "entity-collection-collectionName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml new file mode 100644 index 0000000000..97e1616207 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml @@ -0,0 +1,18 @@ +description: "entity-collection-collectionName-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json new file mode 100644 index 0000000000..ccad66aac9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-collection-collectionName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml new file mode 100644 index 0000000000..46bcb473f8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml @@ -0,0 +1,19 @@ +description: "entity-collection-collectionName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json new file mode 100644 index 0000000000..52220c1cd1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json @@ -0,0 +1,32 @@ +{ + "description": "entity-collection-collectionOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "collectionOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml new file mode 100644 index 0000000000..58d101981d --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml @@ -0,0 +1,20 @@ +description: "entity-collection-collectionOptions-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: "foo" + collectionOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-required.json b/source/unified-test-format/tests/invalid/entity-collection-database-required.json new file mode 100644 index 0000000000..ba96b43f76 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-required.json @@ -0,0 +1,30 @@ +{ + "description": "entity-collection-database-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-required.yml b/source/unified-test-format/tests/invalid/entity-collection-database-required.yml new file mode 100644 index 0000000000..6a584cd9b6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-required.yml @@ -0,0 +1,18 @@ +description: "entity-collection-database-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-type.json b/source/unified-test-format/tests/invalid/entity-collection-database-type.json new file mode 100644 index 0000000000..b87134498d --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-collection-database-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": 0, + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-type.yml b/source/unified-test-format/tests/invalid/entity-collection-database-type.yml new file mode 100644 index 0000000000..8ed3c743e9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-type.yml @@ -0,0 +1,19 @@ +description: "entity-collection-database-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: 0 + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-required.json b/source/unified-test-format/tests/invalid/entity-collection-id-required.json new file mode 100644 index 0000000000..84e5352ead --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-required.json @@ -0,0 +1,30 @@ +{ + "description": "entity-collection-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "database": "database0", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-required.yml b/source/unified-test-format/tests/invalid/entity-collection-id-required.yml new file mode 100644 index 0000000000..83de062263 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-required.yml @@ -0,0 +1,18 @@ +description: "entity-collection-id-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + database: *database0 + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-type.json b/source/unified-test-format/tests/invalid/entity-collection-id-type.json new file mode 100644 index 0000000000..f0821e5250 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-collection-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": 0, + "database": "database0", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-type.yml b/source/unified-test-format/tests/invalid/entity-collection-id-type.yml new file mode 100644 index 0000000000..c7bfa502ee --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-type.yml @@ -0,0 +1,19 @@ +description: "entity-collection-id-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: 0 + database: *database0 + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.json new file mode 100644 index 0000000000..964cd27966 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.json @@ -0,0 +1,25 @@ +{ + "description": "entity-database-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml new file mode 100644 index 0000000000..dcfdc048dc --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml @@ -0,0 +1,16 @@ +description: "entity-database-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-client-required.json b/source/unified-test-format/tests/invalid/entity-database-client-required.json new file mode 100644 index 0000000000..54f99cf13e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-required.json @@ -0,0 +1,23 @@ +{ + "description": "entity-database-client-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-client-required.yml b/source/unified-test-format/tests/invalid/entity-database-client-required.yml new file mode 100644 index 0000000000..6e271b1859 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-required.yml @@ -0,0 +1,14 @@ +description: "entity-database-client-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-client-type.json b/source/unified-test-format/tests/invalid/entity-database-client-type.json new file mode 100644 index 0000000000..ff4584c405 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-database-client-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": 0, + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-client-type.yml b/source/unified-test-format/tests/invalid/entity-database-client-type.yml new file mode 100644 index 0000000000..fe04610863 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-type.yml @@ -0,0 +1,15 @@ +description: "entity-database-client-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: 0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-required.json b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.json new file mode 100644 index 0000000000..64cca95c49 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.json @@ -0,0 +1,23 @@ +{ + "description": "entity-database-databaseName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml new file mode 100644 index 0000000000..cd2c6e77a0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml @@ -0,0 +1,14 @@ +description: "entity-database-databaseName-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-type.json b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.json new file mode 100644 index 0000000000..bd01aef781 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-database-databaseName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml new file mode 100644 index 0000000000..34ec075b64 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml @@ -0,0 +1,15 @@ +description: "entity-database-databaseName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json new file mode 100644 index 0000000000..bc22ad3129 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json @@ -0,0 +1,25 @@ +{ + "description": "entity-database-databaseOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml new file mode 100644 index 0000000000..77828e5390 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml @@ -0,0 +1,16 @@ +description: "entity-database-databaseOptions-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-id-required.json b/source/unified-test-format/tests/invalid/entity-database-id-required.json new file mode 100644 index 0000000000..0b65cf1159 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-required.json @@ -0,0 +1,23 @@ +{ + "description": "entity-database-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "client": "client0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-id-required.yml b/source/unified-test-format/tests/invalid/entity-database-id-required.yml new file mode 100644 index 0000000000..51e63b4ab1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-required.yml @@ -0,0 +1,14 @@ +description: "entity-database-id-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + client: *client0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-id-type.json b/source/unified-test-format/tests/invalid/entity-database-id-type.json new file mode 100644 index 0000000000..98b5789d04 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-database-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": 0, + "client": "client0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-id-type.yml b/source/unified-test-format/tests/invalid/entity-database-id-type.yml new file mode 100644 index 0000000000..1174c36ca4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-type.yml @@ -0,0 +1,15 @@ +description: "entity-database-id-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: 0 + client: *client0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-maxProperties.json b/source/unified-test-format/tests/invalid/entity-maxProperties.json new file mode 100644 index 0000000000..f4a6b7c914 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-maxProperties.json @@ -0,0 +1,22 @@ +{ + "description": "entity-maxProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + }, + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-maxProperties.yml b/source/unified-test-format/tests/invalid/entity-maxProperties.yml new file mode 100644 index 0000000000..c6f8bcffc5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-maxProperties.yml @@ -0,0 +1,15 @@ +description: "entity-maxProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-minProperties.json b/source/unified-test-format/tests/invalid/entity-minProperties.json new file mode 100644 index 0000000000..d89949ce30 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-minProperties.json @@ -0,0 +1,13 @@ +{ + "description": "entity-minProperties", + "schemaVersion": "1.0", + "createEntities": [ + {} + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-minProperties.yml b/source/unified-test-format/tests/invalid/entity-minProperties.yml new file mode 100644 index 0000000000..d758bd5745 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-minProperties.yml @@ -0,0 +1,10 @@ +description: "entity-minProperties" + +schemaVersion: "1.0" + +createEntities: + - {} + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.json new file mode 100644 index 0000000000..ab4cd2014f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.json @@ -0,0 +1,24 @@ +{ + "description": "entity-session-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0", + "client": "client0", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml new file mode 100644 index 0000000000..1bb2233043 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml @@ -0,0 +1,15 @@ +description: "entity-session-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + client: *client0 + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-client-required.json b/source/unified-test-format/tests/invalid/entity-session-client-required.json new file mode 100644 index 0000000000..8c9ed72e99 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-required.json @@ -0,0 +1,22 @@ +{ + "description": "entity-session-client-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-client-required.yml b/source/unified-test-format/tests/invalid/entity-session-client-required.yml new file mode 100644 index 0000000000..3c1fe4e3c0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-required.yml @@ -0,0 +1,13 @@ +description: "entity-session-client-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-client-type.json b/source/unified-test-format/tests/invalid/entity-session-client-type.json new file mode 100644 index 0000000000..b5ccc3f60f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-type.json @@ -0,0 +1,23 @@ +{ + "description": "entity-session-client-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0", + "client": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-client-type.yml b/source/unified-test-format/tests/invalid/entity-session-client-type.yml new file mode 100644 index 0000000000..60353c08db --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-type.yml @@ -0,0 +1,14 @@ +description: "entity-session-client-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + client: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-id-required.json b/source/unified-test-format/tests/invalid/entity-session-id-required.json new file mode 100644 index 0000000000..3e5d5c5439 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-required.json @@ -0,0 +1,22 @@ +{ + "description": "entity-session-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "client": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-id-required.yml b/source/unified-test-format/tests/invalid/entity-session-id-required.yml new file mode 100644 index 0000000000..2aa4269d16 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-required.yml @@ -0,0 +1,13 @@ +description: "entity-session-id-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + client: *client0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-id-type.json b/source/unified-test-format/tests/invalid/entity-session-id-type.json new file mode 100644 index 0000000000..dcd46e5be7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-type.json @@ -0,0 +1,23 @@ +{ + "description": "entity-session-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": 0, + "client": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-id-type.yml b/source/unified-test-format/tests/invalid/entity-session-id-type.yml new file mode 100644 index 0000000000..058dcdb1b7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-type.yml @@ -0,0 +1,14 @@ +description: "entity-session-id-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: 0 + client: *client0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json new file mode 100644 index 0000000000..0ee15891eb --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-session-sessionOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0", + "client": "client0", + "sessionOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml new file mode 100644 index 0000000000..2d651f690b --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml @@ -0,0 +1,15 @@ +description: "entity-session-sessionOptions-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + client: *client0 + sessionOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json new file mode 100644 index 0000000000..c8e76e9985 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json @@ -0,0 +1,19 @@ +{ + "description": "entity-stream-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0", + "hexBytes": "FF", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml new file mode 100644 index 0000000000..e2f4f8e93d --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml @@ -0,0 +1,13 @@ +description: "entity-stream-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + hexBytes: "FF" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json new file mode 100644 index 0000000000..7381893b55 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json @@ -0,0 +1,18 @@ +{ + "description": "entity-stream-hexBytes-pattern", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0", + "hexBytes": "FFF" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml new file mode 100644 index 0000000000..3e1aa8e45d --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml @@ -0,0 +1,12 @@ +description: "entity-stream-hexBytes-pattern" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + hexBytes: "FFF" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json new file mode 100644 index 0000000000..cc3bf09b20 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json @@ -0,0 +1,17 @@ +{ + "description": "entity-stream-hexBytes-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml new file mode 100644 index 0000000000..b1a4b98561 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml @@ -0,0 +1,11 @@ +description: "entity-stream-hexBytes-required" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json new file mode 100644 index 0000000000..e6e2299eac --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-stream-hexBytes-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0", + "hexBytes": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml new file mode 100644 index 0000000000..79b8188c5b --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml @@ -0,0 +1,12 @@ +description: "entity-stream-hexBytes-type" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + hexBytes: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-required.json b/source/unified-test-format/tests/invalid/entity-stream-id-required.json new file mode 100644 index 0000000000..ff814d4e9c --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-required.json @@ -0,0 +1,17 @@ +{ + "description": "entity-stream-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "hexBytes": "FF" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-required.yml b/source/unified-test-format/tests/invalid/entity-stream-id-required.yml new file mode 100644 index 0000000000..0ca388aade --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-required.yml @@ -0,0 +1,11 @@ +description: "entity-stream-id-required" + +schemaVersion: "1.0" + +createEntities: + - stream: + hexBytes: "FF" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-type.json b/source/unified-test-format/tests/invalid/entity-stream-id-type.json new file mode 100644 index 0000000000..5fc654d97e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-stream-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": 0, + "hexBytes": "FF" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-type.yml b/source/unified-test-format/tests/invalid/entity-stream-id-type.yml new file mode 100644 index 0000000000..d7b17952cb --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-type.yml @@ -0,0 +1,12 @@ +description: "entity-stream-id-type" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: 0 + hexBytes: "FF" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/expectedError-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.json new file mode 100644 index 0000000000..3a79df8e34 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "foo": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml new file mode 100644 index 0000000000..6146367962 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml @@ -0,0 +1,15 @@ +description: "expectedError-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCode-type.json b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.json new file mode 100644 index 0000000000..b6b6f5d05a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorCode-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorCode": "foo" + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml new file mode 100644 index 0000000000..2110e9c6ad --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorCode-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorCode: "foo" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json new file mode 100644 index 0000000000..3ac5e43045 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorCodeName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorCodeName": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml new file mode 100644 index 0000000000..c35ec3ed4a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorCodeName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorCodeName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorContains-type.json b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.json new file mode 100644 index 0000000000..847a987dff --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorContains-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorContains": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml new file mode 100644 index 0000000000..f4989b61e3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorContains-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorContains: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json new file mode 100644 index 0000000000..4eab56ad18 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json @@ -0,0 +1,27 @@ +{ + "description": "expectedError-errorLabelsContain-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsContain": [ + 0 + ] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml new file mode 100644 index 0000000000..4382fa645a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorLabelsContain-items" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsContain: [0] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json new file mode 100644 index 0000000000..48162110aa --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsContain-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsContain": [] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml new file mode 100644 index 0000000000..ebd650e11a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorLabelsContain-minItems" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsContain: [] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json new file mode 100644 index 0000000000..a0aba918b5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsContain-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsContain": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml new file mode 100644 index 0000000000..537cfb9cfe --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorLabelsContain-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsContain: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json new file mode 100644 index 0000000000..6c94d07135 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json @@ -0,0 +1,27 @@ +{ + "description": "expectedError-errorLabelsOmit-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsOmit": [ + 0 + ] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml new file mode 100644 index 0000000000..d80711ef0f --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorLabelsOmit-items" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsOmit: [0] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json new file mode 100644 index 0000000000..88c6582028 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsOmit-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsOmit": [] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml new file mode 100644 index 0000000000..205162cf67 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorLabelsOmit-minItems" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsOmit: [] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json new file mode 100644 index 0000000000..5f57114fea --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsOmit-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsOmit": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml new file mode 100644 index 0000000000..6c1947b043 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-errorLabelsOmit-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsOmit: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-isClientError-type.json b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.json new file mode 100644 index 0000000000..bfcc06679b --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-isClientError-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isClientError": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml new file mode 100644 index 0000000000..8b8afc1061 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-isClientError-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isClientError: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-const.json b/source/unified-test-format/tests/invalid/expectedError-isError-const.json new file mode 100644 index 0000000000..6a398bbf22 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-const.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-isError-const", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": false + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-const.yml b/source/unified-test-format/tests/invalid/expectedError-isError-const.yml new file mode 100644 index 0000000000..347ae2965a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-const.yml @@ -0,0 +1,15 @@ +description: "expectedError-isError-const" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: false diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-type.json b/source/unified-test-format/tests/invalid/expectedError-isError-type.json new file mode 100644 index 0000000000..354aff31f4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-isError-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-type.yml b/source/unified-test-format/tests/invalid/expectedError-isError-type.yml new file mode 100644 index 0000000000..3e53f4af90 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-type.yml @@ -0,0 +1,15 @@ +description: "expectedError-isError-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-minProperties.json b/source/unified-test-format/tests/invalid/expectedError-minProperties.json new file mode 100644 index 0000000000..10e0b89ab7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-minProperties.json @@ -0,0 +1,23 @@ +{ + "description": "expectedError-minProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": {} + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-minProperties.yml b/source/unified-test-format/tests/invalid/expectedError-minProperties.yml new file mode 100644 index 0000000000..5292d85008 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-minProperties.yml @@ -0,0 +1,14 @@ +description: "expectedError-minProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: {} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json new file mode 100644 index 0000000000..2c4f7d27e7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json @@ -0,0 +1,32 @@ +{ + "description": "expectedEvent-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "foo": 0 + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml new file mode 100644 index 0000000000..c2c1cb5a7c --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml @@ -0,0 +1,17 @@ +description: "expectedEvent-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json new file mode 100644 index 0000000000..ea6078faae --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandFailedEvent-commandName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandFailedEvent": { + "commandName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml new file mode 100644 index 0000000000..57504b78f8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandFailedEvent-commandName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandFailedEvent: + commandName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json new file mode 100644 index 0000000000..ee6eb50658 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "foo": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml new file mode 100644 index 0000000000..6ea724ddfb --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandStartedEvent-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json new file mode 100644 index 0000000000..4c9483caf3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-command-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml new file mode 100644 index 0000000000..91ba129869 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandStartedEvent-command-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json new file mode 100644 index 0000000000..a5a66096a0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-commandName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml new file mode 100644 index 0000000000..07c968cdd4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandStartedEvent-commandName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json new file mode 100644 index 0000000000..dc040ec108 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-databaseName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml new file mode 100644 index 0000000000..355d90d6f7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandStartedEvent-databaseName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + databaseName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json new file mode 100644 index 0000000000..4a20e906b9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandSucceededEvent-commandName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandSucceededEvent": { + "commandName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml new file mode 100644 index 0000000000..740b377fa2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandSucceededEvent-commandName-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandSucceededEvent: + commandName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json new file mode 100644 index 0000000000..5464542751 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandSucceededEvent-reply-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandSucceededEvent": { + "reply": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml new file mode 100644 index 0000000000..5a4b35e272 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-commandSucceededEvent-reply-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandSucceededEvent: + reply: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json new file mode 100644 index 0000000000..f01441946f --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json @@ -0,0 +1,33 @@ +{ + "description": "expectedEvent-maxProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": {}, + "commandSucceededEvent": {} + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml new file mode 100644 index 0000000000..d349133ea1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml @@ -0,0 +1,18 @@ +description: "expectedEvent-maxProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: {} + commandSucceededEvent: {} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-minProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.json new file mode 100644 index 0000000000..ebcc494894 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.json @@ -0,0 +1,30 @@ +{ + "description": "expectedEvent-minProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + {} + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml new file mode 100644 index 0000000000..88de63a898 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml @@ -0,0 +1,17 @@ +description: "expectedEvent-minProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - {} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json new file mode 100644 index 0000000000..6ecf5931fb --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json @@ -0,0 +1,29 @@ +{ + "description": "expectedEventsForClient-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [], + "foo": 0 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml new file mode 100644 index 0000000000..a15835d1ca --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml @@ -0,0 +1,17 @@ +description: "expectedEventsForClient-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: [] + foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json new file mode 100644 index 0000000000..b879db8598 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json @@ -0,0 +1,27 @@ +{ + "description": "expectedEventsForClient-client-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "events": [] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml new file mode 100644 index 0000000000..57db7b07e2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml @@ -0,0 +1,15 @@ +description: "expectedEventsForClient-client-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - events: [] diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json new file mode 100644 index 0000000000..4ee5427df1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json @@ -0,0 +1,28 @@ +{ + "description": "expectedEventsForClient-client-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": 0, + "events": [] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml new file mode 100644 index 0000000000..015fd7849b --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml @@ -0,0 +1,16 @@ +description: "expectedEventsForClient-client-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: 0 + events: [] diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json new file mode 100644 index 0000000000..ee8ce4a403 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json @@ -0,0 +1,30 @@ +{ + "description": "expectedEventsForClient-events-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + 0 + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml new file mode 100644 index 0000000000..e5a6f4606b --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml @@ -0,0 +1,16 @@ +description: "expectedEventsForClient-events-items" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: [0] diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json new file mode 100644 index 0000000000..7f1bc6fb53 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json @@ -0,0 +1,27 @@ +{ + "description": "expectedEventsForClient-events-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0" + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml new file mode 100644 index 0000000000..dd14eb2b8e --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml @@ -0,0 +1,15 @@ +description: "expectedEventsForClient-events-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json new file mode 100644 index 0000000000..f171fc2b93 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json @@ -0,0 +1,28 @@ +{ + "description": "expectedEventsForClient-events-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": 0 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml new file mode 100644 index 0000000000..b51730f4ff --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml @@ -0,0 +1,16 @@ +description: "expectedEventsForClient-events-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: 0 diff --git a/source/unified-test-format/tests/invalid/initialData-items.json b/source/unified-test-format/tests/invalid/initialData-items.json new file mode 100644 index 0000000000..9c27d554f9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-items.json @@ -0,0 +1,13 @@ +{ + "description": "initialData-items", + "schemaVersion": "1.0", + "initialData": [ + 0 + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/initialData-items.yml b/source/unified-test-format/tests/invalid/initialData-items.yml new file mode 100644 index 0000000000..87fa3e562b --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-items.yml @@ -0,0 +1,9 @@ +description: "initialData-items" + +schemaVersion: "1.0" + +initialData: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/initialData-minItems.json b/source/unified-test-format/tests/invalid/initialData-minItems.json new file mode 100644 index 0000000000..984100a2be --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "initialData-minItems", + "schemaVersion": "1.0", + "initialData": [], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/initialData-minItems.yml b/source/unified-test-format/tests/invalid/initialData-minItems.yml new file mode 100644 index 0000000000..dc1afde97f --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-minItems.yml @@ -0,0 +1,9 @@ +description: "initialData-minItems" + +schemaVersion: "1.0" + +initialData: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/initialData-type.json b/source/unified-test-format/tests/invalid/initialData-type.json new file mode 100644 index 0000000000..c33585e03a --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-type.json @@ -0,0 +1,11 @@ +{ + "description": "initialData-type", + "schemaVersion": "1.0", + "initialData": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/initialData-type.yml b/source/unified-test-format/tests/invalid/initialData-type.yml new file mode 100644 index 0000000000..946c0e4364 --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-type.yml @@ -0,0 +1,9 @@ +description: "initialData-type" + +schemaVersion: "1.0" + +initialData: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/operation-additionalProperties.json b/source/unified-test-format/tests/invalid/operation-additionalProperties.json new file mode 100644 index 0000000000..8f2f1434ec --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-additionalProperties.json @@ -0,0 +1,23 @@ +{ + "description": "operation-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "foo": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-additionalProperties.yml b/source/unified-test-format/tests/invalid/operation-additionalProperties.yml new file mode 100644 index 0000000000..d5dbef8e8f --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-additionalProperties.yml @@ -0,0 +1,14 @@ +description: "operation-additionalProperties" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + foo: 0 diff --git a/source/unified-test-format/tests/invalid/operation-arguments-type.json b/source/unified-test-format/tests/invalid/operation-arguments-type.json new file mode 100644 index 0000000000..a22f3921c3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-arguments-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-arguments-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "arguments": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-arguments-type.yml b/source/unified-test-format/tests/invalid/operation-arguments-type.yml new file mode 100644 index 0000000000..1564f21601 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-arguments-type.yml @@ -0,0 +1,14 @@ +description: "operation-arguments-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + arguments: 0 diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json new file mode 100644 index 0000000000..bc15fbac76 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json @@ -0,0 +1,26 @@ +{ + "description": "operation-expectError-conflicts_with_expectResult", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": true + }, + "expectResult": {} + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml new file mode 100644 index 0000000000..aa9430fdb1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml @@ -0,0 +1,16 @@ +description: "operation-expectError-conflicts_with_expectResult" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: true + expectResult: {} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json new file mode 100644 index 0000000000..dead4a3b9d --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json @@ -0,0 +1,26 @@ +{ + "description": "operation-expectError-conflicts_with_saveResultAsEntity", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": true + }, + "saveResultAsEntity": "foo" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml new file mode 100644 index 0000000000..6c369598fe --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml @@ -0,0 +1,16 @@ +description: "operation-expectError-conflicts_with_saveResultAsEntity" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: true + saveResultAsEntity: "foo" diff --git a/source/unified-test-format/tests/invalid/operation-expectError-type.json b/source/unified-test-format/tests/invalid/operation-expectError-type.json new file mode 100644 index 0000000000..b224ba3535 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-expectError-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-type.yml b/source/unified-test-format/tests/invalid/operation-expectError-type.yml new file mode 100644 index 0000000000..55588d4359 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-type.yml @@ -0,0 +1,14 @@ +description: "operation-expectError-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: 0 diff --git a/source/unified-test-format/tests/invalid/operation-expectEvents-type.json b/source/unified-test-format/tests/invalid/operation-expectEvents-type.json new file mode 100644 index 0000000000..ecd4c011a9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectEvents-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-expectEvents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml b/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml new file mode 100644 index 0000000000..2c9936ed28 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml @@ -0,0 +1,14 @@ +description: "operation-expectEvents-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: 0 diff --git a/source/unified-test-format/tests/invalid/operation-name-required.json b/source/unified-test-format/tests/invalid/operation-name-required.json new file mode 100644 index 0000000000..42fcb3a308 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-required.json @@ -0,0 +1,21 @@ +{ + "description": "operation-name-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "object": "client0" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-name-required.yml b/source/unified-test-format/tests/invalid/operation-name-required.yml new file mode 100644 index 0000000000..253cb579b9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-required.yml @@ -0,0 +1,12 @@ +description: "operation-name-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - object: *client0 diff --git a/source/unified-test-format/tests/invalid/operation-name-type.json b/source/unified-test-format/tests/invalid/operation-name-type.json new file mode 100644 index 0000000000..2f91da078a --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-type.json @@ -0,0 +1,22 @@ +{ + "description": "operation-name-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": 0, + "object": "client0" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-name-type.yml b/source/unified-test-format/tests/invalid/operation-name-type.yml new file mode 100644 index 0000000000..f9323050b5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-type.yml @@ -0,0 +1,13 @@ +description: "operation-name-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: 0 + object: *client0 diff --git a/source/unified-test-format/tests/invalid/operation-object-required.json b/source/unified-test-format/tests/invalid/operation-object-required.json new file mode 100644 index 0000000000..c0410ce3fd --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-required.json @@ -0,0 +1,21 @@ +{ + "description": "operation-object-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-object-required.yml b/source/unified-test-format/tests/invalid/operation-object-required.yml new file mode 100644 index 0000000000..eec6101bbe --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-required.yml @@ -0,0 +1,12 @@ +description: "operation-object-required" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" diff --git a/source/unified-test-format/tests/invalid/operation-object-type.json b/source/unified-test-format/tests/invalid/operation-object-type.json new file mode 100644 index 0000000000..edb0a0b51a --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-type.json @@ -0,0 +1,22 @@ +{ + "description": "operation-object-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-object-type.yml b/source/unified-test-format/tests/invalid/operation-object-type.yml new file mode 100644 index 0000000000..fb8e10267b --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-type.yml @@ -0,0 +1,13 @@ +description: "operation-object-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: 0 diff --git a/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json new file mode 100644 index 0000000000..65ead94c7a --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-saveResultAsEntity-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "saveResultAsEntity": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml new file mode 100644 index 0000000000..7d20938f32 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml @@ -0,0 +1,14 @@ +description: "operation-saveResultAsEntity-type" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + saveResultAsEntity: 0 diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json new file mode 100644 index 0000000000..79fa687e45 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json @@ -0,0 +1,16 @@ +{ + "description": "runOnRequirement-additionalProperties", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "foo": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml new file mode 100644 index 0000000000..044b803d29 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml @@ -0,0 +1,11 @@ +description: "runOnRequirement-additionalProperties" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json new file mode 100644 index 0000000000..78766eb925 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-maxServerVersion-pattern", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "maxServerVersion": "1.2.3.4" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml new file mode 100644 index 0000000000..ce5af27df5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-maxServerVersion-pattern" + +schemaVersion: "1.0" + +runOnRequirements: + - maxServerVersion: "1.2.3.4" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json new file mode 100644 index 0000000000..ffc9118ba2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-maxServerVersion-type", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "maxServerVersion": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml new file mode 100644 index 0000000000..a63df28232 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-maxServerVersion-type" + +schemaVersion: "1.0" + +runOnRequirements: + - maxServerVersion: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json new file mode 100644 index 0000000000..c2bfed3be7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json @@ -0,0 +1,13 @@ +{ + "description": "runOnRequirement-minProperties", + "schemaVersion": "1.0", + "runOnRequirements": [ + {} + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml new file mode 100644 index 0000000000..e8b91fd4f5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-minProperties" + +schemaVersion: "1.0" + +runOnRequirements: + - {} + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json new file mode 100644 index 0000000000..19abc1755f --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-minServerVersion-pattern", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "1.2.3.4" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml new file mode 100644 index 0000000000..69d4986665 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-minServerVersion-pattern" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "1.2.3.4" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json new file mode 100644 index 0000000000..688d1c67ee --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-minServerVersion-type", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml new file mode 100644 index 0000000000..e2727c5800 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-minServerVersion-type" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json new file mode 100644 index 0000000000..f62e5040d4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json @@ -0,0 +1,17 @@ +{ + "description": "runOnRequirement-topologies-enum", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": [ + "foo" + ] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml new file mode 100644 index 0000000000..924a67515d --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-topologies-enum" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: ["foo"] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json new file mode 100644 index 0000000000..a205b3293d --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json @@ -0,0 +1,17 @@ +{ + "description": "runOnRequirement-topologies-items", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": [ + 0 + ] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml new file mode 100644 index 0000000000..7b2265fb26 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-topologies-items" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json new file mode 100644 index 0000000000..16f29b3f4b --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-topologies-minItems", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml new file mode 100644 index 0000000000..3e6d11ab8f --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-topologies-minItems" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json new file mode 100644 index 0000000000..f6d147cd6f --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-topologies-type", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml new file mode 100644 index 0000000000..5a1652d8d9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml @@ -0,0 +1,10 @@ +description: "runOnRequirement-topologies-type" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-items.json b/source/unified-test-format/tests/invalid/runOnRequirements-items.json new file mode 100644 index 0000000000..40ec84a3f3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-items.json @@ -0,0 +1,13 @@ +{ + "description": "runOnRequirements-items", + "schemaVersion": "1.0", + "runOnRequirements": [ + 0 + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-items.yml b/source/unified-test-format/tests/invalid/runOnRequirements-items.yml new file mode 100644 index 0000000000..a86bbb1318 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-items.yml @@ -0,0 +1,9 @@ +description: "runOnRequirements-items" + +schemaVersion: "1.0" + +runOnRequirements: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-minItems.json b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.json new file mode 100644 index 0000000000..4ca9f99b5d --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "runOnRequirements-minItems", + "schemaVersion": "1.0", + "runOnRequirements": [], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml new file mode 100644 index 0000000000..034786a220 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml @@ -0,0 +1,9 @@ +description: "runOnRequirements-minItems" + +schemaVersion: "1.0" + +runOnRequirements: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-type.json b/source/unified-test-format/tests/invalid/runOnRequirements-type.json new file mode 100644 index 0000000000..98b859f3ea --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-type.json @@ -0,0 +1,11 @@ +{ + "description": "runOnRequirements-type", + "schemaVersion": "1.0", + "runOnRequirements": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-type.yml b/source/unified-test-format/tests/invalid/runOnRequirements-type.yml new file mode 100644 index 0000000000..42afc21ced --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-type.yml @@ -0,0 +1,9 @@ +description: "runOnRequirements-type" + +schemaVersion: "1.0" + +runOnRequirements: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/schemaVersion-pattern.json b/source/unified-test-format/tests/invalid/schemaVersion-pattern.json new file mode 100644 index 0000000000..bcb8980516 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-pattern.json @@ -0,0 +1,10 @@ +{ + "description": "schemaVersion-pattern", + "schemaVersion": "1.2.3.4", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml b/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml new file mode 100644 index 0000000000..3d36ee64b5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml @@ -0,0 +1,7 @@ +description: "schemaVersion-pattern" + +schemaVersion: "1.2.3.4" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/schemaVersion-required.json b/source/unified-test-format/tests/invalid/schemaVersion-required.json new file mode 100644 index 0000000000..7388ff0bf1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-required.json @@ -0,0 +1,9 @@ +{ + "description": "schemaVersion-required", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/schemaVersion-required.yml b/source/unified-test-format/tests/invalid/schemaVersion-required.yml new file mode 100644 index 0000000000..14a13872d8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-required.yml @@ -0,0 +1,5 @@ +description: "schemaVersion-required" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/schemaVersion-type.json b/source/unified-test-format/tests/invalid/schemaVersion-type.json new file mode 100644 index 0000000000..646473a209 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-type.json @@ -0,0 +1,10 @@ +{ + "description": "schemaVersion-type", + "schemaVersion": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/schemaVersion-type.yml b/source/unified-test-format/tests/invalid/schemaVersion-type.yml new file mode 100644 index 0000000000..9dfeed1e78 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-type.yml @@ -0,0 +1,7 @@ +description: "schemaVersion-type" + +schemaVersion: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/test-additionalProperties.json b/source/unified-test-format/tests/invalid/test-additionalProperties.json new file mode 100644 index 0000000000..a699319c30 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-additionalProperties.json @@ -0,0 +1,11 @@ +{ + "description": "test-additionalProperties", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "foo": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-additionalProperties.yml b/source/unified-test-format/tests/invalid/test-additionalProperties.yml new file mode 100644 index 0000000000..65a2e2aa64 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-additionalProperties.yml @@ -0,0 +1,8 @@ +description: "test-additionalProperties" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + foo: 0 diff --git a/source/unified-test-format/tests/invalid/test-description-required.json b/source/unified-test-format/tests/invalid/test-description-required.json new file mode 100644 index 0000000000..8bf23014d4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-required.json @@ -0,0 +1,9 @@ +{ + "description": "test-description-required", + "schemaVersion": "1.0", + "tests": [ + { + "operation": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-description-required.yml b/source/unified-test-format/tests/invalid/test-description-required.yml new file mode 100644 index 0000000000..d1c725b69a --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-required.yml @@ -0,0 +1,6 @@ +description: "test-description-required" + +schemaVersion: "1.0" + +tests: + - operation: [] diff --git a/source/unified-test-format/tests/invalid/test-description-type.json b/source/unified-test-format/tests/invalid/test-description-type.json new file mode 100644 index 0000000000..bba3690449 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-type.json @@ -0,0 +1,10 @@ +{ + "description": "test-description-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": 0, + "operation": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-description-type.yml b/source/unified-test-format/tests/invalid/test-description-type.yml new file mode 100644 index 0000000000..891db9bbc1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-type.yml @@ -0,0 +1,7 @@ +description: "test-description-type" + +schemaVersion: "1.0" + +tests: + - description: 0 + operation: [] diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-items.json b/source/unified-test-format/tests/invalid/test-expectEvents-items.json new file mode 100644 index 0000000000..394f74746c --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-items.json @@ -0,0 +1,13 @@ +{ + "description": "test-expectEvents-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-items.yml b/source/unified-test-format/tests/invalid/test-expectEvents-items.yml new file mode 100644 index 0000000000..1a889b1b4f --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-items.yml @@ -0,0 +1,8 @@ +description: "test-expectEvents-items" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + expectEvents: [0] diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-type.json b/source/unified-test-format/tests/invalid/test-expectEvents-type.json new file mode 100644 index 0000000000..1569f0a0d7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-expectEvents-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-type.yml b/source/unified-test-format/tests/invalid/test-expectEvents-type.yml new file mode 100644 index 0000000000..0a1e9d97d5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-type.yml @@ -0,0 +1,8 @@ +description: "test-expectEvents-type" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + expectEvents: 0 diff --git a/source/unified-test-format/tests/invalid/test-operations-items.json b/source/unified-test-format/tests/invalid/test-operations-items.json new file mode 100644 index 0000000000..00af8e7453 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-items.json @@ -0,0 +1,12 @@ +{ + "description": "test-operations-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-operations-items.yml b/source/unified-test-format/tests/invalid/test-operations-items.yml new file mode 100644 index 0000000000..6bb097fa3c --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-items.yml @@ -0,0 +1,7 @@ +description: "test-operations-items" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [0] diff --git a/source/unified-test-format/tests/invalid/test-operations-required.json b/source/unified-test-format/tests/invalid/test-operations-required.json new file mode 100644 index 0000000000..67c6f83044 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-required.json @@ -0,0 +1,9 @@ +{ + "description": "test-operations-required", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo" + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-operations-required.yml b/source/unified-test-format/tests/invalid/test-operations-required.yml new file mode 100644 index 0000000000..80af1a900a --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-required.yml @@ -0,0 +1,6 @@ +description: "test-operations-required" + +schemaVersion: "1.0" + +tests: + - description: "foo" diff --git a/source/unified-test-format/tests/invalid/test-operations-type.json b/source/unified-test-format/tests/invalid/test-operations-type.json new file mode 100644 index 0000000000..1e8b5b2496 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-type.json @@ -0,0 +1,10 @@ +{ + "description": "test-operations-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-operations-type.yml b/source/unified-test-format/tests/invalid/test-operations-type.yml new file mode 100644 index 0000000000..713c7ff0ec --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-type.yml @@ -0,0 +1,7 @@ +description: "test-operations-type" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: 0 diff --git a/source/unified-test-format/tests/invalid/test-outcome-items.json b/source/unified-test-format/tests/invalid/test-outcome-items.json new file mode 100644 index 0000000000..cf6bb54f87 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-items.json @@ -0,0 +1,13 @@ +{ + "description": "test-outcome-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "outcome": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-outcome-items.yml b/source/unified-test-format/tests/invalid/test-outcome-items.yml new file mode 100644 index 0000000000..bd5565e4fb --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-items.yml @@ -0,0 +1,8 @@ +description: "test-outcome-items" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + outcome: [0] diff --git a/source/unified-test-format/tests/invalid/test-outcome-minItems.json b/source/unified-test-format/tests/invalid/test-outcome-minItems.json new file mode 100644 index 0000000000..aadf8e514a --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "test-outcome-minItems", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "outcome": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-outcome-minItems.yml b/source/unified-test-format/tests/invalid/test-outcome-minItems.yml new file mode 100644 index 0000000000..f8d87d1f56 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-minItems.yml @@ -0,0 +1,8 @@ +description: "test-outcome-minItems" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + outcome: [] diff --git a/source/unified-test-format/tests/invalid/test-outcome-type.json b/source/unified-test-format/tests/invalid/test-outcome-type.json new file mode 100644 index 0000000000..e60c119d7e --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-outcome-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "outcome": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-outcome-type.yml b/source/unified-test-format/tests/invalid/test-outcome-type.yml new file mode 100644 index 0000000000..cfc44183a5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-type.yml @@ -0,0 +1,8 @@ +description: "test-outcome-type" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + outcome: 0 diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-items.json b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.json new file mode 100644 index 0000000000..866bebb51f --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.json @@ -0,0 +1,13 @@ +{ + "description": "test-runOnRequirements-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "runOnRequirements": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml new file mode 100644 index 0000000000..2324fa8045 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml @@ -0,0 +1,8 @@ +description: "test-runOnRequirements-items" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + runOnRequirements: [0] diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json new file mode 100644 index 0000000000..d61f063849 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "test-runOnRequirements-minItems", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "runOnRequirements": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml new file mode 100644 index 0000000000..2f3e0391ad --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml @@ -0,0 +1,8 @@ +description: "test-runOnRequirements-minItems" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + runOnRequirements: [] diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-type.json b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.json new file mode 100644 index 0000000000..5b25b1005d --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-runOnRequirements-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "runOnRequirements": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml new file mode 100644 index 0000000000..12bd90e859 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml @@ -0,0 +1,8 @@ +description: "test-runOnRequirements-type" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + runOnRequirements: 0 diff --git a/source/unified-test-format/tests/invalid/test-skipReason-type.json b/source/unified-test-format/tests/invalid/test-skipReason-type.json new file mode 100644 index 0000000000..0408e76834 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-skipReason-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-skipReason-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "skipReason": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-skipReason-type.yml b/source/unified-test-format/tests/invalid/test-skipReason-type.yml new file mode 100644 index 0000000000..7921aefc79 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-skipReason-type.yml @@ -0,0 +1,8 @@ +description: "test-skipReason-type" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + skipReason: 0 diff --git a/source/unified-test-format/tests/invalid/tests-items.json b/source/unified-test-format/tests/invalid/tests-items.json new file mode 100644 index 0000000000..11f37469e4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-items.json @@ -0,0 +1,7 @@ +{ + "description": "tests-items", + "schemaVersion": "1.0", + "tests": [ + 0 + ] +} diff --git a/source/unified-test-format/tests/invalid/tests-items.yml b/source/unified-test-format/tests/invalid/tests-items.yml new file mode 100644 index 0000000000..8c9829634c --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-items.yml @@ -0,0 +1,5 @@ +description: "tests-items" + +schemaVersion: "1.0" + +tests: [0] diff --git a/source/unified-test-format/tests/invalid/tests-minItems.json b/source/unified-test-format/tests/invalid/tests-minItems.json new file mode 100644 index 0000000000..3f74f94af7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-minItems.json @@ -0,0 +1,5 @@ +{ + "description": "tests-minItems", + "schemaVersion": "1.0", + "tests": [] +} diff --git a/source/unified-test-format/tests/invalid/tests-minItems.yml b/source/unified-test-format/tests/invalid/tests-minItems.yml new file mode 100644 index 0000000000..834120f835 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-minItems.yml @@ -0,0 +1,5 @@ +description: "tests-minItems" + +schemaVersion: "1.0" + +tests: [] diff --git a/source/unified-test-format/tests/invalid/tests-required.json b/source/unified-test-format/tests/invalid/tests-required.json new file mode 100644 index 0000000000..de4b2fd063 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-required.json @@ -0,0 +1,4 @@ +{ + "description": "tests-required", + "schemaVersion": "1.0" +} diff --git a/source/unified-test-format/tests/invalid/tests-required.yml b/source/unified-test-format/tests/invalid/tests-required.yml new file mode 100644 index 0000000000..41c5072acf --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-required.yml @@ -0,0 +1,3 @@ +description: "tests-required" + +schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/tests-type.json b/source/unified-test-format/tests/invalid/tests-type.json new file mode 100644 index 0000000000..62d8194a41 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-type.json @@ -0,0 +1,5 @@ +{ + "description": "tests-type", + "schemaVersion": "1.0", + "tests": 0 +} diff --git a/source/unified-test-format/tests/invalid/tests-type.yml b/source/unified-test-format/tests/invalid/tests-type.yml new file mode 100644 index 0000000000..c16781a727 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-type.yml @@ -0,0 +1,5 @@ +description: "tests-type" + +schemaVersion: "1.0" + +tests: 0 diff --git a/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json new file mode 100644 index 0000000000..7f7f1978c3 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json @@ -0,0 +1,18 @@ +{ + "description": "entity-bucket-database-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "bucket": { + "id": "bucket0", + "database": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml new file mode 100644 index 0000000000..7aeda8e1ac --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml @@ -0,0 +1,12 @@ +description: "entity-bucket-database-undefined" + +schemaVersion: "1.0" + +createEntities: + - bucket: + id: &bucket0 "bucket0" + database: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json new file mode 100644 index 0000000000..20b0733e34 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json @@ -0,0 +1,19 @@ +{ + "description": "entity-collection-database-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "collection": { + "id": "collection0", + "database": "foo", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml new file mode 100644 index 0000000000..a66b6ed292 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml @@ -0,0 +1,13 @@ +description: "entity-collection-database-undefined" + +schemaVersion: "1.0" + +createEntities: + - collection: + id: &collection0 "collection0" + database: "foo" + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json new file mode 100644 index 0000000000..0f8110e6d3 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json @@ -0,0 +1,19 @@ +{ + "description": "entity-database-client-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "database": { + "id": "database0", + "client": "foo", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml new file mode 100644 index 0000000000..34b58cece5 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml @@ -0,0 +1,13 @@ +description: "entity-database-client-undefined" + +schemaVersion: "1.0" + +createEntities: + - database: + id: &database0 "database0" + client: "foo" + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json new file mode 100644 index 0000000000..260356436a --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json @@ -0,0 +1,18 @@ +{ + "description": "entity-session-client-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "session": { + "id": "session0", + "client": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml new file mode 100644 index 0000000000..16eaea46b7 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml @@ -0,0 +1,12 @@ +description: "entity-session-client-undefined" + +schemaVersion: "1.0" + +createEntities: + - session: + id: &session0 "session0" + client: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json new file mode 100644 index 0000000000..ea425fb568 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json @@ -0,0 +1,66 @@ +{ + "description": "returnDocument-enum-invalid", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + } + ], + "tests": [ + { + "description": "FindOneAndReplace returnDocument invalid enum value", + "operations": [ + { + "name": "findOneAndReplace", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "_id": 1, + "x": 111 + }, + "returnDocument": "invalid" + } + } + ] + }, + { + "description": "FindOneAndUpdate returnDocument invalid enum value", + "operations": [ + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "invalid" + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml new file mode 100644 index 0000000000..b877f5e965 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml @@ -0,0 +1,34 @@ +description: "returnDocument-enum-invalid" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName test + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName coll + +tests: + - description: "FindOneAndReplace returnDocument invalid enum value" + operations: + - name: findOneAndReplace + object: *collection0 + arguments: + filter: { _id: 1 } + replacement: { _id: 1, x: 111 } + returnDocument: invalid + + - description: "FindOneAndUpdate returnDocument invalid enum value" + operations: + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 }} + returnDocument: invalid diff --git a/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json new file mode 100644 index 0000000000..ceb5532917 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json @@ -0,0 +1,10 @@ +{ + "description": "schemaVersion-unsupported", + "schemaVersion": "0.1", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml new file mode 100644 index 0000000000..0cb6994d24 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml @@ -0,0 +1,7 @@ +description: "schemaVersion-unsupported" + +schemaVersion: "0.1" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.json b/source/unified-test-format/tests/valid-pass/poc-change-streams.json new file mode 100644 index 0000000000..dc6e332e3e --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.json @@ -0,0 +1,410 @@ +{ + "description": "poc-change-streams", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ], + "ignoreCommandMonitoringEvents": [ + "getMore", + "killCursors" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "change-stream-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "client": { + "id": "client1", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "change-stream-tests" + } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "change-stream-tests-2" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "test" + } + }, + { + "collection": { + "id": "collection2", + "database": "database1", + "collectionName": "test2" + } + }, + { + "collection": { + "id": "collection3", + "database": "database2", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "change-stream-tests", + "documents": [] + }, + { + "collectionName": "test2", + "databaseName": "change-stream-tests", + "documents": [] + }, + { + "collectionName": "test", + "databaseName": "change-stream-tests-2", + "documents": [] + } + ], + "tests": [ + { + "description": "Executing a watch helper on a MongoClient results in notifications for changes to all collections in all databases in the cluster.", + "runOnRequirements": [ + { + "minServerVersion": "3.8.0", + "topologies": [ + "replicaset" + ] + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "client0", + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection2", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "insertOne", + "object": "collection3", + "arguments": { + "document": { + "y": 1 + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "z": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test2" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests-2", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "y": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "z": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": 1, + "cursor": {}, + "pipeline": [ + { + "$changeStream": { + "allChangesForCluster": true, + "fullDocument": { + "$$unsetOrMatches": "default" + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "Test consecutive resume", + "runOnRequirements": [ + { + "minServerVersion": "4.1.7", + "topologies": [ + "replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "getMore" + ], + "closeConnection": true + } + } + } + }, + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "batchSize": 1 + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "x": 2 + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "x": 3 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 2 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 3 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "test", + "cursor": { + "batchSize": 1 + }, + "pipeline": [ + { + "$changeStream": { + "fullDocument": { + "$$unsetOrMatches": "default" + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "change-stream-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "aggregate": "test", + "cursor": { + "batchSize": 1 + }, + "pipeline": [ + { + "$changeStream": { + "fullDocument": { + "$$unsetOrMatches": "default" + }, + "resumeAfter": { + "$$exists": true + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "change-stream-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "aggregate": "test", + "cursor": { + "batchSize": 1 + }, + "pipeline": [ + { + "$changeStream": { + "fullDocument": { + "$$unsetOrMatches": "default" + }, + "resumeAfter": { + "$$exists": true + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "change-stream-tests" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml new file mode 100644 index 0000000000..bdf83fdc60 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -0,0 +1,217 @@ +description: "poc-change-streams" + +schemaVersion: "1.0" + +createEntities: + # Entities for creating changeStreams + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + # Original tests do not observe getMore commands but only because event + # assertions ignore extra events. killCursors is explicitly ignored. + ignoreCommandMonitoringEvents: [ getMore, killCursors ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name change-stream-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + # Entities for executing insert operations + - client: + id: &client1 client1 + useMultipleMongoses: false + - database: + id: &database1 database1 + client: *client1 + databaseName: &database1Name change-stream-tests + - database: + id: &database2 database2 + client: *client1 + databaseName: &database2Name change-stream-tests-2 + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: &collection1Name test + - collection: + id: &collection2 collection2 + database: *database1 + collectionName: &collection2Name test2 + - collection: + id: &collection3 collection3 + database: *database2 + collectionName: &collection3Name test + +initialData: + - collectionName: *collection1Name + databaseName: *database1Name + documents: [] + - collectionName: *collection2Name + databaseName: *database1Name + documents: [] + - collectionName: *collection3Name + databaseName: *database2Name + documents: [] + +tests: + - description: "Executing a watch helper on a MongoClient results in notifications for changes to all collections in all databases in the cluster." + runOnRequirements: + - minServerVersion: "3.8.0" + topologies: [ replicaset ] + operations: + - name: createChangeStream + object: *client0 + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection2 + arguments: + document: { x: 1 } + - name: insertOne + object: *collection3 + arguments: + document: { y: 1 } + - name: insertOne + object: *collection1 + arguments: + document: { z: 1 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database1Name + coll: *collection2Name + fullDocument: + _id: { $$type: objectId } + x: 1 + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database2Name + coll: *collection3Name + fullDocument: + # Original tests did not include _id, but matching now only permits + # extra keys for root-level documents. + _id: { $$type: objectId } + y: 1 + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database1Name + coll: *collection1Name + fullDocument: + _id: { $$type: objectId } + z: 1 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: 1 + cursor: {} + pipeline: + - $changeStream: + allChangesForCluster: true + # Some drivers may send a default value for fullDocument + # or omit it entirely (see: SPEC-1350). + fullDocument: { $$unsetOrMatches: default } + commandName: aggregate + databaseName: admin + + - description: "Test consecutive resume" + runOnRequirements: + - minServerVersion: "4.1.7" + topologies: [ replicaset ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ getMore ] + closeConnection: true + - name: createChangeStream + object: *collection0 + arguments: + batchSize: 1 + saveResultAsEntity: *changeStream0 + - name: insertOne + object: *collection1 + arguments: + document: { x: 1 } + - name: insertOne + object: *collection1 + arguments: + document: { x: 2 } + - name: insertOne + object: *collection1 + arguments: + document: { x: 3 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database1Name + coll: *collection1Name + fullDocument: + _id: { $$type: objectId } + x: 1 + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database1Name + coll: *collection1Name + fullDocument: + _id: { $$type: objectId } + x: 2 + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database1Name + coll: *collection1Name + fullDocument: + _id: { $$type: objectId } + x: 3 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection1Name + cursor: { batchSize: 1 } + pipeline: + - $changeStream: + fullDocument: { $$unsetOrMatches: default } + commandName: aggregate + databaseName: *database1Name + # The original test only asserted the first command, since expected + # events were only an ordered subset. This file does ignore getMore + # commands but we must expect the subsequent aggregate commands, since + # each failed getMore will resume. While doing so we can also assert + # that those commands include a resume token. + - &resumingAggregate + commandStartedEvent: + command: + aggregate: *collection1Name + cursor: { batchSize: 1 } + pipeline: + - $changeStream: + fullDocument: { $$unsetOrMatches: default } + resumeAfter: { $$exists: true } + commandName: aggregate + databaseName: *database0Name + - *resumingAggregate diff --git a/source/unified-test-format/tests/valid-pass/poc-command-monitoring.json b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.json new file mode 100644 index 0000000000..499396e0ba --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.json @@ -0,0 +1,222 @@ +{ + "description": "poc-command-monitoring", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "tests": [ + { + "description": "A successful find event with a getmore and the server kills the cursor", + "runOnRequirements": [ + { + "minServerVersion": "3.1", + "topologies": [ + "single", + "replicaset" + ] + } + ], + "operations": [ + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3, + "limit": 4 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3, + "limit": 4 + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": { + "$$type": [ + "int", + "long" + ] + }, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "test", + "batchSize": 1 + }, + "commandName": "getMore", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "nextBatch": [ + { + "_id": 4, + "x": 44 + } + ] + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "A failed find event", + "operations": [ + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "$or": true + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml new file mode 100644 index 0000000000..19a282327c --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml @@ -0,0 +1,101 @@ +description: "poc-command-monitoring" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + - commandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name command-monitoring-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } + +tests: + - description: "A successful find event with a getmore and the server kills the cursor" + runOnRequirements: + - minServerVersion: "3.1" + topologies: [ single, replicaset ] + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gte: 1 }} + sort: { _id: 1 } + batchSize: 3 + limit: 4 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: { $gte : 1 } } + sort: { _id: 1 } + batchSize: 3 + limit: 4 + commandName: find + databaseName: *database0Name + - commandSucceededEvent: + reply: + ok: 1 + cursor: + id: { $$type: [ int, long ] } + ns: &namespace command-monitoring-tests.test + firstBatch: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [ int, long ] } + collection: *collection0Name + batchSize: 1 + commandName: getMore + databaseName: *database0Name + - commandSucceededEvent: + reply: + ok: 1 + cursor: + id: 0 + ns: *namespace + nextBatch: + - { _id: 4, x: 44 } + commandName: getMore + + - description: "A failed find event" + operations: + - name: find + object: *collection0 + arguments: + filter: { $or: true } + expectError: { isError: true } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collection0Name + filter: { $or: true } + commandName: find + databaseName: *database0Name + - commandFailedEvent: + commandName: find diff --git a/source/unified-test-format/tests/valid-pass/poc-crud.json b/source/unified-test-format/tests/valid-pass/poc-crud.json new file mode 100644 index 0000000000..2ed86d6150 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-crud.json @@ -0,0 +1,446 @@ +{ + "description": "poc-crud", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "admin" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "coll1" + } + }, + { + "collection": { + "id": "collection2", + "database": "database0", + "collectionName": "coll2", + "collectionOptions": { + "readConcern": { + "level": "majority" + } + } + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + }, + { + "collectionName": "coll1", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + }, + { + "collectionName": "coll2", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + }, + { + "collectionName": "aggregate_out", + "databaseName": "crud-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "BulkWrite with mixed ordered operations", + "operations": [ + { + "name": "bulkWrite", + "object": "collection0", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + { + "updateOne": { + "filter": { + "_id": 2 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "insertOne": { + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "deleteMany": { + "filter": { + "x": { + "$nin": [ + 24, + 34 + ] + } + } + } + }, + { + "replaceOne": { + "filter": { + "_id": 4 + }, + "replacement": { + "_id": 4, + "x": 44 + }, + "upsert": true + } + } + ], + "ordered": true + }, + "expectResult": { + "deletedCount": 2, + "insertedCount": 2, + "insertedIds": { + "$$unsetOrMatches": { + "0": 3, + "3": 4 + } + }, + "matchedCount": 3, + "modifiedCount": 3, + "upsertedCount": 1, + "upsertedIds": { + "5": 4 + } + } + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 34 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "InsertMany continue-on-error behavior with unordered (duplicate key in requests)", + "operations": [ + { + "name": "insertMany", + "object": "collection1", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ], + "ordered": false + }, + "expectError": { + "expectResult": { + "deletedCount": 0, + "insertedCount": 2, + "matchedCount": 0, + "modifiedCount": 0, + "upsertedCount": 0, + "upsertedIds": {} + } + } + } + ], + "outcome": [ + { + "collectionName": "coll1", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "ReplaceOne prohibits atomic modifiers", + "operations": [ + { + "name": "replaceOne", + "object": "collection1", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll1", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ] + }, + { + "description": "readConcern majority with out stage", + "runOnRequirements": [ + { + "minServerVersion": "4.1.0", + "topologies": [ + "replicaset", + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "aggregate", + "object": "collection2", + "arguments": { + "pipeline": [ + { + "$sort": { + "x": 1 + } + }, + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$out": "aggregate_out" + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "coll2", + "pipeline": [ + { + "$sort": { + "x": 1 + } + }, + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$out": "aggregate_out" + } + ], + "readConcern": { + "level": "majority" + } + }, + "commandName": "aggregate", + "databaseName": "crud-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "aggregate_out", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "Aggregate with $listLocalSessions", + "runOnRequirements": [ + { + "minServerVersion": "3.6.0" + } + ], + "operations": [ + { + "name": "aggregate", + "object": "database1", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + }, + { + "$addFields": { + "dummy": "dummy field" + } + }, + { + "$project": { + "_id": 0, + "dummy": 1 + } + } + ] + }, + "expectResult": [ + { + "dummy": "dummy field" + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-crud.yml b/source/unified-test-format/tests/valid-pass/poc-crud.yml new file mode 100644 index 0000000000..7d101a077a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-crud.yml @@ -0,0 +1,183 @@ +description: "poc-crud" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name admin + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name coll1 + - collection: + id: &collection2 collection2 + database: *database0 + collectionName: &collection2Name coll2 + collectionOptions: + readConcern: { level: majority } + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - collectionName: *collection1Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - collectionName: *collection2Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - collectionName: &out aggregate_out + databaseName: *database0Name + documents: [] + +tests: + - description: "BulkWrite with mixed ordered operations" + operations: + - name: bulkWrite + object: *collection0 + arguments: + requests: + - insertOne: + document: { _id: 3, x: 33 } + - updateOne: + filter: { _id: 2 } + update: { $inc: { x: 1 } } + - updateMany: + filter: { _id: { $gt: 1 } } + update: { $inc: { x: 1 } } + - insertOne: + document: { _id: 4, x: 44 } + - deleteMany: + filter: { x: { $nin: [ 24, 34 ] } } + - replaceOne: + filter: { _id: 4 } + replacement: { _id: 4, x: 44 } + upsert: true + ordered: true + expectResult: + deletedCount: 2 + insertedCount: 2 + insertedIds: { $$unsetOrMatches: { 0: 3, 3: 4 } } + matchedCount: 3 + modifiedCount: 3 + upsertedCount: 1 + upsertedIds: { 5: 4 } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 2, x: 24 } + - {_id: 3, x: 34 } + - {_id: 4, x: 44 } + + - description: "InsertMany continue-on-error behavior with unordered (duplicate key in requests)" + operations: + - name: insertMany + object: *collection1 + arguments: + documents: + - { _id: 2, x: 22 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + ordered: false + expectError: + expectResult: + deletedCount: 0 + insertedCount: 2 + # Since the map of insertedIds is generated before execution it + # could indicate inserts that did not actually succeed. We omit + # this field rather than expect drivers to provide an accurate + # map filtered by write errors. + matchedCount: 0 + modifiedCount: 0 + upsertedCount: 0 + upsertedIds: { } + outcome: + - collectionName: *collection1Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + + - description: "ReplaceOne prohibits atomic modifiers" + operations: + - name: replaceOne + object: *collection1 + arguments: + filter: { _id: 1 } + replacement: { $set: { x: 22 }} + expectError: + isClientError: true + expectEvents: + - client: *client0 + events: [] + outcome: + - collectionName: *collection1Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + + - description: "readConcern majority with out stage" + runOnRequirements: + - minServerVersion: "4.1.0" + topologies: [ replicaset, sharded-replicaset ] + operations: + - name: aggregate + object: *collection2 + arguments: + pipeline: &pipeline + - $sort: { x : 1 } + - $match: { _id: { $gt: 1 } } + - $out: *out + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection2Name + pipeline: *pipeline + readConcern: { level: majority } + # The following two assertions were not in the original test + commandName: aggregate + databaseName: *database0Name + outcome: + - collectionName: *out + databaseName: *database0Name + documents: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + + - description: "Aggregate with $listLocalSessions" + runOnRequirements: + - minServerVersion: "3.6.0" + operations: + - name: aggregate + object: *database1 + arguments: + pipeline: + - $listLocalSessions: { } + - $limit: 1 + - $addFields: { dummy: "dummy field"} + - $project: { _id: 0, dummy: 1} + expectResult: + - { dummy: "dummy field" } diff --git a/source/unified-test-format/tests/valid-pass/poc-gridfs.json b/source/unified-test-format/tests/valid-pass/poc-gridfs.json new file mode 100644 index 0000000000..c04ed89a7c --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-gridfs.json @@ -0,0 +1,299 @@ +{ + "description": "poc-gridfs", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 10, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "57d83cd477bfb1ccd975ab33d827a92b", + "filename": "length-10", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000006" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000007" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2, + "data": { + "$binary": { + "base64": "mao=", + "subType": "00" + } + } + } + ] + } + ], + "tests": [ + { + "description": "Delete when length is 10", + "operations": [ + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + } + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [] + } + ] + }, + { + "description": "Download when there are three chunks", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectResult": { + "$$matchesHexBytes": "112233445566778899aa" + } + } + ] + }, + { + "description": "Download when files entry does not exist", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000000" + } + }, + "expectError": { + "isError": true + } + } + ] + }, + { + "description": "Download when an intermediate chunk is missing", + "operations": [ + { + "name": "deleteOne", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 + } + }, + "expectResult": { + "deletedCount": 1 + } + }, + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectError": { + "isError": true + } + } + ] + }, + { + "description": "Upload when length is 5", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "1122334455" + }, + "chunkSizeBytes": 4 + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "oid0" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {}, + "sort": { + "uploadDate": -1 + }, + "limit": 1 + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "oid0" + }, + "length": 5, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": "283d4fea5dded59cf837d3047328f5af", + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "_id": { + "$gt": { + "$oid": "000000000000000000000007" + } + } + }, + "sort": { + "n": 1 + } + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "oid0" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } + }, + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "oid0" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VQ==", + "subType": "00" + } + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-gridfs.yml b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml new file mode 100644 index 0000000000..8a0478658b --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml @@ -0,0 +1,154 @@ +description: "poc-gridfs" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name gridfs-tests + - bucket: + id: &bucket0 bucket0 + database: *database0 + - collection: + id: &bucket0_files_collection bucket0_files_collection + database: *database0 + collectionName: &bucket0_files_collectionName fs.files + - collection: + id: &bucket0_chunks_collection bucket0_chunks_collection + database: *database0 + collectionName: &bucket0_chunks_collectionName fs.chunks + +initialData: + - collectionName: *bucket0_files_collectionName + databaseName: *database0Name + documents: + - _id: { $oid: "000000000000000000000005" } + length: 10 + chunkSize: 4 + uploadDate: { $date: "1970-01-01T00:00:00.000Z" } + md5: "57d83cd477bfb1ccd975ab33d827a92b" + filename: "length-10" + contentType: "application/octet-stream" + aliases: [] + metadata: {} + - collectionName: *bucket0_chunks_collectionName + databaseName: *database0Name + documents: + - _id: { $oid: "000000000000000000000005" } + files_id: { $oid: "000000000000000000000005" } + n: 0 + data: { $binary: { base64: "ESIzRA==", subType: "00" } } # hex: 11223344 + - _id: { $oid: "000000000000000000000006" } + files_id: { $oid: "000000000000000000000005" } + n: 1 + data: { $binary: { base64: "VWZ3iA==", subType: "00" } } # hex: 55667788 + - _id: { $oid: "000000000000000000000007" } + files_id: { $oid: "000000000000000000000005" } + n: 2 + data: { $binary: { base64: "mao=", subType: "00" } } # hex: 99aa + +tests: + # Changed from original test ("length is 8") to operate on same initialData + - description: "Delete when length is 10" + operations: + - name: delete + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000005" } + # Original test uses "assert.data" syntax to modify outcome collection for + # comparison. This can be accomplished using "outcome" directly. + outcome: + - collectionName: *bucket0_files_collectionName + databaseName: *database0Name + documents: [] + - collectionName: *bucket0_chunks_collectionName + databaseName: *database0Name + documents: [] + + - description: "Download when there are three chunks" + operations: + # Original test uses "download" operation. We use an explicit operation + # that returns a stream and then assert the contents of that stream. + - name: download + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000005" } + expectResult: { $$matchesHexBytes: "112233445566778899aa" } + + - description: "Download when files entry does not exist" + operations: + - name: download + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000000" } + # Original test expects "FileNotFound" error, which isn't specified + expectError: { isError: true } + + - description: "Download when an intermediate chunk is missing" + operations: + # Original test uses "arrange" syntax to modify initialData. This can be + # accomplished as a delete operation on the chunks collection. + - name: deleteOne + object: *bucket0_chunks_collection + arguments: + filter: + files_id: { $oid: "000000000000000000000005" } + n: 1 + expectResult: + deletedCount: 1 + - name: download + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000005" } + # Original test expects "ChunkIsMissing" error, which isn't specified + expectError: { isError: true } + + - description: "Upload when length is 5" + operations: + # Original test uses "upload" operation. We use an explicit operation + # that takes a stream, which has been created from the expected hex bytes. + - name: upload + object: *bucket0 + arguments: + filename: filename + source: { $$hexBytes: "1122334455" } + chunkSizeBytes: 4 + # Original test references the result directly in "assert.data". Here, + # we need to save the result as an entity, which we can later reference. + expectResult: { $$type: objectId } + saveResultAsEntity: &oid0 oid0 + # "outcome" does not allow operators, but we can perform the assertions + # with separate find operations. + - name: find + object: *bucket0_files_collection + arguments: + filter: {} + sort: { uploadDate: -1 } + limit: 1 + expectResult: + - _id: { $$matchesEntity: *oid0 } + length: 5 + chunkSize: 4 + uploadDate: { $$type: date } + md5: "283d4fea5dded59cf837d3047328f5af" + filename: filename + - name: find + object: *bucket0_chunks_collection + arguments: + # We cannot use the saved ObjectId when querying, but filtering by a + # non-zero timestamp will exclude initialData and sort can return the + # expected chunks in order. + filter: { _id: { $gt: { $oid: "000000000000000000000007" } } } + sort: { n: 1 } + expectResult: + - _id: { $$type: objectId } + files_id: { $$matchesEntity: *oid0 } + n: 0 + data: { $binary: { base64: "ESIzRA==", subType: "00" } } # hex 11223344 + - _id: { $$type: objectId } + files_id: { $$matchesEntity: *oid0 } + n: 1 + data: { $binary: { base64: "VQ==", subType: "00" } } # hex 55 diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-reads.json b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.json new file mode 100644 index 0000000000..2b65d501a7 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.json @@ -0,0 +1,433 @@ +{ + "description": "poc-retryable-reads", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "uriOptions": { + "retryReads": false + }, + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-reads-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "retryable-reads-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-reads-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "Aggregate succeeds after InterruptedAtShutdown", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorCode": 11600 + } + } + } + }, + { + "name": "aggregate", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$sort": { + "x": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "coll", + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$sort": { + "x": 1 + } + } + ] + }, + "databaseName": "retryable-reads-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "aggregate": "coll", + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$sort": { + "x": 1 + } + } + ] + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "Find succeeds on second attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true + } + } + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": {}, + "sort": { + "_id": 1 + }, + "limit": 2 + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {}, + "sort": { + "_id": 1 + }, + "limit": 2 + }, + "databaseName": "retryable-reads-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {}, + "sort": { + "_id": 1 + }, + "limit": 2 + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "Find fails on first attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true + } + } + } + }, + { + "name": "find", + "object": "collection1", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client1", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {} + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "Find fails on second attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true + } + } + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {} + }, + "databaseName": "retryable-reads-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {} + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "ListDatabases succeeds on second attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "closeConnection": true + } + } + } + }, + { + "name": "listDatabases", + "object": "client0" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listDatabases": 1 + } + } + }, + { + "commandStartedEvent": { + "command": { + "listDatabases": 1 + } + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml new file mode 100644 index 0000000000..c1ea7ec696 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml @@ -0,0 +1,193 @@ +description: "poc-retryable-reads" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + topologies: [ single, replicaset ] + - minServerVersion: "4.1.7" + topologies: [ sharded ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + uriOptions: { retryReads: false } + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName retryable-reads-tests + - database: + id: &database1 database1 + client: *client1 + databaseName: *databaseName + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName coll + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: *collectionName + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + +tests: + - description: "Aggregate succeeds after InterruptedAtShutdown" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ aggregate ] + errorCode: 11600 # InterruptedAtShutdown + - name: aggregate + object: *collection0 + arguments: + pipeline: &pipeline + - $match: { _id: { $gt: 1 } } + - $sort: { x: 1 } + expectResult: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collectionName + pipeline: *pipeline + databaseName: *databaseName + - commandStartedEvent: + command: + aggregate: *collectionName + pipeline: *pipeline + databaseName: *databaseName + + - description: "Find succeeds on second attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ find ] + closeConnection: true + # Find options and expected result changed to use common initialData + - name: find + object: collection0 + arguments: + filter: {} + sort: { _id: 1 } + limit: 2 + expectResult: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + expectEvents: + - client: *client0 + events: + - &findAttempt + commandStartedEvent: + command: + find: *collectionName + filter: {} + sort: { _id: 1 } + limit: 2 + databaseName: *databaseName + - *findAttempt + + - description: "Find fails on first attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ find ] + closeConnection: true + - name: find + object: collection1 # client uses retryReads=false + arguments: + filter: {} + # Other arguments in the original test are not relevant + expectError: { isError: true } + expectEvents: + - client: *client1 + events: + - commandStartedEvent: + command: + find: *collectionName + filter: {} + databaseName: *databaseName + + - description: "Find fails on second attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ find ] + closeConnection: true + - name: find + object: collection0 + arguments: + filter: {} + # Other arguments in the original test are not relevant + expectError: { isError: true } + expectEvents: + - client: *client0 + events: + - &findAttempt + commandStartedEvent: + command: + find: *collectionName + filter: {} + databaseName: *databaseName + - *findAttempt + + - description: "ListDatabases succeeds on second attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ listDatabases ] + closeConnection: true + - name: listDatabases + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: { listDatabases: 1 } + - commandStartedEvent: + command: { listDatabases: 1 } diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-writes.json b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.json new file mode 100644 index 0000000000..e64ce1bcee --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.json @@ -0,0 +1,481 @@ +{ + "description": "poc-retryable-writes", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "3.6", + "topologies": [ + "replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "uriOptions": { + "retryWrites": false + }, + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "FindOneAndUpdate is committed on first attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 1 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + }, + "expectResult": { + "_id": 1, + "x": 11 + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "FindOneAndUpdate is not committed on first attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 1 + }, + "data": { + "failBeforeCommitExceptionCode": 1 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + }, + "expectResult": { + "_id": 1, + "x": 11 + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "FindOneAndUpdate is never committed", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 2 + }, + "data": { + "failBeforeCommitExceptionCode": 1 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + }, + "expectError": { + "isError": true + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "InsertMany succeeds after PrimarySteppedDown", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "name": "insertMany", + "object": "collection0", + "arguments": { + "documents": [ + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ], + "ordered": true + }, + "expectResult": { + "insertedCount": 2, + "insertedIds": { + "$$unsetOrMatches": { + "0": 3, + "1": 4 + } + } + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "InsertOne fails after connection failure when retryWrites option is false", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client1", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + }, + "expectError": { + "errorLabelsOmit": [ + "RetryableWriteError" + ] + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "InsertOne fails after multiple retryable writeConcernErrors", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + }, + "expectError": { + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml new file mode 100644 index 0000000000..0b94fbd803 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml @@ -0,0 +1,212 @@ +description: "poc-retryable-writes" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "3.6" + topologies: [ replicaset ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + uriOptions: { retryWrites: false } + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName retryable-writes-tests + - database: + id: &database1 database1 + client: *client1 + databaseName: *databaseName + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName coll + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: *collectionName + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + - description: "FindOneAndUpdate is committed on first attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 1 } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 } } + returnDocument: Before + expectResult: { _id: 1, x: 11 } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + + - description: "FindOneAndUpdate is not committed on first attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 1 } + data: { failBeforeCommitExceptionCode: 1 } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 } } + returnDocument: Before + expectResult: { _id: 1, x: 11 } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + + - description: "FindOneAndUpdate is never committed" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 2 } + data: { failBeforeCommitExceptionCode: 1 } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 } } + returnDocument: Before + expectError: { isError: true } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + + - description: "InsertMany succeeds after PrimarySteppedDown" + runOnRequirements: &failCommand_requirements + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.7" + # Original test uses "sharded", but retryable writes requires a sharded + # cluster backed by replica sets + topologies: [ sharded-replicaset ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: 189 # PrimarySteppedDown + errorLabels: [ RetryableWriteError ] + - name: insertMany + object: *collection0 + arguments: + documents: + # Documents are modified from original test for "initialData" + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + ordered: true + expectResult: + # insertMany returns a BulkWriteResult, but there is no reason to + # assert other fields + insertedCount: 2 + insertedIds: { $$unsetOrMatches: { 0: 3, 1: 4 } } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + + - description: "InsertOne fails after connection failure when retryWrites option is false" + runOnRequirements: *failCommand_requirements + operations: + - name: failPoint + object: testRunner + arguments: + client: *client1 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - name: insertOne + object: *collection1 + arguments: + document: { _id: 3, x: 33 } + expectError: + # If retryWrites is false, the driver should not add the + # RetryableWriteError label to the error. + errorLabelsOmit: [ RetryableWriteError ] + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + + - description: "InsertOne fails after multiple retryable writeConcernErrors" + runOnRequirements: *failCommand_requirements + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ insert ] + writeConcernError: + code: 91 # ShutdownInProgress + errmsg: "Replication is being shut down" + - name: insertOne + object: *collection0 + arguments: + document: { _id: 3, x: 33 } + expectError: + errorLabelsContain: [ RetryableWriteError ] + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } # The write was still applied diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.json b/source/unified-test-format/tests/valid-pass/poc-sessions.json new file mode 100644 index 0000000000..75f3489428 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.json @@ -0,0 +1,466 @@ +{ + "description": "poc-sessions", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "3.6.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "session-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ], + "tests": [ + { + "description": "Server supports explicit sessions", + "operations": [ + { + "name": "assertSessionNotDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + }, + { + "name": "assertSessionNotDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "endSession", + "object": "session0" + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": -1 + } + }, + "expectResult": [] + }, + { + "name": "assertSameLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client0" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + } + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": -1 + }, + "lsid": { + "$$sessionLsid": "session0" + } + }, + "commandName": "find", + "databaseName": "session-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "Server supports implicit sessions", + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 2 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": -1 + } + }, + "expectResult": [] + }, + { + "name": "assertSameLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client0" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$type": "object" + } + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": -1 + }, + "lsid": { + "$$type": "object" + } + }, + "commandName": "find", + "databaseName": "session-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "Dirty explicit session is discarded", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + } + } + }, + { + "name": "assertSessionNotDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + }, + { + "name": "assertSessionDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "assertSessionDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "endSession", + "object": "session0" + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": -1 + } + }, + "expectResult": [] + }, + { + "name": "assertDifferentLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client0" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1 + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1 + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 3 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 2 + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": -1 + }, + "lsid": { + "$$type": "object" + } + }, + "commandName": "find", + "databaseName": "session-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.yml b/source/unified-test-format/tests/valid-pass/poc-sessions.yml new file mode 100644 index 0000000000..cb16657da3 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.yml @@ -0,0 +1,214 @@ +description: "poc-sessions" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "3.6.0" + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name session-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - session: + id: &session0 session0 + client: *client0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + +tests: + - description: "Server supports explicit sessions" + operations: + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 2 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 2 } } } + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + - name: endSession + object: *session0 + - &find_with_implicit_session + name: find + object: *collection0 + arguments: + filter: { _id: -1 } + expectResult: [] + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collection0Name + documents: [ { _id: 2 } ] + ordered: true + lsid: { $$sessionLsid: *session0 } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: -1 } + lsid: { $$sessionLsid: *session0 } + commandName: find + databaseName: *database0Name + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + + - description: "Server supports implicit sessions" + operations: + - name: insertOne + object: *collection0 + arguments: + document: { _id: 2 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 2 } } } + - *find_with_implicit_session + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collection0Name + documents: + - { _id: 2 } + ordered: true + # Original test did not include any assertion, but we can use + # $$type to expect an arbitrary lsid document + lsid: { $$type: object } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: -1 } + lsid: { $$type: object } + commandName: find + databaseName: *database0Name + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + + - description: "Dirty explicit session is discarded" + # Original test specified retryWrites=true, but that is now the default. + # Retryable writes will require a sharded-replicaset, though. + runOnRequirements: + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 2 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 2 } } } + - name: assertSessionDirty + object: testRunner + arguments: + session: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 3 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 3 } } } + - name: assertSessionDirty + object: testRunner + arguments: + session: *session0 + - name: endSession + object: *session0 + - *find_with_implicit_session + - name: assertDifferentLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + expectEvents: + - client: *client0 + events: + # ajv's YAML parser is unable to handle anchors on array elements, so + # we define an anchor on the commandStartedEvent object instead + - commandStartedEvent: &insert_attempt + command: + insert: *collection0Name + documents: + - { _id: 2 } + ordered: true + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + commandName: insert + databaseName: *database0Name + - commandStartedEvent: *insert_attempt + - commandStartedEvent: + command: + insert: *collection0Name + documents: + - { _id: 3 } + ordered: true + lsid: { $$sessionLsid: *session0 } + txnNumber: 2 + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: -1 } + lsid: { $$type: object } + commandName: find + databaseName: *database0Name + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json new file mode 100644 index 0000000000..820ed65927 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json @@ -0,0 +1,505 @@ +{ + "description": "poc-transactions-convenient-api", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "uriOptions": { + "readConcernLevel": "local", + "w": 1 + }, + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + }, + { + "session": { + "id": "session1", + "client": "client1" + } + }, + { + "session": { + "id": "session2", + "client": "client0", + "sessionOptions": { + "defaultTransactionOptions": { + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "w": 1 + } + } + } + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "withTransaction and no transaction options set", + "operations": [ + { + "name": "withTransaction", + "object": "session0", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "$$exists": false + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "autocommit": false, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "withTransaction inherits transaction options from client", + "operations": [ + { + "name": "withTransaction", + "object": "session1", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "session": "session1", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client1", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session1" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "level": "local" + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session1" + }, + "txnNumber": 1, + "autocommit": false, + "writeConcern": { + "w": 1 + }, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "withTransaction inherits transaction options from defaultTransactionOptions", + "operations": [ + { + "name": "withTransaction", + "object": "session2", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session2", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session2" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session2" + }, + "txnNumber": 1, + "autocommit": false, + "writeConcern": { + "w": 1 + }, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "withTransaction explicit transaction options", + "operations": [ + { + "name": "withTransaction", + "object": "session0", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ], + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "w": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "autocommit": false, + "writeConcern": { + "w": 1 + }, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml new file mode 100644 index 0000000000..4f981d15dd --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml @@ -0,0 +1,235 @@ +description: "poc-transactions-convenient-api" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + uriOptions: + readConcernLevel: local + w: 1 + useMultipleMongoses: true + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName transaction-tests + - database: + id: &database1 database1 + client: *client1 + databaseName: *databaseName + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName test + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: *collectionName + - session: + id: &session0 session0 + client: *client0 + - session: + id: &session1 session1 + client: *client1 + - session: + id: &session2 session2 + client: *client0 + sessionOptions: + defaultTransactionOptions: + readConcern: { level: majority } + writeConcern: { w: 1 } + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: [] + +tests: + - description: "withTransaction and no transaction options set" + operations: + - name: withTransaction + object: *session0 + arguments: + callback: + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + # omitted fields + readConcern: { $$exists: false } + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + autocommit: false + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + writeConcern: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: &outcome + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1 } + + - description: "withTransaction inherits transaction options from client" + operations: + - name: withTransaction + object: *session1 + arguments: + callback: + - name: insertOne + object: *collection1 + arguments: + session: *session1 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + expectEvents: + - client: *client1 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session1 } + txnNumber: 1 + startTransaction: true + autocommit: false + readConcern: { level: local } + # omitted fields + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session1 } + txnNumber: 1 + autocommit: false + writeConcern: { w: 1 } + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: *outcome + + - description: "withTransaction inherits transaction options from defaultTransactionOptions" + operations: + - name: withTransaction + object: *session2 + arguments: + callback: + - name: insertOne + object: *collection0 + arguments: + session: *session2 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session2 } + txnNumber: 1 + startTransaction: true + autocommit: false + readConcern: { level: majority } + # omitted fields + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session2 } + txnNumber: 1 + autocommit: false + writeConcern: { w: 1 } + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: *outcome + + - description: "withTransaction explicit transaction options" + operations: + - name: withTransaction + object: *session0 + arguments: + callback: + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + readConcern: { level: majority } + writeConcern: { w: 1 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + readConcern: { level: majority } + # omitted fields + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + autocommit: false + writeConcern: { w: 1 } + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: *outcome diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json new file mode 100644 index 0000000000..a0b297d59a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json @@ -0,0 +1,409 @@ +{ + "description": "poc-transactions-mongos-pin-auto", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "remain pinned after non-transient Interrupted error on insertOne", + "operations": [ + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 11601 + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 4 + } + }, + "expectError": { + "errorLabelsOmit": [ + "TransientTransactionError", + "UnknownTransactionCommitResult" + ], + "errorCodeName": "Interrupted" + } + }, + { + "name": "assertSessionPinned", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "commitTransaction", + "object": "session0" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 3 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 4 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "recoveryToken": { + "$$type": "object" + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + } + ] + } + ] + }, + { + "description": "unpin after transient error within a transaction", + "operations": [ + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 4 + } + }, + "expectError": { + "errorLabelsContain": [ + "TransientTransactionError" + ], + "errorLabelsOmit": [ + "UnknownTransactionCommitResult" + ] + } + }, + { + "name": "assertSessionUnpinned", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "abortTransaction", + "object": "session0" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 3 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 4 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "recoveryToken": { + "$$type": "object" + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml new file mode 100644 index 0000000000..47db7c3188 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml @@ -0,0 +1,169 @@ +description: "poc-transactions-mongos-pin-auto" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name transaction-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - session: + id: &session0 session0 + client: *client0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + +tests: + - description: "remain pinned after non-transient Interrupted error on insertOne" + operations: + - &startTransaction + name: startTransaction + object: *session0 + - &firstInsert + name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 3 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 3 } } } + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: 11601 # Interrupted + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 4 } + expectError: + errorLabelsOmit: [ TransientTransactionError, UnknownTransactionCommitResult ] + errorCodeName: Interrupted + - name: assertSessionPinned + object: testRunner + arguments: + session: *session0 + - name: commitTransaction + object: *session0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: &firstInsertEvent + command: + insert: *collection0Name + documents: [ { _id: 3 } ] + ordered: true + readConcern: { $$exists: false } + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: &secondInsertEvent + command: + insert: *collection0Name + documents: [ { _id: 4 } ] + ordered: true + readConcern: { $$exists: false } + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + # Original test expected any value, but we can assert an object + recoveryToken: { $$type: object } + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } + + - description: "unpin after transient error within a transaction" + operations: + - *startTransaction + - *firstInsert + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 4 } + expectError: + errorLabelsContain: [ TransientTransactionError ] + errorLabelsOmit: [ UnknownTransactionCommitResult ] + - name: assertSessionUnpinned + object: testRunner + arguments: + session: *session0 + - name: abortTransaction + object: *session0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: *firstInsertEvent + - commandStartedEvent: *secondInsertEvent + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + # Original test expected any value, but we can assert an object + recoveryToken: { $$type: object } + commandName: abortTransaction + databaseName: admin + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions.json b/source/unified-test-format/tests/valid-pass/poc-transactions.json new file mode 100644 index 0000000000..62528f9ce1 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions.json @@ -0,0 +1,322 @@ +{ + "description": "poc-transactions", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "Client side error in command starting transaction", + "operations": [ + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": { + ".": "." + } + } + }, + "expectError": { + "isClientError": true + } + }, + { + "name": "assertSessionTransactionState", + "object": "testRunner", + "arguments": { + "session": "session0", + "state": "starting" + } + } + ] + }, + { + "description": "explicitly create collection using create command", + "runOnRequirements": [ + { + "minServerVersion": "4.3.4", + "topologies": [ + "replicaset", + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "dropCollection", + "object": "database0", + "arguments": { + "collection": "test" + } + }, + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "createCollection", + "object": "database0", + "arguments": { + "session": "session0", + "collection": "test" + } + }, + { + "name": "assertCollectionNotExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test" + } + }, + { + "name": "commitTransaction", + "object": "session0" + }, + { + "name": "assertCollectionExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "test", + "writeConcern": { + "$$exists": false + } + }, + "commandName": "drop", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "create": "test", + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "create", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "create index on a non-existing collection", + "runOnRequirements": [ + { + "minServerVersion": "4.3.4", + "topologies": [ + "replicaset", + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "dropCollection", + "object": "database0", + "arguments": { + "collection": "test" + } + }, + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "createIndex", + "object": "collection0", + "arguments": { + "session": "session0", + "name": "x_1", + "keys": { + "x": 1 + } + } + }, + { + "name": "assertIndexNotExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test", + "indexName": "x_1" + } + }, + { + "name": "commitTransaction", + "object": "session0" + }, + { + "name": "assertIndexExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test", + "indexName": "x_1" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "test", + "writeConcern": { + "$$exists": false + } + }, + "commandName": "drop", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "createIndexes": "test", + "indexes": [ + { + "name": "x_1", + "key": { + "x": 1 + } + } + ], + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "createIndexes", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions.yml b/source/unified-test-format/tests/valid-pass/poc-transactions.yml new file mode 100644 index 0000000000..5f229e464c --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions.yml @@ -0,0 +1,170 @@ +description: "poc-transactions" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name transaction-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - session: + id: &session0 session0 + client: *client0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + +tests: + - description: "Client side error in command starting transaction" + operations: + - name: startTransaction + object: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: { .: . } } + # Original test only asserted a generic error + expectError: { isClientError: true } + - name: assertSessionTransactionState + object: testRunner + arguments: + session: *session0 + state: starting + + - description: "explicitly create collection using create command" + runOnRequirements: + - minServerVersion: "4.3.4" + topologies: [ replicaset, sharded-replicaset ] + operations: + - name: dropCollection + object: *database0 + arguments: + collection: *collection0Name + - name: startTransaction + object: *session0 + - name: createCollection + object: *database0 + arguments: + session: *session0 + collection: *collection0Name + - name: assertCollectionNotExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + - name: commitTransaction + object: *session0 + - name: assertCollectionExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + drop: *collection0Name + writeConcern: { $$exists: false } + commandName: drop + databaseName: *database0Name + - commandStartedEvent: + command: + create: *collection0Name + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + commandName: create + databaseName: *database0Name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + commandName: commitTransaction + databaseName: admin + + - description: "create index on a non-existing collection" + runOnRequirements: + - minServerVersion: "4.3.4" + topologies: [ replicaset, sharded-replicaset ] + operations: + - name: dropCollection + object: *database0 + arguments: + collection: *collection0Name + - name: startTransaction + object: *session0 + - name: createIndex + object: *collection0 + arguments: + session: *session0 + name: &indexName "x_1" + keys: { x: 1 } + - name: assertIndexNotExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + indexName: *indexName + - name: commitTransaction + object: *session0 + - name: assertIndexExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + indexName: *indexName + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + drop: *collection0Name + writeConcern: { $$exists: false } + commandName: drop + databaseName: *database0Name + - commandStartedEvent: + command: + createIndexes: *collection0Name + indexes: + - name: *indexName + key: { x: 1 } + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + commandName: createIndexes + databaseName: *database0Name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + commandName: commitTransaction + databaseName: admin diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst new file mode 100644 index 0000000000..b6b1e5e860 --- /dev/null +++ b/source/unified-test-format/unified-test-format.rst @@ -0,0 +1,2638 @@ +=================== +Unified Test Format +=================== + +:Spec Title: Unified Test Format +:Spec Version: 1.0.0 +:Author: Jeremy Mikola +:Advisors: Prashant Mital, Isabel Atkinson, Thomas Reggi +:Status: Draft +:Type: Standards +:Minimum Server Version: N/A +:Last Modified: 2020-10-08 + +.. contents:: + +-------- + +Abstract +======== + +This project defines a unified schema for YAML and JSON specification tests, +which run operations against a MongoDB deployment. By conforming various spec +tests to a single schema, drivers can implement a single test runner to execute +acceptance tests for multiple specifications, thereby reducing maintenance of +existing specs and implementation time for new specifications. + + +META +==== + +The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", +"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be +interpreted as described in `RFC 2119 `__. + +This document tends to use "SHOULD" more frequently than other specifications, +but mainly in the context of providing guidance on writing test files. This is +discussed in more detail in `Design Rationale`_. + + +Goals +===== + +This test format can be used to define tests for the following specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `Command Monitoring <../command-monitoring/command-monitoring.rst>`__ +- `CRUD <../crud/crud.rst>`__ +- `GridFS <../gridfs/gridfs-spec.rst>`__ +- `Retryable Reads <../retryable-reads/retryable-reads.rst>`__ +- `Retryable Writes <../retryable-writes/retryable-writes.rst>`__ +- `Sessions <../sessions/driver-sessions.rst>`__ +- `Transactions <../transactions/transactions.rst>`__ +- `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ + +This is not an exhaustive list. Specifications that are known to not be +supported by this format may be discussed under `Future Work`_. + + +Specification +============= + + +Terms +----- + +Entity + Any object or value that is indexed by a unique name and stored in the + `Entity Map`_. This will typically be a driver object (e.g. client, session) + defined in `createEntities`_ but may also be a + `saved operation result `_. A exhaustive list + of supported types is presented in `Supported Entity Types`_. Entities are + referenced by name throughout the test file (e.g. `Entity Test Operations`_). + +Internal MongoClient + A MongoClient created specifically for use with internal test operations, such + as inserting collection data before a test, performing special assertions + during a test, or asserting collection data after a test. + +Iterable + This term is used by various specifications as the return type for operations + that return a sequence of items, which may be iterated. For example, the CRUD + spec uses this as the return value for ``find`` and permit API flexibility + rather than stipulate that a cursor object be returned directly. + + +Schema Version +-------------- + +This specification and the `Test Format`_ follow +`semantic versioning `__. The version is primarily used to +validate test files with a `JSON schema `__ and also +allow test runners to determine whether a particular test file is supported. + +New tests files SHOULD always be written using the latest major version of this +specification; however, test files SHOULD be conservative in the minor version +they specify (as noted in `schemaVersion`_). + + +JSON Schema Validation +~~~~~~~~~~~~~~~~~~~~~~ + +Each major version of this specification will have a corresponding JSON schema +for its most recent minor version (e.g. ``schema-1.1.json``). A JSON schema for +a particular minor version MUST be capable of validating any and all test files +in that major version series up to and including the minor version. For example, +``schema-2.1.json`` should validate test files with `schemaVersion`_ "2.0" and +"2.1", but would not be expected to validate "1.0", "2.2", or "3.0". + +The JSON schema MUST remain consistent with the `Test Format`_ section. If and +when a new major version is introduced, the `Breaking Changes`_ section MUST be +updated and any JSON schema(s) for a previous major version(s) MUST remain +available so that older test files can still be validated. + +`Ajv `__ MAY be used to programmatically validate both YAML +and JSON files using the JSON schema. The JSON schema MUST NOT use syntax that +is unsupported by this tool, which bears mentioning because there are multiple +versions of the +`JSON schema specification `__. + + +Test Runner Support +~~~~~~~~~~~~~~~~~~~ + +Each test file defines a `schemaVersion`_, which test runners will use to +determine compatibility (i.e. whether and how the test file will be +interpreted). Test files are considered compatible with a test runner if their +`schemaVersion`_ is less than or equal to a supported version in the test +runner, given the same major version component. For example: + +- A test runner supporting version 1.5.1 could execute test files with versions + 1.0 and 1.5 but *not* 1.6 and 2.0. +- A test runner supporting version 2.1 could execute test files with versions + 2.0 and 2.1 but *not* 1.0 and 1.5. +- A test runner supporting *both* versions 1.5.1 and 2.0 could execute test + files with versions 1.4, 1.5, and 2.0, but *not* 1.6, 2.1, or 3.0. +- A test runner supporting version 2.0.1 could execute test files with versions + 2.0 and 2.0.1 but *not* 2.0.2 or 2.1. This example is provided for + completeness, but test files SHOULD NOT need to refer to patch versions (as + previously mentioned). + +Test runners MUST NOT process incompatible files and MUST raise an error if they +encounter an incompatible file (as discussed in `Executing a Test File`_). Test +runners MAY support multiple schema versions (as demonstrated in the example +above). + + +Impact of Spec Changes on Schema Version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Backwards-breaking changes SHALL warrant a new major version. These changes +include, but are not limited to: + +- Subtractive changes, such as removing a field, operation, or type of supported + entity or event +- Changing an existing field from optional to required +- Introducing a new, required field in the test format +- Significant changes to test file execution (not BC) + +Backwards-compatible changes SHALL warrant a new minor version. These changes +include, but are not limited to: + +- Additive changes, such as a introducing a new `Special Test Operations`_ or + type of supported entity or event +- Changing an existing field from required to optional +- Introducing a new, optional field in the test format +- Minor changes to test file execution (BC) + +Small fixes and internal spec changes (e.g. grammar, adding clarifying text to +the spec) MAY warrant a new patch version; however, patch versions SHOULD NOT +alter the structure of the test format and thus SHOULD NOT be relevant to test +files (as noted in `schemaVersion`_). + + +Entity Map +---------- + +The entity map indexes arbitrary objects and values by unique names, so that +they can be referenced from test constructs (e.g. +`operation.object `_). To ensure each test is executed in +isolation, test runners MUST NOT share entity maps between tests. Most entities +will be driver objects created by the `createEntities`_ directive during test +setup, but the entity map may also be modified during test execution via the +`operation.saveResultAsEntity `_ directive. + +Test runners MAY choose to implement the entity map in a fashion most suited to +their language, but implementations MUST enforce both uniqueness of entity names +and referential integrity when fetching an entity. Test runners MUST raise an +error if an attempt is made to store an entity with a name that already exists +in the map and MUST raise an error if an entity is not found for a name or is +found but has an unexpected type. + +Consider the following examples:: + + # Error due to a duplicate name (client0 was already defined) + createEntities: + - client: { id: client0 } + - client: { id: client0 } + + # Error due to a missing entity (client1 is undefined) + createEntities: + - client: { id: client0 } + - session: { id: session0, client: client1 } + + # Error due to an unexpected entity type (session instead of client) + createEntities: + - client: { id: client0 } + - session: { id: session0, client: client0 } + - session: { id: session1, client: session0 } + + +Supported Entity Types +~~~~~~~~~~~~~~~~~~~~~~ + +Test runners MUST support the following types of entities: + +- MongoClient. See `entity_client`_ and `Client Operations`_. +- Database. See `entity_database`_ and `Database Operations`_. +- Collection. See `entity_collection`_ and `Collection Operations`_ +- ClientSession. See `entity_session`_ and `Session Operations`_. +- GridFS Bucket. See `entity_bucket`_ and `Bucket Operations`_. +- ChangeStream. See `ChangeStream Operations`_. +- All known BSON types and/or equivalent language types for the target driver. + For the present version of the spec, the following BSON types are known: + 0x01-0x13, 0x7F, 0xFF. + + Tests SHOULD NOT utilize deprecated types (e.g. 0x0E: Symbol), since they may + not be supported by all drivers and could yield runtime errors (e.g. while + loading a test file with an Extended JSON parser). + +This is an exhaustive list of supported types for the entity map. Test runners +MUST raise an error if an attempt is made to store an unsupported type in the +entity map. + +Adding new entity types (including known BSON types) to this list will require +a minor version bump to the spec and schema version. Removing entity types will +require a major version bump. See `Impact of Spec Changes on Schema Version`_ +for more information. + + +Test Format +----------- + +Each specification test file can define one or more tests, which inherit some +top-level configuration (e.g. namespace, initial data). YAML and JSON test files +are parsed as an object by the test runner. This section defines the top-level +keys for that object and links to various sub-sections for definitions of +nested structures (e.g. individual `test`_, `operation`_). + +Although test runners are free to process YAML or JSON files, YAML is the +canonical format for writing tests. YAML files may be converted to JSON using a +tool such as `js-yaml `__ . + + +Top-level Fields +~~~~~~~~~~~~~~~~ + +The top-level fields of a test file are as follows: + +- ``description``: Required string. The name of the test file. + + This SHOULD describe the common purpose of tests in this file and MAY refer to + the filename (e.g. "updateOne-hint"). + +.. _schemaVersion: + +- ``schemaVersion``: Required string. Version of this specification with which + the test file complies. + + Test files SHOULD be conservative when specifying a schema version. For + example, if the latest schema version is 1.1 but the test file complies with + schema version 1.0, the test file should specify 1.0. + + Test runners will use this to determine compatibility (i.e. whether and how + the test file will be interpreted). The format of this string is defined in + `Version String`_; however, test files SHOULD NOT need to refer to specific + patch versions since patch-level changes SHOULD NOT alter the structure of the + test format (as previously noted in `Schema Version`_). + +.. _runOnRequirements: + +- ``runOnRequirements``: Optional array of one or more `runOnRequirement`_ + objects. List of server version and/or topology requirements for which the + tests in this file can be run. If no requirements are met, the test runner + MUST skip this test file. + +.. _createEntities: + +- ``createEntities``: Optional array of one or more `entity`_ objects. List of + entities (e.g. client, collection, session objects) that SHALL be created + before each test case is executed. + + Test files SHOULD define entities in dependency order, such that all + referenced entities (e.g. client) are defined before any of their dependent + entities (e.g. database, session). + +.. _initialData: + +- ``initialData``: Optional array of one or more `collectionData`_ objects. Data + that will exist in collections before each test case is executed. + + Before each test and for each `collectionData`_, the test runner MUST drop the + collection and insert the specified documents (if any) using a "majority" + write concern. If no documents are specified, the test runner MUST create the + collection with a "majority" write concern. + +.. _tests: + +- ``tests``: Required array of one or more `test`_ objects. List of test cases + to be executed independently of each other. + + +runOnRequirement +~~~~~~~~~~~~~~~~ + +A combination of server version and/or topology requirements for running the +test(s). + +The format of server version strings is defined in `Version String`_. When +comparing server version strings, each component SHALL be compared numerically. +For example, "4.0.10" is greater than "4.0.9" and "3.6" and less than "4.2.0". + +The structure of this object is as follows: + +- ``minServerVersion``: Optional string. The minimum server version (inclusive) + required to successfully run the tests. If this field is omitted, there is no + lower bound on the required server version. The format of this string is + defined in `Version String`_. + +- ``maxServerVersion``: Optional string. The maximum server version (inclusive) + against which the tests can be run successfully. If this field is omitted, + there is no upper bound on the required server version. The format of this + string is defined in `Version String`_. + +- ``topologies``: Optional array of one or more strings. Server topologies + against which the tests can be run successfully. Valid topologies are + "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded + cluster backed by replica sets). If this field is omitted, there is no + topology requirement for the test. + + When matching a "sharded-replicaset" topology, test runners MUST ensure that + all shards are backed by a replica set. The process for doing so is described + in `Determining if a Sharded Cluster Uses Replica Sets`_. When matching a + "sharded" topology, test runners MUST accept any type of sharded cluster (i.e. + "sharded" implies "sharded-replicaset", but not vice versa). + + +entity +~~~~~~ + +An entity (e.g. client, collection, session object) that will be created in the +`Entity Map`_ before each test is executed. + +This object MUST contain **exactly one** top-level key that identifies the +entity type and maps to a nested object, which specifies a unique name for the +entity (``id`` key) and any other parameters necessary for its construction. +Tests SHOULD use sequential names based on the entity type (e.g. "session0", +"session1"). + +When defining an entity object in YAML, a `node anchor`_ SHOULD be created on +the entity's ``id`` key. This anchor will allow the unique name to be referenced +with an `alias node`_ later in the file (e.g. from another entity or +`operation`_ document) and also leverage YAML's parser for reference validation. + +.. _node anchor: https://yaml.org/spec/1.2/spec.html#id2785586 +.. _alias node: https://yaml.org/spec/1.2/spec.html#id2786196 + +The structure of this object is as follows: + +.. _entity_client: + +- ``client``: Optional object. Defines a MongoClient object. + + The structure of this object is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. ``id: &client0 client0``). + + - ``uriOptions``: Optional object. Additional URI options to apply to the + test suite's connection string that is used to create this client. Any keys + in this object MUST override conflicting keys in the connection string. + + Documentation for supported options may be found in the + `URI Options <../uri-options/uri-options.rst>`__ spec, with one notable + exception: if ``readPreferenceTags`` is specified in this object, the key + will map to an array of strings, each representing a tag set, since it is + not feasible to define multiple ``readPreferenceTags`` keys in the object. + + .. _entity_client_useMultipleMongoses: + + - ``useMultipleMongoses``: Optional boolean. If true and the topology is a + sharded cluster, the test runner MUST assert that this MongoClient connects + to multiple mongos hosts (e.g. by inspecting the connection string). If + false and the topology is a sharded cluster, the test runner MUST ensure + that this MongoClient connects to only a single mongos host (e.g. by + modifying the connection string). + + If this option is not specified and the topology is a sharded cluster, the + test runner MUST NOT enforce any limit on the number of mongos hosts in the + connection string and any tests using this client SHOULD NOT depend on a + particular number of mongos hosts. + + This option has no effect for non-sharded topologies. + + .. _entity_client_observeEvents: + + - ``observeEvents``: Optional array of one or more strings. Types of events + that can be observed for this client. Unspecified event types MUST be + ignored by this client's event listeners and SHOULD NOT be included in + `test.expectEvents `_ for this client. + + Test files SHOULD NOT observe events from multiple specs (e.g. command + monitoring *and* SDAM events) for a single client. See + `Mixing event types in observeEvents and expectEvents`_ for more + information. + + Supported types correspond to the top-level keys (strings) documented in + `expectedEvent`_ and are as follows: + + - `commandStartedEvent `_ + + - `commandSucceededEvent `_ + + - `commandFailedEvent `_ + + .. _entity_client_ignoreCommandMonitoringEvents: + + - ``ignoreCommandMonitoringEvents``: Optional array of one or more strings. + Command names for which the test runner MUST ignore any observed command + monitoring events. The command(s) will be ignored in addition to + ``configureFailPoint`` and any commands containing sensitive information + (per the + `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ + spec). + + Test files SHOULD NOT use this option unless one or more command monitoring + events are specified in `observeEvents `_. + +.. _entity_database: + +- ``database``: Optional object. Defines a Database object. + + The structure of this object is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. ``id: &database0 database0``). + + - ``client``: Required string. Client entity from which this database will be + created. The YAML file SHOULD use an `alias node`_ for a client entity's + ``id`` field (e.g. ``client: *client0``). + + - ``databaseName``: Required string. Database name. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. + ``databaseName: &database0Name foo``). + + - ``databaseOptions``: Optional `collectionOrDatabaseOptions`_ object. + +.. _entity_collection: + +- ``collection``: Optional object. Defines a Collection object. + + The structure of this object is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. + ``id: &collection0 collection0``). + + - ``database``: Required string. Database entity from which this collection + will be created. The YAML file SHOULD use an `alias node`_ for a database + entity's ``id`` field (e.g. ``database: *database0``). + + - ``collectionName``: Required string. Collection name. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. + ``collectionName: &collection0Name foo``). + + - ``collectionOptions``: Optional `collectionOrDatabaseOptions`_ object. + +.. _entity_session: + +- ``session``: Optional object. Defines an explicit ClientSession object. + + The structure of this object is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. ``id: &session0 session0``). + + - ``client``: Required string. Client entity from which this session will be + created. The YAML file SHOULD use an `alias node`_ for a client entity's + ``id`` field (e.g. ``client: *client0``). + + - ``sessionOptions``: Optional object. Map of parameters to pass to + `MongoClient.startSession <../source/sessions/driver-sessions.rst#startsession>`__ + when creating the session. Supported options are defined in the following + specifications: + + - `Causal Consistency <../causal-consistency/causal-consistency.rst#sessionoptions-changes>`__ + - `Transactions <../transactions/transactions.rst#sessionoptions-changes>`__ + + When specifying TransactionOptions for ``defaultTransactionOptions``, the + transaction options MUST remain nested under ``defaultTransactionOptions`` + and MUST NOT be flattened into ``sessionOptions``. + +.. _entity_bucket: + +- ``bucket``: Optional object. Defines a Bucket object, as defined in the + `GridFS <../gridfs/gridfs-spec.rst>`__ spec. + + The structure of this object is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. ``id: &bucket0 bucket0``). + + - ``database``: Required string. Database entity from which this bucket will + be created. The YAML file SHOULD use an `alias node`_ for a database + entity's ``id`` field (e.g. ``database: *database0``). + + - ``bucketOptions``: Optional object. Additional options used to construct + the bucket object. Supported options are defined in the + `GridFS <../source/gridfs/gridfs-spec.rst#configurable-gridfsbucket-class>`__ + specification. The ``readConcern``, ``readPreference``, and ``writeConcern`` + options use the same structure as defined in `Common Options`_. + + +collectionData +~~~~~~~~~~~~~~ + +List of documents corresponding to the contents of a collection. This structure +is used by both `initialData`_ and `test.outcome `_, which insert +and read documents, respectively. + +The structure of this object is as follows: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +- ``documents``: Required array of objects. List of documents corresponding to + the contents of the collection. This list may be empty. + + +test +~~~~ + +Test case consisting of a sequence of operations to be executed. + +The structure of this object is as follows: + +- ``description``: Required string. The name of the test. + + This SHOULD describe the purpose of this test (e.g. "insertOne is retried"). + +.. _test_runOnRequirements: + +- ``runOnRequirements``: Optional array of one or more `runOnRequirement`_ + objects. List of server version and/or topology requirements for which this + test can be run. If specified, these requirements are evaluated independently + and in addition to any top-level `runOnRequirements`_. If no requirements in + this array are met, the test runner MUST skip this test. + + These requirements SHOULD be more restrictive than those specified in the + top-level `runOnRequirements`_ (if any) and SHOULD NOT be more permissive. + This is advised because both sets of requirements MUST be satisified in order + for a test to be executed and more permissive requirements at the test-level + could be taken out of context on their own. + +.. _test_skipReason: + +- ``skipReason``: Optional string. If set, the test will be skipped. The string + SHOULD explain the reason for skipping the test (e.g. JIRA ticket). + +.. _test_operations: + +- ``operations``: Required array of one or more `operation`_ objects. List of + operations to be executed for the test case. + +.. _test_expectEvents: + +- ``expectEvents``: Optional array of one or more `expectedEventsForClient`_ + objects. For one or more clients, a list of events that are expected to be + observed in a particular order. + + If a driver only supports configuring event listeners globally (for all + clients), the test runner SHOULD associate each observed event with a client + in order to perform these assertions. + + Test files SHOULD NOT expect events from multiple specs (e.g. command + monitoring *and* SDAM events) for a single client. See + `Mixing event types in observeEvents and expectEvents`_ for more + information. + +.. _test_outcome: + +- ``outcome``: Optional array of one or more `collectionData`_ objects. Data + that is expected to exist in collections after each test case is executed. + + The list of documents herein SHOULD be sorted ascendingly by the ``_id`` field + to allow for deterministic comparisons. The procedure for asserting collection + contents is discussed in `Executing a Test`_. + + +operation +~~~~~~~~~ + +An operation to be executed as part of the test. + +The structure of this object is as follows: + +.. _operation_name: + +- ``name``: Required string. Name of the operation (e.g. method) to perform on + the object. + +.. _operation_object: + +- ``object``: Required string. Name of the object on which to perform the + operation. This SHOULD correspond to either an `entity`_ name (for + `Entity Test Operations`_) or "testRunner" (for `Special Test Operations`_). + If the object is an entity, The YAML file SHOULD use an `alias node`_ for its + ``id`` field (e.g. ``object: *collection0``). + +.. _operation_arguments: + +- ``arguments``: Optional object. Map of parameter names and values for the + operation. The structure of this object will vary based on the operation. + See `Entity Test Operations`_ and `Special Test Operations`_. + + The ``session`` parameter is handled specially (see `commonOptions_session`_). + +.. _operation_expectError: + +- ``expectError``: Optional `expectedError`_ object. One or more assertions for + an error expected to be raised by the operation. + + This field is mutually exclusive with + `expectResult `_ and + `saveResultAsEntity `_. + + This field SHOULD NOT be used for `Special Test Operations`_ (i.e. + ``object: testRunner``). + +.. _operation_expectResult: + +- ``expectResult``: Optional mixed type. A value corresponding to the expected + result of the operation. This field may be a scalar value, a single document, + or an array of values. Test runners MUST follow the rules in + `Evaluating Matches`_ when processing this assertion. + + This field is mutually exclusive with `expectError `_. + + This field SHOULD NOT be used for `Special Test Operations`_ (i.e. + ``object: testRunner``). + +.. _operation_saveResultAsEntity: + +- ``saveResultAsEntity``: Optional string. If specified, the actual result + returned by the operation (if any) will be saved with this name in the + `Entity Map`_. The test runner MUST raise an error if the name is already in + use or if the result does not comply with `Supported Entity Types`_. + + This field is mutually exclusive with `expectError `_. + + This field SHOULD NOT be used for `Special Test Operations`_ (i.e. + ``object: testRunner``). + + +expectedError +~~~~~~~~~~~~~ + +One or more assertions for an error/exception, which is expected to be raised by +an executed operation. At least one key is required in this object. + +The structure of this object is as follows: + +- ``isError``: Optional boolean. If true, the test runner MUST assert that an + error was raised. This is primarily used when no other error assertions apply + but the test still needs to assert an expected error. Test files MUST NOT + specify false, as `expectedError`_ is only applicable when an operation is + expected to raise an error. + +- ``isClientError``: Optional boolean. If true, the test runner MUST assert that + the error originates from the client (i.e. it is not derived from a server + response). If false, the test runner MUST assert that the error does not + originate from the client. + + Client errors include, but are not limited to: parameter validation errors + before a command is sent to the server; network errors. + +- ``errorContains``: Optional string. A substring of the expected error message + (e.g. "errmsg" field in a server error document). The test runner MUST assert + that the error message contains this string using a case-insensitive match. + + See `bulkWrite`_ for special considerations for BulkWriteExceptions. + +- ``errorCode``: Optional integer. The expected "code" field in the + server-generated error response. The test runner MUST assert that the error + includes a server-generated response whose "code" field equals this value. + In the interest of readability, YAML files SHOULD use a comment to note the + corresponding code name (e.g. ``errorCode: 26 # NamespaceNotFound``). + + Server error codes are defined in + `error_codes.yml `__. + + Test files SHOULD NOT assert error codes for client errors, as specifications + do not define standardized codes for client errors. + +- ``errorCodeName``: Optional string. The expected "codeName" field in the + server-generated error response. The test runner MUST assert that the error + includes a server-generated response whose "codeName" field equals this value + using a case-insensitive comparison. + + See `bulkWrite`_ for special considerations for BulkWriteExceptions. + + Server error codes are defined in + `error_codes.yml `__. + + Test files SHOULD NOT assert error codes for client errors, as specifications + do not define standardized codes for client errors. + +- ``errorLabelsContain``: Optional array of one or more strings. A list of error + label strings that the error is expected to have. The test runner MUST assert + that the error contains all of the specified labels (e.g. using the + ``hasErrorLabel`` method). + +- ``errorLabelsOmit``: Optional array of one or more strings. A list of error + label strings that the error is expected not to have. The test runner MUST + assert that the error does not contain any of the specified labels (e.g. using + the ``hasErrorLabel`` method). + +.. _expectedError_expectResult: + +- ``expectResult``: Optional mixed type. This field follows the same rules as + `operation.expectResult `_ and is only used in cases + where the error includes a result (e.g. `bulkWrite`_). If specified, the test + runner MUST assert that the error includes a result and that it matches this + value. + + +expectedEventsForClient +~~~~~~~~~~~~~~~~~~~~~~~ + +A list of events that are expected to be observed (in that order) for a client +while executing `operations `_. + +The structure of each object is as follows: + +- ``client``: Required string. Client entity on which the events are expected + to be observed. See `commonOptions_client`_. + +- ``events``: Required array of `expectedEvent`_ objects. List of events, which + are expected to be observed (in this order) on the corresponding client while + executing `operations`_. If the array is empty, the test runner MUST assert + that no events were observed on the client (excluding ignored events). + + +expectedEvent +~~~~~~~~~~~~~ + +An event (e.g. APM), which is expected to be observed while executing the test's +operations. + +This object MUST contain **exactly one** top-level key that identifies the +event type and maps to a nested object, which contains one or more assertions +for the event's properties. + +Some event properties are omitted in the following structures because they +cannot be reliably tested. Taking command monitoring events as an example, +``requestId`` and ``operationId`` are nondeterministic and types for +``connectionId`` and ``failure`` can vary by implementation. + +The structure of this object is as follows: + +.. _expectedEvent_commandStartedEvent: + +- ``commandStartedEvent``: Optional object. Assertions for one or more + `CommandStartedEvent <../command-monitoring/command-monitoring.rst#api>`__ + fields. + + The structure of this object is as follows: + + - ``command``: Optional document. A value corresponding to the expected + command document. Test runners MUST follow the rules in + `Evaluating Matches`_ when processing this assertion. + + - ``commandName``: Optional string. Test runners MUST assert that the command + name matches this value. + + - ``databaseName``: Optional string. Test runners MUST assert that the + database name matches this value. The YAML file SHOULD use an `alias node`_ + for this value (e.g. ``databaseName: *database0Name``). + +.. _expectedEvent_commandSucceededEvent: + +- ``commandSucceededEvent``: Optional object. Assertions for one or more + `CommandSucceededEvent <../command-monitoring/command-monitoring.rst#api>`__ + fields. + + The structure of this object is as follows: + + - ``reply``: Optional document. A value corresponding to the expected + reply document. Test runners MUST follow the rules in `Evaluating Matches`_ + when processing this assertion. + + - ``commandName``: Optional string. Test runners MUST assert that the command + name matches this value. + +.. _expectedEvent_commandFailedEvent: + +- ``commandFailedEvent``: Optional object. Assertions for one or more + `CommandFailedEvent <../command-monitoring/command-monitoring.rst#api>`__ + fields. + + The structure of this object is as follows: + + - ``commandName``: Optional string. Test runners MUST assert that the command + name matches this value. + + +collectionOrDatabaseOptions +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Map of parameters used to construct a collection or database object. + +The structure of this object is as follows: + + - ``readConcern``: Optional object. See `commonOptions_readConcern`_. + + - ``readPreference``: Optional object. See `commonOptions_readPreference`_. + + - ``writeConcern``: Optional object. See `commonOptions_writeConcern`_. + + +Common Options +-------------- + +This section defines the structure of common options that are referenced from +various contexts in the test format. Comprehensive documentation for some of +these types and their parameters may be found in the following specifications: + +- `Read and Write Concern <../read-write-concern/read-write-concern.rst>`__. +- `Server Selection: Read Preference <../server-selection/server-selection.rst#read-preference>`__. + +The structure of these common options is as follows: + +.. _commonOptions_collectionName: + +- ``collectionName``: String. Collection name. The YAML file SHOULD use an + `alias node`_ for a collection entity's ``collectionName`` field (e.g. + ``collectionName: *collection0Name``). + +.. _commonOptions_databaseName: + +- ``databaseName``: String. Database name. The YAML file SHOULD use an + `alias node`_ for a database entity's ``databaseName`` field (e.g. + ``databaseName: *database0Name``). + +.. _commonOptions_readConcern: + +- ``readConcern``: Object. Map of parameters to construct a read concern. + + The structure of this object is as follows: + + - ``level``: Required string. + +.. _commonOptions_readPreference: + +- ``readPreference``: Object. Map of parameters to construct a read + preference. + + The structure of this object is as follows: + + - ``mode``: Required string. + + - ``tagSets``: Optional array of objects. + + - ``maxStalenessSeconds``: Optional integer. + + - ``hedge``: Optional object. + +.. _commonOptions_client: + +- ``client``: String. Client entity name, which the test runner MUST resolve + to a MongoClient object. The YAML file SHOULD use an `alias node`_ for a + client entity's ``id`` field (e.g. ``client: *client0``). + +.. _commonOptions_session: + +- ``session``: String. Session entity name, which the test runner MUST resolve + to a ClientSession object. The YAML file SHOULD use an `alias node`_ for a + session entity's ``id`` field (e.g. ``session: *session0``). + +.. _commonOptions_writeConcern: + +- ``writeConcern``: Object. Map of parameters to construct a write concern. + + The structure of this object is as follows: + + - ``journal``: Optional boolean. + + - ``w``: Optional integer or string. + + - ``wtimeoutMS``: Optional integer. + + +Version String +-------------- + +Version strings, which are used for `schemaVersion`_ and `runOnRequirement`_, +MUST conform to one of the following formats, where each component is a +non-negative integer: + +- ``..`` +- ``.`` (```` is assumed to be zero) +- ```` (```` and ```` are assumed to be zero) + + +Entity Test Operations +---------------------- + +Entity operations correspond to an API method on a driver object. If +`operation.object `_ refers to an `entity`_ name (e.g. +"collection0") then `operation.name `_ is expected to reference +an API method on that class. + +Test files SHALL use camelCase when referring to API methods and parameters, +even if the defining specifications use other forms (e.g. snake_case in GridFS). + +This spec does not provide exhaustive documentation for all possible API methods +that may appear in a test; however, the following sections discuss all supported +entities and their operations in some level of detail. Special handling for +certain operations is also discussed as needed. + + +Expressing Required and Optional Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some specifications group optional parameters for API methods under an +``options`` parameter (e.g. ``options: Optional`` in the CRUD +spec); however, driver APIs vary in how they accept options (e.g. Python's +keyword/named arguments, ``session`` as either an option or required parameter +depending on whether a language supports method overloading). Therefore, test +files SHALL declare all required and optional parameters for an API method +directly within `operation.arguments `_ (e.g. ``upsert`` +for ``updateOne`` is *not* nested under an ``options`` key). + + +Special Handling for Arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If ``session`` is specified in `operation.arguments`_, it is defined according +to `commonOptions_session`_. Test runners MUST resolve the ``session`` argument +to `session `_ entity *before* passing it as a parameter to any +API method. + +If ``readConcern``, ``readPreference``, or ``writeConcern`` are specified in +`operation.arguments`_, test runners MUST interpret them according to the +corresponding definition in `Common Options`_ and MUST convert the value into +the appropriate object *before* passing it as a parameter to any API method. + + +Converting Returned Model Objects to Documents +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For operations that return a model object (e.g. ``BulkWriteResult`` for +``bulkWrite``), the test runner MUST convert the model object to a document when +evaluating `expectResult `_ or +`saveResultAsEntity `_. Similarly, for operations +that may return iterables of model objects (e.g. ``DatabaseInfo`` for +``listDatabases``), the test runner MUST convert the iterable to an array of +documents when evaluating `expectResult`_ or `saveResultAsEntity`_. + + +Iterating Returned Iterables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unless otherwise stated by an operation below, test runners MUST fully iterate +any iterable returned by an operation as part of that operation's execution. +This is necessary to ensure consistent behavior among drivers, as discussed in +`aggregate`_ and `find`_, and also ensures that error and event assertions can +be evaluated consistently. + + +Client Operations +----------------- + +These operations and their arguments may be documented in the following +specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `Enumerating Databases <../enumerate-databases.rst>`__ + +Client operations that require special handling or are not documented by an +existing specification are described below. + + +.. _client_createChangeStream: + +createChangeStream +~~~~~~~~~~~~~~~~~~ + +Creates a cluster-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the client's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the client's ``watch`` +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. + +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. + + +watch +~~~~~ + +This operation SHOULD NOT be used in test files. See +`client_createChangeStream`_. + + +Database Operations +------------------- + +These operations and their arguments may be documented in the following +specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `CRUD <../crud/crud.rst>`__ +- `Enumerating Collections <../enumerate-collections.rst>`__ + +Database operations that require special handling or are not documented by an +existing specification are described below. + + +.. _database_createChangeStream: + +createChangeStream +~~~~~~~~~~~~~~~~~~ + +Creates a database-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the database's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the database's ``watch`` +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. + +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. + + +runCommand +~~~~~~~~~~ + +Generic command runner. + +This method does not inherit a read concern or write concern (per the +`Read and Write Concern <../read-write-concern/read-write-concern.rst#generic-command-method>`__ +spec), nor does it inherit a read preference (per the +`Server Selection <../server-selection/server-selection.rst#use-of-read-preferences-with-commands>`__ +spec); however, they may be specified as arguments. + +The following arguments are supported: + +- ``command``: Required document. The command to be executed. + +- ``commandName``: Required string. The name of the command to run. This is used + by languages that are unable preserve the order of keys in the ``command`` + argument when parsing YAML/JSON. + +- ``readConcern``: Optional object. See `commonOptions_readConcern`_. + +- ``readPreference``: Optional object. See `commonOptions_readPreference`_. + +- ``session``: Optional string. See `commonOptions_session`_. + +- ``writeConcern``: Optional object. See `commonOptions_writeConcern`_. + + +watch +~~~~~ + +This operation SHOULD NOT be used in test files. See +`database_createChangeStream`_. + + +Collection Operations +--------------------- + +These operations and their arguments may be documented in the following +specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `CRUD <../crud/crud.rst>`__ +- `Enumerating Indexes <../enumerate-indexes.rst>`__ +- `Index Management <../index-management.rst>`__ + +Collection operations that require special handling or are not documented by an +existing specification are described below. + + +aggregate +~~~~~~~~~ + +When executing an ``aggregate`` operation, the test runner MUST fully iterate +the result. This will ensure consistent behavior between drivers that eagerly +create a server-side cursor and those that do so lazily when iteration begins. + + +bulkWrite +~~~~~~~~~ + +The ``requests`` parameter for ``bulkWrite`` is documented as a list of +WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) +provides important context to the method, but that type information is not +easily expressed in YAML and JSON. To account for this, test files MUST nest +each WriteModel object in a single-key object, where the key identifies the +request type (e.g. "insertOne") and its value is an object expressing the +parameters, as in the following example:: + + arguments: + requests: + - insertOne: + document: { _id: 1, x: 1 } + - replaceOne: + filter: { _id: 2 } + replacement: { x: 2 } + upsert: true + - updateOne: + filter: { _id: 3 } + update: { $set: { x: 3 } } + upsert: true + - updateMany: + filter: { } + update: { $inc: { x: 1 } } + - deleteOne: + filter: { x: 2 } + - deleteMany: + filter: { x: { $gt: 2 } } + ordered: true + +Because the ``insertedIds`` field of BulkWriteResult is optional for drivers to +implement, assertions for that field SHOULD utilize the `$$unsetOrMatches`_ +operator. + +While operations typically raise an error *or* return a result, the +``bulkWrite`` operation is unique in that it may report both via the +``writeResult`` property of a BulkWriteException. In this case, the intermediary +write result may be matched with `expectedError_expectResult`_. Because +``writeResult`` is optional for drivers to implement, such assertions SHOULD +utilize the `$$unsetOrMatches`_ operator. + +Additionally, BulkWriteException is unique in that it aggregates one or more +server errors in its ``writeConcernError`` and ``writeErrors`` properties. +When test runners evaluate `expectedError`_ assertions for ``errorContains`` and +``errorCodeName``, they MUST examine the aggregated errors and consider any +match therein to satisfy the assertion(s). Drivers that concatenate all write +and write concern error messages into the BulkWriteException message MAY +optimize the check for ``errorContains`` by examining the concatenated message. +Drivers that expose ``code`` but not ``codeName`` through BulkWriteException MAY +translate the expected code name to a number (see: +`error_codes.yml `__) +and compare with ``code`` instead, but MUST raise an error if the comparison +cannot be attempted (e.g. ``code`` is also not available, translation fails). + + +.. _collection_createChangeStream: + +createChangeStream +~~~~~~~~~~~~~~~~~~ + +Creates a collection-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the collection's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the collection's ``watch`` +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. + +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. + + +find +~~~~ + +When executing a ``find`` operation, the test runner MUST fully iterate the +result. This will ensure consistent behavior between drivers that eagerly create +a server-side cursor and those that do so lazily when iteration begins. + + +findOneAndReplace and findOneAndUpdate +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``returnDocument`` option for ``findOneAndReplace`` and ``findOneAndUpdate`` +is documented as an enum with possible values "Before" and "After". Test files +SHOULD express ``returnDocument`` as a string and test runners MUST raise an +error if its value does not case-insensitively match either enum value. + + +insertMany +~~~~~~~~~~ + +The CRUD spec documents ``insertMany`` as returning a BulkWriteResult. Because +the ``insertedIds`` field of BulkWriteResult is optional for drivers to +implement, assertions for that field SHOULD utilize the `$$unsetOrMatches`_ +operator. + + +insertOne +~~~~~~~~~ + +The CRUD spec documents ``insertOne`` as returning an InsertOneResult; however, +because all fields InsertOneResult are optional drivers are permitted to forgo +it entirely and have ``insertOne`` return nothing (i.e. void method). Tests +asserting InsertOneResult SHOULD utilize the `$$unsetOrMatches`_ operator for +*both* the result object and any optional fields within, as in the following +examples:: + + - name: insertOne + object: *collection0 + arguments: + document: { _id: 2 } + expectResult: + $$unsetOrMatches: + insertedId: { $$unsetOrMatches: 2 } + + +watch +~~~~~ + +This operation SHOULD NOT be used in test files. See +`collection_createChangeStream`_. + + +Session Operations +------------------ + +These operations and their arguments may be documented in the following +specifications: + +- `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ +- `Driver Sessions <../sessions/driver-sessions.rst>`__ + +Session operations that require special handling or are not documented by an +existing specification are described below. + + +withTransaction +~~~~~~~~~~~~~~~ + +The ``withTransaction`` operation's ``callback`` parameter is a function and not +easily expressed in YAML/JSON. For ease of testing, this parameter is expressed +as an array of `operation`_ objects (analogous to +`test.operations `_). Test runners MUST evaluate error and +result assertions when executing these operations in the callback. + + +Bucket Operations +----------------- + +These operations and their arguments may be documented in the following +specifications: + +- `GridFS <../gridfs/gridfs-spec.rst>`__ + +Bucket operations that require special handling or are not documented by an +existing specification are described below. + + +.. _download: +.. _downloadByName: + +download and downloadByName +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These operations proxy the bucket's ``openDownloadStream`` and +``openDownloadStreamByName`` methods and support the same parameters and +options, but return a string containing the stream's contents instead of the +stream itself. Test runners MUST fully read the stream to yield the returned +string. This is also necessary to ensure that any expected errors are raised +(e.g. missing chunks). Test files SHOULD use `$$matchesHexBytes`_ in +`expectResult `_ to assert the contents of the returned +string. + + +downloadToStream and downloadToStreamByName +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These operations SHOULD NOT be used in test files. See +`IO operations for GridFS streams`_ in `Future Work`_. + + +openDownloadStream and openDownloadStreamByName +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These operations SHOULD NOT be used in test files. See +`download and downloadByName`_. + + +.. _openUploadStream: +.. _openUploadStreamWithId: + +openUploadStream and openUploadStreamWithId +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These operations SHOULD NOT be used in test files. See +`IO operations for GridFS streams`_ in `Future Work`_. + + +.. _upload: +.. _uploadWithId: + +upload and uploadWithId +~~~~~~~~~~~~~~~~~~~~~~~ + +These operations proxy the bucket's ``uploadFromStream`` and +``uploadFromStreamWithId`` methods and support the same parameters and options +with one exception: the ``source`` parameter is an object specifying hex bytes +from which test runners MUST construct a readable stream for the underlying +methods. The structure of ``source`` is as follows:: + + { $$hexBytes: } + +The string MUST contain an even number of hexademical characters +(case-insensitive) and MAY be empty. The test runner MUST raise an error if the +structure of ``source`` or its string is malformed. The test runner MUST convert +the string to a byte sequence denoting the stream's readable data (if any). For +example, "12ab" would denote a stream with two bytes: "0x12" and "0xab". + + + +uploadFromStream and uploadFromStreamWithId +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These operations SHOULD NOT be used in test files. See +`upload and uploadWithId`_. + + +ChangeStream Operations +----------------------- + +Change stream entities are special in that they are not defined in +`createEntities`_ but are instead created by using +`operation.saveResultAsEntity `_ with a +`client_createChangeStream`_, `database_createChangeStream`_, or +`collection_createChangeStream`_ operation. + +Test files SHOULD NOT use a ``watch`` operation to create a change stream, as +the implementation of that method may vary among drivers. For example, some +implementations of ``watch`` immediately execute ``aggregate`` and construct the +server-side cursor, while others may defer ``aggregate`` until the change stream +object is iterated. + +The `Change Streams <../change-streams/change-streams.rst>`__ spec does not +define a consistent API for the ChangeStream class, since the mechanisms for +iteration and capturing a resume token may differ between synchronous and +asynchronous drivers. To account for this, this section explicitly defines the +supported operations for change stream entities. + + +iterateUntilDocumentOrError +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Iterates the change stream until either a single document is returned or an +error is raised. This operation takes no arguments. If +`expectResult `_ is specified, it SHOULD be a single +document. + +`Iterating the Change Stream <../change-streams/tests#iterating-the-change-stream>`__ +in the change stream spec cautions drivers that implement a blocking mode of +iteration (e.g. asynchronous drivers) not to iterate the change stream +unnecessarily, as doing so could cause the test runner to block indefinitely. +This should not be a concern for ``iterateUntilDocumentOrError`` as iteration +only continues until either a document or error is encountered. + +Test runners MUST ensure that this operation will not inadvertently skip the +first document in a change stream. Albeit rare, this could happen if +``iterateUntilDocumentOrError`` were to blindly invoke ``next`` (or equivalent) +on a change stream in a driver where newly created change streams are already +positioned at their first element and the change stream cursor had a non-empty +``firstBatch`` (i.e. ``resumeAfter`` or ``startAfter`` used). Alternatively, +some drivers may use a different iterator method for advancing a change stream +to its first position (e.g. ``rewind`` in PHP). + + +Special Test Operations +----------------------- + +Certain operations do not correspond to API methods but instead represent +special test operations (e.g. assertions). These operations are distinguished by +`operation.object `_ having a value of "testRunner". The +`operation.name `_ field will correspond to an operation +defined below. + +Special test operations return no result and are always expected to succeed. +These operations SHOULD NOT be combined with +`expectError `_, +`expectResult `_, or +`saveResultAsEntity `_. + + +failPoint +~~~~~~~~~ + +The ``failPoint`` operation instructs the test runner to configure a fail point +using a "primary" read preference using the specified client entity (fail points +are not configured using the internal MongoClient). + +The following arguments are supported: + +- ``failPoint``: Required document. The ``configureFailPoint`` command to be + executed. + +- ``client``: Required string. See `commonOptions_client`_. + + The client entity SHOULD specify false for + `useMultipleMongoses `_ if this operation + could be executed on a sharded topology (according to `runOnRequirements`_ or + `test.runOnRequirements `_). This is advised because + server selection rules for mongos could lead to unpredictable behavior if + different servers were selected for configuring the fail point and executing + subsequent operations. + +When executing this operation, the test runner MUST keep a record of the fail +point so that it can be disabled after the test. The test runner MUST also +ensure that the ``configureFailPoint`` command is excluded from the list of +observed command monitoring events for this client (if applicable). + +An example of this operation follows:: + + # Enable the fail point on the server selected with a primary read preference + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + closeConnection: true + + +targetedFailPoint +~~~~~~~~~~~~~~~~~ + +The ``targetedFailPoint`` operation instructs the test runner to configure a +fail point on a specific mongos using the client entity associated with a +pinned session. + +The following arguments are supported: + +- ``failPoint``: Required document. The ``configureFailPoint`` command to be + executed. + +- ``session``: Required string. See `commonOptions_session`_. + + The client entity associated with this session SHOULD specify true for + `useMultipleMongoses `_. This is advised + because multiple mongos connections are necessary to test session pinning. + +The MongoClient and mongos on which to run the ``configureFailPoint`` command is +determined by the ``session`` argument (after resolution to a session entity). +Test runners MUST error if the session is not pinned to a mongos server at the +time this operation is executed. + +When executing this operation, the test runner MUST keep a record of both the +fail point and session (or pinned mongos server) so that the fail point can be +disabled on the same mongos server after the test. The test runner MUST also +ensure that the ``configureFailPoint`` command is excluded from the list of +observed command monitoring events for this client (if applicable). + +An example of this operation follows:: + + # Enable the fail point on the mongos to which session0 is pinned + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["commitTransaction"] + closeConnection: true + + +assertSessionTransactionState +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionTransactionState`` operation instructs the test runner to +assert that the given session has a particular transaction state. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +- ``state``: Required string. Expected transaction state for the session. + Possible values are as follows: ``none``, ``starting``, ``in_progress``, + ``committed``, and ``aborted``. + +An example of this operation follows:: + + - name: assertSessionTransactionState + object: testRunner + arguments: + session: *session0 + state: in_progress + + +assertSessionPinned +~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionPinned`` operation instructs the test runner to assert that +the given session is pinned to a mongos server. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +An example of this operation follows:: + + - name: assertSessionPinned + object: testRunner + arguments: + session: *session0 + + +assertSessionUnpinned +~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionUnpinned`` operation instructs the test runner to assert that +the given session is not pinned to a mongos server. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +An example of this operation follows:: + + - name: assertSessionUnpinned + object: testRunner + arguments: + session: *session0 + + +assertDifferentLsidOnLastTwoCommands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertDifferentLsidOnLastTwoCommands`` operation instructs the test runner +to assert that the last two CommandStartedEvents observed on the client have +different ``lsid`` fields. This assertion is primarily used to test that dirty +server sessions are discarded from the pool. + +The following arguments are supported: + +- ``client``: Required string. See `commonOptions_client`_. + + The client entity SHOULD include "commandStartedEvent" in + `observeEvents `_. + +The test runner MUST fail this assertion if fewer than two CommandStartedEvents +have been observed on the client or if either command does not include an +``lsid`` field. + +An example of this operation follows:: + + - name: assertDifferentLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + + +assertSameLsidOnLastTwoCommands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSameLsidOnLastTwoCommands`` operation instructs the test runner to +assert that the last two CommandStartedEvents observed on the client have +identical ``lsid`` fields. This assertion is primarily used to test that +non-dirty server sessions are not discarded from the pool. + +The following arguments are supported: + +- ``client``: Required string. See `commonOptions_client`_. + + The client entity SHOULD include "commandStartedEvent" in + `observeEvents `_. + +The test runner MUST fail this assertion if fewer than two CommandStartedEvents +have been observed on the client or if either command does not include an +``lsid`` field. + +An example of this operation follows:: + + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + + +assertSessionDirty +~~~~~~~~~~~~~~~~~~ + +The ``assertSessionDirty`` operation instructs the test runner to assert that +the given session is marked dirty. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +An example of this operation follows:: + + - name: assertSessionDirty + object: testRunner + arguments: + session: *session0 + + +assertSessionNotDirty +~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionNotDirty`` operation instructs the test runner to assert that +the given session is not marked dirty. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +An example of this operation follows:: + + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + + +assertCollectionExists +~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertCollectionExists`` operation instructs the test runner to assert +that the given collection exists in the database. The test runner MUST use the +internal MongoClient for this operation. + +The following arguments are supported: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +An example of this operation follows:: + + - name: assertCollectionExists + object: testRunner + arguments: + collectionName: *collection0Name + databaseName: *database0Name + +Use a ``listCollections`` command to check whether the collection exists. Note +that it is currently not possible to run ``listCollections`` from within a +transaction. + + +assertCollectionNotExists +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertCollectionNotExists`` operation instructs the test runner to assert +that the given collection does not exist in the database. The test runner MUST +use the internal MongoClient for this operation. + +The following arguments are supported: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +An example of this operation follows:: + + - name: assertCollectionNotExists + object: testRunner + arguments: + collectionName: *collection0Name + databaseName: *database0Name + +Use a ``listCollections`` command to check whether the collection exists. Note +that it is currently not possible to run ``listCollections`` from within a +transaction. + + +assertIndexExists +~~~~~~~~~~~~~~~~~ + +The ``assertIndexExists`` operation instructs the test runner to assert that an +index with the given name exists on the collection. The test runner MUST use the +internal MongoClient for this operation. + +The following arguments are supported: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +- ``indexName``: Required string. Index name. + +An example of this operation follows:: + + - name: assertIndexExists + object: testRunner + arguments: + collectionName: *collection0Name + databaseName: *database0Name + indexName: t_1 + +Use a ``listIndexes`` command to check whether the index exists. Note that it is +currently not possible to run ``listIndexes`` from within a transaction. + + +assertIndexNotExists +~~~~~~~~~~~~~~~~~~~~ + +The ``assertIndexNotExists`` operation instructs the test runner to assert that +an index with the given name does not exist on the collection. The test runner +MUST use the internal MongoClient for this operation. + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +- ``indexName``: Required string. Index name. + +An example of this operation follows:: + + - name: assertIndexNotExists + object: testRunner + arguments: + collectionName: *collection0Name + databaseName: *database0Name + indexName: t_1 + +Use a ``listIndexes`` command to check whether the index exists. Note that it is +currently not possible to run ``listIndexes`` from within a transaction. + + +Evaluating Matches +------------------ + +Expected values in tests (e.g. +`operation.expectResult `_) are expressed as either +relaxed or canonical `Extended JSON <../extended-json.rst>`_. + +The algorithm for matching expected and actual values is specified with the +following pseudo-code:: + + function match (expected, actual): + if expected is a document: + // handle special operators (e.g. $$type) + if first and only key of expected starts with "$$": + execute any assertion(s) for the special operator + return + + assert that actual is a document + + for every key/value in expected: + // handle key-based operators (e.g. $$exists, $$unsetOrMatches) + if value is a document and its first and only key starts with "$$": + execute any assertion(s) for the special operator + continue to the next iteration unless actual value must be matched + + assert that actual[key] exists + assert that actual[key] matches value + + if expected is not the root document: + assert that actual does not contain additional keys + + return + + if expected is an array: + assert that actual is an array + assert that actual and expected have the same number of elements + + for every index/value in expected: + assert that actual[index] matches value + + return + + // expected is neither a document nor array + assert that actual and expected are the same type, noting flexible numerics + assert that actual and expected are equal + +The rules for comparing documents and arrays are discussed in more detail in +subsequent sections. When comparing types *other* than documents and arrays, +test runners MAY adopt any of the following approaches to compare expected and +actual values, as long as they are consistent: + +- Convert both values to relaxed or canonical `Extended JSON`_ and compare + strings +- Convert both values to BSON, and compare bytes +- Convert both values to native representations, and compare accordingly + +When comparing types that contain documents as internal properties (e.g. +CodeWScope), the rules in `Evaluating Matches`_ do not apply and the documents +MUST match exactly; however, test runners MUST permit variation in document key +order or otherwise normalize the documents before comparison. + + +Flexible Numeric Comparisons +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When comparing numeric types (excluding Decimal128), test runners MUST consider +32-bit, 64-bit, and floating point numbers to be equal if their values are +numerically equivalent. For example, an expected value of ``1`` would match an +actual value of ``1.0`` (e.g. ``ok`` field in a server response) but would not +match ``1.5``. + + +Allowing Extra Fields in Root-level Documents +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When matching root-level documents, test runners MUST permit the actual document +to contain additional fields not present in the expected document. Examples of +root-level documents include, but are not limited to: + +- ``command`` for `CommandStartedEvent `_ +- ``reply`` for `CommandSucceededEvent `_ +- `expectResult`_ for ``findOneAndUpdate`` `Collection Operations`_ +- `expectResult`_ for `iterateUntilDocumentOrError`_ `ChangeStream Operations`_ +- each array element in `expectResult`_ for `find`_ or `aggregate`_ + `Collection Operations`_ + +For example, the following documents match:: + + expected: { x: 1 } + actual: { x: 1, y: 1 } + +The inverse is not true. For example, the following documents do not match:: + + expected: { x: 1, y: 1 } + actual: { x: 1 } + +Test runners MUST NOT permit additional fields in nested documents. For example, +the following documents do not match:: + + expected: { x: { y: 1 } } + actual: { x: { y: 1, z: 1 } } + +It may be helpful to think of expected documents as a form of query criteria. +The intention behind this rule is that it is not always feasible or relevant for +a test to exhaustively specify all fields in an expected document (e.g. cluster +time in ``command`` for `CommandStartedEvent`_). + +When the expected value is an array, test runners MUST differentiate between +an array of values, which may be documents, (e.g. ``distinct``) and an array of +root-level documents (e.g. ``find``, ``aggregate``). For example, the following +array of documents would not match if returned by ``distinct``, but would match +if returned via ``find`` (after iterating the cursor): + + expected: [ { x: 1 }, { x: 2 } ] + actual: [ { x: 1, y: 1 }, { x: 2, y: 2 } ] + + +Document Key Order Variation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When matching documents, test runners MUST NOT require keys in the expected and +actual document to appear in the same order. For example, the following +documents would match: + + expected: { x: 1, y: 1 } + actual: { y: 1, x: 1 } + + +Arrays Must Contain the Same Number of Elements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When comparing arrays, expected and actual values MUST contain the same number +of elements. For example, the following arrays corresponding to a ``distinct`` +operation result would not match:: + + expected: [ 1, 2, 3 ] + actual: [ 1, 2, 3, 4 ] + + +Special Operators for Matching Assertions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When matching expected and actual values, an equality comparison is not always +sufficient. For instance, a test file cannot anticipate what a session ID will +be at runtime, but may still want to analyze the contents of an ``lsid`` field +in a command document. To address this need, special operators can be used. + +These operators are objects with a single key identifying the operator. Such +keys are prefixed with ``$$`` to ease in detecting an operator (test runners +need only inspect the first key of each object) and differentiate the object +from MongoDB query operators, which use a single `$` prefix. The key will map to +some value that influences the operator's behavior (if applicable). + +When examining the structure of an expected value during a comparison, test +runners MUST check if the value is an object whose first and only key starts +with ``$$`` and, if so, defer to the special logic defined in this section. + + +$$exists +```````` + +Syntax:: + + { field: { $$exists: } } + +This operator can be used anywhere the value for a key might be specified in an +expected document. If true, the test runner MUST assert that the key exists in +the actual document, irrespective of its value (e.g. a key with a ``null`` value +would match). If false, the test runner MUST assert that the key does not exist +in the actual document. This operator is modeled after the +`$exists `__ +query operator. + +An example of this operator checking for a field's presence follows:: + + command: + getMore: { $$exists: true } + collection: *collectionName, + batchSize: 5 + +An example of this operator checking for a field's absence follows:: + + command: + update: *collectionName + updates: [ { q: {}, u: { $set: { x: 1 } } } ] + ordered: true + writeConcern: { $$exists: false } + + +$$type +`````` + +Syntax:: + + { $$type: } + { $$type: [ , , ... ] } + +This operator can be used anywhere a matched value is expected (including +`expectResult `_). The test runner MUST assert that the +actual value exists and matches one of the expected types, which correspond to +the documented string types for the +`$type `__ +query operator. + +An example of this operator follows:: + + command: + getMore: { $$type: [ "int", "long" ] } + collection: { $$type: "string" } + +When the actual value is an array, test runners MUST NOT examine types of the +array's elements. Only the type of actual field SHALL be checked. This is +admittedly inconsistent with the behavior of the +`$type `__ +query operator, but there is presently no need for this behavior in tests. + + +$$matchesEntity +``````````````` + +Syntax, where ``entityName`` is a string:: + + { $$matchesEntity: } + +This operator can be used anywhere a matched value is expected (including +`expectResult `_). If the entity name is defined in the +current test's `Entity Map`_, the test runner MUST fetch that entity and assert +that the actual value matches the entity using the standard rules in +`Evaluating Matches`_; otherwise, the test runner MUST raise an error for an +undefined entity. The YAML file SHOULD use an `alias node`_ for the entity name. + +This operator is primarily used to assert identifiers for uploaded GridFS files. + +An example of this operator follows:: + + operations: + - + object: *bucket0 + name: upload + arguments: + filename: "filename" + source: { $$hexBytes: "12AB" } + expectResult: { $$type: "objectId" } + saveResultAsEntity: &objectid0 "objectid0" + - object: *filesCollection + name: find + arguments: + sort: { uploadDate: -1 } + limit: 1 + expectResult: + - _id: { $$matchesEntity: *objectid0 } + + +$$matchesHexBytes +````````````````` + +Syntax, where ``hexBytes`` is an even number of hexademical characters +(case-insensitive) and MAY be empty:: + + { $$matchesHexBytes: } + +This operator can be used anywhere a matched value is expected (including +`expectResult `_) and the actual value is a string. +The test runner MUST raise an error if the ``hexBytes`` string is malformed. +This operator is primarily used to assert the results of `download`_ and +`downloadByName`_, which return stream contents as a string. + + +$$unsetOrMatches +```````````````` + +Syntax:: + + { $$unsetOrMatches: } + +This operator can be used anywhere a matched value is expected (including +`expectResult `_), excluding an array element because +`Arrays Must Contain the Same Number of Elements`_. The test runner MUST assert +that the actual value either does not exist or matches the expected value. +Matching the expected value MUST use the standard rules in +`Evaluating Matches`_, which means that it may contain special operators. + +This operator is primarily used to assert driver-optional fields from the CRUD +spec (e.g. ``insertedId`` for InsertOneResult, ``writeResult`` for +BulkWriteException). + +An example of this operator used for a result's field follows:: + + expectResult: + insertedId: { $$unsetOrMatches: 2 } + +An example of this operator used for an entire result follows:: + + expectError: + expectResult: + $$unsetOrMatches: + deletedCount: 0 + insertedCount: 2 + matchedCount: 0 + modifiedCount: 0 + upsertedCount: 0 + upsertedIds: { } + + +$$sessionLsid +````````````` + +Syntax:: + + { $$sessionLsid: } + +This operator can be used anywhere a matched value is expected (including +`expectResult `_). If the +`session entity `_ is defined in the current test's +`Entity Map`_, the test runner MUST assert that the actual value equals its +logical session ID; otherwise, the test runner MUST raise an error for an +undefined or mistyped entity. The YAML file SHOULD use an `alias node`_ for a +session entity's ``id`` field (e.g. ``session: *session0``). + +An example of this operator follows:: + + command: + ping: 1 + lsid: { $$sessionLsid: *session0 } + + +Test Runner Implementation +-------------------------- + +The sections below describe instructions for instantiating the test runner, +loading each test file, and executing each test within a test file. Test runners +MUST NOT share state created by processing a test file with the processing of +subsequent test files, and likewise for tests within a test file. + + +Initializing the Test Runner +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The test runner MUST be configurable with a connection string (or equivalent +configuration), which will be used to initialize the internal MongoClient and +any `client entities `_ (in combination with other URI options). +This specification is not prescriptive about how this information is provided. +For example, it may be read from an environment variable or configuration file. + +Create a new MongoClient, which will be used for internal operations (e.g. +processing `initialData`_ and `test.outcome `_). This is referred +to elsewhere in the specification as the internal MongoClient. + +Determine the server version and topology type using the internal MongoClient. +This information will be used to evaluate any future `runOnRequirement`_ checks. + +The test runner SHOULD terminate any open transactions (see: +`Terminating Open Transactions`_) using the internal MongoClient before +executing any tests. + + +Executing a Test File +~~~~~~~~~~~~~~~~~~~~~ + +The instructions in this section apply for each test file loaded by the test +runner. + +Test files, which may be YAML or JSON files, MUST be interpreted using an +`Extended JSON`_ parser. The parser MUST accept relaxed and canonical Extended +JSON (per `Extended JSON: Parsers <../extended-json.rst#parsers>`__), as test +files may use either. + +Upon loading a file, the test runner MUST read the `schemaVersion`_ field and +determine if the test file can be processed further. Test runners MAY support +multiple versions and MUST NOT process incompatible files (as discussed in +`Test Runner Support`_). If a test file is incompatible, test runners MUST raise +an error and MAY do so by reporting a test failure. Test runners MAY make an +effort to infer the number of tests (and their descriptions) from an +incompatible file and report a failure for each test. + +If `runOnRequirements`_ is specified, the test runner MUST skip the test file +unless one or more `runOnRequirement`_ objects are satisfied. + +For each element in `tests`_, follow the process in `Executing a Test`_. + + +Executing a Test +~~~~~~~~~~~~~~~~ + +The instructions in this section apply for each `test`_ occuring in a test file +loaded by the test runner. After processing a test, test runners MUST reset +any internal state that resulted from doing so. For example, the `Entity Map`_ +created for one test MUST NOT be shared with another. + +If at any point while executing this test an unexpected error is encountered or +an assertion fails, the test runner MUST consider this test to have failed and +SHOULD continue with the instructions in this section to ensure that the test +environment is cleaned up (e.g. disable fail points, kill sessions) while also +forgoing any additional assertions. + +If `test.skipReason `_ is specified, the test runner MUST skip +this test and MAY use the string value to log a message. + +If `test.runOnRequirements `_ is specified, the test +runner MUST skip the test unless one or more `runOnRequirement`_ objects are +satisfied. + +If `initialData`_ is specified, for each `collectionData`_ therein the test +runner MUST drop the collection and insert the specified documents (if any) +using a "majority" write concern. If no documents are specified, the test runner +MUST create the collection with a "majority" write concern. The test runner +MUST use the internal MongoClient for these operations. + +Create a new `Entity Map`_ that will be used for this test. If `createEntities`_ +is specified, the test runner MUST create each `entity`_ accordingly and add it +to the map. If the topology is a sharded cluster, the test runner MUST handle +`useMultipleMongoses`_ accordingly if it is specified for any client entities. + +If the test might execute a ``distinct`` command within a sharded transaction, +for each target collection the test runner SHOULD execute a non-transactional +``distinct`` command on each mongos server using the internal MongoClient. See +`StaleDbVersion Errors on Sharded Clusters`_ for more information. + +If the test might execute a ``configureFailPoint`` command, for each target +client the test runner MAY specify a reduced value for ``heartbeatFrequencyMS`` +(and ``minHeartbeatFrequencyMS`` if possible) to speed up SDAM recovery time and +server selection after a failure; however, test runners MUST NOT do so for any +client that specifies ``heartbeatFrequencyMS`` in its ``uriOptions``. + +For each client entity where `observeEvents `_ +has been specified, the test runner MUST enable all event listeners necessary to +collect the specified event types. Test runners MAY leave event listeners +disabled for tests that do not assert events (i.e. tests that omit both +`test.expectEvents `_ and special operations such as +`assertSameLsidOnLastTwoCommands`_). + +For each client with command monitoring enabled, the test runner MUST ignore +events for the following: + +- Any command(s) specified in + `ignoreCommandMonitoringEvents `_. + +- Any ``configureFailPoint`` commands executed for `failPoint`_ and + `targetedFailPoint`_ operations. + +- Any commands containing sensitive information (per the + `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ + spec). + +For each element in `test.operations `_, follow the process +in `Executing an Operation`_. If an unexpected error is encountered or an +assertion fails, the test runner MUST consider this test to have failed. + +If any event listeners were enabled on any client entities, the test runner MUST +now disable those event listeners. + +If any fail points were configured, the test runner MUST now disable those fail +points (on the same server) to avoid spurious failures in subsequent tests. For +any fail points configured using `targetedFailPoint`_, the test runner MUST +disable the fail point on the same mongos server on which it was originally +configured. See `Disabling Fail Points`_ for more information. + +If `test.expectEvents `_ is specified, for each object +therein the test runner MUST assert that the number and sequence of expected +events match the number and sequence of actual events observed on the specified +client. If the list of expected events is empty, the test runner MUST assert +that no events were observed on the client. The process for matching events is +described in `expectedEvent`_. + +If `test.outcome `_ is specified, for each `collectionData`_ +therein the test runner MUST assert that the collection contains exactly the +expected data. The test runner MUST query each collection using the internal +MongoClient, an ascending sort order on the ``_id`` field (i.e. ``{ _id: 1 }``), +a "primary" read preference, a "local" read concern. When comparing collection +data, the rules in `Evaluating Matches`_ do not apply and the documents MUST +match exactly; however, test runners MUST permit variations in document key +order or otherwise normalize the documents before comparison. If the list of +documents is empty, the test runner MUST assert that the collection is empty. + +Clear the entity map for this test. For each ClientSession in the entity map, +the test runner MUST end the session (e.g. call +`endSession <../sessions/driver-sessions.rst#endsession>`_). + +If the test failed, the test runner MUST terminate any open transactions (see: +`Terminating Open Transactions`_). + +Proceed to the subsequent test. + + +Executing an Operation +~~~~~~~~~~~~~~~~~~~~~~ + +The instructions in this section apply for each `operation`_ occuring in a +`test`_ contained within a test file loaded by the test runner. + +If at any point while executing an operation an unexpected error is encountered +or an assertion fails, the test runner MUST consider the parent test to have +failed and proceed from `Executing a Test`_ accordingly. + +If `operation.object `_ is "testRunner", this is a special +operation. If `operation.name `_ is defined in +`Special Test Operations`_, the test runner MUST execute the operation +accordingly and, if successful, proceed to the next operation in the test; +otherwise, the test runner MUST raise an error for an undefined operation. The +test runner MUST keep a record of any fail points configured by special +operations so that they may be disabled after the current test. + +If `operation.object`_ is not "testRunner", this is an entity operation. If +`operation.object`_ is defined in the current test's `Entity Map`_, the test +runner MUST fetch that entity and note its type; otherwise, the test runner +MUST raise an error for an undefined entity. If `operation.name`_ does not +correspond to a known operation for the entity type (per +`Entity Test Operations`_), the test runner MUST raise an error for an +unsupported operation. Test runners MAY skip tests that include operations that +are intentionally unimplemented (e.g. +``listCollectionNames``). + +Proceed with preparing the operation's arguments. If ``session`` is specified in +`operation.arguments `_, the test runner MUST resolve it +to a session entity and MUST raise an error if the name is undefined or maps to +an unexpected type. If a key in `operation.arguments`_ does not correspond to a +known parameter/option for the operation, the test runner MUST raise an error +for an unsupported argument. + +Before executing the operation, the test runner MUST be prepared to catch a +potential error from the operation (e.g. enter a ``try`` block). Proceed with +executing the operation and capture its result or error. + +Note that some operations require special handling, as discussed in +`Entity Test Operations`_. For example, model objects may need to be converted +to documents (before matching or saving in the entity map) and returned +iterables may need to be fully iterated. + +If `operation.expectError `_ is specified, the test +runner MUST assert that the operation yielded an error; otherwise, the test +runner MUST assert that the operation did not yield an error. If an error was +expected, the test runner MUST evaluate any assertions in `expectedError`_ +accordingly. + +If `operation.expectResult `_ is specified, the test +MUST assert that it matches the actual result of the operation according to the +rules outlined in `Evaluating Matches`_. + +If `operation.saveResultAsEntity `_ is specified, +the test runner MUST store the result in the current test's entity map using the +specified name. If the operation did not return a result or the result does not +comply with `Supported Entity Types`_ then the test runner MUST raise an error. + +After asserting the operation's error and/or result and optionally saving the +result, proceed to the subsequent operation. + + +Special Procedures +~~~~~~~~~~~~~~~~~~ + +This section describes some procedures that may be referenced from earlier +sections. + + +Terminating Open Transactions +````````````````````````````` + +Open transactions can cause tests to block indiscriminately. Test runners SHOULD +terminate all open transactions at the start of a test suite and after each +failed test by killing all sessions in the cluster. Using the internal +MongoClient, execute the ``killAllSessions`` command on either the primary or, +if connected to a sharded cluster, all mongos servers. + +For example:: + + db.adminCommand({ + killAllSessions: [] + }); + +The test runner MAY ignore any command failure with error Interrupted(11601) to +work around `SERVER-38335`_. + +.. _SERVER-38335: https://jira.mongodb.org/browse/SERVER-38335 + + +StaleDbVersion Errors on Sharded Clusters +````````````````````````````````````````` + +When a shard receives its first command that contains a ``databaseVersion``, the +shard returns a StaleDbVersion error and mongos retries the operation. In a +sharded transaction, mongos does not retry these operations and instead returns +the error to the client. For example:: + + Command distinct failed: Transaction aa09e296-472a-494f-8334-48d57ab530b6:1 was aborted on statement 0 due to: an error from cluster data placement change :: caused by :: got stale databaseVersion response from shard sh01 at host localhost:27217 :: caused by :: don't know dbVersion. + +To workaround this limitation, a test runners MUST execute a non-transactional +``distinct`` command on each mongos server before running any test that might +execute ``distinct`` within a transaction. To ease the implementation, test +runners MAY execute ``distinct`` before *every* test. + +Test runners can remove this workaround once `SERVER-39704`_ is resolved, after +which point mongos should retry the operation transparently. The ``distinct`` +command is the only command allowed in a sharded transaction that uses the +``databaseVersion`` concept so it is the only command affected. + +.. _SERVER-39704: https://jira.mongodb.org/browse/SERVER-39704 + + +Server Fail Points +------------------ + +Many tests utilize the ``configureFailPoint`` command to trigger server-side +errors such as dropped connections or command errors. Tests can configure fail +points using the special `failPoint`_ or `targetedFailPoint`_ opertions. + +This internal command is not documented in the MongoDB manual (pending +`DOCS-10784`_); however, there is scattered documentation available on the +server wiki (`The "failCommand" Fail Point `_) and employee +blogs (e.g. `Intro to Fail Points `_, +`Testing Network Errors with MongoDB `_). Documentation can +also be gleaned from JIRA tickets (e.g. `SERVER-35004`_, `SERVER-35083`_). This +specification does not aim to provide comprehensive documentation for all fail +points available for driver testing, but some fail points are documented in +`Fail Points Commonly Used in Tests`_. + +.. _failpoint-wiki: https://github.com/mongodb/mongo/wiki/The-%22failCommand%22-fail-point +.. _failpoint-blog1: https://kchodorow.com/2013/01/15/intro-to-fail-points/ +.. _failpoint-blog2: https://emptysqua.re/blog/mongodb-testing-network-errors/ +.. _DOCS-10784: https://jira.mongodb.org/browse/DOCS-10784 +.. _SERVER-35004: https://jira.mongodb.org/browse/SERVER-35004 +.. _SERVER-35083: https://jira.mongodb.org/browse/SERVER-35083 + + +Configuring Fail Points +~~~~~~~~~~~~~~~~~~~~~~~ + +The ``configureFailPoint`` command is executed on the ``admin`` database and has +the following structure:: + + db.adminCommand({ + configureFailPoint: , + mode: , + data: + }); + +The value of ``configureFailPoint`` is a string denoting the fail point to be +configured (e.g. "failCommand"). + +The ``mode`` option is a generic fail point option and may be assigned a string +or object value. The string values "alwaysOn" and "off" may be used to enable or +disable the fail point, respectively. An object may be used to specify either +``times`` or ``skip``, which are mutually exclusive: + +- ``{ times: }`` may be used to limit the number of times the fail + point may trigger before transitioning to "off". +- ``{ skip: }`` may be used to defer the first trigger of a fail + point, after which it will transition to "alwaysOn". + +The ``data`` option is an object that may be used to specify any options that +control the particular fail point's behavior. + +In order to use ``configureFailPoint``, the undocumented ``enableTestCommands`` +`server parameter `_ must +be enabled by either the configuration file or command line option (e.g. +``--setParameter enableTestCommands=1``). It cannot be enabled at runtime via +the `setParameter `_ +command). This parameter should already be enabled for most configuration files +in `mongo-orchestration `_. + + +Disabling Fail Points +~~~~~~~~~~~~~~~~~~~~~ + +A fail point may be disabled like so:: + + db.adminCommand({ + configureFailPoint: , + mode: "off" + }); + + +Fail Points Commonly Used in Tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +failCommand +``````````` + +The ``failCommand`` fail point allows the client to force the server to return +an error for commands listed in the ``data.failCommands`` field. Additionally, +this fail point is documented in server wiki: +`The failCommand Fail Point `__. + +The ``failCommand`` fail point may be configured like so:: + + db.adminCommand({ + configureFailPoint: "failCommand", + mode: , + data: { + failCommands: [, ...], + closeConnection: , + errorCode: , + writeConcernError: , + appName: , + blockConnection: , + blockTimeMS: , + } + }); + +``failCommand`` supports the following ``data`` options, which may be combined +if desired: + +* ``failCommands``: Required array of strings. Lists the command names to fail. +* ``closeConnection``: Optional boolean, which defaults to ``false``. If + ``true``, the command will not be executed, the connection will be closed, and + the client will see a network error. +* ``errorCode``: Optional integer, which is unset by default. If set, the + command will not be executed and the specified command error code will be + returned as a command error. +* ``appName``: Optional string, which is unset by default. If set, the fail + point will only apply to connections for MongoClients created with this + ``appname``. New in server 4.4.0-rc2 + (`SERVER-47195 `_). +* ``blockConnection``: Optional boolean, which defaults to ``false``. If + ``true``, the server should block the affected commands for ``blockTimeMS``. + New in server 4.3.4 + (`SERVER-41070 `_). +* ``blockTimeMS``: Optional integer, which is required when ``blockConnection`` + is ``true``. The number of milliseconds for which the affected commands should + be blocked. New in server 4.3.4 + (`SERVER-41070 `_). + + +Determining if a Sharded Cluster Uses Replica Sets +-------------------------------------------------- + +When connected to a mongos server, the test runner can query the +`config.shards `__ +collection. Each shard in the cluster is represented by a document in this +collection. If the shard is backed by a single server, the ``host`` field will +contain a single host. If the shard is backed by a replica set, the ``host`` +field contain the name of the replica followed by a forward slash and a +comma-delimited list of hosts. + + +Design Rationale +================ + +This specification was primarily derived from the test formats used by the +`Transactions <../transactions/transactions.rst>`__ and +`CRUD <../crud/crud.rst>`__ specs, which have served models or other specs. + +This specification commonly uses "SHOULD" when providing guidance on writing +test files. While this may appear contradictory to the driver mantra preferring +"MUST", it is intentional. Some of this guidance addresses style (e.g. adding +comments, using YAML anchors) and cannot be enforced with a JSON schema. Other +guidance needs to be purposefully ignored in order to test the test runner +implementation (e.g. defining entities out of order to trigger runtime errors). +The specification does prefer "MUST" in other contexts, such as discussing parts +of the test file format that *are* enforceable by the JSON schema or the test +runner implementation. + + +Breaking Changes +================ + +This section is reserved for future use. Any breaking changes to the test format +SHOULD be described here in detail for historical reference, in addition to any +shorter description that may be added to the `Change Log`_. + + +Future Work +=========== + + +Mixing event types in observeEvents and expectEvents +---------------------------------------------------- + +The test format advises against mixing events from different specs (e.g. command +monitoring *and* SDAM) in `observeEvents`_ and `test.expectEvents`_. While +registering event listeners is trivial, determining how to collate events of +multiple types can be a challenge, particularly when some events may not be +predictable (e.g. ServerHeartbeatStartedEvent, CommandStartedEvent for +``getMore`` issued during change stream iteration). If the need arises to expect +multiple types of events in the same test, a future version of this spec can +define an approach for doing so. + + +Support events types beyond command monitoring +---------------------------------------------- + +The spec currently only supports command monitoring events in `observeEvents`_ +and `test.expectEvents`_, as those are the only kind of events used in tests for +specifications that will initially adopt the unified test format. New event +types (e.g. SDAM) can be added in future versions of the spec as needed, which +will also require `Mixing event types in observeEvents and expectEvents`_ to be +addressed. + + +Allow extra observed events to be ignored +----------------------------------------- + +While command monitoring events for specific commands can be ignored (e.g. +killCursors for change streams), the sequence of observed events must otherwise +match the sequence of expected events (including length). The present design +would not support expecting an event for a command while also ignoring extra +events for the same command (e.g. change stream iteration on a sharded cluster +where multiple getMore commands may be issued). No spec tests currently require +this functionality, but that may change in the future. + + +Assert expected log messages +---------------------------- + +When drivers support standardized logging, the test format may need to support +assertions for messages expected to be logged while executing operations. Since +log messages are strings, this may require an operator to match regex patterns +within strings. Additionally, the test runner may need to support ignoring extra +log output, similar to `Allow extra observed events to be ignored`_. + + +Target failPoint by read preference +----------------------------------- + +The `failPoint`_ operation currently uses a "primary" read preference. To date, +no spec has needed behavior to configure a fail point on a non-primary node. If +the need does arise, `failPoint`_ can be enhanced to support a +``readPreference`` argument. + + +IO operations for GridFS streams +-------------------------------- + +Original GridFS spec tests refer to "upload", "download", and "download_by_name" +methods, which allow the tests to abstract stream IO and either upload a byte +sequence or assert a downloaded byte sequence. These operations correspond to +the `download`_, `downloadByName`_, `upload`_, and `uploadWithId`_ +`Bucket Operations`_. + +In order to support methods such as ``downloadToStream``, ``openUploadStream``, +and ``openUploadStreamWithId``, test runners would need to represent streams as +entities and support IO operations to directly read from and write to a stream +entity. This may not be worth the added complexity if the existing operations +provide adequate test coverage for GridFS implementations. + + +Support Client-side Encryption integration tests +------------------------------------------------ + +Supporting client-side encryption spec tests will require the following changes +to the test format: + +- ``json_schema`` will need to be specified when creating a collection, via + either the collection entity definition or `initialData`_. +- ``key_vault_data`` can be expressed via `initialData`_ +- ``autoEncryptOpts`` will need to be specified when defining a client entity. + Preparation of this field may require reading AWS credentials from environment + variables. + +The process for executing tests should not require significant changes, but test +files will need to express a dependency on mongocryptd. + + +Support SDAM integration tests +------------------------------ + +SDAM integration tests should not require test format changes, but will +introduce several new special test operations for the "testRunner" object. While +the tests themselves only define expectations for command monitoring events, +some special operations may require observing additional event types. There are +also special operations for defining threads and executing operations within +threads, which may warrant introducing a new "thread" entity type. + + +Incorporate referenced entity operations into the schema version +---------------------------------------------------------------- + +The `Schema Version`_ is not impacted by changes to operations defined in other +specs and referenced in `Entity Test Operations` (e.g. ``find`` for CRUD). The +`operation.name `_ and +`operation.arguments `_ fields are loosely defined in the +JSON schema as string and object types, respectively. + +Ideally, all operations (and their arguments) would be enforced by the JSON +schema *and* any changes to operations would affect the schema version +accordingly. For example, a new ``find`` option would warrant a minor version +bump both for the CRUD spec and this spec and its schema. + +As discussed in `Executing an Operation`_, test runners MUST raise errors for +unsupported operations and arguments. This is a concession until such time that +better processes can be established for versioning other specs *and* collating +spec changes developed in parallel or during the same release cycle. + + +Change Log +==========