From 90fcf6cd3c9c6fdcebe6ed1ac00c2b1f0c01d85f Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 22 Jul 2020 10:04:53 -0400 Subject: [PATCH 01/90] DRIVERS-709 wip --- source/Makefile | 6 + source/unified-test-format/example.json | 63 + source/unified-test-format/schema.json | 285 +++++ .../unified-test-format.rst | 1060 +++++++++++++++++ 4 files changed, 1414 insertions(+) create mode 100644 source/unified-test-format/example.json create mode 100644 source/unified-test-format/schema.json create mode 100644 source/unified-test-format/unified-test-format.rst diff --git a/source/Makefile b/source/Makefile index eaa584eb6b..74f89ba425 100644 --- a/source/Makefile +++ b/source/Makefile @@ -17,3 +17,9 @@ HAS_JSYAML: echo 'Error: need "npm install -g js-yaml"' 1>&2; \ exit 1; \ fi + +HAS_AJV: + @if ! command -v ajv > /dev/null; then \ + echo 'Error: need "npm install -g ajv"' 1>&2; \ + exit 1; \ + fi diff --git a/source/unified-test-format/example.json b/source/unified-test-format/example.json new file mode 100644 index 0000000000..eabb41674f --- /dev/null +++ b/source/unified-test-format/example.json @@ -0,0 +1,63 @@ +{ + "tests": [ + { + "description": "valid because error:true and result has only expected fields", + "operations": [ + { + "object": "collection", + "name": "runCommand", + "error": true, + "result": { + "errorContains": "string" + } + } + ] + }, + { + "description": "valid because error == false. result can be any type (non-object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "error": false, + "result": 1 + } + ] + }, + { + "description": "valid because error == false. result can be any type (object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "error": false, + "result": { + "foo": "bar" + } + } + ] + }, + { + "description": "valid because error != true. result can be any type (non-object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "result": 1 + } + ] + }, + { + "description": "valid because error != true. result can be any type (object)", + "operations": [ + { + "object": "collection", + "name": "operation", + "result": { + "foo": "bar" + } + } + ] + } + ] +} diff --git a/source/unified-test-format/schema.json b/source/unified-test-format/schema.json new file mode 100644 index 0000000000..b94bdc7b67 --- /dev/null +++ b/source/unified-test-format/schema.json @@ -0,0 +1,285 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "type": "object", + "title": "Unified Test Format", + "description": "Schema definition for unified test format.", + "additionalProperties": false, + "required": ["tests"], + "properties": { + "runOn": { + "type": "array", + "description": "List of server version and/or topology requirements for which the tests can be run. If the test environment satisfies one or more of these requirements, the tests may be executed; otherwise, this file should be skipped. If this field is omitted, the tests can be assumed to have no particular requirements and should be executed.", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + + "databaseName": { + "type": "string", + "description": "Name of database under test. This is primarily useful when the database name must be referenced in an assertion." + }, + + "collectionName": { + "type": "string", + "description": "Name of collection under test. This is primarily useful when the collection name must be referenced in an assertion." + }, + + "initialData": { + "type": "array", + "description": "Data that should exist in collections before each test case is executed.", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + }, + + "tests": { + "type": "array", + "description": "Test cases.", + "minItems": 1, + "items": { "$ref": "#/definitions/test" } + } + }, + + "definitions": { + "runOnRequirement": { + "type": "object", + "description": "A combination of server version and/or topology requirements for running the tests.", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { + "type": "string", + "description": "The maximum server version (inclusive) against which the tests can be run successfully. If this field is omitted, it should be assumed that there is no upper bound on the required server version.", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$", + "examples": ["4.0", "4.2.0"] + }, + "minServerVersion": { + "type": "string", + "description": "The minimum server version (inclusive) required to successfully run the tests. If this field is omitted, it should be assumed that there is no lower bound on the required server version.", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$", + "examples": ["4.0", "4.2.0"] + }, + "topology": { + "type": "array", + "description": "List of server topologies against which the tests can be run successfully. Valid topologies are \"single\", \"replicaset\", and \"sharded\". If this field is omitted, the default is all topologies.", + "default": ["single", "replicaset", "sharded"], + "minItems": 1, + "items": { + "type": "string", + "enum": ["single", "replicaset", "sharded"] + } + } + } + }, + + "collectionData": { + "type": "object", + "description": "List of documents that should correspond to the contents of a collection.", + "additionalProperties": false, + "required": ["documents"], + "properties": { + "collection": { + "type": "string", + "description": "Collection name. If omitted, this defaults to the collection under test." + }, + "database": { + "type": "string", + "description": "Database name. If omitted, this defaults to the database under test." + }, + "documents": { + "type": "array", + "description": "List of documents corresponding to the contents of the collection. This list may be empty.", + "items": { "type": "object" } + } + } + }, + + "event": { + "type": "object", + "description": "An event (e.g. APM, SDAM) to be observed while running the test operations.", + "additionalProperties": false, + "minProperties": 1, + "maxProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "command": { "type": "object" }, + "commandName": { "type": "string" }, + "databaseName": { "type": "string" } + } + } + } + }, + + "collectionOrDatabaseOptions": { + "type": "object", + "description": "Map of parameters to pass when creating a collection or database.", + "additionalProperties": false, + "properties": { + "readConcern": { "type": "object" }, + "readPreference": { "type": "object" }, + "writeConcern": { "type": "object" } + } + }, + + "operation": { + "type": "object", + "description": "An operation to be executed as part of the test.", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { + "type": "string", + "description": "Name of the operation (e.g. method) to perform on the object." + }, + "object": { + "type": "string", + "description": "Name of the object on which to perform the operation.", + "default": "collection", + "enum": ["collection", "database", "session0", "session1", "testRunner"] + }, + "collectionOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" }, + "databaseOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" }, + "command_name": { + "type": "string", + "description": "Required only when name is \"runCommand\". The name of the command to run. This may be used by languages that are unable preserve the order of keys in the command argument when parsing YAML or JSON." + }, + "arguments": { + "type": "object", + "description": "Map of parameter names and values for the operation." + }, + "error": { + "type": "boolean", + "description": "If true, the test should expect the operation to raise an error/exception. This could be either a server-generated or a driver-generated error.", + "default": false + }, + "result": { + "description": "The return value from the operation, if any. This field may be a scalar value, a single document, or an array of documents in the case of a multi-document read. If the operation is expected to return an error, the result is a single document containing several \"error\" fields defined in a conditional schema." + } + }, + "allOf": [ + { + "$comment": "Require command_name for runCommand operations", + "oneOf": [ + { + "properties": { "name": { "const": "runCommand" }}, + "required": ["command_name"] + }, + { + "properties": { "name": { "not": { "const": "runCommand" }}} + } + ] + }, + { + "$comment": "Conditional result schema when error is true", + "anyOf": [ + { "properties": { "error": false }}, + { "properties": { "error": { "const": false }}}, + { + "properties": { + "error": { "const": true }, + "result": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "errorContains": { + "type": "string", + "description": "A substring of the expected error message." + }, + "errorCodeName": { + "type": "string", + "description": "The expected \"codeName\" field in the server error response." + }, + "errorLabelsContain": { + "type": "array", + "description": "A list of error label strings that the error is expected to have.", + "items": { "type": "string" } + }, + "errorLabelsOmit": { + "type": "array", + "description": "A list of error label strings that the error is expected not to have.", + "items": { "type": "string" } + } + } + } + } + } + ] + } + ] + }, + + "test": { + "type": "object", + "description": "Test case consisting of a sequence of operations to be executed. The test may optionally include configuration directives and event/outcome assertions.", + "required": ["description", "operations"], + "properties": { + "description": { + "type": "string", + "description": "The name of the test." + }, + "skipReason": { + "type": "string", + "description": "If set, the test will be skipped. The string should explain the reason for skipping the test (e.g. JIRA ticket)." + }, + "useMultipleMongoses": { + "type": "boolean", + "description": "If true, the MongoClient for this test should be initialized with multiple mongos seed addresses. If false or omitted, only a single mongos address should be specified. This field has no effect for non-sharded topologies." + }, + "clientOptions": { + "$comment": "TODO: this schema is not sufficient for expressing multiple readPreferenceTags, since keys would repeat", + "type": "object", + "description": "Additional connection string options to pass to the MongoClient constructor.", + "additionalProperties": { + "type": ["number", "string"] + } + }, + "failPoint": { + "type": "object", + "description": "A server failpoint to enable expressed as a complete configureFailPoint command to run on the admin database. This option and useMultipleMongoses:true are mutually exclusive.", + "additionalProperties": false, + "required": ["configureFailPoint", "mode"], + "properties": { + "configureFailPoint": { "type": "string" }, + "mode": { "type": ["object", "string"] }, + "data": { "type": "object" } + } + }, + "sessionOptions": { + "type": "object", + "description": " Map of session names (e.g. \"session0\") to documents, each of which denotes parameters to pass to MongoClient.startSession() when creating that session.", + "additionalProperties": false, + "patternProperties": { + "^session[0-1]$": { "type": "object" } + } + }, + "operations": { + "type": "array", + "description": "List of operations to be executed for the test case.", + "items": { "$ref": "#/definitions/operation" } + }, + "expectedEvents": { + "type": "array", + "description": "List of events, which are expected to be observed in this order by running the operations.", + "items": { "$ref": "#/definitions/event" } + }, + "outcome": { + "type": "array", + "description": "Data that should exist in collections after all operations have been executed. The list of documents should be sorted ascendingly by the \"_id\" field to allow for deterministic comparisons.", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + } + }, + "allOf": [ + { + "$comment": "useMultipleMongoses:true and failPoint are mutually exclusive", + "if": { "properties": { "useMultipleMongoses": { "const": true }}}, + "then": { "properties": { "failPoint": { "not": { "type": "object" }}}} + } + ] + } + } +} diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst new file mode 100644 index 0000000000..978d5684f2 --- /dev/null +++ b/source/unified-test-format/unified-test-format.rst @@ -0,0 +1,1060 @@ +=================== +Unified Test Format +=================== + +:Spec Title: Unified Test Format +:Spec Version: 1.0 +:Author: Jeremy Mikola +:Advisors: Prashant Mital +:Status: Draft +:Type: Standards +:Minimum Server Version: N/A +:Last Modified: 2020-08-15 + +.. contents:: + +-------- + +Abstract +======== + +This project defines a unified schema for YAML and JSON specification tests, +which run operations against a MongoDB deployment. By conforming various spec +tests to a single schema, drivers can implement a single test runner to execute +acceptance tests for multiple specifications, thereby reducing maintenance of +existing specs and implementation time for new specifications. + +META +==== + +The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", +"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be +interpreted as described in `RFC 2119 `_. + +Specification +============= + + +Test Format +----------- + +Each specification test file can define one or more tests, which inherit some +top-level configuration (e.g. namespace, initial data). YAML and JSON test files +are parsed as a document by the test runner. This section defines the top-level +keys for that document and links to various sub-sections for definitions of +nested structures (e.g. individual `test`_, `operation`_). + +The test format is also defined in the accompanying `schema.json `_ +file, which may be used to programmatically validate YAML and JSON files using +a tool such as `Ajv `_. + +The top-level fields of a test file are as follows: + +- ``runOn``: Optional array of documents. List of server version and/or topology + requirements for which the tests can be run. If the test environment satisfies + one or more of these requirements, the tests may be executed; otherwise, this + file should be skipped. If this field is omitted, the tests can be assumed to + have no particular requirements and should be executed. + + If set, the array should contain at least one document. The structure of each + document is defined in `runOnRequirement`_. + +- ``createEntities``: Optional array of documents. List of entities (e.g. + client, collection, session objects) that should be created before each test + case is executed. The structure of each document is defined in `entity`_. + +- ``collectionName``: Optional string. Name of collection under test. This is + primarily useful when the collection name must be referenced in an assertion. + If unset, drivers may use whatever value they prefer. + +- ``databaseName``: Optional string. Name of database under test. This is + primarily useful when the database name must be referenced in an assertion. + If unset, drivers may use whatever value they prefer. + +.. _initialData: + +- ``initialData``: Optional array of documents. Data that should exist in + collections before each test case is executed. + + If set, the array should contain at least one document. The structure of each + document is defined in `collectionData`_. + +- ``tests``: Required array of documents. List of test cases to be executed + independently of each other. + + The array should contain at least one document. The structure of each + document is defined in `test`_. + + +runOnRequirement +~~~~~~~~~~~~~~~~ + +A combination of server version and/or topology requirements for running the +test(s). The structure of this document is as follows: + +- ``minServerVersion``: Optional string. The minimum server version (inclusive) + required to successfully run the tests. If this field is omitted, it should be + assumed that there is no lower bound on the required server version. + +- ``maxServerVersion``: Optional string. The maximum server version (inclusive) + against which the tests can be run successfully. If this field is omitted, it + should be assumed that there is no upper bound on the required server version. + +- ``topology``: Optional array of strings. List of server topologies against + which the tests can be run successfully. Valid topologies are "single", + "replicaset", and "sharded". If this field is omitted, the default is all + topologies (i.e. ``["single", "replicaset", "sharded"]``). + + **TODO**: Consider a sharded-replicaset topology. + + +entity +~~~~~~ + +An entity (e.g. client, collection, session object) that will be created +before each test is executed. This document must contain exactly one top-level +key that identifies the entity type and maps to a nested document, which +specifies a unique name for the entity (``id`` key) and any other parameters +necessary for its construction. Tests SHOULD use sequential names based on the +entity type (e.g. "session0", "session1"). + +When defining an entity document in YAML, a +`node anchor `_ SHOULD be created +on the entity's ``id`` key. This anchor will allow the unique name to be +referenced with an `alias node `_ +later in the file (e.g. from another entity or `operation`_ document) and also +leverage YAML's parser for reference validation. + +The structure of this document is as follows: + +- ``client``: Optional document. Corresponds with a MongoClient object. The + structure of this document is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor `_ + for this field (e.g. ``id: &client0 client0``). + + - ``uriOptions``: Optional document. Additional URI options to apply to the + test suite's connection string that is used to create this client. Any keys + in this document MUST override conflicting keys in the connection string. + + Documentation for supported options may be found in the + `URI Options <../uri-options/uri-options.rst>`__ spec, with one notable + exception: if ``readPreferenceTags`` is specified in this document, the key + will map to an array of strings, each representing a tag set, since it is + not feasible to define multiple ``readPreferenceTags`` keys in the document. + + **TODO**: Consider moving the ``useMultipleMongoses`` `test`_ option here. + +- ``database``: Optional document. Corresponds with a Database object. The + structure of this document is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor `_ + for this field (e.g. ``id: &database0 database0``). + + - ``client``: Required string. Client entity from which this database will be + created. The YAML file SHOULD use an + `alias node `_ for a client + entity's ``id`` field (e.g. ``client: *client0``). + + - ``databaseName``: Optional string. Database name. If omitted, this defaults + to the name of the database under test. + + - ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. + +- ``collection``: Optional document. Corresponds with a Collection object. The + structure of this document is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor `_ + for this field (e.g. ``id: &collection0 collection0``). + + - ``database``: Required string. Database entity from which this collection + will be created. The YAML file SHOULD use an + `alias node `_ for a database + entity's ``id`` field (e.g. ``database: *database0``). + + - ``collectionName``: Optional string. Collection name. If omitted, this + defaults to the name of the collection under test. + + - ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. + +- ``session``: Optional document. Corresponds with an explicit ClientSession + object. The structure of this document is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor `_ + for this field (e.g. ``id: &session0 session0``). + + - ``client``: Required string. Client entity from which this session will be + created. The YAML file SHOULD use an + `alias node `_ for a client + entity's ``id`` field (e.g. ``client: *client0``). + + - ``sessionOptions``: Optional document. Map of parameters to pass to + `MongoClient.startSession <../source/sessions/driver-sessions.rst#startsession>`__ + when creating the session. Supported options are defined in the following + specifications: + + - `Causal Consistency <../causal-consistency/causal-consistency.rst#sessionoptions-changes>`__ + - `Transactions <../transactions/transactions.rst#sessionoptions-changes>`__ + +- ``bucket``: Optional document. Corresponds with a GridFS Bucket object. The + structure of this document is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor `_ + for this field (e.g. ``id: &bucket0 bucket0``). + + - ``database``: Required string. Database entity from which this bucket will + be created. The YAML file SHOULD use an + `alias node `_ for a database + entity's ``id`` field (e.g. ``database: *database0``). + + - ``bucketOptions``: Optional document. Additional options used to construct + the bucket object. Supported options are defined in the + `GridFS <../source/gridfs/gridfs-spec.rst#configurable-gridfsbucket-class>`__ + specification. The ``readConcern``, ``readPreference``, and ``writeConcern`` + options use the same structure as defined in `Common Options`_. + + +collectionData +~~~~~~~~~~~~~~ + +List of documents that should correspond to the contents of a collection. This +structure is used by both `initialData`_ and `test.outcome `_, +which insert and read documents, respectively. Both of those directives +directives may operate on the collection under test, they do not share the +same Collection object(s) as test `operations `_. The structure of +this document is as follows: + +- ``collectionName``: Optional string. Collection name (not an `entity`_). + Defaults to the name of the collection under test. + +- ``databaseName``: Optional string. Database name (not an `entity`_). Defaults + to the name of the database under test. + +- ``documents``: Required array of documents. List of documents corresponding to + the contents of the collection. This list may be empty. + + +test +~~~~ + +Test case consisting of a sequence of operations to be executed. The test may +optionally include configuration directives and event/outcome assertions. The +structure of each document is as follows: + +- ``description``: Required string. The name of the test. + +- ``skipReason``: Optional string. If set, the test will be skipped. The string + should explain the reason for skipping the test (e.g. JIRA ticket). + +- ``useMultipleMongoses``. Optional boolean. If true, the MongoClient for this + test should be initialized with multiple mongos seed addresses. If false or + omitted, only a single mongos address should be specified. This field has no + effect for non-sharded topologies. + + Note: ``useMultipleMongoses:true`` is mutually exclusive with ``failPoint``. + +- ``clientOptions``: Optional document. Additional connection string options + to pass to the MongoClient constructor. + + Note: this option does not support expressing multiple read preference tags, + tags, since keys would repeat. + +.. _test_failPoint: + +- ``failPoint``: Optional document. A server failpoint to enable expressed as + a complete ``configureFailPoint`` command to run on the admin database. + + Note: This option is mutually exclusive with ``useMultipleMongoses:true``. + +- ``sessionOptions``: Optional document. Map of session names (e.g. "session0") + to documents, each of which denotes parameters to pass to + ``MongoClient.startSession()`` when creating that session. + + **TODO**: remove this in favor of the session `entity`_ + +.. _test_operations: + +- ``operations``: Required array of documents. List of operations to be executed + for the test case. + + The array should contain at least one document. The structure of each + document is defined in `operation`_. + +- ``expectedEvents``: Optional array of documents. List of events, which are + expected to be observed in this order by running the operations. + + The array should contain at least one document. The structure of each + document is defined in `expectedEvent`_. + +.. _test_outcome: + +- ``outcome``: Optional array of documents. Data that should exist in + collections after all operations have been executed. The list of documents + should be sorted ascendingly by the ``_id`` field to allow for deterministic + comparisons. + + If set, the array should contain at least one document. The structure of each + document is defined in `collectionData`_. + + +operation +~~~~~~~~~ + +An operation to be executed as part of the test. The structure of this document +is as follows: + +.. _operation_name: + +- ``name``: Required string. Name of the operation (e.g. method) to perform on + the object. + +.. _operation_object: + +- ``object``: Optional string. Name of the object on which to perform the + operation. Can be "collection", "database", "session0", "session1", or + "testRunner". Defaults to "collection". + + **TODO**: link to special operations section + + **TODO**: update this to refer to an entity identifier + +- ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. + +- ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. + +.. _operation_commandName: + +- ``commandName``: Optional string. Required only when ``name`` is "runCommand". + The name of the command to run. This may be used by languages that are unable + preserve the order of keys in the ``command`` argument when parsing YAML/JSON. + +.. _operation_arguments: + +- ``arguments``: Optional document. Map of parameter names and values for the + operation. The structure of this document will vary based on the operation. + + **TODO**: link to operation section + +- ``error``: Optional boolean. If true, the test should expect the operation to + raise an error/exception. This could be either a server-generated or a + driver-generated error. Defaults to false. + +- ``result``: Optional mixed type. The return value from the operation, if any. + This field may be a scalar value, a single document, or an array of documents + in the case of a multi-document read. If the operation is expected to raise an + error/exception (i.e. ``error:true``), the structure of this document is + defined in `expectedError`_. + + **TODO**: link to section on matching logic + + +expectedError +~~~~~~~~~~~~~ + +One or more assertions for an error/exception, which is expected to be raised by +an executed operation. At least one key is required in this document. The +structure of this document is as follows: + +- ``errorContains``: Optional string. A substring of the expected error message. + +- ``errorCodeName``: Optional string. The expected "codeName" field in the + server-generated error response. + +- ``errorLabelsContain``: Optional array of strings. A list of error label + strings that the error is expected to have. + +- ``errorLabelsOmit``: Optional array of strings. A list of error label strings + that the error is expected not to have. + + +expectedEvent +~~~~~~~~~~~~~ + +An event (e.g. APM, SDAM), which is expected to be observed while executing +operations. This document must contain exactly one top-level key that identifies +the event type and maps to a nested document, which contains one or more +assertions for the event's properties. The structure of this document is as +follows: + +.. _expectedEvent_commandStartedEvent: + +- ``commandStartedEvent``: Optional document. Assertions for a one or more + `CommandStartedEvent <../command-monitoring/command-monitoring.rst#api>`__ + fields. The structure of this document is as follows: + + - ``command``: Optional document. + + **TODO**: link to section on matching logic + + - ``commandName``: Optional string. + + - ``databaseName``: Optional string. + + +collectionOrDatabaseOptions +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Map of parameters used to construct a collection or database object. The +structure of this document is as follows: + + - ``readConcern``: Optional document. See `commonOptions_readConcern`_. + + - ``readPreference``: Optional document. See `commonOptions_readPreference`_. + + - ``writeConcern``: Optional document. See `commonOptions_writeConcern`_. + + +Common Options +~~~~~~~~~~~~~~ + +This section defines the structure of common options that are referenced from +various contexts in the test format. Comprehensive documentation for some of +these types and their parameters may be found in the following specifications: + +- `Read and Write Concern <../read-write-concern/read-write-concern.rst>`__. +- `Server Selection: Read Preference <../server-selection/server-selection.rst#read-preference>`__. + +The structure of these common options is as follows: + +.. _commonOptions_readConcern: + +- ``readConcern``: Optional document. Map of parameters to construct a read + concern. The structure of this document is as follows: + + - ``level``: Required string. + +.. _commonOptions_readPreference: + +- ``readPreference``: Optional document. Map of parameters to construct a read + preference. The structure of this document is as follows: + + - ``mode``: Required string. + + - ``tagSets``: Optional array of documents. + + - ``maxStalenessSeconds``: Optional integer. + + - ``hedge``: Optional document. + +.. _commonOptions_session: + +- ``session``: Optional string. Session entity which will be resolved to a + ClientSession object. The YAML file SHOULD use an + `alias node `_ for a session + entity's ``id`` field (e.g. ``session: *session0``). + +.. _commonOptions_writeConcern: + +- ``writeConcern``: Optional document. Map of parameters to construct a write + concern. The structure of this document is as follows: + + - ``journal``: Optional boolean. + + - ``w``: Optional integer or string. + + - ``wtimeoutMS``: Optional integer. + + +Entity Test Operations +---------------------- + +Most operations correspond to an API method on a driver object. If +`operation.object `_ refers to an `entity`_ name (e.g. +"collection0") then `operation.name `_ is expected to reference +an API method on that class. Required and optional parameters for API methods +are both specified directly within `operation.arguments `_ +(e.g. ``upsert`` for ``updateOne`` is *not* nested under an ``options`` key). + +This spec does not provide exhaustive documentation for all possible API methods +that may appear in a test; however, the following sections discuss all supported +entities and their operations in some level of detail. + +**TODO**: While CRUD methods tend to flatten options into ``arguments``, session +methods often leave those options nested within an ``options`` key. We should +pick one of these conventions for consistency. + + +client +~~~~~~ + +These operations and their arguments may be documented in the following +specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `Enumerating Databases <../enumerate-databases.rst>`__ + + +database +~~~~~~~~ + +These operations and their arguments may be documented in the following +specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `CRUD <../crud/crud.rst>`__ +- `Enumerating Collections <../enumerate-collections.rst>`__ + +Other database operations not documented by an existing specification follow. + +runCommand +`````````` + +Generic command runner. This operation requires that +`operation.commandName `_ also be specified. + +This method does not inherit a read concern or write concern (per the +`Read and Write Concern <../read-write-concern/read-write-concern.rst#generic-command-method>`__ +spec), nor does it inherit a read preference (per the +`Server Selection <../server-selection/server-selection.rst#use-of-read-preferences-with-commands>`__ +spec); however, they may be specified as arguments. + +The following arguments are supported: + +- ``command``: Required document. The command to be executed. + +- ``readConcern``: Optional document. See `commonOptions_readConcern`_. + +- ``readPreference``: Optional document. See `commonOptions_readPreference`_. + +- ``session``: Optional string. See `commonOptions_session`_. + +- ``writeConcern``: Optional document. See `commonOptions_writeConcern`_. + + +collection +~~~~~~~~~~ + +These operations and their arguments may be documented in the following +specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `CRUD <../crud/crud.rst>`__ +- `Enumerating Indexes <../enumerate-indexes.rst>`__ +- `Index Management <../index-management.rst>`__ + + +session +~~~~~~~ + +These operations and their arguments may be documented in the following +specifications: + +- `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ +- `Driver Sessions <../sessions/driver-sessions.rst>`__ + +withTransaction +``````````````` + +The ``withTransaction`` operation is unique in that its ``callback`` parameter +is a function and not easily expressed in YAML/JSON. For ease of testing, this +parameter is defined as an array of `operation`_ documents (analogous to +`test.operations `_). Drivers MUST evaluate error and result +assertions when executing these operations. + + +bucket +~~~~~~ + +These operations and their arguments may be documented in the following +specifications: + +- `GridFS <../gridfs/gridfs-spec.rst>`__ + + +Special Test Operations +----------------------- + +Certain operations do not correspond to API methods but instead represent +special test operations (e.g. assertions). These operations are distinguished by +`operation.object `_ having a value of "testRunner". The +`operation.name `_ field will correspond to an operation +defined below. + + +targetedFailPoint +~~~~~~~~~~~~~~~~~ + +The ``targetedFailPoint`` operation instructs the test runner to configure a +fail point on a specific mongos. The mongos on which to run the +``configureFailPoint`` command is determined by the ``session`` argument. The +session must already be pinned to a mongos server. The "failPoint" argument is +the ``configureFailPoint`` command to run. + +If a test uses ``targetedFailPoint``, drivers SHOULD disable the fail point +after running all `test.operations `_ to avoid spurious +failures in subsequent tests. The fail point may be disabled like so:: + + db.adminCommand({ + configureFailPoint: , + mode: "off" + }); + +An example instructing the test runner to enable the `failCommand`_ fail point +on the mongos server to which "session0" is pinned follows:: + + # Enable the fail point only on the mongos to which session0 is pinned + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["commitTransaction"] + closeConnection: true + +Tests that use the ``targetedFailPoint`` operation do not include +``configureFailPoint`` commands in their command expectations. Drivers MUST +ensure that ``configureFailPoint`` commands do not appear in the list of logged +commands, either by manually filtering it from the list of observed commands or +by using a different MongoClient to execute ``configureFailPoint``. + + +assertSessionTransactionState +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionTransactionState`` operation instructs the test runner to +assert that the transaction state of the given session is equal to the specified +value. The possible values are as follows: ``none``, ``starting``, +``in_progress``, ``committed``, ``aborted``. + +An example of this operation follows:: + + - name: assertSessionTransactionState + object: testRunner + arguments: + session: *session0 + state: in_progress + + +assertSessionPinned +~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionPinned`` operation instructs the test runner to assert that +the given session is pinned to a mongos. + +An example of this operation follows:: + + - name: assertSessionPinned + object: testRunner + arguments: + session: *session0 + + +assertSessionUnpinned +~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionUnpinned`` operation instructs the test runner to assert that +the given session is not pinned to a mongos. + +An example of this operation follows:: + + - name: assertSessionPinned + object: testRunner + arguments: + session: *session0 + + +assertCollectionExists +~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertCollectionExists`` operation instructs the test runner to assert +that the given collection exists in the database. + +An example of this operation follows:: + + - name: assertCollectionExists + object: testRunner + arguments: + database: db + collection: test + +Use a ``listCollections`` command to check whether the collection exists. Note +that it is currently not possible to run ``listCollections`` from within a +transaction. + +**TODO**: If this will refer to a collection entity, database is redundant. +Otherwise, consider renaming the arguments to databaseName and collectionName as +was done in `collectionData`_. + + +assertCollectionNotExists +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertCollectionNotExists`` operation instructs the test runner to assert +that the given collection does not exist in the database. + +An example of this operation follows:: + + - name: assertCollectionNotExists + object: testRunner + arguments: + database: db + collection: test + +Use a ``listCollections`` command to check whether the collection exists. Note +that it is currently not possible to run ``listCollections`` from within a +transaction. + +**TODO**: If this will refer to a collection entity, database is redundant. +Otherwise, consider renaming the arguments to databaseName and collectionName as +was done in `collectionData`_. + + +assertIndexExists +~~~~~~~~~~~~~~~~~ + +The ``assertIndexExists`` operation instructs the test runner to assert that the +index with the given name exists on the collection. + +An example of this operation follows:: + + - name: assertIndexExists + object: testRunner + arguments: + database: db + collection: test + index: t_1 + +Use a ``listIndexes`` command to check whether the index exists. Note that it is +currently not possible to run ``listIndexes`` from within a transaction. + +**TODO**: If this will refer to a collection entity, database is redundant. +Otherwise, consider renaming the arguments to databaseName and collectionName as +was done in `collectionData`_. + +assertIndexNotExists +~~~~~~~~~~~~~~~~~~~~ + +The ``assertIndexNotExists`` operation instructs the test runner to assert that +the index with the given name does not exist on the collection. + +An example of this operation follows:: + + - name: assertIndexNotExists + object: testRunner + arguments: + database: db + collection: test + index: t_1 + +Use a ``listIndexes`` command to check whether the index exists. Note that it is +currently not possible to run ``listIndexes`` from within a transaction. + +**TODO**: If this will refer to a collection entity, database is redundant. +Otherwise, consider renaming the arguments to databaseName and collectionName as +was done in `collectionData`_. + + +TBD: Command Started Events +--------------------------- + +The event listener used for these tests MUST ignore the security commands listed +in the `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ +spec. + + +Logical Session Id +~~~~~~~~~~~~~~~~~~ + +Each `expectedEvent_commandStartedEvent`_ that includes an ``lsid`` with the +value "session0" or "session1". Tests MUST assert that the command's actual +``lsid`` matches the id of the correct ClientSession named ``session0`` or +``session1``. + +**TODO**: Define a custom matcher for comparing with a session entity's LSID +(e.g. ``lsid: { $$sessionLsid: *session0 }``). + + +Null Values +~~~~~~~~~~~ + +Some command-started events in ``expectations`` include ``null`` values for +fields such as ``txnNumber``, ``autocommit``, and ``writeConcern``. +Tests MUST assert that the actual command **omits** any field that has a +``null`` value in the expected command. + +**TODO**: Define a custom matcher for checking whether a field exists or not +(e.g. ``txnNumber: { $$exists: false }``). + + +Cursor Id +~~~~~~~~~ + +A ``getMore`` value of ``"42"`` in a command-started event is a fake cursorId +that MUST be ignored. (In the Command Monitoring Spec tests, fake cursorIds are +correlated with real ones, but that is not necessary for Transactions Spec +tests.) + +**TODO**: Define a custom matcher for checking whether a field exists or not +(e.g. ``getMore: { $$exists: true }``) or matches a particular BSON type (e.g. +``getMore: { $$type: "int" }``, ``getMore: { $$type: ["int", "long"] }``). + + +afterClusterTime +~~~~~~~~~~~~~~~~ + +A ``readConcern.afterClusterTime`` value of ``42`` in a command-started event +is a fake cluster time. Drivers MUST assert that the actual command includes an +afterClusterTime. + +**TODO**: Define a custom matcher for checking whether a field exists or not +(e.g. ``afterClusterTime: { $$exists: true }``). + + +recoveryToken +~~~~~~~~~~~~~ + +A ``recoveryToken`` value of ``42`` in a command-started event is a +placeholder for an arbitrary recovery token. Drivers MUST assert that the +actual command includes a "recoveryToken" field and SHOULD assert that field +is a BSON document. + +**TODO**: Define a custom matcher for checking whether a field matches a +particular BSON type (e.g. ``recoveryToken: { $$type: "object" }``). + + +TBD: Use as Integration Tests +----------------------------- + +Run a MongoDB replica set with a primary, a secondary, and an arbiter, +**server version 4.0.0 or later**. (Including a secondary ensures that +server selection in a transaction works properly. Including an arbiter helps +ensure that no new bugs have been introduced related to arbiters.) + +A driver that implements support for sharded transactions MUST also run these +tests against a MongoDB sharded cluster with multiple mongoses and +**server version 4.2 or later**. Some tests require +initializing the MongoClient with multiple mongos seeds to ensures that mongos +transaction pinning and the recoveryToken works properly. + +Load each YAML (or JSON) file using a Canonical Extended JSON parser. + +Then for each element in ``tests``: + +#. If the ``skipReason`` field is present, skip this test completely. +#. Create a MongoClient and call + ``client.admin.runCommand({killAllSessions: []})`` to clean up any open + transactions from previous test failures. Ignore a command failure with + error code 11601 ("Interrupted") to work around `SERVER-38335`_. + + - Running ``killAllSessions`` cleans up any open transactions from + a previously failed test to prevent the current test from blocking. + It is sufficient to run this command once before starting the test suite + and once after each failed test. + - When testing against a sharded cluster run this command on ALL mongoses. + +#. Create a collection object from the MongoClient, using the ``database_name`` + and ``collection_name`` fields of the YAML file. +#. Drop the test collection, using writeConcern "majority". +#. Execute the "create" command to recreate the collection, using writeConcern + "majority". (Creating the collection inside a transaction is prohibited, so + create it explicitly.) +#. If the YAML file contains a ``data`` array, insert the documents in ``data`` + into the test collection, using writeConcern "majority". +#. When testing against a sharded cluster run a ``distinct`` command on the + newly created collection on all mongoses. For an explanation see, + Why do tests that run distinct sometimes fail with StaleDbVersion? +#. If ``failPoint`` is specified, its value is a configureFailPoint command. + Run the command on the admin database to enable the fail point. +#. Create a **new** MongoClient ``client``, with Command Monitoring listeners + enabled. (Using a new MongoClient for each test ensures a fresh session pool + that hasn't executed any transactions previously, so the tests can assert + actual txnNumbers, starting from 1.) Pass this test's ``clientOptions`` if + present. + + - When testing against a sharded cluster and ``useMultipleMongoses`` is + ``true`` the client MUST be created with multiple (valid) mongos seed + addreses. + +#. Call ``client.startSession`` twice to create ClientSession objects + ``session0`` and ``session1``, using the test's "sessionOptions" if they + are present. Save their lsids so they are available after calling + ``endSession``, see `Logical Session Id`_. +#. For each element in ``operations``: + + - If the operation ``name`` is a special test operation type, execute it and + go to the next operation, otherwise proceed to the next step. + - Enter a "try" block or your programming language's closest equivalent. + - Create a Database object from the MongoClient, using the ``database_name`` + field at the top level of the test file. + - Create a Collection object from the Database, using the + ``collection_name`` field at the top level of the test file. + If ``collectionOptions`` or ``databaseOptions`` is present, create the + Collection or Database object with the provided options, respectively. + Otherwise create the object with the default options. + - Execute the named method on the provided ``object``, passing the + arguments listed. Pass ``session0`` or ``session1`` to the method, + depending on which session's name is in the arguments list. + If ``arguments`` contains no "session", pass no explicit session to the + method. + - If the driver throws an exception / returns an error while executing this + series of operations, store the error message and server error code. + - If the operation's ``error`` field is ``true``, verify that the method + threw an exception or returned an error. + - If the result document has an "errorContains" field, verify that the + method threw an exception or returned an error, and that the value of the + "errorContains" field matches the error string. "errorContains" is a + substring (case-insensitive) of the actual error message. + + If the result document has an "errorCodeName" field, verify that the + method threw a command failed exception or returned an error, and that + the value of the "errorCodeName" field matches the "codeName" in the + server error response. + + If the result document has an "errorLabelsContain" field, verify that the + method threw an exception or returned an error. Verify that all of the + error labels in "errorLabelsContain" are present in the error or exception + using the ``hasErrorLabel`` method. + + If the result document has an "errorLabelsOmit" field, verify that the + method threw an exception or returned an error. Verify that none of the + error labels in "errorLabelsOmit" are present in the error or exception + using the ``hasErrorLabel`` method. + - If the operation returns a raw command response, eg from ``runCommand``, + then compare only the fields present in the expected result document. + Otherwise, compare the method's return value to ``result`` using the same + logic as the CRUD Spec Tests runner. + +#. Call ``session0.endSession()`` and ``session1.endSession``. +#. If the test includes a list of command-started events in ``expectations``, + compare them to the actual command-started events using the + same logic as the Command Monitoring Spec Tests runner, plus the rules in + the Command-Started Events instructions below. +#. If ``failPoint`` is specified, disable the fail point to avoid spurious + failures in subsequent tests. The fail point may be disabled like so:: + + db.adminCommand({ + configureFailPoint: , + mode: "off" + }); + +#. For each element in ``outcome``: + + - If ``name`` is "collection", verify that the test collection contains + exactly the documents in the ``data`` array. Ensure this find reads the + latest data by using **primary read preference** with + **local read concern** even when the MongoClient is configured with + another read preference or read concern. + Note the server does not guarantee that documents returned by a find + command will be in inserted order. This find MUST sort by ``{_id:1}``. + +.. _SERVER-38335: https://jira.mongodb.org/browse/SERVER-38335 + + +Server Fail Points +================== + +Many tests utilize the ``configureFailPoint`` command to trigger server-side +errors such as dropped connections or command errors. Tests refer to this by the +`test.failPoint `_ field or the the special ``targetedFailPoint`` operation +type on the ``testRunner`` object (in ``tests[].operations[]``). + +This internal command is not documented in the MongoDB manual (pending +`DOCS-10784`_); however, there is scattered documentation available on the +server wiki (`The failCommand Fail Point `__) +and employee blogs (e.g. `Intro to Fail Points `__, +`Testing Network Errors with MongoDB `__). +Documentation can also be gleaned from JIRA tickets (e.g. `SERVER-35004`_, +`SERVER-35083`_). This specification does not aim to provide comprehensive +documentation for all fail points available for driver testing. + +.. _DOCS-10784: https://jira.mongodb.org/browse/DOCS-10784 +.. _SERVER-35004: https://jira.mongodb.org/browse/SERVER-35004 +.. _SERVER-35083: https://jira.mongodb.org/browse/SERVER-35083 + +The ``configureFailPoint`` command should be executed on the ``admin`` database +and has the following structure:: + + db.adminCommand({ + configureFailPoint: , + mode: , + data: + }); + +The value of ``configureFailPoint`` is a string denoting the fail point to be +configured (e.g. "failCommand"). + +The ``mode`` option is a generic fail point option and may be assigned a string +or document value. The string values "alwaysOn" and "off" may be used to +enable or disable the fail point, respectively. A document may be used to +specify either ``times`` or ``skip``, which are mutually exclusive: + +- ``{ times: }`` may be used to limit the number of times the fail + point may trigger before transitioning to "off". +- ``{ skip: }`` may be used to defer the first trigger of a fail + point, after which it will transition to "alwaysOn". + +The ``data`` option is a document that may be used to specify any options that +control the particular fail point's behavior. + +In order to use ``configureFailPoint``, the undocumented ``enableTestCommands`` +`server parameter `_ must +be enabled by either the configuration file or command line option (e.g. +``--setParameter enableTestCommands=1``). It cannot be enabled at runtime via +the `setParameter `_ +command). This parameter should already be enabled for most configuration files +in `mongo-orchestration `_. + + +failCommand +----------- + +The ``failCommand`` fail point allows the client to force the server to return +an error for commands listed in the ``data.failCommands`` field. Additionally, +this fail point is documented in server wiki: +`The failCommand Fail Point `_. + +The ``failCommand`` fail point may be configured like so:: + + db.adminCommand({ + configureFailPoint: "failCommand", + mode: , + data: { + failCommands: ["commandName", "commandName2"], + closeConnection: , + errorCode: , + writeConcernError: , + appName: , + blockConnection: , + blockTimeMS: , + } + }); + +``failCommand`` supports the following ``data`` options, which may be combined +if desired: + +* ``failCommands``: Required array of strings. Lists the command names to fail. +* ``closeConnection``: Optional boolean, which defaults to ``false``. If + ``true``, the command will not be executed, the connection will be closed, and + the client will see a network error. +* ``errorCode``: Optional integer, which is unset by default. If set, the + command will not be executed and the specified command error code will be + returned as a command error. +* ``appName``: Optional string, which is unset by default. If set, the fail + point will only apply to connections for MongoClients created with this + ``appname``. New in server 4.4.0-rc2 (`SERVER-47195 `_). +* ``blockConnection``: Optional boolean, which defaults to ``false``. If + ``true``, the server should block the affected commands for ``blockTimeMS``. + New in server 4.3.4 (`SERVER-41070 `_). +* ``blockTimeMS``: Integer, which is required when ``blockConnection`` is + ``true``. The number of milliseconds for which the affected commands should be + blocked. New in server 4.3.4 (`SERVER-41070 `_). + + +Design Rationale +================ + +This specification was primarily derived from the test formats used by the +`Transactions <../transactions/transactions.rst>`__ and +`CRUD <../crud/crud.rst>`__ specs, which have served models or other specs. + + +Change Log +========== From f2c6bb04378c1859cc63e1612cf48037d0579fa9 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 19 Aug 2020 19:54:32 -0400 Subject: [PATCH 02/90] consolidate test format, simplify assert syntax, docs for matching --- .../unified-test-format.rst | 637 +++++++++++++----- 1 file changed, 482 insertions(+), 155 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 978d5684f2..83dc89bd99 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -35,6 +35,43 @@ Specification ============= +Entity Map +---------- + +The entity map indexes arbitrary objects and values by unique names, so that +they can be referenced from test constructs (e.g. +`operation.object `_). To ensure each test is executed in +isolation, test runners MUST NOT share entity maps between tests. Most entities +will be driver objects created by the `createEntities`_ directive during test +setup, but the entity map may also be modified during test execution via the +`operation.saveResultAsEntity `_ directive. + +Test runners may choose to implement the entity map in a fashion most suited to +their language, but implementations MUST enforce both uniqueness of entity names +and referential integrity when fetching an entity. Test runners MUST raise an +error if an attempt is made to store an entity with a name that already exists +in the map and MUST raise an error if an entity is not found for a name or is +found but has an unexpected type. + +Consider the following examples:: + + # Error due to a duplicate name (client0 was already defined) + createEntities: + - client: { id: client0 } + - client: { id: client0 } + + # Error due to a missing entity (client1 is undefined) + createEntities: + - client: { id: client0 } + - session: { id: session0, client: client1 } + + # Error due to an unexpected entity type (session instead of client) + createEntities: + - client: { id: client0 } + - session: { id: session0, client: client0 } + - session: { id: session1, client: session0 } + + Test Format ----------- @@ -50,26 +87,33 @@ a tool such as `Ajv `_. The top-level fields of a test file are as follows: +.. _runOn: + - ``runOn``: Optional array of documents. List of server version and/or topology - requirements for which the tests can be run. If the test environment satisfies - one or more of these requirements, the tests may be executed; otherwise, this - file should be skipped. If this field is omitted, the tests can be assumed to - have no particular requirements and should be executed. + requirements for which the tests in this file can be run. These requirements + may be overridden on a per-test basis by `test.runOn `_. Test + runners MUST skip a test if its requirements are not met. If set, the array should contain at least one document. The structure of each document is defined in `runOnRequirement`_. +.. _createEntities: + - ``createEntities``: Optional array of documents. List of entities (e.g. client, collection, session objects) that should be created before each test case is executed. The structure of each document is defined in `entity`_. +.. _collectionName: + - ``collectionName``: Optional string. Name of collection under test. This is primarily useful when the collection name must be referenced in an assertion. - If unset, drivers may use whatever value they prefer. + If unset, test runners may use whatever value they prefer. + +.. _databaseName: - ``databaseName``: Optional string. Name of database under test. This is primarily useful when the database name must be referenced in an assertion. - If unset, drivers may use whatever value they prefer. + If unset, test runners may use whatever value they prefer. .. _initialData: @@ -100,23 +144,22 @@ test(s). The structure of this document is as follows: against which the tests can be run successfully. If this field is omitted, it should be assumed that there is no upper bound on the required server version. -- ``topology``: Optional array of strings. List of server topologies against - which the tests can be run successfully. Valid topologies are "single", - "replicaset", and "sharded". If this field is omitted, the default is all - topologies (i.e. ``["single", "replicaset", "sharded"]``). - - **TODO**: Consider a sharded-replicaset topology. +- ``topology``: Optional string or array of strings. One or more of server + topologies against which the tests can be run successfully. Valid topologies + are "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded + cluster backed by replica sets). If this field is omitted, it should be + assumed that there is no topology requirement for the test. entity ~~~~~~ -An entity (e.g. client, collection, session object) that will be created -before each test is executed. This document must contain exactly one top-level -key that identifies the entity type and maps to a nested document, which -specifies a unique name for the entity (``id`` key) and any other parameters -necessary for its construction. Tests SHOULD use sequential names based on the -entity type (e.g. "session0", "session1"). +An entity (e.g. client, collection, session object) that will be created in the +`Entity Map`_ before each test is executed. This document must contain exactly +one top-level key that identifies the entity type and maps to a nested document, +which specifies a unique name for the entity (``id`` key) and any other +parameters necessary for its construction. Tests SHOULD use sequential names +based on the entity type (e.g. "session0", "session1"). When defining an entity document in YAML, a `node anchor `_ SHOULD be created @@ -144,7 +187,12 @@ The structure of this document is as follows: will map to an array of strings, each representing a tag set, since it is not feasible to define multiple ``readPreferenceTags`` keys in the document. - **TODO**: Consider moving the ``useMultipleMongoses`` `test`_ option here. + .. _client_allowMultipleMongoses: + + - ``allowMultipleMongoses``: Optional boolean. If false, the MongoClient + MUST be initialized with only a single mongos host. If true or omitted, or + if the topology is non-sharded, this option has no effect. This option is + primarily used in conjunction with tests that set non-targeted fail points. - ``database``: Optional document. Corresponds with a Database object. The structure of this document is as follows: @@ -159,7 +207,7 @@ The structure of this document is as follows: entity's ``id`` field (e.g. ``client: *client0``). - ``databaseName``: Optional string. Database name. If omitted, this defaults - to the name of the database under test. + to the name of the database under test (see: `databaseName`_). - ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. @@ -176,10 +224,12 @@ The structure of this document is as follows: entity's ``id`` field (e.g. ``database: *database0``). - ``collectionName``: Optional string. Collection name. If omitted, this - defaults to the name of the collection under test. + defaults to the name of the collection under test (see: `collectionName`_). - ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. +.. _entity_session: + - ``session``: Optional document. Corresponds with an explicit ClientSession object. The structure of this document is as follows: @@ -224,16 +274,16 @@ collectionData List of documents that should correspond to the contents of a collection. This structure is used by both `initialData`_ and `test.outcome `_, -which insert and read documents, respectively. Both of those directives -directives may operate on the collection under test, they do not share the -same Collection object(s) as test `operations `_. The structure of +which insert and read documents, respectively. Both of those directives may +operate on the collection(s) under test, they do not share the same Collection +and MongoClient object(s) as test `operations `_. The structure of this document is as follows: - ``collectionName``: Optional string. Collection name (not an `entity`_). - Defaults to the name of the collection under test. + Defaults to the name of the collection under test (see: `collectionName`_). - ``databaseName``: Optional string. Database name (not an `entity`_). Defaults - to the name of the database under test. + to the name of the database under test (see: `databaseName`_). - ``documents``: Required array of documents. List of documents corresponding to the contents of the collection. This list may be empty. @@ -248,34 +298,18 @@ structure of each document is as follows: - ``description``: Required string. The name of the test. -- ``skipReason``: Optional string. If set, the test will be skipped. The string - should explain the reason for skipping the test (e.g. JIRA ticket). +.. _test_runOn: -- ``useMultipleMongoses``. Optional boolean. If true, the MongoClient for this - test should be initialized with multiple mongos seed addresses. If false or - omitted, only a single mongos address should be specified. This field has no - effect for non-sharded topologies. - - Note: ``useMultipleMongoses:true`` is mutually exclusive with ``failPoint``. - -- ``clientOptions``: Optional document. Additional connection string options - to pass to the MongoClient constructor. - - Note: this option does not support expressing multiple read preference tags, - tags, since keys would repeat. - -.. _test_failPoint: - -- ``failPoint``: Optional document. A server failpoint to enable expressed as - a complete ``configureFailPoint`` command to run on the admin database. - - Note: This option is mutually exclusive with ``useMultipleMongoses:true``. +- ``runOn``: Optional array of documents. List of server version and/or topology + requirements for which the tests in this file can be run. If specified, these + requirements override any top-level requirements in `runOn`_. Test runners + MUST skip a test if its requirements are not met. -- ``sessionOptions``: Optional document. Map of session names (e.g. "session0") - to documents, each of which denotes parameters to pass to - ``MongoClient.startSession()`` when creating that session. + If set, the array should contain at least one document. The structure of each + document is defined in `runOnRequirement`_. - **TODO**: remove this in favor of the session `entity`_ +- ``skipReason``: Optional string. If set, the test will be skipped. The string + should explain the reason for skipping the test (e.g. JIRA ticket). .. _test_operations: @@ -291,6 +325,11 @@ structure of each document is as follows: The array should contain at least one document. The structure of each document is defined in `expectedEvent`_. + **TODO**: Determine if an empty array should test that no events are observed. + Decide if event types (e.g. APM, SDAM) should be mixed in the same array and + whether tests should be able to filter out certain types (assuming the test + runner observes any supported type). + .. _test_outcome: - ``outcome``: Optional array of documents. Data that should exist in @@ -315,42 +354,42 @@ is as follows: .. _operation_object: -- ``object``: Optional string. Name of the object on which to perform the - operation. Can be "collection", "database", "session0", "session1", or - "testRunner". Defaults to "collection". - - **TODO**: link to special operations section - - **TODO**: update this to refer to an entity identifier - -- ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. - -- ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. - -.. _operation_commandName: - -- ``commandName``: Optional string. Required only when ``name`` is "runCommand". - The name of the command to run. This may be used by languages that are unable - preserve the order of keys in the ``command`` argument when parsing YAML/JSON. +- ``object``: Required string. Name of the object on which to perform the + operation. This should correspond to either an `entity`_ name (for + `Entity Test Operations`_) or "testRunner" (for `Special Test Operations`_). + If the object is an entity, The YAML file SHOULD use an + `alias node `_ for its ``id`` + field (e.g. ``object: *collection0``). .. _operation_arguments: - ``arguments``: Optional document. Map of parameter names and values for the operation. The structure of this document will vary based on the operation. + See `Entity Test Operations`_ and `Special Test Operations`_. - **TODO**: link to operation section +.. _operation_expectedError: -- ``error``: Optional boolean. If true, the test should expect the operation to - raise an error/exception. This could be either a server-generated or a - driver-generated error. Defaults to false. - -- ``result``: Optional mixed type. The return value from the operation, if any. - This field may be a scalar value, a single document, or an array of documents - in the case of a multi-document read. If the operation is expected to raise an - error/exception (i.e. ``error:true``), the structure of this document is +- ``expectedError``: Optional document. One or more assertions for an expected + error raised by the operation. The structure of this document is defined in `expectedError`_. - **TODO**: link to section on matching logic +.. _operation_expectedResult: + +- ``expectedResult``: Optional mixed type. A value corresponding to the expected + result of the operation. This field may be a scalar value, a single document, + or an array of documents in the case of a multi-document read. Test runners + MUST follow the rules in `Evaluating Matches`_ when processing this assertion. + This field is mutually exclusive with + `expectedError `_. + +.. _operation_saveResultAsEntity: + +- ``saveResultAsEntity``: Optional string. If specified, the actual result + returned by the operation (if any) will be saved with this name in the + `Entity Map`_. The test runner MUST raise an error if the name is already in + use. + + This is primarily used for change streams. expectedError @@ -360,6 +399,16 @@ One or more assertions for an error/exception, which is expected to be raised by an executed operation. At least one key is required in this document. The structure of this document is as follows: +- ``type``: Optional string or array of strings. One or more classifications of + errors, at least one of which should apply to the expected error. Valid types + are as follows: + + - ``client``: client-generated error (e.g. parameter validation error before + a command is sent to the server). + + - ``server``: server-generated error (e.g. error derived from a server + response). + - ``errorContains``: Optional string. A substring of the expected error message. - ``errorCodeName``: Optional string. The expected "codeName" field in the @@ -371,6 +420,12 @@ structure of this document is as follows: - ``errorLabelsOmit``: Optional array of strings. A list of error label strings that the error is expected not to have. +.. _expectedError_expectedResult: + +- ``expectedResult``: Optional mixed type. This field follows the same rules as + `operation.expectedResult `_ and is only used in + cases where the error includes a result (e.g. `bulkWrite`_). + expectedEvent ~~~~~~~~~~~~~ @@ -387,9 +442,8 @@ follows: `CommandStartedEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. The structure of this document is as follows: - - ``command``: Optional document. - - **TODO**: link to section on matching logic + - ``command``: Optional document. Test runners MUST follow the rules in + `Evaluating Matches`_ when processing this assertion. - ``commandName``: Optional string. @@ -501,11 +555,11 @@ specifications: Other database operations not documented by an existing specification follow. + runCommand `````````` -Generic command runner. This operation requires that -`operation.commandName `_ also be specified. +Generic command runner. This method does not inherit a read concern or write concern (per the `Read and Write Concern <../read-write-concern/read-write-concern.rst#generic-command-method>`__ @@ -517,6 +571,10 @@ The following arguments are supported: - ``command``: Required document. The command to be executed. +- ``commandName``: Required string. The name of the command to run. This is used + by languages that are unable preserve the order of keys in the ``command`` + argument when parsing YAML/JSON. + - ``readConcern``: Optional document. See `commonOptions_readConcern`_. - ``readPreference``: Optional document. See `commonOptions_readPreference`_. @@ -538,6 +596,17 @@ specifications: - `Index Management <../index-management.rst>`__ +bulkWrite +````````` + +While operations typically raise an error *or* return a result, the +``bulkWrite`` operation is unique in that it may report both via the +``writeResult`` property of a BulkWriteException. In this case, the intermediary +write result may be matched with `expectedError_expectedResult`_. Because +``writeResult`` is optional for drivers to implement, such assertions should +utilize the `$$unsetOrMatches`` operator. + + session ~~~~~~~ @@ -547,14 +616,15 @@ specifications: - `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ - `Driver Sessions <../sessions/driver-sessions.rst>`__ + withTransaction ``````````````` The ``withTransaction`` operation is unique in that its ``callback`` parameter is a function and not easily expressed in YAML/JSON. For ease of testing, this parameter is defined as an array of `operation`_ documents (analogous to -`test.operations `_). Drivers MUST evaluate error and result -assertions when executing these operations. +`test.operations `_). Test runners MUST evaluate error and +result assertions when executing these operations. bucket @@ -576,28 +646,48 @@ special test operations (e.g. assertions). These operations are distinguished by defined below. +failPoint +~~~~~~~~~ + +The ``failPoint`` operation instructs the test runner to configure a fail point. +The ``failPoint`` argument is the ``configureFailPoint`` command to run. Tests +using this operation SHOULD also specify +`allowMultipleMongoses `_ for any client +entity(ies) used in the test, as failure to do so could cause unpredictable +behavior when running the test on sharded topologies. + +An example of this operation follows:: + + # Enable the fail point only on the mongos to which session0 is pinned + - name: failPoint + object: testRunner + arguments: + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: ["insert"] + closeConnection: true + +See also: + +- `Clearing Fail Points After Tests`_ +- `Ignoring APM Events for configureFailPoint`_ + + targetedFailPoint ~~~~~~~~~~~~~~~~~ The ``targetedFailPoint`` operation instructs the test runner to configure a fail point on a specific mongos. The mongos on which to run the -``configureFailPoint`` command is determined by the ``session`` argument. The -session must already be pinned to a mongos server. The "failPoint" argument is -the ``configureFailPoint`` command to run. - -If a test uses ``targetedFailPoint``, drivers SHOULD disable the fail point -after running all `test.operations `_ to avoid spurious -failures in subsequent tests. The fail point may be disabled like so:: - - db.adminCommand({ - configureFailPoint: , - mode: "off" - }); +``configureFailPoint`` command is determined by the ``session`` argument. Test +runners MUST error if the session is not pinned to a mongos server at the time +this operation is executed. The ``failPoint`` argument is the +``configureFailPoint`` command to run. An example instructing the test runner to enable the `failCommand`_ fail point on the mongos server to which "session0" is pinned follows:: - # Enable the fail point only on the mongos to which session0 is pinned - name: targetedFailPoint object: testRunner arguments: @@ -609,11 +699,10 @@ on the mongos server to which "session0" is pinned follows:: failCommands: ["commitTransaction"] closeConnection: true -Tests that use the ``targetedFailPoint`` operation do not include -``configureFailPoint`` commands in their command expectations. Drivers MUST -ensure that ``configureFailPoint`` commands do not appear in the list of logged -commands, either by manually filtering it from the list of observed commands or -by using a different MongoClient to execute ``configureFailPoint``. +See also: + +- `Clearing Fail Points After Tests`_ +- `Ignoring APM Events for configureFailPoint`_ assertSessionTransactionState @@ -752,76 +841,235 @@ Otherwise, consider renaming the arguments to databaseName and collectionName as was done in `collectionData`_. -TBD: Command Started Events ---------------------------- +Evaluating Matches +------------------ -The event listener used for these tests MUST ignore the security commands listed -in the `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ -spec. +Expected values in tests (e.g. +`operation.expectedResult `_) are expressed as +`Extended JSON <../extended-json.rst>`__. +The algorithm for matching expected and actual values is specified with the +following pseudo-code:: -Logical Session Id -~~~~~~~~~~~~~~~~~~ + function match (expected, actual): + if expected is a document: + if first key of expected starts with "$$": + assert that the special operator (identified by key) matches + return -Each `expectedEvent_commandStartedEvent`_ that includes an ``lsid`` with the -value "session0" or "session1". Tests MUST assert that the command's actual -``lsid`` matches the id of the correct ClientSession named ``session0`` or -``session1``. + assert that actual is a document -**TODO**: Define a custom matcher for comparing with a session entity's LSID -(e.g. ``lsid: { $$sessionLsid: *session0 }``). + for every key/value in expected: + assert that actual[key] exists + assert that actual[key] matches value + return -Null Values -~~~~~~~~~~~ + if expected is an array: + assert that actual is an array + assert that actual and expected have the same number of elements -Some command-started events in ``expectations`` include ``null`` values for -fields such as ``txnNumber``, ``autocommit``, and ``writeConcern``. -Tests MUST assert that the actual command **omits** any field that has a -``null`` value in the expected command. + for every index/value in expected: + assert that actual[index] matches value -**TODO**: Define a custom matcher for checking whether a field exists or not -(e.g. ``txnNumber: { $$exists: false }``). + return + // expected is neither a document nor array + assert that actual and expected are the same type + assert that actual and expected are equal -Cursor Id -~~~~~~~~~ +The rules for comparing documents and arrays are discussed in more detail in +subsequent sections. When comparing types *other* than documents and arrays, +test runners MAY adopt any of the following approaches to compare expected and +actual values, as long as they are consistent: -A ``getMore`` value of ``"42"`` in a command-started event is a fake cursorId -that MUST be ignored. (In the Command Monitoring Spec tests, fake cursorIds are -correlated with real ones, but that is not necessary for Transactions Spec -tests.) +- Convert both values to Extended JSON and compare strings +- Convert both values to BSON, and compare bytes +- Convert both values to native representations, and compare accordingly -**TODO**: Define a custom matcher for checking whether a field exists or not -(e.g. ``getMore: { $$exists: true }``) or matches a particular BSON type (e.g. -``getMore: { $$type: "int" }``, ``getMore: { $$type: ["int", "long"] }``). +When comparing types that may contain documents (e.g. CodeWScope), test runners +MUST follow standard document matching rules when comparing those properties. -afterClusterTime -~~~~~~~~~~~~~~~~ +Extra Fields in Actual Documents +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A ``readConcern.afterClusterTime`` value of ``42`` in a command-started event -is a fake cluster time. Drivers MUST assert that the actual command includes an -afterClusterTime. +When matching expected and actual *documents*, test runners MUST permit the +actual documents to contain additional fields not present in the expected +document. For example, the following documents match:: -**TODO**: Define a custom matcher for checking whether a field exists or not -(e.g. ``afterClusterTime: { $$exists: true }``). + expected: { x: 1 } + actual: { x: 1, y: 1 } +The inverse is not true. For example, the following documents would not match:: -recoveryToken -~~~~~~~~~~~~~ + expected: { x: 1, y: 1 } + actual: { x: 1 } + +It may be helpful to think of expected documents as a form of query criteria. +The intention behind this rule is that it is not always feasible or relevant for +a test to exhaustively specify all fields in an expected document (e.g. cluster +time in a `CommandStartedEvent `_ command). -A ``recoveryToken`` value of ``42`` in a command-started event is a -placeholder for an arbitrary recovery token. Drivers MUST assert that the -actual command includes a "recoveryToken" field and SHOULD assert that field -is a BSON document. +Note that this rule for allowing extra fields in actual values only applies when +matching *documents* documents. When comparing arrays, expected and actual +values must contain the same number of elements. For example, the following +arrays corresponding to a ``distinct`` operation result would not match:: -**TODO**: Define a custom matcher for checking whether a field matches a -particular BSON type (e.g. ``recoveryToken: { $$type: "object" }``). + expected: [ 1, 2, 3 ] + actual: [ 1, 2, 3, 4 ] +That said, any individual documents *within* an array or list (e.g. result of a +``find`` operation) may be matched according to the rules in this section. For +example, the following arrays would match:: -TBD: Use as Integration Tests ------------------------------ + expected: [ { x: 1 }, { x: 2 } ] + actual: [ { x: 1, y: 1 }, { x: 2, y: 2 } ] + + +Special Operators for Matching Assertions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When matching expected and actual values, an equality comparison is not always +sufficient. For instance, a test file cannot anticipate what a session ID will +be at runtime, but may still want to analyze the contents of an ``lsid`` field +in a command document. To address this need, special operators can be used. + +These operators are documents with a single key identifying the operator. Such +keys are prefixed with ``$$`` to ease in detecting an operator (test runners +need only inspect the first key of each document) and differentiate the document +from MongoDB query operators, which use a single `$` prefix. The key will map to +some value that influences the operator's behavior (if applicable). + +When examining the structure of an expected value during a comparison, test +runners MUST examine the first key of any document for a ``$$`` prefix and, if +present, defer to the special logic defined in this section. + + +$$exists +```````` + +Syntax:: + + { field: { $$exists: } } + +This operator can be used anywhere the value for a key might be specified in an +expected dcoument. If true, the test runner MUST assert that the key exists in +the actual document, irrespective of its value (e.g. a key with a ``null`` value +would match). If false, the test runner MUST assert that the key does not exist +in the actual document. This operator is modeled after the +`$exists `__ +query operator. + +An example of this operator checking for a field's presence follows:: + + command: + getMore: { $$exists: true } + collection: *collectionName, + batchSize: 5 + +An example of this operator checking for a field's absence follows:: + + command: + update: *collectionName + updates: [ { q: {}, u: { $set: { x: 1 } } } ] + ordered: true + writeConcern: { $$exists: false } + + +$$type +`````` + +Syntax, where ``bsonType`` is a string or integer:: + + { $$type: } + { $$type: [ , , ... ] } + +This operator can be used anywhere a matched value is expected (including an +`expectedResult `_). The test runner MUST assert that +the actual value exists and matches one of the expected types, which correspond +to the documented types for the +`$type `__ +query operator. + +An example of this operator follows:: + + command: + getMore: { $$type: [ int, long ] } + collection: { $$type: 2 } # string + +When the actual value is an array, test runners MUST NOT examine types of the +array's elements. Only the type of actual field should be checked. This is +admittedly inconsistent with the behavior of the +`$type `__ +query operator, but there is presently no need for this behavior in tests. + + +$$unsetOrMatches +```````````````` + +Syntax:: + + { $$unsetOrMatches: } + +This operator can be used anywhere a matched value is expected (including an +`expectedResult `_). The test runner MUST assert that +actual value either does not exist or and matches the expected value. Matching +the expected value should use the standard rules in `Evaluating Matches`_, which +means that it may contain special operators. + +This operator is primarily used to assert driver-optional fields from the CRUD +spec (e.g. ``insertedId`` for InsertOneResult, ``writeResult`` for +BulkWriteException). + +An example of this operator used for a result's field follows:: + + expectedResult: + insertedId: { $$unsetOrMatches: 2 } + +An example of this operator used for an entire result follows:: + + expectedError: + expectedResult: + $$unsetOrMatches: + deletedCount: 0 + insertedCount: 2 + matchedCount: 0 + modifiedCount: 0 + upsertedCount: 0 + upsertedIds: { } + + +$$sessionLsid +````````````` + +Syntax:: + + { $$sessionLsid: } + +This operation is used for matching any value with the logical session ID of a +`session entity `_. The value will refer to a unique name of a +session entity. The YAML file SHOULD use an +`alias node `_ for a session +entity's ``id`` field (e.g. ``session: *session0``). + +An example of this operator follows:: + + command: + ping: 1 + lsid: { $$sessionLsid: *session0 } + + +TBD: Command Started Events +--------------------------- + +The event listener used for these tests MUST ignore the security commands listed +in the `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ +spec. + + +Running Tests +------------- Run a MongoDB replica set with a primary, a secondary, and an arbiter, **server version 4.0.0 or later**. (Including a secondary ensures that @@ -876,7 +1124,7 @@ Then for each element in ``tests``: #. Call ``client.startSession`` twice to create ClientSession objects ``session0`` and ``session1``, using the test's "sessionOptions" if they are present. Save their lsids so they are available after calling - ``endSession``, see `Logical Session Id`_. + ``endSession``, see `Logical Session Id`. #. For each element in ``operations``: - If the operation ``name`` is a special test operation type, execute it and @@ -953,8 +1201,7 @@ Server Fail Points Many tests utilize the ``configureFailPoint`` command to trigger server-side errors such as dropped connections or command errors. Tests refer to this by the -`test.failPoint `_ field or the the special ``targetedFailPoint`` operation -type on the ``testRunner`` object (in ``tests[].operations[]``). +special `failPoint`_ or `targetedFailPoint`_ opertions. This internal command is not documented in the MongoDB manual (pending `DOCS-10784`_); however, there is scattered documentation available on the @@ -963,12 +1210,16 @@ and employee blogs (e.g. `Intro to Fail Points `__). Documentation can also be gleaned from JIRA tickets (e.g. `SERVER-35004`_, `SERVER-35083`_). This specification does not aim to provide comprehensive -documentation for all fail points available for driver testing. +documentation for all fail points available for driver testing, but some fail +points are documented in `Fail Points Commonly Used in Tests`_. .. _DOCS-10784: https://jira.mongodb.org/browse/DOCS-10784 .. _SERVER-35004: https://jira.mongodb.org/browse/SERVER-35004 .. _SERVER-35083: https://jira.mongodb.org/browse/SERVER-35083 +Configuring Fail Points +----------------------- + The ``configureFailPoint`` command should be executed on the ``admin`` database and has the following structure:: @@ -1002,9 +1253,39 @@ the `setParameter `_. +Clearing Fail Points After Tests +-------------------------------- + +If a test configures one or more fail points, test runners MUST disable those +fail points after running all `test.operations `_ to avoid +spurious failures in subsequent tests. For tests using `targetedFailPoint`_, the +test runner MUST disable the fail point on the same mongos node on which it was +originally configured. + +A fail point may be disabled like so:: + + db.adminCommand({ + configureFailPoint: , + mode: "off" + }); + +Ignoring APM Events for configureFailPoint +------------------------------------------ + +Tests that use `failPoint`_ and `targetedFailPoint`_ operations SHOULD not +include ``configureFailPoint`` commands in their command expectations. Test +runners MUST ensure that ``configureFailPoint`` commands executed for +`failPoint`_ and `targetedFailPoint`_ operations do not appear in the list of +logged commands, either by manually filtering it from the list of observed +commands or by using a different MongoClient to execute ``configureFailPoint``. + + +Fail Points Commonly Used in Tests +---------------------------------- + failCommand ------------ +~~~~~~~~~~~ The ``failCommand`` fail point allows the client to force the server to return an error for commands listed in the ``data.failCommands`` field. Additionally, @@ -1058,3 +1339,49 @@ This specification was primarily derived from the test formats used by the Change Log ========== + +Note: this will be cleared when publishing version 1.0 of the spec + +2020-08-19: + +* added test.runOn and clarified that it can override top-level runOn requirements + +* runOn.topology can be a single string in addition to array of strings + +* added "sharded-replicaset" topology type, which will be relevant for change + streams, transactions, retryable writes. + +* removed top-level collectionName and databaseName fields, since they can be + specified when creating collection and database entities. + +* removed test.clientOptions, since client entities can specify their own options + +* moved operation.failPoint to failPoint special operation + +* operation.object is now required and takes either an entity name (e.g. + "collection0") or "testRunner" + +* operation.commandName moved to an argument of the runCommand database + operation. Since that method is documented entirely in this spec, I didn't + see an issue with consolidating. + +* renamed operation.result to expectedResult and noted that it may co-exist with + error assertions in special cases (e.g. BulkWriteException). + +* remove error assertions from operation.result. These are now specified under + operation.expectedError, which replaces the error boolean and requires at + least one assertion. Added a type assertion (e.g. client, server), which + should be useful for discerning client-side and server-side errors (currently + achieved with APM assertions). + +* added operation.saveResultAsEntity to capture a result in the entity map + (primarily for use with change streams) + +* consolidated documentation for ignoring configureFailPoint commands in APM and + also disabling fail points after a test, which is now referenced from the + failPoint and targetedFailPoint operations + +* removed $$assert nesting in favor of $$, since test runners can + easily check the first document key for a ``$$`` prefix. + +* completed section on evaluating matches and added pseudo-code From ecdf667b2ab4044cdd256a33aaeb78db5ecb196b Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 21 Aug 2020 13:23:04 -0400 Subject: [PATCH 03/90] bulkWrite errors, schemaVersion, and configureFailPoint APM exclusion --- .../unified-test-format.rst | 198 ++++++++++++------ 1 file changed, 136 insertions(+), 62 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 83dc89bd99..267830d27b 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -3,7 +3,7 @@ Unified Test Format =================== :Spec Title: Unified Test Format -:Spec Version: 1.0 +:Spec Version: 1.0.0 :Author: Jeremy Mikola :Advisors: Prashant Mital :Status: Draft @@ -29,12 +29,38 @@ META The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 `_. +interpreted as described in `RFC 2119 `__. Specification ============= +Schema Version +-------------- + +Each test file notes a `schemaVersion`_, which test runners will use to +determine compatibility (i.e. whether and how the test file will be +interpreted). Test runners are free to determine how to handle incompatible test +files (e.g. skip and log a notice, fail and raise an error). + +Each major version of this specification will have a corresponding JSON schema +(e.g. `schema-1.json `__), which may be used to programmatically +validate YAML and JSON files using a tool such as `Ajv `__. + +This specification and the `Test Format`_ follow +`semantic versioning `__. Backwards breaking changes (e.g. +removing a field, introducing a required field) will warrant a new major +version. Backwards compatible changes (e.g. introducing an optional field) will +warrant a new minor version. Small bug fixes and internal changes (e.g. grammar) +will warrant a new patch version. + +The latest JSON schema MUST remain consistent with the `Test Format`_ section. +If and when a new major version is introduced, the `Breaking Changes`_ section +must be updated and JSON schema(s) for any previous major version(s) MUST remain +available so that older test files can still be validated. New tests files +SHOULD always be written using the latest version of this specification. + + Entity Map ---------- @@ -81,12 +107,23 @@ are parsed as a document by the test runner. This section defines the top-level keys for that document and links to various sub-sections for definitions of nested structures (e.g. individual `test`_, `operation`_). -The test format is also defined in the accompanying `schema.json `_ -file, which may be used to programmatically validate YAML and JSON files using -a tool such as `Ajv `_. +Although test runners are free to process YAML or JSON files, YAML is the +canonical format for writing tests. YAML files may be converted to JSON using a +tool such as `js-yaml `__ . + + +Top-level Fields +~~~~~~~~~~~~~~~~ The top-level fields of a test file are as follows: +.. _schemaVersion: + +- ``schemaVersion``: Required string. Version of this specification to which the + test file complies. Test runners will use this to determine compatibility + (i.e. whether and how the test file will be interpreted). The format of this + string is defined in `Version String`_. + .. _runOn: - ``runOn``: Optional array of documents. List of server version and/or topology @@ -138,11 +175,13 @@ test(s). The structure of this document is as follows: - ``minServerVersion``: Optional string. The minimum server version (inclusive) required to successfully run the tests. If this field is omitted, it should be - assumed that there is no lower bound on the required server version. + assumed that there is no lower bound on the required server version. The + format of this string is defined in `Version String`_. - ``maxServerVersion``: Optional string. The maximum server version (inclusive) against which the tests can be run successfully. If this field is omitted, it - should be assumed that there is no upper bound on the required server version. + should be assumed that there is no upper bound on the required server version. + The format of this string is defined in `Version String`_. - ``topology``: Optional string or array of strings. One or more of server topologies against which the tests can be run successfully. Valid topologies @@ -161,12 +200,13 @@ which specifies a unique name for the entity (``id`` key) and any other parameters necessary for its construction. Tests SHOULD use sequential names based on the entity type (e.g. "session0", "session1"). -When defining an entity document in YAML, a -`node anchor `_ SHOULD be created -on the entity's ``id`` key. This anchor will allow the unique name to be -referenced with an `alias node `_ -later in the file (e.g. from another entity or `operation`_ document) and also -leverage YAML's parser for reference validation. +When defining an entity document in YAML, a `node anchor`_ SHOULD be created on +the entity's ``id`` key. This anchor will allow the unique name to be referenced +with an `alias node`_ later in the file (e.g. from another entity or +`operation`_ document) and also leverage YAML's parser for reference validation. + +.. _node anchor: https://yaml.org/spec/1.2/spec.html#id2785586 +.. _alias node: https://yaml.org/spec/1.2/spec.html#id2786196 The structure of this document is as follows: @@ -174,8 +214,7 @@ The structure of this document is as follows: structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD - define a `node anchor `_ - for this field (e.g. ``id: &client0 client0``). + define a `node anchor`_ for this field (e.g. ``id: &client0 client0``). - ``uriOptions``: Optional document. Additional URI options to apply to the test suite's connection string that is used to create this client. Any keys @@ -198,13 +237,11 @@ The structure of this document is as follows: structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD - define a `node anchor `_ - for this field (e.g. ``id: &database0 database0``). + define a `node anchor`_ for this field (e.g. ``id: &database0 database0``). - ``client``: Required string. Client entity from which this database will be - created. The YAML file SHOULD use an - `alias node `_ for a client - entity's ``id`` field (e.g. ``client: *client0``). + created. The YAML file SHOULD use an `alias node`_ for a client entity's + ``id`` field (e.g. ``client: *client0``). - ``databaseName``: Optional string. Database name. If omitted, this defaults to the name of the database under test (see: `databaseName`_). @@ -215,12 +252,11 @@ The structure of this document is as follows: structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD - define a `node anchor `_ - for this field (e.g. ``id: &collection0 collection0``). + define a `node anchor`_ for this field (e.g. + ``id: &collection0 collection0``). - ``database``: Required string. Database entity from which this collection - will be created. The YAML file SHOULD use an - `alias node `_ for a database + will be created. The YAML file SHOULD use an `alias node`_ for a database entity's ``id`` field (e.g. ``database: *database0``). - ``collectionName``: Optional string. Collection name. If omitted, this @@ -234,13 +270,11 @@ The structure of this document is as follows: object. The structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD - define a `node anchor `_ - for this field (e.g. ``id: &session0 session0``). + define a `node anchor`_ for this field (e.g. ``id: &session0 session0``). - ``client``: Required string. Client entity from which this session will be - created. The YAML file SHOULD use an - `alias node `_ for a client - entity's ``id`` field (e.g. ``client: *client0``). + created. The YAML file SHOULD use an `alias node`_ for a client entity's + ``id`` field (e.g. ``client: *client0``). - ``sessionOptions``: Optional document. Map of parameters to pass to `MongoClient.startSession <../source/sessions/driver-sessions.rst#startsession>`__ @@ -254,12 +288,10 @@ The structure of this document is as follows: structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD - define a `node anchor `_ - for this field (e.g. ``id: &bucket0 bucket0``). + define a `node anchor`_ for this field (e.g. ``id: &bucket0 bucket0``). - ``database``: Required string. Database entity from which this bucket will - be created. The YAML file SHOULD use an - `alias node `_ for a database + be created. The YAML file SHOULD use an `alias node`_ for a database entity's ``id`` field (e.g. ``database: *database0``). - ``bucketOptions``: Optional document. Additional options used to construct @@ -319,6 +351,8 @@ structure of each document is as follows: The array should contain at least one document. The structure of each document is defined in `operation`_. +.. _test_expectedEvents: + - ``expectedEvents``: Optional array of documents. List of events, which are expected to be observed in this order by running the operations. @@ -357,9 +391,8 @@ is as follows: - ``object``: Required string. Name of the object on which to perform the operation. This should correspond to either an `entity`_ name (for `Entity Test Operations`_) or "testRunner" (for `Special Test Operations`_). - If the object is an entity, The YAML file SHOULD use an - `alias node `_ for its ``id`` - field (e.g. ``object: *collection0``). + If the object is an entity, The YAML file SHOULD use an `alias node`_ for its + ``id`` field (e.g. ``object: *collection0``). .. _operation_arguments: @@ -410,9 +443,11 @@ structure of this document is as follows: response). - ``errorContains``: Optional string. A substring of the expected error message. + See `bulkWrite`_ for special considerations for BulkWriteExceptions. - ``errorCodeName``: Optional string. The expected "codeName" field in the - server-generated error response. + server-generated error response. See `bulkWrite`_ for special considerations + for BulkWriteExceptions. - ``errorLabelsContain``: Optional array of strings. A list of error label strings that the error is expected to have. @@ -498,8 +533,7 @@ The structure of these common options is as follows: .. _commonOptions_session: - ``session``: Optional string. Session entity which will be resolved to a - ClientSession object. The YAML file SHOULD use an - `alias node `_ for a session + ClientSession object. The YAML file SHOULD use an `alias node`_ for a session entity's ``id`` field (e.g. ``session: *session0``). .. _commonOptions_writeConcern: @@ -514,6 +548,17 @@ The structure of these common options is as follows: - ``wtimeoutMS``: Optional integer. +Version String +-------------- + +Version strings, which are used for `schemaVersion`_ and `runOn`_, MUST conform +to one of the following formats, where each component is an integer: + +- ``..`` +- ``.`` (``>`` is assumed to be zero) +- ```` (```` and ``>`` are assumed to be zero) + + Entity Test Operations ---------------------- @@ -606,6 +651,14 @@ write result may be matched with `expectedError_expectedResult`_. Because ``writeResult`` is optional for drivers to implement, such assertions should utilize the `$$unsetOrMatches`` operator. +Additionally, BulkWriteException is unique in that it aggregates one or more +server errors in its ``writeConcernError`` and ``writeErrors`` properties. +When test runners evaluate `expectedError`_ assertions for ``errorContains`` and +``errorCodeName``, they MUST examine the aggregated errors and consider any +match therein to satisfy the assertion(s). Drivers that concatenate all write +and write concern error messages into the BulkWriteException message MAY +optimize the check for ``errorContains`` by examining the concatenated message. + session ~~~~~~~ @@ -672,7 +725,7 @@ An example of this operation follows:: See also: - `Clearing Fail Points After Tests`_ -- `Ignoring APM Events for configureFailPoint`_ +- `Excluding configureFailPoint from APM`_ targetedFailPoint @@ -702,7 +755,7 @@ on the mongos server to which "session0" is pinned follows:: See also: - `Clearing Fail Points After Tests`_ -- `Ignoring APM Events for configureFailPoint`_ +- `Excluding configureFailPoint from APM`_ assertSessionTransactionState @@ -1049,9 +1102,8 @@ Syntax:: This operation is used for matching any value with the logical session ID of a `session entity `_. The value will refer to a unique name of a -session entity. The YAML file SHOULD use an -`alias node `_ for a session -entity's ``id`` field (e.g. ``session: *session0``). +session entity. The YAML file SHOULD use an `alias node`_ for a session entity's +``id`` field (e.g. ``session: *session0``). An example of this operator follows:: @@ -1200,19 +1252,22 @@ Server Fail Points ================== Many tests utilize the ``configureFailPoint`` command to trigger server-side -errors such as dropped connections or command errors. Tests refer to this by the -special `failPoint`_ or `targetedFailPoint`_ opertions. +errors such as dropped connections or command errors. Tests can configure fail +points using the special `failPoint`_ or `targetedFailPoint`_ opertions. This internal command is not documented in the MongoDB manual (pending `DOCS-10784`_); however, there is scattered documentation available on the -server wiki (`The failCommand Fail Point `__) -and employee blogs (e.g. `Intro to Fail Points `__, -`Testing Network Errors with MongoDB `__). -Documentation can also be gleaned from JIRA tickets (e.g. `SERVER-35004`_, -`SERVER-35083`_). This specification does not aim to provide comprehensive -documentation for all fail points available for driver testing, but some fail -points are documented in `Fail Points Commonly Used in Tests`_. - +server wiki (`The "failCommand" Fail Point `_) and employee blogs +(e.g. `Intro to Fail Points `_, +`Testing Network Errors with MongoDB `_). Documentation can +also be gleaned from JIRA tickets (e.g. `SERVER-35004`_, `SERVER-35083`_). This +specification does not aim to provide comprehensive documentation for all fail +points available for driver testing, but some fail points are documented in +`Fail Points Commonly Used in Tests`_. + +.. _failpoint-wiki: https://github.com/mongodb/mongo/wiki/The-%22failCommand%22-fail-point +.. _failpoint-blog1: https://kchodorow.com/2013/01/15/intro-to-fail-points/ +.. _failpoint-blog2: https://emptysqua.re/blog/mongodb-testing-network-errors/ .. _DOCS-10784: https://jira.mongodb.org/browse/DOCS-10784 .. _SERVER-35004: https://jira.mongodb.org/browse/SERVER-35004 .. _SERVER-35083: https://jira.mongodb.org/browse/SERVER-35083 @@ -1269,15 +1324,14 @@ A fail point may be disabled like so:: mode: "off" }); -Ignoring APM Events for configureFailPoint ------------------------------------------- +Excluding configureFailPoint from APM +------------------------------------- -Tests that use `failPoint`_ and `targetedFailPoint`_ operations SHOULD not -include ``configureFailPoint`` commands in their command expectations. Test -runners MUST ensure that ``configureFailPoint`` commands executed for +Test runners MUST ensure that ``configureFailPoint`` commands executed for `failPoint`_ and `targetedFailPoint`_ operations do not appear in the list of -logged commands, either by manually filtering it from the list of observed -commands or by using a different MongoClient to execute ``configureFailPoint``. +logged commands used to assert `test.expectedEvents `_, +either by manually filtering it from the list of observed commands or by using a +different MongoClient to execute ``configureFailPoint``. Fail Points Commonly Used in Tests @@ -1290,7 +1344,7 @@ failCommand The ``failCommand`` fail point allows the client to force the server to return an error for commands listed in the ``data.failCommands`` field. Additionally, this fail point is documented in server wiki: -`The failCommand Fail Point `_. +`The failCommand Fail Point `__. The ``failCommand`` fail point may be configured like so:: @@ -1337,11 +1391,31 @@ This specification was primarily derived from the test formats used by the `CRUD <../crud/crud.rst>`__ specs, which have served models or other specs. +Breaking Changes +================ + +This section is reserved for future use. Any breaking changes to the test format +should be described here in detail for historical reference, in addition to any +shorter description that may be added to the `Change Log`_. + + Change Log ========== Note: this will be cleared when publishing version 1.0 of the spec +2020-08-21: + +* clarify error assertions for BulkWriteException + +* note that YAML is the canonical format and discuss js-yaml + +* note that configureFailPoint must be excluded from APM + +* reformat external links to YAML spec and fail point docs + +* add schemaVersion field and document how the spec will handle versions + 2020-08-19: * added test.runOn and clarified that it can override top-level runOn requirements From c2089ca17251f9568a869bde499726369fc4d6cd Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 21 Aug 2020 20:17:01 -0400 Subject: [PATCH 04/90] Revise schema version and move allowMultipleMongoses to top-level --- .../unified-test-format.rst | 94 +++++++++++++------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 267830d27b..39917abe09 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -5,11 +5,11 @@ Unified Test Format :Spec Title: Unified Test Format :Spec Version: 1.0.0 :Author: Jeremy Mikola -:Advisors: Prashant Mital +:Advisors: Prashant Mital, Isabel Atkinson :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-08-15 +:Last Modified: 2020-08-21 .. contents:: @@ -38,15 +38,6 @@ Specification Schema Version -------------- -Each test file notes a `schemaVersion`_, which test runners will use to -determine compatibility (i.e. whether and how the test file will be -interpreted). Test runners are free to determine how to handle incompatible test -files (e.g. skip and log a notice, fail and raise an error). - -Each major version of this specification will have a corresponding JSON schema -(e.g. `schema-1.json `__), which may be used to programmatically -validate YAML and JSON files using a tool such as `Ajv `__. - This specification and the `Test Format`_ follow `semantic versioning `__. Backwards breaking changes (e.g. removing a field, introducing a required field) will warrant a new major @@ -54,6 +45,17 @@ version. Backwards compatible changes (e.g. introducing an optional field) will warrant a new minor version. Small bug fixes and internal changes (e.g. grammar) will warrant a new patch version. +Each test file defines a `schemaVersion`_, which test runners will use to +determine compatibility (i.e. whether and how the test file will be +interpreted). Test runners MAY support multiple versions of the test format. +Test runners MUST NOT process incompatible files but are otherwise free to +determine how to handle such files (e.g. skip and log a notice, fail and raise +an error). + +Each major version of this specification will have a corresponding JSON schema +(e.g. `schema-1.json `__), which may be used to programmatically +validate YAML and JSON files using a tool such as `Ajv `__. + The latest JSON schema MUST remain consistent with the `Test Format`_ section. If and when a new major version is introduced, the `Breaking Changes`_ section must be updated and JSON schema(s) for any previous major version(s) MUST remain @@ -134,6 +136,16 @@ The top-level fields of a test file are as follows: If set, the array should contain at least one document. The structure of each document is defined in `runOnRequirement`_. +.. _allowMultipleMongoses: + +- ``allowMultipleMongoses``: Optional boolean. If false, all MongoClients + created for this test file (both internal and `entities `_) + that could connect to a sharded cluster MUST be initialized with only a single + mongos host. Defaults to true. If true or the topology is non-sharded, this + option has no effect. Test files that include tests with a `failPoint`_ + operation that may run on sharded topologies MUST specify false for this + option. + .. _createEntities: - ``createEntities``: Optional array of documents. List of entities (e.g. @@ -160,6 +172,8 @@ The top-level fields of a test file are as follows: If set, the array should contain at least one document. The structure of each document is defined in `collectionData`_. +.. _tests: + - ``tests``: Required array of documents. List of test cases to be executed independently of each other. @@ -210,6 +224,8 @@ with an `alias node`_ later in the file (e.g. from another entity or The structure of this document is as follows: +.. _entity_client: + - ``client``: Optional document. Corresponds with a MongoClient object. The structure of this document is as follows: @@ -226,12 +242,7 @@ The structure of this document is as follows: will map to an array of strings, each representing a tag set, since it is not feasible to define multiple ``readPreferenceTags`` keys in the document. - .. _client_allowMultipleMongoses: - - - ``allowMultipleMongoses``: Optional boolean. If false, the MongoClient - MUST be initialized with only a single mongos host. If true or omitted, or - if the topology is non-sharded, this option has no effect. This option is - primarily used in conjunction with tests that set non-targeted fail points. +.. _entity_database: - ``database``: Optional document. Corresponds with a Database object. The structure of this document is as follows: @@ -248,6 +259,8 @@ The structure of this document is as follows: - ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. +.. _entity_collection: + - ``collection``: Optional document. Corresponds with a Collection object. The structure of this document is as follows: @@ -698,20 +711,26 @@ special test operations (e.g. assertions). These operations are distinguished by `operation.name `_ field will correspond to an operation defined below. +These operations MUST use an internal MongoClient + failPoint ~~~~~~~~~ -The ``failPoint`` operation instructs the test runner to configure a fail point. -The ``failPoint`` argument is the ``configureFailPoint`` command to run. Tests -using this operation SHOULD also specify -`allowMultipleMongoses `_ for any client -entity(ies) used in the test, as failure to do so could cause unpredictable -behavior when running the test on sharded topologies. +The ``failPoint`` operation instructs the test runner to configure a fail point +on the server selected with a ``primary`` read preference. The ``failPoint`` +argument is the ``configureFailPoint`` command to run. + +Test files using this operation MUST also specify false for +`allowMultipleMongoses`_ if they could be executed on sharded topologies +(according to `runOn`_ or `test.runOn `_). This is necessary +because server selection rules for mongos could lead to unpredictable behavior +if different servers were selected for configuring the fail point and executing +subsequent operations. An example of this operation follows:: - # Enable the fail point only on the mongos to which session0 is pinned + # Enable the fail point on the server selected with a primary read preference - name: failPoint object: testRunner arguments: @@ -727,6 +746,9 @@ See also: - `Clearing Fail Points After Tests`_ - `Excluding configureFailPoint from APM`_ +**TODO**: Consider supporting a readPreference argument to target nodes other +than the primary. + targetedFailPoint ~~~~~~~~~~~~~~~~~ @@ -738,9 +760,15 @@ runners MUST error if the session is not pinned to a mongos server at the time this operation is executed. The ``failPoint`` argument is the ``configureFailPoint`` command to run. -An example instructing the test runner to enable the `failCommand`_ fail point -on the mongos server to which "session0" is pinned follows:: +This operation SHOULD NOT be used in test files that specify false for +`allowMultipleMongoses`_ because session pinning cannot be meaningfully tested +without connecting to multiple mongos servers. In practice, this means that +`failPoint`_ and `targetedFailPoint`_ SHOULD NOT be utilized in the same test +file. + +An example of this operation follows:: + # Enable the fail point on the mongos to which session0 is pinned - name: targetedFailPoint object: testRunner arguments: @@ -871,6 +899,7 @@ currently not possible to run ``listIndexes`` from within a transaction. Otherwise, consider renaming the arguments to databaseName and collectionName as was done in `collectionData`_. + assertIndexNotExists ~~~~~~~~~~~~~~~~~~~~ @@ -898,8 +927,8 @@ Evaluating Matches ------------------ Expected values in tests (e.g. -`operation.expectedResult `_) are expressed as -`Extended JSON <../extended-json.rst>`__. +`operation.expectedResult `_) are expressed as either +relaxed or canonical `Extended JSON <../extended-json.rst>`_. The algorithm for matching expected and actual values is specified with the following pseudo-code:: @@ -936,7 +965,8 @@ subsequent sections. When comparing types *other* than documents and arrays, test runners MAY adopt any of the following approaches to compare expected and actual values, as long as they are consistent: -- Convert both values to Extended JSON and compare strings +- Convert both values to relaxed or canonical `Extended JSON`_ and compare + strings - Convert both values to BSON, and compare bytes - Convert both values to native representations, and compare accordingly @@ -1416,6 +1446,12 @@ Note: this will be cleared when publishing version 1.0 of the spec * add schemaVersion field and document how the spec will handle versions +* move client.allowMultipleMongoses to top-level option, since it should apply + to all clients (internal and entities). also note that targetedFailPoint and + failPoint should not be used in the same test file, since the latter requires + allowMultipleMongoses:false and would not provide meaningful test coverage of + mongos pinning for sessions. + 2020-08-19: * added test.runOn and clarified that it can override top-level runOn requirements From 27823444aa71167fb11c3a24e4a74dea39f28287 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 21 Aug 2020 20:56:31 -0400 Subject: [PATCH 05/90] Define terms and start test execution steps --- .../unified-test-format.rst | 97 +++++++++++++------ 1 file changed, 70 insertions(+), 27 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 39917abe09..73fd0dccd8 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -35,6 +35,22 @@ Specification ============= +Terms +----- + +Entity + Any object or value that is indexed by a unique name and stored in the + `Entity Map`_. This will typically be a driver object (e.g. client, session) + defined in `createEntities`_ but may also be a + `saved operation result `_. Entities are + referenced throughout the test file (e.g. `Entity Test Operations`_). + +Internal MongoClient + A MongoClient created specifically for use with internal test operations, such + as inserting collection data before a test, configuring fail points during a + test, or asserting collection data after a test. + + Schema Version -------------- @@ -139,7 +155,7 @@ The top-level fields of a test file are as follows: .. _allowMultipleMongoses: - ``allowMultipleMongoses``: Optional boolean. If false, all MongoClients - created for this test file (both internal and `entities `_) + created for this test file (internal and any ` entities `_) that could connect to a sharded cluster MUST be initialized with only a single mongos host. Defaults to true. If true or the topology is non-sharded, this option has no effect. Test files that include tests with a `failPoint`_ @@ -319,10 +335,8 @@ collectionData List of documents that should correspond to the contents of a collection. This structure is used by both `initialData`_ and `test.outcome `_, -which insert and read documents, respectively. Both of those directives may -operate on the collection(s) under test, they do not share the same Collection -and MongoClient object(s) as test `operations `_. The structure of -this document is as follows: +which insert and read documents, respectively. The structure of this document is +as follows: - ``collectionName``: Optional string. Collection name (not an `entity`_). Defaults to the name of the collection under test (see: `collectionName`_). @@ -711,15 +725,13 @@ special test operations (e.g. assertions). These operations are distinguished by `operation.name `_ field will correspond to an operation defined below. -These operations MUST use an internal MongoClient - failPoint ~~~~~~~~~ The ``failPoint`` operation instructs the test runner to configure a fail point -on the server selected with a ``primary`` read preference. The ``failPoint`` -argument is the ``configureFailPoint`` command to run. +using a ``primary`` read preference and the internal MongoClient. The +``failPoint`` argument is the ``configureFailPoint`` command to run. Test files using this operation MUST also specify false for `allowMultipleMongoses`_ if they could be executed on sharded topologies @@ -754,7 +766,7 @@ targetedFailPoint ~~~~~~~~~~~~~~~~~ The ``targetedFailPoint`` operation instructs the test runner to configure a -fail point on a specific mongos. The mongos on which to run the +fail point on a specific mongos. The MongoClient and mongos on which to run the ``configureFailPoint`` command is determined by the ``session`` argument. Test runners MUST error if the session is not pinned to a mongos server at the time this operation is executed. The ``failPoint`` argument is the @@ -1150,25 +1162,48 @@ in the `Command Monitoring <../command-monitoring/command-monitoring.rst#securit spec. -Running Tests -------------- +Test Runner Implementation +-------------------------- -Run a MongoDB replica set with a primary, a secondary, and an arbiter, -**server version 4.0.0 or later**. (Including a secondary ensures that -server selection in a transaction works properly. Including an arbiter helps -ensure that no new bugs have been introduced related to arbiters.) -A driver that implements support for sharded transactions MUST also run these -tests against a MongoDB sharded cluster with multiple mongoses and -**server version 4.2 or later**. Some tests require -initializing the MongoClient with multiple mongos seeds to ensures that mongos -transaction pinning and the recoveryToken works properly. +Configuring the Test Runner +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The test runner MUST be provided with a connection string (or equivalent +configuration), which will be used to initialize the internal MongoClient and +any `client entities `_ (in combination with other options). +This specification is not prescriptive about how this information is provided. +For example, it may be read from an environment variable or configuration file. -Load each YAML (or JSON) file using a Canonical Extended JSON parser. -Then for each element in ``tests``: +Loading a Test File +~~~~~~~~~~~~~~~~~~~ + +Test files, which may be YAML or JSON files, MUST be interpreted using an +`Extended JSON`_ parser. The parser MUST accept relaxed and canonical Extended +JSON, as test files may use either. + +Upon loading a file, the test runner MUST read the `schemaVersion`_ field and +determine if the test file can be processed further. Test runners MAY support +multiple versions and MUST NOT process incompatible files (as discussed in +`Schema Version`_). + +If the test file is compatible, the test runner shall proceed with determining +default names for the database and collection under test, which may be used by +`database `_ and `collection `_ entities +and `collectionData`_. The test runner MUST use the values from `databaseName`_ +and `collectionName`_ fields if set. If a field is omitted, the test runner MUST +generate a name. This spec is not prescriptive about the logic for doing so. + +Create a MongoClient, which will be used for internal operations (e.g. +`failPoint`_, processing `initialData`_ and `test.outcome `_). +This is referred to elsewhere in the specification as the internal MongoClient. + +For each element in ``tests``: + +#. If the ``skipReason`` field is present, the test runner MUST skip this test + and MAY use the string value to log a message. -#. If the ``skipReason`` field is present, skip this test completely. #. Create a MongoClient and call ``client.admin.runCommand({killAllSessions: []})`` to clean up any open transactions from previous test failures. Ignore a command failure with @@ -1359,9 +1394,10 @@ Excluding configureFailPoint from APM Test runners MUST ensure that ``configureFailPoint`` commands executed for `failPoint`_ and `targetedFailPoint`_ operations do not appear in the list of -logged commands used to assert `test.expectedEvents `_, -either by manually filtering it from the list of observed commands or by using a -different MongoClient to execute ``configureFailPoint``. +logged commands used to assert `test.expectedEvents `_. +This may require manually filtering ``configureFailPoint`` from the list of +observed commands (particularly in the case of `targetedFailPoint`_, which uses +a `client entity `_). Fail Points Commonly Used in Tests @@ -1452,6 +1488,13 @@ Note: this will be cleared when publishing version 1.0 of the spec allowMultipleMongoses:false and would not provide meaningful test coverage of mongos pinning for sessions. +* add terms and define Entity and Internal MongoClient + +* note that failPoint always uses internal MongoClient and targetedFailPoint + uses the client of its session argument + +* start writing steps for test execution + 2020-08-19: * added test.runOn and clarified that it can override top-level runOn requirements From ece67f2e97732d74db80c25ca93c0d8db1cc2baa Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 22 Aug 2020 13:34:58 -0400 Subject: [PATCH 06/90] Readability improvements and TODO items --- .../unified-test-format.rst | 134 +++++++++++------- 1 file changed, 83 insertions(+), 51 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 73fd0dccd8..c2b37cfac2 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -201,7 +201,9 @@ runOnRequirement ~~~~~~~~~~~~~~~~ A combination of server version and/or topology requirements for running the -test(s). The structure of this document is as follows: +test(s). + +The structure of this document is as follows: - ``minServerVersion``: Optional string. The minimum server version (inclusive) required to successfully run the tests. If this field is omitted, it should be @@ -224,11 +226,13 @@ entity ~~~~~~ An entity (e.g. client, collection, session object) that will be created in the -`Entity Map`_ before each test is executed. This document must contain exactly -one top-level key that identifies the entity type and maps to a nested document, -which specifies a unique name for the entity (``id`` key) and any other -parameters necessary for its construction. Tests SHOULD use sequential names -based on the entity type (e.g. "session0", "session1"). +`Entity Map`_ before each test is executed. + +This document MUST contain *exactly one* top-level key that identifies the +entity type and maps to a nested document, which specifies a unique name for the +entity (``id`` key) and any other parameters necessary for its construction. +Tests SHOULD use sequential names based on the entity type (e.g. "session0", +"session1"). When defining an entity document in YAML, a `node anchor`_ SHOULD be created on the entity's ``id`` key. This anchor will allow the unique name to be referenced @@ -242,8 +246,9 @@ The structure of this document is as follows: .. _entity_client: -- ``client``: Optional document. Corresponds with a MongoClient object. The - structure of this document is as follows: +- ``client``: Optional document. Corresponds with a MongoClient object. + + The structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &client0 client0``). @@ -260,8 +265,9 @@ The structure of this document is as follows: .. _entity_database: -- ``database``: Optional document. Corresponds with a Database object. The - structure of this document is as follows: +- ``database``: Optional document. Corresponds with a Database object. + + The structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &database0 database0``). @@ -277,8 +283,9 @@ The structure of this document is as follows: .. _entity_collection: -- ``collection``: Optional document. Corresponds with a Collection object. The - structure of this document is as follows: +- ``collection``: Optional document. Corresponds with a Collection object. + + The structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. @@ -296,7 +303,9 @@ The structure of this document is as follows: .. _entity_session: - ``session``: Optional document. Corresponds with an explicit ClientSession - object. The structure of this document is as follows: + object. + + The structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &session0 session0``). @@ -313,8 +322,9 @@ The structure of this document is as follows: - `Causal Consistency <../causal-consistency/causal-consistency.rst#sessionoptions-changes>`__ - `Transactions <../transactions/transactions.rst#sessionoptions-changes>`__ -- ``bucket``: Optional document. Corresponds with a GridFS Bucket object. The - structure of this document is as follows: +- ``bucket``: Optional document. Corresponds with a GridFS Bucket object. + + The structure of this document is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &bucket0 bucket0``). @@ -335,8 +345,9 @@ collectionData List of documents that should correspond to the contents of a collection. This structure is used by both `initialData`_ and `test.outcome `_, -which insert and read documents, respectively. The structure of this document is -as follows: +which insert and read documents, respectively. + +The structure of this document is as follows: - ``collectionName``: Optional string. Collection name (not an `entity`_). Defaults to the name of the collection under test (see: `collectionName`_). @@ -352,8 +363,9 @@ test ~~~~ Test case consisting of a sequence of operations to be executed. The test may -optionally include configuration directives and event/outcome assertions. The -structure of each document is as follows: +optionally include configuration directives and event/outcome assertions. + +The structure of each document is as follows: - ``description``: Required string. The name of the test. @@ -368,7 +380,7 @@ structure of each document is as follows: document is defined in `runOnRequirement`_. - ``skipReason``: Optional string. If set, the test will be skipped. The string - should explain the reason for skipping the test (e.g. JIRA ticket). + SHOULD explain the reason for skipping the test (e.g. JIRA ticket). .. _test_operations: @@ -391,6 +403,9 @@ structure of each document is as follows: whether tests should be able to filter out certain types (assuming the test runner observes any supported type). + **TODO**: Since event logging is configured per client, each sequence of event + expectations will need to target a specific client entity. + .. _test_outcome: - ``outcome``: Optional array of documents. Data that should exist in @@ -405,8 +420,9 @@ structure of each document is as follows: operation ~~~~~~~~~ -An operation to be executed as part of the test. The structure of this document -is as follows: +An operation to be executed as part of the test. + +The structure of this document is as follows: .. _operation_name: @@ -449,19 +465,23 @@ is as follows: `Entity Map`_. The test runner MUST raise an error if the name is already in use. - This is primarily used for change streams. + **TODO**: This is primarily used for change streams. Once an operation for + iterating a change stream is added, it should link to ``saveResultAsEntity`` + as this will be the only way to add a change stream object to the entity map. expectedError ~~~~~~~~~~~~~ One or more assertions for an error/exception, which is expected to be raised by -an executed operation. At least one key is required in this document. The -structure of this document is as follows: +an executed operation. At least one key is required in this document. + +The structure of this document is as follows: - ``type``: Optional string or array of strings. One or more classifications of - errors, at least one of which should apply to the expected error. Valid types - are as follows: + errors, at least one of which should apply to the expected error. + + Valid types are as follows: - ``client``: client-generated error (e.g. parameter validation error before a command is sent to the server). @@ -493,16 +513,21 @@ expectedEvent ~~~~~~~~~~~~~ An event (e.g. APM, SDAM), which is expected to be observed while executing -operations. This document must contain exactly one top-level key that identifies -the event type and maps to a nested document, which contains one or more -assertions for the event's properties. The structure of this document is as -follows: +operations. + +This document MUST contain *exactly one* top-level key that identifies the event +type and maps to a nested document, which contains one or more assertions for +the event's properties. + +The structure of this document is as follows: .. _expectedEvent_commandStartedEvent: - ``commandStartedEvent``: Optional document. Assertions for a one or more `CommandStartedEvent <../command-monitoring/command-monitoring.rst#api>`__ - fields. The structure of this document is as follows: + fields. + + The structure of this document is as follows: - ``command``: Optional document. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. @@ -515,8 +540,9 @@ follows: collectionOrDatabaseOptions ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Map of parameters used to construct a collection or database object. The -structure of this document is as follows: +Map of parameters used to construct a collection or database object. + +The structure of this document is as follows: - ``readConcern``: Optional document. See `commonOptions_readConcern`_. @@ -540,14 +566,18 @@ The structure of these common options is as follows: .. _commonOptions_readConcern: - ``readConcern``: Optional document. Map of parameters to construct a read - concern. The structure of this document is as follows: + concern. + + The structure of this document is as follows: - ``level``: Required string. .. _commonOptions_readPreference: - ``readPreference``: Optional document. Map of parameters to construct a read - preference. The structure of this document is as follows: + preference. + + The structure of this document is as follows: - ``mode``: Required string. @@ -566,7 +596,9 @@ The structure of these common options is as follows: .. _commonOptions_writeConcern: - ``writeConcern``: Optional document. Map of parameters to construct a write - concern. The structure of this document is as follows: + concern. + + The structure of this document is as follows: - ``journal``: Optional boolean. @@ -704,7 +736,7 @@ The ``withTransaction`` operation is unique in that its ``callback`` parameter is a function and not easily expressed in YAML/JSON. For ease of testing, this parameter is defined as an array of `operation`_ documents (analogous to `test.operations `_). Test runners MUST evaluate error and -result assertions when executing these operations. +result assertions when executing these operations in the callback. bucket @@ -730,15 +762,14 @@ failPoint ~~~~~~~~~ The ``failPoint`` operation instructs the test runner to configure a fail point -using a ``primary`` read preference and the internal MongoClient. The -``failPoint`` argument is the ``configureFailPoint`` command to run. +using a ``primary`` read preference and the internal MongoClient. -Test files using this operation MUST also specify false for -`allowMultipleMongoses`_ if they could be executed on sharded topologies -(according to `runOn`_ or `test.runOn `_). This is necessary -because server selection rules for mongos could lead to unpredictable behavior -if different servers were selected for configuring the fail point and executing -subsequent operations. +The ``failPoint`` argument is the ``configureFailPoint`` command to run. Test +files using this operation MUST also specify false for `allowMultipleMongoses`_ +if they could be executed on sharded topologies (according to `runOn`_ or +`test.runOn `_). This is necessary because server selection rules +for mongos could lead to unpredictable behavior if different servers were +selected for configuring the fail point and executing subsequent operations. An example of this operation follows:: @@ -766,11 +797,12 @@ targetedFailPoint ~~~~~~~~~~~~~~~~~ The ``targetedFailPoint`` operation instructs the test runner to configure a -fail point on a specific mongos. The MongoClient and mongos on which to run the -``configureFailPoint`` command is determined by the ``session`` argument. Test -runners MUST error if the session is not pinned to a mongos server at the time -this operation is executed. The ``failPoint`` argument is the -``configureFailPoint`` command to run. +fail point on a specific mongos. + +The MongoClient and mongos on which to run the ``configureFailPoint`` command is +determined by the ``session`` argument. Test runners MUST error if the session +is not pinned to a mongos server at the time this operation is executed. The +``failPoint`` argument is the ``configureFailPoint`` command to run. This operation SHOULD NOT be used in test files that specify false for `allowMultipleMongoses`_ because session pinning cannot be meaningfully tested From 16a0bb791e907c4104daa752500887f0750932c1 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 22 Aug 2020 14:01:59 -0400 Subject: [PATCH 07/90] Suggest YAML nodes and anchors for collection/database names --- .../unified-test-format.rst | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index c2b37cfac2..a966eaebf1 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -172,13 +172,17 @@ The top-level fields of a test file are as follows: - ``collectionName``: Optional string. Name of collection under test. This is primarily useful when the collection name must be referenced in an assertion. - If unset, test runners may use whatever value they prefer. + The YAML file SHOULD define a `node anchor`_ for this field (e.g. + ``collectionName: &collectionName foo``). If unset, test runners may use + whatever value they prefer. .. _databaseName: - ``databaseName``: Optional string. Name of database under test. This is primarily useful when the database name must be referenced in an assertion. - If unset, test runners may use whatever value they prefer. + The YAML file SHOULD define a `node anchor`_ for this field (e.g. + ``databaseName: &databaseName foo``). If unset, test runners may use whatever + value they prefer. .. _initialData: @@ -276,8 +280,10 @@ The structure of this document is as follows: created. The YAML file SHOULD use an `alias node`_ for a client entity's ``id`` field (e.g. ``client: *client0``). - - ``databaseName``: Optional string. Database name. If omitted, this defaults - to the name of the database under test (see: `databaseName`_). + - ``databaseName``: Optional string. Database name. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. + ``databaseName: &database0Name foo``). If omitted, this defaults to the name + of the database under test (see: `databaseName`_). - ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. @@ -295,8 +301,10 @@ The structure of this document is as follows: will be created. The YAML file SHOULD use an `alias node`_ for a database entity's ``id`` field (e.g. ``database: *database0``). - - ``collectionName``: Optional string. Collection name. If omitted, this - defaults to the name of the collection under test (see: `collectionName`_). + - ``collectionName``: Optional string. Collection name. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. + ``collectionName: &collection0Name foo``). If omitted, this defaults to the + name of the collection under test (see: `collectionName`_). - ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. @@ -349,11 +357,15 @@ which insert and read documents, respectively. The structure of this document is as follows: -- ``collectionName``: Optional string. Collection name (not an `entity`_). - Defaults to the name of the collection under test (see: `collectionName`_). +- ``collectionName``: Optional string. Collection name (not an `entity`_). The + YAML file SHOULD use an `alias node`_ for this value (e.g. + ``collectionName: *collection0Name``). Defaults to the name of the collection + under test (see: `collectionName`_). -- ``databaseName``: Optional string. Database name (not an `entity`_). Defaults - to the name of the database under test (see: `databaseName`_). +- ``databaseName``: Optional string. Database name (not an `entity`_). The + YAML file SHOULD use an `alias node`_ for this value (e.g. + ``databaseName: *database0Name``). Defaults to the name of the database under + test (see: `databaseName`_). - ``documents``: Required array of documents. List of documents corresponding to the contents of the collection. This list may be empty. From acde870a0e384ecedbea4b0d750804a465bbe68e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 22 Aug 2020 14:45:10 -0400 Subject: [PATCH 08/90] Fix link text and version formatting --- source/unified-test-format/unified-test-format.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index a966eaebf1..c7abdce0b8 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -155,7 +155,7 @@ The top-level fields of a test file are as follows: .. _allowMultipleMongoses: - ``allowMultipleMongoses``: Optional boolean. If false, all MongoClients - created for this test file (internal and any ` entities `_) + created for this test file (internal and any `entities `_) that could connect to a sharded cluster MUST be initialized with only a single mongos host. Defaults to true. If true or the topology is non-sharded, this option has no effect. Test files that include tests with a `failPoint`_ @@ -626,8 +626,8 @@ Version strings, which are used for `schemaVersion`_ and `runOn`_, MUST conform to one of the following formats, where each component is an integer: - ``..`` -- ``.`` (``>`` is assumed to be zero) -- ```` (```` and ``>`` are assumed to be zero) +- ``.`` (```` is assumed to be zero) +- ```` (```` and ```` are assumed to be zero) Entity Test Operations From 5e96571f6826f8412345a7a163ddb038577752db Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 22 Aug 2020 15:34:55 -0400 Subject: [PATCH 09/90] indent Server Fail Points and describe process for sharded-replicaset --- .../unified-test-format.rst | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index c7abdce0b8..e928da37fb 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -225,6 +225,10 @@ The structure of this document is as follows: cluster backed by replica sets). If this field is omitted, it should be assumed that there is no topology requirement for the test. + When matching a "sharded-replicaset" topology, test runners MUST ensure that + all shards are backed by a replica set. The process for doing so is described + in `Determining if a Sharded Cluster Uses Replica Sets`_. + entity ~~~~~~ @@ -1358,7 +1362,7 @@ For each element in ``tests``: Server Fail Points -================== +------------------ Many tests utilize the ``configureFailPoint`` command to trigger server-side errors such as dropped connections or command errors. Tests can configure fail @@ -1381,8 +1385,9 @@ points available for driver testing, but some fail points are documented in .. _SERVER-35004: https://jira.mongodb.org/browse/SERVER-35004 .. _SERVER-35083: https://jira.mongodb.org/browse/SERVER-35083 + Configuring Fail Points ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ The ``configureFailPoint`` command should be executed on the ``admin`` database and has the following structure:: @@ -1417,8 +1422,9 @@ the `setParameter `_. + Clearing Fail Points After Tests --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If a test configures one or more fail points, test runners MUST disable those fail points after running all `test.operations `_ to avoid @@ -1433,8 +1439,9 @@ A fail point may be disabled like so:: mode: "off" }); + Excluding configureFailPoint from APM -------------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Test runners MUST ensure that ``configureFailPoint`` commands executed for `failPoint`_ and `targetedFailPoint`_ operations do not appear in the list of @@ -1445,11 +1452,11 @@ a `client entity `_). Fail Points Commonly Used in Tests ----------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ failCommand -~~~~~~~~~~~ +``````````` The ``failCommand`` fail point allows the client to force the server to return an error for commands listed in the ``data.failCommands`` field. Additionally, @@ -1493,6 +1500,18 @@ if desired: blocked. New in server 4.3.4 (`SERVER-41070 `_). +Determining if a Sharded Cluster Uses Replica Sets +-------------------------------------------------- + +When connected to a mongos server, the test runner can query the +`config.shards `__ +collection. Each shard in the cluster is represented by a document in this +collection. If the shard is backed by a single servers, the ``host`` field will +contain a single host. If the shard is backed by a replica set, the ``host``For +field contain the name of the replica followed by a forward slash and a +comma-delimited list of hosts. + + Design Rationale ================ @@ -1514,6 +1533,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-08-22: + +* describe how to determine if sharded clusters use replica sets + 2020-08-21: * clarify error assertions for BulkWriteException From cf3193c0f441fec62286ce49fdcb8c525f4b4523 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Mon, 24 Aug 2020 02:16:43 -0400 Subject: [PATCH 10/90] Test implementation and various other clarifications --- .../unified-test-format.rst | 687 +++++++++++++----- 1 file changed, 488 insertions(+), 199 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index e928da37fb..c88a0bb4e7 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -90,7 +90,7 @@ will be driver objects created by the `createEntities`_ directive during test setup, but the entity map may also be modified during test execution via the `operation.saveResultAsEntity `_ directive. -Test runners may choose to implement the entity map in a fashion most suited to +Test runners MAY choose to implement the entity map in a fashion most suited to their language, but implementations MUST enforce both uniqueness of entity names and referential integrity when fetching an entity. Test runners MUST raise an error if an attempt is made to store an entity with a name that already exists @@ -152,6 +152,9 @@ The top-level fields of a test file are as follows: If set, the array should contain at least one document. The structure of each document is defined in `runOnRequirement`_. + **TODO**: Consider whether these requirements should allow an entire test file + to be skipped up front, instead of allowing per-test requirements to override. + .. _allowMultipleMongoses: - ``allowMultipleMongoses``: Optional boolean. If false, all MongoClients @@ -190,7 +193,9 @@ The top-level fields of a test file are as follows: collections before each test case is executed. If set, the array should contain at least one document. The structure of each - document is defined in `collectionData`_. + document is defined in `collectionData`_. Before each test and for each + `collectionData`_, the test runner MUST drop the collection and insert the + specified documents (if any) using a "majority" write concern. .. _tests: @@ -312,6 +317,20 @@ The structure of this document is as follows: - ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. + .. _entity_collection_createOnServer: + + - ``createOnServer``: Optional boolean. If true, when creating this entity the + test runner MUST also create the collection on the server using a "majority" + write concern. Defaults to false. + + This is primarily used when testing transactions, since collections cannot + be created within transactions. + + **TODO**: This may be redundant if we require initialData to ensure a + collection exists (i.e. create it explicitly if the list of documents would + be empty); however, if that is too obtuse to expect it to be used by + transaction tests then we can keep this as-is. + .. _entity_session: - ``session``: Optional document. Corresponds with an explicit ClientSession @@ -334,6 +353,10 @@ The structure of this document is as follows: - `Causal Consistency <../causal-consistency/causal-consistency.rst#sessionoptions-changes>`__ - `Transactions <../transactions/transactions.rst#sessionoptions-changes>`__ + When specifying TransactionOptions for ``defaultTransactionOptions``, the + transaction options MUST remain nested under ``defaultTransactionOptions`` + and MUST NOT be flattened into ``sessionOptions``. + - ``bucket``: Optional document. Corresponds with a GridFS Bucket object. The structure of this document is as follows: @@ -395,6 +418,8 @@ The structure of each document is as follows: If set, the array should contain at least one document. The structure of each document is defined in `runOnRequirement`_. +.. _test_skipReason: + - ``skipReason``: Optional string. If set, the test will be skipped. The string SHOULD explain the reason for skipping the test (e.g. JIRA ticket). @@ -408,26 +433,36 @@ The structure of each document is as follows: .. _test_expectedEvents: -- ``expectedEvents``: Optional array of documents. List of events, which are - expected to be observed in this order by running the operations. +- ``expectedEvents``: Optional array of documents. Each document will specify a + client entity and a list of events that are expected to be observed (in that + order) on that client while executing `operations `_. - The array should contain at least one document. The structure of each - document is defined in `expectedEvent`_. + The array should contain at least one document. The structure of each document + is as follows: + + - ``client``: Required string. Client entity on which the events are expected + to be observed. The YAML file SHOULD use an `alias node`_ for a client + entity's ``id`` field (e.g. ``client: *client0``). - **TODO**: Determine if an empty array should test that no events are observed. - Decide if event types (e.g. APM, SDAM) should be mixed in the same array and - whether tests should be able to filter out certain types (assuming the test - runner observes any supported type). + - ``events``: Required array of documents. List of events, which are expected + to be observed (in this order) on the corresponding client while executing + `operations`_. If the array is empty, the test runner MUST assert that no + events were observed on the client. - **TODO**: Since event logging is configured per client, each sequence of event - expectations will need to target a specific client entity. + The structure of each document is defined in `expectedEvent`_. + + **TODO**: Decide if event types (e.g. APM, SDAM) should be mixed in the same + array and whether tests should be able to filter out certain types (assuming + the test runner observes any supported type). This will not become relevant + until SDAM integration tests are implemented (not in initial project scope), + but we should anticipate the syntax now to avoid potential BC breaks. .. _test_outcome: -- ``outcome``: Optional array of documents. Data that should exist in - collections after all operations have been executed. The list of documents - should be sorted ascendingly by the ``_id`` field to allow for deterministic - comparisons. +- ``outcome``: Optional array of documents. Each document will specify expected + contents of a collection after all operations have been executed. The list of + documents therein SHOULD be sorted ascendingly by the ``_id`` field to allow + for deterministic comparisons. If set, the array should contain at least one document. The structure of each document is defined in `collectionData`_. @@ -459,18 +494,25 @@ The structure of this document is as follows: operation. The structure of this document will vary based on the operation. See `Entity Test Operations`_ and `Special Test Operations`_. + The ``session`` parameter is handled specially (see `commonOptions_session`_). + .. _operation_expectedError: - ``expectedError``: Optional document. One or more assertions for an expected error raised by the operation. The structure of this document is defined in `expectedError`_. + This field is mutually exclusive with + `expectedResult `_ and + `saveResultAsEntity `_. + .. _operation_expectedResult: - ``expectedResult``: Optional mixed type. A value corresponding to the expected result of the operation. This field may be a scalar value, a single document, or an array of documents in the case of a multi-document read. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. + This field is mutually exclusive with `expectedError `_. @@ -478,8 +520,13 @@ The structure of this document is as follows: - ``saveResultAsEntity``: Optional string. If specified, the actual result returned by the operation (if any) will be saved with this name in the - `Entity Map`_. The test runner MUST raise an error if the name is already in - use. + `Entity Map`_. The test runner MUST raise an error if the name is already in + use. If the operation does not return a value (e.g. void method), the test + runner MAY choose to store an empty value (e.g. ``null``) or do nothing and + leave the entity name undefined. + + This field is mutually exclusive with + `expectedError `_. **TODO**: This is primarily used for change streams. Once an operation for iterating a change stream is added, it should link to ``saveResultAsEntity`` @@ -506,29 +553,41 @@ The structure of this document is as follows: response). - ``errorContains``: Optional string. A substring of the expected error message. + The test runner MUST assert that the error message contains this string using + a case-insensitive match. + See `bulkWrite`_ for special considerations for BulkWriteExceptions. - ``errorCodeName``: Optional string. The expected "codeName" field in the - server-generated error response. See `bulkWrite`_ for special considerations - for BulkWriteExceptions. + server-generated error response. The test runner MUST assert that the error + includes a server-generated response whose "codeName" field equals this value + using a case-insensitive comparison. + + See `bulkWrite`_ for special considerations for BulkWriteExceptions. - ``errorLabelsContain``: Optional array of strings. A list of error label - strings that the error is expected to have. + strings that the error is expected to have. The test runner MUST assert that + the error contains all of the specified labels (e.g. using the + ``hasErrorLabel`` method). - ``errorLabelsOmit``: Optional array of strings. A list of error label strings - that the error is expected not to have. + that the error is expected not to have. The test runner MUST assert that + the error does not contain any of the specified labels (e.g. using the + ``hasErrorLabel`` method). .. _expectedError_expectedResult: - ``expectedResult``: Optional mixed type. This field follows the same rules as `operation.expectedResult `_ and is only used in - cases where the error includes a result (e.g. `bulkWrite`_). + cases where the error includes a result (e.g. `bulkWrite`_). If specified, the + test runner MUST assert that the error includes a result and that it matches + this value. expectedEvent ~~~~~~~~~~~~~ -An event (e.g. APM, SDAM), which is expected to be observed while executing +An event (e.g. APM), which is expected to be observed while executing the test's operations. This document MUST contain *exactly one* top-level key that identifies the event @@ -548,9 +607,38 @@ The structure of this document is as follows: - ``command``: Optional document. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. - - ``commandName``: Optional string. + - ``commandName``: Optional string. Test runners MUST assert that the actual + command name matches this value using a case-insensitive comparison. + + - ``databaseName``: Optional string. Test runners MUST assert that the actual + command name matches this value using a case-insensitive comparison. THe + YAML file SHOULD use an `alias node`_ for this value (e.g. + ``databaseName: *database0Name``). + +.. _expectedEvent_commandSucceededEvent: + +- ``commandSucceededEvent``: Optional document. Assertions for a one or more + `CommandSucceededEvent <../command-monitoring/command-monitoring.rst#api>`__ + fields. + + The structure of this document is as follows: + + - ``reply``: Optional document. Test runners MUST follow the rules in + `Evaluating Matches`_ when processing this assertion. + + - ``commandName``: Optional string. Test runners MUST assert that the actual + command name matches this value using a case-insensitive comparison. - - ``databaseName``: Optional string. +.. _expectedEvent_commandFailedEvent: + +- ``commandFailedEvent``: Optional document. Assertions for a one or more + `CommandFailedEvent <../command-monitoring/command-monitoring.rst#api>`__ + fields. + + The structure of this document is as follows: + + - ``commandName``: Optional string. Test runners MUST assert that the actual + command name matches this value using a case-insensitive comparison. collectionOrDatabaseOptions @@ -637,20 +725,33 @@ to one of the following formats, where each component is an integer: Entity Test Operations ---------------------- -Most operations correspond to an API method on a driver object. If +Entity operations correspond to an API method on a driver object. If `operation.object `_ refers to an `entity`_ name (e.g. "collection0") then `operation.name `_ is expected to reference -an API method on that class. Required and optional parameters for API methods -are both specified directly within `operation.arguments `_ -(e.g. ``upsert`` for ``updateOne`` is *not* nested under an ``options`` key). +an API method on that class. + +Some specifications group optional parameters for API methods under an +``options`` parameter (e.g. ``options: Optional`` in the CRUD +spec). While this improves readability of the spec document(s) by allowing +option documentation to be expanded and reused, it would add unnecessary +verbosity to test files. Therefore, test files SHALL declare all required and +optional parameters for an API method directly within +`operation.arguments `_ (e.g. ``upsert`` for ``updateOne`` +is *not* nested under an ``options`` key), unless otherwise stated below. + +If ``session`` is specified in ``operation.arguments`_, it is defined according +to `commonOptions_session`_. Test runners MUST resolve the ``session`` argument +to session entity *before* passing it as a parameter to any API method. This spec does not provide exhaustive documentation for all possible API methods that may appear in a test; however, the following sections discuss all supported -entities and their operations in some level of detail. +entities and their operations in some level of detail. Special handling for +certain operations is also discussed below. -**TODO**: While CRUD methods tend to flatten options into ``arguments``, session -methods often leave those options nested within an ``options`` key. We should -pick one of these conventions for consistency. +**TODO**: While CRUD methods tend to flatten options into ``arguments`` +(possibly for consistency with bulk write models, which omit ``options`` keys), +session methods often leave those options nested within an ``options`` key. We +should pick one of these conventions for consistency. client @@ -699,7 +800,7 @@ The following arguments are supported: - ``readPreference``: Optional document. See `commonOptions_readPreference`_. -- ``session``: Optional string. See `commonOptions_session`_. +- ``session``: Optional string. See `commonOptions_session`_. - ``writeConcern``: Optional document. See `commonOptions_writeConcern`_. @@ -715,10 +816,40 @@ specifications: - `Enumerating Indexes <../enumerate-indexes.rst>`__ - `Index Management <../index-management.rst>`__ +Collection operations that require special handling are described below. + bulkWrite ````````` +The ``requests`` parameter for ``bulkWrite`` is documented as a list of +WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) +provides important context to the method, but that type information is not +easily expressed in YAML and JSON. To account for this, test files MUST nest +each WriteModel document in a single-key document, where the key identifies the +request type (e.g. "insertOne"), as in the following example:: + + arguments: + requests: + - insertOne: + document: { _id: 1, x: 1 } + - replaceOne: + filter: { _id: 2 } + replacement: { x: 2 } + upsert: true + - updateOne: + filter: { _id: 3 } + update: { $set: { x: 3 } } + upsert: true + - updateMany: + filter: { } + update: { $inc: { x: 1 } } + - deleteOne: + filter: { x: 2 } + - deleteMany: + filter: { x: { $gt: 2 } } + ordered: true + While operations typically raise an error *or* return a result, the ``bulkWrite`` operation is unique in that it may report both via the ``writeResult`` property of a BulkWriteException. In this case, the intermediary @@ -733,6 +864,11 @@ When test runners evaluate `expectedError`_ assertions for ``errorContains`` and match therein to satisfy the assertion(s). Drivers that concatenate all write and write concern error messages into the BulkWriteException message MAY optimize the check for ``errorContains`` by examining the concatenated message. +Drivers that expose ``code`` but not ``codeName`` through BulkWriteException MAY +translate the expected code name to a number (see: +`error_codes.yml `_) +and compare with ``code`` instead, but MUST raise an error if the comparison +cannot be attempted (e.g. ``code`` is also not available, translation fails). session @@ -744,6 +880,8 @@ specifications: - `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ - `Driver Sessions <../sessions/driver-sessions.rst>`__ +Session operations that require special handling are described below. + withTransaction ``````````````` @@ -780,12 +918,20 @@ failPoint The ``failPoint`` operation instructs the test runner to configure a fail point using a ``primary`` read preference and the internal MongoClient. -The ``failPoint`` argument is the ``configureFailPoint`` command to run. Test -files using this operation MUST also specify false for `allowMultipleMongoses`_ -if they could be executed on sharded topologies (according to `runOn`_ or -`test.runOn `_). This is necessary because server selection rules -for mongos could lead to unpredictable behavior if different servers were -selected for configuring the fail point and executing subsequent operations. +The following arguments are supported: + +- ``failPoint``: Required document. The ``configureFailPoint`` command to be + executed. + +Test files using this operation MUST also specify false for +`allowMultipleMongoses`_ if they could be executed on sharded topologies +(according to `runOn`_ or `test.runOn `_). This is necessary +because server selection rules for mongos could lead to unpredictable behavior +if different servers were selected for configuring the fail point and executing +subsequent operations. + +When executing this operation, the test runner MUST keep a record of the fail +point so that it can be disabled after the test. An example of this operation follows:: @@ -800,13 +946,9 @@ An example of this operation follows:: failCommands: ["insert"] closeConnection: true -See also: - -- `Clearing Fail Points After Tests`_ -- `Excluding configureFailPoint from APM`_ - **TODO**: Consider supporting a readPreference argument to target nodes other -than the primary. +than the primary. If this is not needed it could be deferred to a future spec +version. targetedFailPoint @@ -815,10 +957,17 @@ targetedFailPoint The ``targetedFailPoint`` operation instructs the test runner to configure a fail point on a specific mongos. +The following arguments are supported: + +- ``failPoint``: Required document. The ``configureFailPoint`` command to be + executed. + +- ``session``: Required string. See `commonOptions_session`_. + The MongoClient and mongos on which to run the ``configureFailPoint`` command is -determined by the ``session`` argument. Test runners MUST error if the session -is not pinned to a mongos server at the time this operation is executed. The -``failPoint`` argument is the ``configureFailPoint`` command to run. +determined by the ``session`` argument (after resolution to a session entity). +Test runners MUST error if the session is not pinned to a mongos server at the +time this operation is executed. This operation SHOULD NOT be used in test files that specify false for `allowMultipleMongoses`_ because session pinning cannot be meaningfully tested @@ -826,6 +975,12 @@ without connecting to multiple mongos servers. In practice, this means that `failPoint`_ and `targetedFailPoint`_ SHOULD NOT be utilized in the same test file. +When executing this operation, the test runner MUST keep a record of both the +fail point and session so that the fail point can be disabled after the test. +The test runner MUST also ensure that the ``configureFailPoint`` command is +omitted (or otherwise filtered out) from the list of observed command monitoring +events for this client. + An example of this operation follows:: # Enable the fail point on the mongos to which session0 is pinned @@ -840,10 +995,15 @@ An example of this operation follows:: failCommands: ["commitTransaction"] closeConnection: true -See also: - -- `Clearing Fail Points After Tests`_ -- `Excluding configureFailPoint from APM`_ +**TODO**: `allowMultipleMongoses`_ only affects the test runner if it is false +(i.e. a connection string to a sharded cluster is reduce to one mongos). The +transaction spec hinted that ``useMultipleMongoses: true`` would force a +connection string to contain multiple mongos servers, but that seems untenable +since single-mongos and standalone connection strings are indistinguishable from +one another. Should we change `allowMultipleMongoses`_ to actively require +multiple mongos servers if true (omitted would still be a no-op)? That may be +possible if we defer enforcement until after the topology is determined at +runtime. assertSessionTransactionState @@ -1030,6 +1190,12 @@ actual values, as long as they are consistent: - Convert both values to BSON, and compare bytes - Convert both values to native representations, and compare accordingly +When comparing numeric types (excluding Decimal128), test runners MUST consider +32-bit, 64-bit, and floating point numbers to be equal if their values are +numerically equivalent. For example, an expected value of ``1`` would match an +actual value of ``1.0`` (e.g. ``ok`` field in a server response) but would not +match ``1.5``. + When comparing types that may contain documents (e.g. CodeWScope), test runners MUST follow standard document matching rules when comparing those properties. @@ -1202,30 +1368,40 @@ An example of this operator follows:: lsid: { $$sessionLsid: *session0 } -TBD: Command Started Events ---------------------------- - -The event listener used for these tests MUST ignore the security commands listed -in the `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ -spec. - - Test Runner Implementation -------------------------- +The sections below describe instructions for instantiating the test runner, +loading each test file, and executing each test within a test file. Test runners +SHOULD NOT share state created by processing a test file with the processing of +subsequent test files, and likewise for tests within a test file. + Configuring the Test Runner ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The test runner MUST be provided with a connection string (or equivalent +The instructions in this section apply once for the test runner. + +The test runner MUST be configurable with a connection string (or equivalent configuration), which will be used to initialize the internal MongoClient and -any `client entities `_ (in combination with other options). +any `client entities `_ (in combination with other URI options). This specification is not prescriptive about how this information is provided. For example, it may be read from an environment variable or configuration file. +The test runner MAY determine the server version and topology type in advance to +optimize any future `runOnRequirement`_ checks. -Loading a Test File -~~~~~~~~~~~~~~~~~~~ +The test runner SHOULD terminate any open transactions (see: +`Terminating Open Transactions`_) before executing any tests. + + +Executing a Test File +~~~~~~~~~~~~~~~~~~~~~ + +The instructions in this section apply for each test file loaded by the test +runner. After processing a test file, test runners SHOULD reset any internal +state that resulted from doing so. For example, an internal MongoClient created +for one test file SHOULD NOT be shared with another. Test files, which may be YAML or JSON files, MUST be interpreted using an `Extended JSON`_ parser. The parser MUST accept relaxed and canonical Extended @@ -1236,131 +1412,241 @@ determine if the test file can be processed further. Test runners MAY support multiple versions and MUST NOT process incompatible files (as discussed in `Schema Version`_). -If the test file is compatible, the test runner shall proceed with determining +If the test file is compatible, the test runner SHALL proceed with determining default names for the database and collection under test, which may be used by `database `_ and `collection `_ entities and `collectionData`_. The test runner MUST use the values from `databaseName`_ and `collectionName`_ fields if set. If a field is omitted, the test runner MUST generate a name. This spec is not prescriptive about the logic for doing so. -Create a MongoClient, which will be used for internal operations (e.g. -`failPoint`_, processing `initialData`_ and `test.outcome `_). -This is referred to elsewhere in the specification as the internal MongoClient. - -For each element in ``tests``: - -#. If the ``skipReason`` field is present, the test runner MUST skip this test - and MAY use the string value to log a message. - -#. Create a MongoClient and call - ``client.admin.runCommand({killAllSessions: []})`` to clean up any open - transactions from previous test failures. Ignore a command failure with - error code 11601 ("Interrupted") to work around `SERVER-38335`_. - - - Running ``killAllSessions`` cleans up any open transactions from - a previously failed test to prevent the current test from blocking. - It is sufficient to run this command once before starting the test suite - and once after each failed test. - - When testing against a sharded cluster run this command on ALL mongoses. - -#. Create a collection object from the MongoClient, using the ``database_name`` - and ``collection_name`` fields of the YAML file. -#. Drop the test collection, using writeConcern "majority". -#. Execute the "create" command to recreate the collection, using writeConcern - "majority". (Creating the collection inside a transaction is prohibited, so - create it explicitly.) -#. If the YAML file contains a ``data`` array, insert the documents in ``data`` - into the test collection, using writeConcern "majority". -#. When testing against a sharded cluster run a ``distinct`` command on the - newly created collection on all mongoses. For an explanation see, - Why do tests that run distinct sometimes fail with StaleDbVersion? -#. If ``failPoint`` is specified, its value is a configureFailPoint command. - Run the command on the admin database to enable the fail point. -#. Create a **new** MongoClient ``client``, with Command Monitoring listeners - enabled. (Using a new MongoClient for each test ensures a fresh session pool - that hasn't executed any transactions previously, so the tests can assert - actual txnNumbers, starting from 1.) Pass this test's ``clientOptions`` if - present. - - - When testing against a sharded cluster and ``useMultipleMongoses`` is - ``true`` the client MUST be created with multiple (valid) mongos seed - addreses. - -#. Call ``client.startSession`` twice to create ClientSession objects - ``session0`` and ``session1``, using the test's "sessionOptions" if they - are present. Save their lsids so they are available after calling - ``endSession``, see `Logical Session Id`. -#. For each element in ``operations``: - - - If the operation ``name`` is a special test operation type, execute it and - go to the next operation, otherwise proceed to the next step. - - Enter a "try" block or your programming language's closest equivalent. - - Create a Database object from the MongoClient, using the ``database_name`` - field at the top level of the test file. - - Create a Collection object from the Database, using the - ``collection_name`` field at the top level of the test file. - If ``collectionOptions`` or ``databaseOptions`` is present, create the - Collection or Database object with the provided options, respectively. - Otherwise create the object with the default options. - - Execute the named method on the provided ``object``, passing the - arguments listed. Pass ``session0`` or ``session1`` to the method, - depending on which session's name is in the arguments list. - If ``arguments`` contains no "session", pass no explicit session to the - method. - - If the driver throws an exception / returns an error while executing this - series of operations, store the error message and server error code. - - If the operation's ``error`` field is ``true``, verify that the method - threw an exception or returned an error. - - If the result document has an "errorContains" field, verify that the - method threw an exception or returned an error, and that the value of the - "errorContains" field matches the error string. "errorContains" is a - substring (case-insensitive) of the actual error message. - - If the result document has an "errorCodeName" field, verify that the - method threw a command failed exception or returned an error, and that - the value of the "errorCodeName" field matches the "codeName" in the - server error response. - - If the result document has an "errorLabelsContain" field, verify that the - method threw an exception or returned an error. Verify that all of the - error labels in "errorLabelsContain" are present in the error or exception - using the ``hasErrorLabel`` method. - - If the result document has an "errorLabelsOmit" field, verify that the - method threw an exception or returned an error. Verify that none of the - error labels in "errorLabelsOmit" are present in the error or exception - using the ``hasErrorLabel`` method. - - If the operation returns a raw command response, eg from ``runCommand``, - then compare only the fields present in the expected result document. - Otherwise, compare the method's return value to ``result`` using the same - logic as the CRUD Spec Tests runner. - -#. Call ``session0.endSession()`` and ``session1.endSession``. -#. If the test includes a list of command-started events in ``expectations``, - compare them to the actual command-started events using the - same logic as the Command Monitoring Spec Tests runner, plus the rules in - the Command-Started Events instructions below. -#. If ``failPoint`` is specified, disable the fail point to avoid spurious - failures in subsequent tests. The fail point may be disabled like so:: +If the test file specifies false for `allowMultipleMongoses`_ and the test +runner was configured with a connection string (or equivalent configuration) +that references multiple mongos servers in its host list, the test runner MUST +modify the host list used by this test file to only refer to a single mongos +server. + +Create a new MongoClient, which will be used for internal operations in this +test file (e.g. `failPoint`_, processing `initialData`_ and +`test.outcome `_). This is referred to elsewhere in the +specification as the internal MongoClient. Internal MongoClients SHOULD NOT be +shared across test files. + +If `runOn`_ is specified, the test runner MUST skip the test file unless one or +more `runOnRequirement`_ documents are satisfied. + +**TODO**: Confirm whether top-level `runOn`_ can be used to skip an entire file +without considering test-level requirements. If so, test-level requirements can +only be more restrictive. + +For each element in `tests`_, follow the process in `Executing a Test`_. + + +Executing a Test +~~~~~~~~~~~~~~~~ + +The instructions in this section apply for each `test`_ occuring in a test file +loaded by the test runner. After processing a test, test runners SHOULD reset +any internal state that resulted from doing so. For example, the entity map +created for one test SHOULD NOT be shared with another. + +If at any point while executing this test an unexpected error is encountered or +an assertion fails, the test runner MUST consider this test to have failed and +SHOULD continue with the instructions in this section to ensure that the test +environment is cleaned up (e.g. disable fail points, kill sessions) while also +forgoing any additional assertions. + +If `test.skipReason `_ is specified, the test runner MUST skip +this test and MAY use the string value to log a message. + +If `test.runOn `_ is specified, the test runner MUST skip the test +unless one or more `runOnRequirement`_ documents are satisfied. + +If `initialData`_ is specified, for each `collectionData`_ therein the test +runner MUST drop the collection using a "majority" write concern and then insert +the specified documents, also using a "majority" write concern. The test runner +MUST use the internal MongoClient for these operations. + +Create a new `Entity Map`_ that will be used for this test. If `createEntities`_ +is specified, the test runner MUST create each `entity`_ accordingly and add it +to the map. + +For any collection entities that specify true for +`createOnServer `_, the test runner MUST +create those collections on the server using a "majority" write concern. Test +runners SHOULD ignore any NamespaceExists (code 28) errors when doing so. As an +optimization, test runners MAY skip this step for any collections that were +created as a result of inserting documents during processing of `initialData`_. + +If the test might execute a ``distinct`` command within a transaction on a +sharded cluster, the test runner SHOULD execute a non-transactional ``distinct`` +command on each collection entity created. See +`StaleDbVersion Errors on Sharded Clusters`_ for more information. Test runners +MAY combine this step with handling for `createOnServer`_, since both are +primarily relevant to transactions. + +If `test.expectedEvents `_ is specified, the test runner +MUST enable command monitoring for any client entities so that events can be +observed while running operations and later used to assert expected events. Test +runners MAY leave command monitoring disabled for tests that do not assert +expected events. + +Test runners MUST ensure that ``configureFailPoint`` commands executed for +`failPoint`_ and `targetedFailPoint`_ operations are excluded from the list of +observed events. This may require manually filtering out ``configureFailPoint`` +from the list of observed commands (particularly in the case of +`targetedFailPoint`_, which uses a `client entity `_). Test +runners MUST also ensure that any commands containing sensitive information are +excluded (per the +`Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ +spec). + +For each element in `test.operations `_, follow the process +in `Executing an Operation`_. If an unexpected error is encountered or an +assertion fails, the test runner MUST consider this test to have failed. + +If command monitoring was enabled on any client entities, the test runner MUST +now disable command monitoring on those clients and, for each client, collect +any events observed thus far. + +If the fail points were configured, the test runner MUST now disable those fail +points to avoid spurious failures in subsequent tests. For any fail points +configured using `targetedFailPoint`_, the test runner MUST disable the fail +point on the same mongos server on which it was originally configured. See +`Disabling Fail Points`_ for more information. + +If `test.expectedEvents `_ is specified, for each document +therein the test runner MUST assert that the number and sequence of expected +events match the number and sequence of actual events observed on the specified +client. If the list of expected events is empty, the test runner MUST assert +that no events were observed on the client. The process for matching events is +described in `expectedEvent`_. The test runner MUST NOT consider any +``configureFailPoint`` + +If `test.outcome `_ is specified, for each `collectionData`_ +therein the test runner MUST assert that the collection contains exactly the +expected data. The test runner MUST query each collection using an ascending +sort order on the ``_id`` field (i.e. ``{ _id: 1 }``), a ``primary`` read +preference, a ``local`` read concern, and the internal MongoClient. If the list +of documents is empty, the test runner MUST assert that the collection is empty. + +Clear the entity map for this test. For each ClientSession in the entity map, +the test runner MUST end the session (e.g. call +`endSession <../sessions/driver-sessions.rst#endsession>`_). + +If the test failed, the test runner MUST terminate any open transactions (see: +`Terminating Open Transactions`_). + +Proceed to the subsequent test. + + +Executing an Operation +~~~~~~~~~~~~~~~~~~~~~~ + +The instructions in this section apply for each `operation`_ occuring in a +`test`_ contained within a test file loaded by the test runner. + +If at any point while executing an operation an unexpected error is encountered +or an assertion fails, the test runner MUST consider the parent test to have +failed and proceed from `Executing a Test`_ accordingly. + +If `operation.object `_ is "testRunner", this is a special +operation. If `operation.name `_ is defined in +`Special Test Operations`_, the test runner MUST execute the operation +accordingly and, if successful, proceed to the next operation in the test; +otherwise, the test runner MUST raise an error for an undefined operation. The +test runner MUST keep a record of any fail points configured by special +operations so that they may be disabled after the current test. + +If `operation.object`_ is not "testRunner", this is an entity operation. If +`operation.object`_ is defined in the current test's `Entity Map`_, the test +runner MUST fetch that entity and note its type; otherwise, the test runner +MUST raise an error for an undefined entity. If `operation.name`_ does not +correspond to an operation for the entity type (per `Entity Test Operations`_), +the test runner MUST raise an error for an undefined operation. + +Proceed with preparing the operation's arguments. If ``session`` is specified in +`operation.arguments `_, the test runner MUST resolve it +to a session entity and MUST raise an error if the name is undefined or maps to +an unexpected type. + +Proceed with executing the operation such that its return value (if any) potential error can be caught +(e.g. using a ``try`` block). + +Before executing the operation, the test runner MUST be prepared to catch a +potential error from the operation (e.g. enter a ``try`` block). Proceed with +executing the operation and capture its result or error. + +If `operation.expectedError `_ is specified, the test +runner MUST assert that the operation yielded an error; otherwise, the test +runner MUST assert that the operation did not yield an error. If an error was +expected, the test runner MUST evaluate any assertions in `expectedError`_ +accordingly. + +If `operation.expectedResult `_ is specified, the test +MUST assert that it matches the actual result of the operation according to the +rules outlined in `Evaluating Matches`_. + +If `operation.saveResultAsEntity `_ is specified, +the test runner MUST store the result (if any) in the current test's entity map +using the specified name. If the operation did not return a result (e.g. +``void`` method), the test runner MAY decide to store an empty value (e.g. +``null``) or do nothing and leave the entity name undefined. + +After asserting the operation's error and/or result and optionally saving the +result, proceed to the subsequent operation. + + +Special Procedures +~~~~~~~~~~~~~~~~~~ + +This section describes some procedures that may be referenced from earlier +sections. + + +Terminating Open Transactions +````````````````````````````` + +Open transactions can cause tests to block indiscriminately. Test runners SHOULD +terminate all open transactions at the start of a test suite and after each +failed test by killing all sessions in the cluster. Using the internal +MongoClient, execute the ``killAllSessions`` command on either the primary or, +if connected to a sharded cluster, all mongos servers. + +For example:: db.adminCommand({ - configureFailPoint: , - mode: "off" + killAllSessions: [] }); -#. For each element in ``outcome``: - - - If ``name`` is "collection", verify that the test collection contains - exactly the documents in the ``data`` array. Ensure this find reads the - latest data by using **primary read preference** with - **local read concern** even when the MongoClient is configured with - another read preference or read concern. - Note the server does not guarantee that documents returned by a find - command will be in inserted order. This find MUST sort by ``{_id:1}``. +The test runner MAY ignore any command failure with error Interrupted(11601) to +work around `SERVER-38335`_. .. _SERVER-38335: https://jira.mongodb.org/browse/SERVER-38335 +StaleDbVersion Errors on Sharded Clusters +````````````````````````````````````````` + +When a shard receives its first command that contains a ``databaseVersion``, the +shard returns a StaleDbVersion error and mongos retries the operation. In a +sharded transaction, mongos does not retry these operations and instead returns +the error to the client. For example:: + + Command distinct failed: Transaction aa09e296-472a-494f-8334-48d57ab530b6:1 was aborted on statement 0 due to: an error from cluster data placement change :: caused by :: got stale databaseVersion response from shard sh01 at host localhost:27217 :: caused by :: don't know dbVersion. + +To workaround this limitation, a test runners MUST execute a non-transactional +``distinct`` command on each mongos server before running any test that might +execute ``distinct`` within a transaction. To ease the implementation, test +runners MAY execute ``distinct`` before *every* test. + +Test runners can remove this workaround once `SERVER-39704`_ is resolved, after +which point mongos should retry the operation transparently. The ``distinct`` +command is the only command allowed in a sharded transaction that uses the +``databaseVersion`` concept so it is the only command affected. + +.. _SERVER-39704: https://jira.mongodb.org/browse/SERVER-39704 + + Server Fail Points ------------------ @@ -1423,14 +1709,8 @@ command). This parameter should already be enabled for most configuration files in `mongo-orchestration `_. -Clearing Fail Points After Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If a test configures one or more fail points, test runners MUST disable those -fail points after running all `test.operations `_ to avoid -spurious failures in subsequent tests. For tests using `targetedFailPoint`_, the -test runner MUST disable the fail point on the same mongos node on which it was -originally configured. +Disabling Fail Points +~~~~~~~~~~~~~~~~~~~~~ A fail point may be disabled like so:: @@ -1440,17 +1720,6 @@ A fail point may be disabled like so:: }); -Excluding configureFailPoint from APM -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Test runners MUST ensure that ``configureFailPoint`` commands executed for -`failPoint`_ and `targetedFailPoint`_ operations do not appear in the list of -logged commands used to assert `test.expectedEvents `_. -This may require manually filtering ``configureFailPoint`` from the list of -observed commands (particularly in the case of `targetedFailPoint`_, which uses -a `client entity `_). - - Fail Points Commonly Used in Tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1506,8 +1775,8 @@ Determining if a Sharded Cluster Uses Replica Sets When connected to a mongos server, the test runner can query the `config.shards `__ collection. Each shard in the cluster is represented by a document in this -collection. If the shard is backed by a single servers, the ``host`` field will -contain a single host. If the shard is backed by a replica set, the ``host``For +collection. If the shard is backed by a single server, the ``host`` field will +contain a single host. If the shard is backed by a replica set, the ``host`` field contain the name of the replica followed by a forward slash and a comma-delimited list of hosts. @@ -1533,10 +1802,30 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec -2020-08-22: +2020-08-23: * describe how to determine if sharded clusters use replica sets +* use flexible comparisons for numeric types (to help with ``ok`` matches) + +* copy special procedures from transaction spec + +* document test runner implementation, including loading test files, executing + tests, and executing operations + +* clarify interactions with fail points and APM + +* better describe argument handling for entity operations and document edge + cases for bulkWrite ops + +* document command started, succeeded, and failed events (per APM spec) + +* clarify logic for expectedError assertions + +* restructure expectedEvents to list events per client + +* createOnServer option for collection entities (mainly for transactions) + 2020-08-21: * clarify error assertions for BulkWriteException From e6cb23e1b557d3c537ec0231d595d3c8c582d2e7 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 25 Aug 2020 00:08:56 -0400 Subject: [PATCH 11/90] Language edits from PR feedback --- .../unified-test-format.rst | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index c88a0bb4e7..5a12baf6c0 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -401,8 +401,7 @@ The structure of this document is as follows: test ~~~~ -Test case consisting of a sequence of operations to be executed. The test may -optionally include configuration directives and event/outcome assertions. +Test case consisting of a sequence of operations to be executed. The structure of each document is as follows: @@ -1323,8 +1322,8 @@ Syntax:: This operator can be used anywhere a matched value is expected (including an `expectedResult `_). The test runner MUST assert that -actual value either does not exist or and matches the expected value. Matching -the expected value should use the standard rules in `Evaluating Matches`_, which +actual value either does not exist or matches the expected value. Matching the +expected value should use the standard rules in `Evaluating Matches`_, which means that it may contain special operators. This operator is primarily used to assert driver-optional fields from the CRUD @@ -1377,10 +1376,8 @@ SHOULD NOT share state created by processing a test file with the processing of subsequent test files, and likewise for tests within a test file. -Configuring the Test Runner -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The instructions in this section apply once for the test runner. +Initializing the Test Runner +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The test runner MUST be configurable with a connection string (or equivalent configuration), which will be used to initialize the internal MongoClient and @@ -1446,7 +1443,7 @@ Executing a Test The instructions in this section apply for each `test`_ occuring in a test file loaded by the test runner. After processing a test, test runners SHOULD reset -any internal state that resulted from doing so. For example, the entity map +any internal state that resulted from doing so. For example, the `Entity Map`_ created for one test SHOULD NOT be shared with another. If at any point while executing this test an unexpected error is encountered or @@ -1569,9 +1566,6 @@ Proceed with preparing the operation's arguments. If ``session`` is specified in to a session entity and MUST raise an error if the name is undefined or maps to an unexpected type. -Proceed with executing the operation such that its return value (if any) potential error can be caught -(e.g. using a ``try`` block). - Before executing the operation, the test runner MUST be prepared to catch a potential error from the operation (e.g. enter a ``try`` block). Proceed with executing the operation and capture its result or error. From 00986eaca5a9fdd051f787924b39e7c46d113c05 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 25 Aug 2020 23:23:45 -0400 Subject: [PATCH 12/90] Require explicit database and collection names and add extra notes --- .../unified-test-format.rst | 261 ++++++++++-------- 1 file changed, 139 insertions(+), 122 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 5a12baf6c0..2050a7e1ad 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -171,22 +171,6 @@ The top-level fields of a test file are as follows: client, collection, session objects) that should be created before each test case is executed. The structure of each document is defined in `entity`_. -.. _collectionName: - -- ``collectionName``: Optional string. Name of collection under test. This is - primarily useful when the collection name must be referenced in an assertion. - The YAML file SHOULD define a `node anchor`_ for this field (e.g. - ``collectionName: &collectionName foo``). If unset, test runners may use - whatever value they prefer. - -.. _databaseName: - -- ``databaseName``: Optional string. Name of database under test. This is - primarily useful when the database name must be referenced in an assertion. - The YAML file SHOULD define a `node anchor`_ for this field (e.g. - ``databaseName: &databaseName foo``). If unset, test runners may use whatever - value they prefer. - .. _initialData: - ``initialData``: Optional array of documents. Data that should exist in @@ -194,8 +178,14 @@ The top-level fields of a test file are as follows: If set, the array should contain at least one document. The structure of each document is defined in `collectionData`_. Before each test and for each - `collectionData`_, the test runner MUST drop the collection and insert the - specified documents (if any) using a "majority" write concern. + `collectionData`_, the test runner MUST drop and the collection and insert the + specified documents (if any) using a "majority" write concern. If no documents + are specified, the test runner MUST create the collection with a "majority" + write concern. + + The behavior to explicitly create a collection when no documents are specified + is primarily used for testing transactions, since collections cannot be + created within transactions. .. _tests: @@ -289,10 +279,9 @@ The structure of this document is as follows: created. The YAML file SHOULD use an `alias node`_ for a client entity's ``id`` field (e.g. ``client: *client0``). - - ``databaseName``: Optional string. Database name. The YAML file SHOULD + - ``databaseName``: Required string. Database name. The YAML file SHOULD define a `node anchor`_ for this field (e.g. - ``databaseName: &database0Name foo``). If omitted, this defaults to the name - of the database under test (see: `databaseName`_). + ``databaseName: &database0Name foo``). - ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. @@ -310,26 +299,12 @@ The structure of this document is as follows: will be created. The YAML file SHOULD use an `alias node`_ for a database entity's ``id`` field (e.g. ``database: *database0``). - - ``collectionName``: Optional string. Collection name. The YAML file SHOULD + - ``collectionName``: Required string. Collection name. The YAML file SHOULD define a `node anchor`_ for this field (e.g. - ``collectionName: &collection0Name foo``). If omitted, this defaults to the - name of the collection under test (see: `collectionName`_). - - - ``collectionOptions``: Optional document. See `collectionOrDatabaseOptions`_. - - .. _entity_collection_createOnServer: - - - ``createOnServer``: Optional boolean. If true, when creating this entity the - test runner MUST also create the collection on the server using a "majority" - write concern. Defaults to false. + ``collectionName: &collection0Name foo``). - This is primarily used when testing transactions, since collections cannot - be created within transactions. - - **TODO**: This may be redundant if we require initialData to ensure a - collection exists (i.e. create it explicitly if the list of documents would - be empty); however, if that is too obtuse to expect it to be used by - transaction tests then we can keep this as-is. + - ``collectionOptions``: Optional document. See + `collectionOrDatabaseOptions`_. .. _entity_session: @@ -384,15 +359,9 @@ which insert and read documents, respectively. The structure of this document is as follows: -- ``collectionName``: Optional string. Collection name (not an `entity`_). The - YAML file SHOULD use an `alias node`_ for this value (e.g. - ``collectionName: *collection0Name``). Defaults to the name of the collection - under test (see: `collectionName`_). +- ``collectionName``: Required string. See `commonOptions_collectionName`_. -- ``databaseName``: Optional string. Database name (not an `entity`_). The - YAML file SHOULD use an `alias node`_ for this value (e.g. - ``databaseName: *database0Name``). Defaults to the name of the database under - test (see: `databaseName`_). +- ``databaseName``: Required string. See `commonOptions_databaseName`_. - ``documents``: Required array of documents. List of documents corresponding to the contents of the collection. This list may be empty. @@ -436,6 +405,10 @@ The structure of each document is as follows: client entity and a list of events that are expected to be observed (in that order) on that client while executing `operations `_. + If a driver only supports configuring event listeners globally (for all + clients), the test runner SHOULD associate each observed event with a client + in order to perform these assertions. + The array should contain at least one document. The structure of each document is as follows: @@ -655,7 +628,7 @@ The structure of this document is as follows: Common Options -~~~~~~~~~~~~~~ +-------------- This section defines the structure of common options that are referenced from various contexts in the test format. Comprehensive documentation for some of @@ -666,10 +639,21 @@ these types and their parameters may be found in the following specifications: The structure of these common options is as follows: +.. _commonOptions_collectionName: + +- ``collectionName``: String. Collection name. The YAML file SHOULD use an + `alias node`_ for a collection entity's ``collectionName`` field (e.g. + ``collectionName: *collection0Name``). + +.. _commonOptions_databaseName: + +- ``databaseName``: String. Database name. The YAML file SHOULD use an + `alias node`_ for a database entity's ``databaseName`` field (e.g. + ``databaseName: *database0Name``). + .. _commonOptions_readConcern: -- ``readConcern``: Optional document. Map of parameters to construct a read - concern. +- ``readConcern``: Document. Map of parameters to construct a read concern. The structure of this document is as follows: @@ -677,7 +661,7 @@ The structure of these common options is as follows: .. _commonOptions_readPreference: -- ``readPreference``: Optional document. Map of parameters to construct a read +- ``readPreference``: Document. Map of parameters to construct a read preference. The structure of this document is as follows: @@ -692,14 +676,13 @@ The structure of these common options is as follows: .. _commonOptions_session: -- ``session``: Optional string. Session entity which will be resolved to a - ClientSession object. The YAML file SHOULD use an `alias node`_ for a session - entity's ``id`` field (e.g. ``session: *session0``). +- ``session``: String. Session entity name, which the test runner MUST resolve + to a ClientSession object. The YAML file SHOULD use an `alias node`_ for a + session entity's ``id`` field (e.g. ``session: *session0``). .. _commonOptions_writeConcern: -- ``writeConcern``: Optional document. Map of parameters to construct a write - concern. +- ``writeConcern``: Document. Map of parameters to construct a write concern. The structure of this document is as follows: @@ -738,20 +721,20 @@ optional parameters for an API method directly within `operation.arguments `_ (e.g. ``upsert`` for ``updateOne`` is *not* nested under an ``options`` key), unless otherwise stated below. -If ``session`` is specified in ``operation.arguments`_, it is defined according +If ``session`` is specified in `operation.arguments`_, it is defined according to `commonOptions_session`_. Test runners MUST resolve the ``session`` argument to session entity *before* passing it as a parameter to any API method. +If ``readConcern``, ``readPreference``, or ``writeConcern`` are specified in +`operation.arguments`_, test runners MUST interpret them according to the +corresponding definition in `Common Options`_ and MUST convert the value into +the appropriate object *before* passing it as a parameter to any API method. + This spec does not provide exhaustive documentation for all possible API methods that may appear in a test; however, the following sections discuss all supported entities and their operations in some level of detail. Special handling for certain operations is also discussed below. -**TODO**: While CRUD methods tend to flatten options into ``arguments`` -(possibly for consistency with bulk write models, which omit ``options`` keys), -session methods often leave those options nested within an ``options`` key. We -should pick one of these conventions for consistency. - client ~~~~~~ @@ -854,7 +837,7 @@ While operations typically raise an error *or* return a result, the ``writeResult`` property of a BulkWriteException. In this case, the intermediary write result may be matched with `expectedError_expectedResult`_. Because ``writeResult`` is optional for drivers to implement, such assertions should -utilize the `$$unsetOrMatches`` operator. +utilize the `$$unsetOrMatches`_ operator. Additionally, BulkWriteException is unique in that it aggregates one or more server errors in its ``writeConcernError`` and ``writeErrors`` properties. @@ -1009,9 +992,15 @@ assertSessionTransactionState ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``assertSessionTransactionState`` operation instructs the test runner to -assert that the transaction state of the given session is equal to the specified -value. The possible values are as follows: ``none``, ``starting``, -``in_progress``, ``committed``, ``aborted``. +assert that the given session has a particular transaction state. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +- ``state``: Required string. Expected transaction state for the session. + Possible values are as follows: ``none``, ``starting``, ``in_progress``, + ``committed``, and ``aborted``. An example of this operation follows:: @@ -1026,7 +1015,11 @@ assertSessionPinned ~~~~~~~~~~~~~~~~~~~ The ``assertSessionPinned`` operation instructs the test runner to assert that -the given session is pinned to a mongos. +the given session is pinned to a mongos server. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. An example of this operation follows:: @@ -1040,11 +1033,15 @@ assertSessionUnpinned ~~~~~~~~~~~~~~~~~~~~~ The ``assertSessionUnpinned`` operation instructs the test runner to assert that -the given session is not pinned to a mongos. +the given session is not pinned to a mongos server. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. An example of this operation follows:: - - name: assertSessionPinned + - name: assertSessionUnpinned object: testRunner arguments: session: *session0 @@ -1054,93 +1051,107 @@ assertCollectionExists ~~~~~~~~~~~~~~~~~~~~~~ The ``assertCollectionExists`` operation instructs the test runner to assert -that the given collection exists in the database. +that the given collection exists in the database. The test runner MUST use the +internal MongoClient for this operation. + +The following arguments are supported: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. An example of this operation follows:: - name: assertCollectionExists object: testRunner arguments: - database: db - collection: test + collectionName: *collection0Name + databaseName: *database0Name Use a ``listCollections`` command to check whether the collection exists. Note that it is currently not possible to run ``listCollections`` from within a transaction. -**TODO**: If this will refer to a collection entity, database is redundant. -Otherwise, consider renaming the arguments to databaseName and collectionName as -was done in `collectionData`_. - assertCollectionNotExists ~~~~~~~~~~~~~~~~~~~~~~~~~ The ``assertCollectionNotExists`` operation instructs the test runner to assert -that the given collection does not exist in the database. +that the given collection does not exist in the database. The test runner MUST +use the internal MongoClient for this operation. + +The following arguments are supported: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. An example of this operation follows:: - name: assertCollectionNotExists object: testRunner arguments: - database: db - collection: test + collectionName: *collection0Name + databaseName: *database0Name Use a ``listCollections`` command to check whether the collection exists. Note that it is currently not possible to run ``listCollections`` from within a transaction. -**TODO**: If this will refer to a collection entity, database is redundant. -Otherwise, consider renaming the arguments to databaseName and collectionName as -was done in `collectionData`_. - assertIndexExists ~~~~~~~~~~~~~~~~~ -The ``assertIndexExists`` operation instructs the test runner to assert that the -index with the given name exists on the collection. +The ``assertIndexExists`` operation instructs the test runner to assert that an +index with the given name exists on the collection. The test runner MUST use the +internal MongoClient for this operation. + +The following arguments are supported: + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +- ``indexName``: Required string. Index name. An example of this operation follows:: - name: assertIndexExists object: testRunner arguments: - database: db - collection: test - index: t_1 + collectionName: *collection0Name + databaseName: *database0Name + indexName: t_1 Use a ``listIndexes`` command to check whether the index exists. Note that it is currently not possible to run ``listIndexes`` from within a transaction. -**TODO**: If this will refer to a collection entity, database is redundant. -Otherwise, consider renaming the arguments to databaseName and collectionName as -was done in `collectionData`_. - assertIndexNotExists ~~~~~~~~~~~~~~~~~~~~ The ``assertIndexNotExists`` operation instructs the test runner to assert that -the index with the given name does not exist on the collection. +an index with the given name does not exist on the collection. The test runner +MUST use the internal MongoClient for this operation. + +- ``collectionName``: Required string. See `commonOptions_collectionName`_. + +- ``databaseName``: Required string. See `commonOptions_databaseName`_. + +- ``indexName``: Required string. Index name. An example of this operation follows:: - name: assertIndexNotExists object: testRunner arguments: - database: db - collection: test - index: t_1 + collectionName: *collection0Name + databaseName: *database0Name + indexName: t_1 Use a ``listIndexes`` command to check whether the index exists. Note that it is currently not possible to run ``listIndexes`` from within a transaction. -**TODO**: If this will refer to a collection entity, database is redundant. -Otherwise, consider renaming the arguments to databaseName and collectionName as -was done in `collectionData`_. - Evaluating Matches ------------------ @@ -1154,7 +1165,7 @@ following pseudo-code:: function match (expected, actual): if expected is a document: - if first key of expected starts with "$$": + if first key of expected starts with "$$": assert that the special operator (identified by key) matches return @@ -1409,13 +1420,6 @@ determine if the test file can be processed further. Test runners MAY support multiple versions and MUST NOT process incompatible files (as discussed in `Schema Version`_). -If the test file is compatible, the test runner SHALL proceed with determining -default names for the database and collection under test, which may be used by -`database `_ and `collection `_ entities -and `collectionData`_. The test runner MUST use the values from `databaseName`_ -and `collectionName`_ fields if set. If a field is omitted, the test runner MUST -generate a name. This spec is not prescriptive about the logic for doing so. - If the test file specifies false for `allowMultipleMongoses`_ and the test runner was configured with a connection string (or equivalent configuration) that references multiple mongos servers in its host list, the test runner MUST @@ -1459,27 +1463,19 @@ If `test.runOn `_ is specified, the test runner MUST skip the test unless one or more `runOnRequirement`_ documents are satisfied. If `initialData`_ is specified, for each `collectionData`_ therein the test -runner MUST drop the collection using a "majority" write concern and then insert -the specified documents, also using a "majority" write concern. The test runner +runner MUST drop the collection and insert the specified documents (if any) +using a "majority" write concern. If no documents are specified, the test runner +MUST create the collection with a "majority" write concern. The test runner MUST use the internal MongoClient for these operations. Create a new `Entity Map`_ that will be used for this test. If `createEntities`_ is specified, the test runner MUST create each `entity`_ accordingly and add it to the map. -For any collection entities that specify true for -`createOnServer `_, the test runner MUST -create those collections on the server using a "majority" write concern. Test -runners SHOULD ignore any NamespaceExists (code 28) errors when doing so. As an -optimization, test runners MAY skip this step for any collections that were -created as a result of inserting documents during processing of `initialData`_. - If the test might execute a ``distinct`` command within a transaction on a -sharded cluster, the test runner SHOULD execute a non-transactional ``distinct`` -command on each collection entity created. See -`StaleDbVersion Errors on Sharded Clusters`_ for more information. Test runners -MAY combine this step with handling for `createOnServer`_, since both are -primarily relevant to transactions. +sharded cluster, for each collection entity the test runner SHOULD execute a +non-transactional ``distinct`` command on each mongos server. See +`StaleDbVersion Errors on Sharded Clusters`_ for more information. If `test.expectedEvents `_ is specified, the test runner MUST enable command monitoring for any client entities so that events can be @@ -1796,6 +1792,27 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-08-25: + +* note that drivers with global event listeners will need to associate events + with clients for executing assertions (copied from change streams spec). + +* note special consideration for drivers with global event listeners + +* require databaseName and collectionName when creating database and collection + entities, respectively. also require those options in initialData, outcome, + and special operations (YAML aliases may be used). remove top-level + databaseName and collectionName fields (again) and any language for test + runners generate their own names. + +* remove createOnServer option for collection entities. initialData will now + explicitly create a collection if the list of documents is empty. + +* note handling of read concern, read preference, and write concern options for + entity operations. + +* document arguments for all special operations + 2020-08-23: * describe how to determine if sharded clusters use replica sets From 8bf4f0bbfb7fefc598ea9cc173953158469c5c22 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 26 Aug 2020 00:47:56 -0400 Subject: [PATCH 13/90] observeEvents and useMultipleMongoses client entity options --- .../unified-test-format.rst | 196 ++++++++++-------- 1 file changed, 107 insertions(+), 89 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2050a7e1ad..4cb3492cff 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-08-21 +:Last Modified: 2020-08-25 .. contents:: @@ -155,16 +155,6 @@ The top-level fields of a test file are as follows: **TODO**: Consider whether these requirements should allow an entire test file to be skipped up front, instead of allowing per-test requirements to override. -.. _allowMultipleMongoses: - -- ``allowMultipleMongoses``: Optional boolean. If false, all MongoClients - created for this test file (internal and any `entities `_) - that could connect to a sharded cluster MUST be initialized with only a single - mongos host. Defaults to true. If true or the topology is non-sharded, this - option has no effect. Test files that include tests with a `failPoint`_ - operation that may run on sharded topologies MUST specify false for this - option. - .. _createEntities: - ``createEntities``: Optional array of documents. List of entities (e.g. @@ -231,7 +221,7 @@ entity An entity (e.g. client, collection, session object) that will be created in the `Entity Map`_ before each test is executed. -This document MUST contain *exactly one* top-level key that identifies the +This document MUST contain **exactly one** top-level key that identifies the entity type and maps to a nested document, which specifies a unique name for the entity (``id`` key) and any other parameters necessary for its construction. Tests SHOULD use sequential names based on the entity type (e.g. "session0", @@ -266,6 +256,30 @@ The structure of this document is as follows: will map to an array of strings, each representing a tag set, since it is not feasible to define multiple ``readPreferenceTags`` keys in the document. + .. _entity_client_useMultipleMongoses: + + - ``useMultipleMongoses``: Optional boolean. If true and the topology is a + sharded cluster, the test runner MUST assert that this MongoClient connects + to multiple mongos hosts (e.g. by inspecting the connection string). If + false and the topology is a sharded cluster, the test runner MUST ensure + that this MongoClient connects to only a single mongos host (e.g. by + modifying the connection string). This option has no effect for non-sharded + topologies. + + .. _entity_client_observeEvents: + + - ``observeEvents``: Optional string or array of strings. One or more types of + events that can be observed for this client. Unspecified event types MUST + be ignored by this client's event listeners and SHOULD NOT be included in + `test.expectedEvents `_ for this client. Supported + types correspond to those documented in `expectedEvent`_ and are as follows: + + - `commandStartedEvent `_ + + - `commandSucceededEvent `_ + + - `commandFailedEvent `_ + .. _entity_database: - ``database``: Optional document. Corresponds with a Database object. @@ -562,9 +576,9 @@ expectedEvent An event (e.g. APM), which is expected to be observed while executing the test's operations. -This document MUST contain *exactly one* top-level key that identifies the event -type and maps to a nested document, which contains one or more assertions for -the event's properties. +This document MUST contain **exactly one** top-level key that identifies the +event type and maps to a nested document, which contains one or more assertions +for the event's properties. The structure of this document is as follows: @@ -898,22 +912,28 @@ failPoint ~~~~~~~~~ The ``failPoint`` operation instructs the test runner to configure a fail point -using a ``primary`` read preference and the internal MongoClient. +using a ``primary`` read preference using the specified client. The following arguments are supported: - ``failPoint``: Required document. The ``configureFailPoint`` command to be executed. -Test files using this operation MUST also specify false for -`allowMultipleMongoses`_ if they could be executed on sharded topologies -(according to `runOn`_ or `test.runOn `_). This is necessary -because server selection rules for mongos could lead to unpredictable behavior -if different servers were selected for configuring the fail point and executing -subsequent operations. +- ``client``: Required string. Client entity used to configure the fail point. + The YAML file SHOULD use an `alias node`_ for a client entity's ``id`` field + (e.g. ``client: *client0``). + + The client entity SHOULD specify false for + `useMultipleMongoses `_ if this operation + could be executed on a sharded topology (according to `runOn`_ or + `test.runOn `_). This is advised because server selection rules + for mongos could lead to unpredictable behavior if different servers were + selected for configuring the fail point and executing subsequent operations. When executing this operation, the test runner MUST keep a record of the fail -point so that it can be disabled after the test. +point so that it can be disabled after the test. The test runner MUST also +ensure that the ``configureFailPoint`` command is excluded from the list of +observed command monitoring events for this client (if applicable). An example of this operation follows:: @@ -921,6 +941,7 @@ An example of this operation follows:: - name: failPoint object: testRunner arguments: + client: *client0 failPoint: configureFailPoint: failCommand mode: { times: 1 } @@ -946,22 +967,20 @@ The following arguments are supported: - ``session``: Required string. See `commonOptions_session`_. + The client entity associated with this session SHOULD specify true for + `useMultipleMongoses `_. This is advised + because multiple mongos connections are necessary to test session pinning. + The MongoClient and mongos on which to run the ``configureFailPoint`` command is determined by the ``session`` argument (after resolution to a session entity). Test runners MUST error if the session is not pinned to a mongos server at the time this operation is executed. -This operation SHOULD NOT be used in test files that specify false for -`allowMultipleMongoses`_ because session pinning cannot be meaningfully tested -without connecting to multiple mongos servers. In practice, this means that -`failPoint`_ and `targetedFailPoint`_ SHOULD NOT be utilized in the same test -file. - When executing this operation, the test runner MUST keep a record of both the -fail point and session so that the fail point can be disabled after the test. -The test runner MUST also ensure that the ``configureFailPoint`` command is -omitted (or otherwise filtered out) from the list of observed command monitoring -events for this client. +fail point and session (or pinned mongos server) so that the fail point can be +disabled on the same mongos server after the test. The test runner MUST also +ensure that the ``configureFailPoint`` command is excluded from the list of +observed command monitoring events for this client (if applicable). An example of this operation follows:: @@ -977,16 +996,6 @@ An example of this operation follows:: failCommands: ["commitTransaction"] closeConnection: true -**TODO**: `allowMultipleMongoses`_ only affects the test runner if it is false -(i.e. a connection string to a sharded cluster is reduce to one mongos). The -transaction spec hinted that ``useMultipleMongoses: true`` would force a -connection string to contain multiple mongos servers, but that seems untenable -since single-mongos and standalone connection strings are indistinguishable from -one another. Should we change `allowMultipleMongoses`_ to actively require -multiple mongos servers if true (omitted would still be a no-op)? That may be -possible if we defer enforcement until after the topology is determined at -runtime. - assertSessionTransactionState ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1231,15 +1240,15 @@ a test to exhaustively specify all fields in an expected document (e.g. cluster time in a `CommandStartedEvent `_ command). Note that this rule for allowing extra fields in actual values only applies when -matching *documents* documents. When comparing arrays, expected and actual -values must contain the same number of elements. For example, the following -arrays corresponding to a ``distinct`` operation result would not match:: +matching documents. When comparing arrays, expected and actual values MUST +contain the same number of elements. For example, the following arrays +corresponding to a ``distinct`` operation result would not match:: expected: [ 1, 2, 3 ] actual: [ 1, 2, 3, 4 ] That said, any individual documents *within* an array or list (e.g. result of a -``find`` operation) may be matched according to the rules in this section. For +``find`` operation) MAY be matched according to the rules in this section. For example, the following arrays would match:: expected: [ { x: 1 }, { x: 2 } ] @@ -1396,11 +1405,16 @@ any `client entities `_ (in combination with other URI options). This specification is not prescriptive about how this information is provided. For example, it may be read from an environment variable or configuration file. -The test runner MAY determine the server version and topology type in advance to -optimize any future `runOnRequirement`_ checks. +Create a new MongoClient, which will be used for internal operations (e.g. +processing `initialData`_ and `test.outcome `_). This is referred +to elsewhere in the specification as the internal MongoClient. + +Determine the server version and topology type using the internal MongoClient. +This information will be used to evaluate any future `runOnRequirement`_ checks. The test runner SHOULD terminate any open transactions (see: -`Terminating Open Transactions`_) before executing any tests. +`Terminating Open Transactions`_) using the internal MongoClient before +executing any tests. Executing a Test File @@ -1413,25 +1427,14 @@ for one test file SHOULD NOT be shared with another. Test files, which may be YAML or JSON files, MUST be interpreted using an `Extended JSON`_ parser. The parser MUST accept relaxed and canonical Extended -JSON, as test files may use either. +JSON (per `Extended JSON: Parsers <../extended-json.rst#parsers>`__), as test +files may use either. Upon loading a file, the test runner MUST read the `schemaVersion`_ field and determine if the test file can be processed further. Test runners MAY support multiple versions and MUST NOT process incompatible files (as discussed in `Schema Version`_). -If the test file specifies false for `allowMultipleMongoses`_ and the test -runner was configured with a connection string (or equivalent configuration) -that references multiple mongos servers in its host list, the test runner MUST -modify the host list used by this test file to only refer to a single mongos -server. - -Create a new MongoClient, which will be used for internal operations in this -test file (e.g. `failPoint`_, processing `initialData`_ and -`test.outcome `_). This is referred to elsewhere in the -specification as the internal MongoClient. Internal MongoClients SHOULD NOT be -shared across test files. - If `runOn`_ is specified, the test runner MUST skip the test file unless one or more `runOnRequirement`_ documents are satisfied. @@ -1470,26 +1473,26 @@ MUST use the internal MongoClient for these operations. Create a new `Entity Map`_ that will be used for this test. If `createEntities`_ is specified, the test runner MUST create each `entity`_ accordingly and add it -to the map. +to the map. If the topology is a sharded cluster, the test runner MUST handle +`useMultipleMongoses`_ accordingly if it is specified for any client entities. -If the test might execute a ``distinct`` command within a transaction on a -sharded cluster, for each collection entity the test runner SHOULD execute a -non-transactional ``distinct`` command on each mongos server. See +If the test might execute a ``distinct`` command within a sharded transaction, +for each target collection the test runner SHOULD execute a non-transactional +``distinct`` command on each mongos server using the internal MongoClient. See `StaleDbVersion Errors on Sharded Clusters`_ for more information. -If `test.expectedEvents `_ is specified, the test runner -MUST enable command monitoring for any client entities so that events can be -observed while running operations and later used to assert expected events. Test -runners MAY leave command monitoring disabled for tests that do not assert -expected events. +If `test.expectedEvents `_ is specified, for each client +entity the test runner MUST enable all event listeners necessary to collect the +event types specified in `observeEvents `_. Test +runners MAY leave event listeners disabled for tests and/or clients that do not +assert expected events. Test runners MUST ensure that ``configureFailPoint`` commands executed for `failPoint`_ and `targetedFailPoint`_ operations are excluded from the list of -observed events. This may require manually filtering out ``configureFailPoint`` -from the list of observed commands (particularly in the case of -`targetedFailPoint`_, which uses a `client entity `_). Test -runners MUST also ensure that any commands containing sensitive information are -excluded (per the +observed command monitoring events (if applicable). This may require manually +filtering out ``configureFailPoint`` command monitoring events from the list of +observed events. Test runners MUST also ensure that any commands containing +sensitive information are excluded (per the `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ spec). @@ -1497,23 +1500,22 @@ For each element in `test.operations `_, follow the process in `Executing an Operation`_. If an unexpected error is encountered or an assertion fails, the test runner MUST consider this test to have failed. -If command monitoring was enabled on any client entities, the test runner MUST -now disable command monitoring on those clients and, for each client, collect -any events observed thus far. +If any event listeners were enabled on any client entities, the test runner MUST +now disable those event listeners and, for each client, collect the ordered list +of events observed thus far. -If the fail points were configured, the test runner MUST now disable those fail -points to avoid spurious failures in subsequent tests. For any fail points -configured using `targetedFailPoint`_, the test runner MUST disable the fail -point on the same mongos server on which it was originally configured. See -`Disabling Fail Points`_ for more information. +If any fail points were configured, the test runner MUST now disable those fail +points (on the same server) to avoid spurious failures in subsequent tests. For +any fail points configured using `targetedFailPoint`_, the test runner MUST +disable the fail point on the same mongos server on which it was originally +configured. See `Disabling Fail Points`_ for more information. If `test.expectedEvents `_ is specified, for each document therein the test runner MUST assert that the number and sequence of expected events match the number and sequence of actual events observed on the specified client. If the list of expected events is empty, the test runner MUST assert that no events were observed on the client. The process for matching events is -described in `expectedEvent`_. The test runner MUST NOT consider any -``configureFailPoint`` +described in `expectedEvent`_. If `test.outcome `_ is specified, for each `collectionData`_ therein the test runner MUST assert that the collection contains exactly the @@ -1797,8 +1799,6 @@ Note: this will be cleared when publishing version 1.0 of the spec * note that drivers with global event listeners will need to associate events with clients for executing assertions (copied from change streams spec). -* note special consideration for drivers with global event listeners - * require databaseName and collectionName when creating database and collection entities, respectively. also require those options in initialData, outcome, and special operations (YAML aliases may be used). remove top-level @@ -1813,6 +1813,24 @@ Note: this will be cleared when publishing version 1.0 of the spec * document arguments for all special operations +* define observeEvents option for client entities to filter observed events + +* revise docs for configuring event listeners and remove wording that assumed + a test runner might only collect command monitoring events. + +* failPoint now takes a client entity and no longer uses internal MongoClient. + this is related to moving the option for multi-mongos and single-mongos to the + client entity. + +* replace top-level allowMultipleMongoses option, which previously applied to + all clients, with a useMultipleMongoses client entity option. This option can + be used to require multiple mongos hosts (desirable for targetedFailPoint) or + modify a connection string to a single host (desirable for failPoint). + +* construct internal MongoClient when initializing the test runner and allow it + to be shared throughout. This is possible because it no longer cares about the + number of mongos hosts (formerly allowMultipleMongoses). + 2020-08-23: * describe how to determine if sharded clusters use replica sets From b223eeb94ee856ddfaa9d3932598ddd3663e6720 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 26 Aug 2020 01:07:25 -0400 Subject: [PATCH 14/90] Example insertOne test --- .../tests/example-insertOne.json | 99 +++++++++++++++++++ .../tests/example-insertOne.yml | 51 ++++++++++ 2 files changed, 150 insertions(+) create mode 100644 source/unified-test-format/tests/example-insertOne.json create mode 100644 source/unified-test-format/tests/example-insertOne.yml diff --git a/source/unified-test-format/tests/example-insertOne.json b/source/unified-test-format/tests/example-insertOne.json new file mode 100644 index 0000000000..d11e1ddce1 --- /dev/null +++ b/source/unified-test-format/tests/example-insertOne.json @@ -0,0 +1,99 @@ +{ + "schemaVersion": "1.0", + "runOn": [ + { + "minServerVersion": "2.6" + } + ], + "createEntites": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "test", + "documents": [ + { + "_id": 1 + } + ] + } + ], + "tests": [ + { + "description": "insertOne", + "operations": [ + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "document": { + "_id": 2 + } + }, + "expectedResult": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + ], + "expectedEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": "insert", + "databaseName": "test", + "command": { + "insert": "coll", + "documents": [ + { + "_id": 2 + } + ] + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "test", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/example-insertOne.yml b/source/unified-test-format/tests/example-insertOne.yml new file mode 100644 index 0000000000..0a902b39bb --- /dev/null +++ b/source/unified-test-format/tests/example-insertOne.yml @@ -0,0 +1,51 @@ +schemaVersion: "1.0" + +runOn: + - minServerVersion: "2.6" + +createEntites: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name test + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + +tests: + - description: "insertOne" + operations: + - + object: *collection0 + name: insertOne + arguments: + document: { _id: 2 } + expectedResult: + insertedId: { $$unsetOrMatches: 2 } + expectedEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: insert + databaseName: *database0Name + command: + insert: *collection0Name + documents: + - { _id: 2 } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } From 0fcd5da7bc1a983d2544db58ef5281c19bc6c51d Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 26 Aug 2020 15:09:57 -0400 Subject: [PATCH 15/90] special operations for sessions spec tests --- .../unified-test-format.rst | 99 ++++++++++++++++++- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 4cb3492cff..e5ece78eb1 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -427,8 +427,7 @@ The structure of each document is as follows: is as follows: - ``client``: Required string. Client entity on which the events are expected - to be observed. The YAML file SHOULD use an `alias node`_ for a client - entity's ``id`` field (e.g. ``client: *client0``). + to be observed. See `commonOptions_client`_. - ``events``: Required array of documents. List of events, which are expected to be observed (in this order) on the corresponding client while executing @@ -688,6 +687,12 @@ The structure of these common options is as follows: - ``hedge``: Optional document. +.. _commonOptions_client: + +- ``client``: String. Client entity name, which the test runner MUST resolve + to a MongoClient object. The YAML file SHOULD use an `alias node`_ for a + client entity's ``id`` field (e.g. ``client: *client0``). + .. _commonOptions_session: - ``session``: String. Session entity name, which the test runner MUST resolve @@ -919,9 +924,7 @@ The following arguments are supported: - ``failPoint``: Required document. The ``configureFailPoint`` command to be executed. -- ``client``: Required string. Client entity used to configure the fail point. - The YAML file SHOULD use an `alias node`_ for a client entity's ``id`` field - (e.g. ``client: *client0``). +- ``client``: Required string. See `commonOptions_client`_. The client entity SHOULD specify false for `useMultipleMongoses `_ if this operation @@ -1056,6 +1059,88 @@ An example of this operation follows:: session: *session0 +assertDifferentLsidOnLastTwoCommands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertDifferentLsidOnLastTwoCommands`` operation instructs the test runner +to assert that the last two CommandStartedEvents observed on the client have +different ``lsid`` fields. This assertion is primarily used to test that dirty +server sessions are discarded from the pool. + +The following arguments are supported: + +- ``client``: Required string. See `commonOptions_client`_. + + The client entity SHOULD include "commandStartedEvent" in + `observeEvents `_. + +An example of this operation follows:: + + - name: assertDifferentLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + + +assertSameLsidOnLastTwoCommands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSameLsidOnLastTwoCommands`` operation instructs the test runner to +assert that the last two CommandStartedEvents observed on the client have +identical ``lsid`` fields. This assertion is primarily used to test that +non-dirty server sessions are not discarded from the pool. + +The following arguments are supported: + +- ``client``: Required string. See `commonOptions_client`_. + + The client entity SHOULD include "commandStartedEvent" in + `observeEvents `_. + +An example of this operation follows:: + + - name: assertSameLsidOnLastTwoCommands + object: testRunner + arguments: + client: *client0 + + +assertSessionDirty +~~~~~~~~~~~~~~~~~~ + +The ``assertSessionDirty`` operation instructs the test runner to assert that +the given session is marked dirty. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +An example of this operation follows:: + + - name: assertSessionDirty + object: testRunner + arguments: + session: *session0 + + +assertSessionNotDirty +~~~~~~~~~~~~~~~~~~~~~ + +The ``assertSessionNotDirty`` operation instructs the test runner to assert that +the given session is not marked dirty. + +The following arguments are supported: + +- ``session``: Required string. See `commonOptions_session`_. + +An example of this operation follows:: + + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + + assertCollectionExists ~~~~~~~~~~~~~~~~~~~~~~ @@ -1794,6 +1879,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-08-26: + +* special operations from sessions spec tests + 2020-08-25: * note that drivers with global event listeners will need to associate events From 62537552e6d86f0a52c252d2249a852b2c32e6ca Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 26 Aug 2020 16:39:22 -0400 Subject: [PATCH 16/90] open questions --- .../unified-test-format.rst | 133 +++++++++++++++--- 1 file changed, 111 insertions(+), 22 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index e5ece78eb1..2fa62e82e9 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -145,16 +145,12 @@ The top-level fields of a test file are as follows: .. _runOn: - ``runOn``: Optional array of documents. List of server version and/or topology - requirements for which the tests in this file can be run. These requirements - may be overridden on a per-test basis by `test.runOn `_. Test - runners MUST skip a test if its requirements are not met. + requirements for which the tests in this file can be run. If no requirements + are met, the test runner MUST skip this test file. If set, the array should contain at least one document. The structure of each document is defined in `runOnRequirement`_. - **TODO**: Consider whether these requirements should allow an entire test file - to be skipped up front, instead of allowing per-test requirements to override. - .. _createEntities: - ``createEntities``: Optional array of documents. List of entities (e.g. @@ -394,8 +390,15 @@ The structure of each document is as follows: - ``runOn``: Optional array of documents. List of server version and/or topology requirements for which the tests in this file can be run. If specified, these - requirements override any top-level requirements in `runOn`_. Test runners - MUST skip a test if its requirements are not met. + requirements are evaluated independently and in addition to any top-level + `runOn`_ requirements. If no requirements in this array are met, the test + runner MUST skip this test. + + These requirements SHOULD be more restrictive than those specified in the + top-level `runOn`_ requirements (if any). They SHOULD NOT be more permissive. + This is advised because both sets of requirements MUST be satisified in order + for a test to be executed and more permissive requirements at the test-level + could be taken out of context on their own. If set, the array should contain at least one document. The structure of each document is defined in `runOnRequirement`_. @@ -436,12 +439,6 @@ The structure of each document is as follows: The structure of each document is defined in `expectedEvent`_. - **TODO**: Decide if event types (e.g. APM, SDAM) should be mixed in the same - array and whether tests should be able to filter out certain types (assuming - the test runner observes any supported type). This will not become relevant - until SDAM integration tests are implemented (not in initial project scope), - but we should anticipate the syntax now to avoid potential BC breaks. - .. _test_outcome: - ``outcome``: Optional array of documents. Each document will specify expected @@ -952,10 +949,6 @@ An example of this operation follows:: failCommands: ["insert"] closeConnection: true -**TODO**: Consider supporting a readPreference argument to target nodes other -than the primary. If this is not needed it could be deferred to a future spec -version. - targetedFailPoint ~~~~~~~~~~~~~~~~~ @@ -1523,10 +1516,6 @@ multiple versions and MUST NOT process incompatible files (as discussed in If `runOn`_ is specified, the test runner MUST skip the test file unless one or more `runOnRequirement`_ documents are satisfied. -**TODO**: Confirm whether top-level `runOn`_ can be used to skip an entire file -without considering test-level requirements. If so, test-level requirements can -only be more restrictive. - For each element in `tests`_, follow the process in `Executing a Test`_. @@ -1874,6 +1863,100 @@ should be described here in detail for historical reference, in addition to any shorter description that may be added to the `Change Log`_. +Open Questions +============== + +Note: these questions should be resolved and this section removed before +publishing version 1.0 of the spec. + + +General +------- + + +Interaction between top-level and test-level runOn +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are at least two ways to handle runOn requirements defined at both the +test and file level: + +* If `test.runOn`_ is specified, those requirements are checked in addition to + any top-level `runOn`_. This behavior is the most straightforward as test + runners will not have to consider inheritance or overriding anything (similar + to reasons top-level ``databaseName`` and ``collectionName`` fields were + ultimately removed). If the top-level `runOn`_ fails, the test runner can skip + the entire file outright (i.e. short-circuit the logical-and of both checks). + +* If `test.runOn`_ is specified, those requirements are checked *instead* of any + top-level `runOn`_. This would mean that a test runner cannot immediately skip + an entire file based on evaluation of the top-level `runOn`_; however, it also + means that humans editing a test file can no longer trust that the top-level + `runOn`_ applies to all tests in the file. + +During review, it was asked if a test runner could enforce that `test.runOn`_ is +only more restrictive (and never more permissive) than the top-level `runOn`_. +While this is technically feasible, it would add considerable complexity to a +test runner to make such comparisons. + + +Mixing event types in expectedEvent +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Can different event types (e.g. command monitoring, SDAM) appear within the +``events`` array within `test.expectedEvents`_? To date, no specs (including +SDAM) expect events multiple types of events in the same test. `observeEvents`_ +does allow tests to filter events per client, so a test could conceivably expect +mixed types (e.g. CommandStartedEvent and ServerDescriptionChangedEvent). + +Perhaps the underlying question is: when collecting observed events on a client, +should the test runner keep them in one ordered list or group them into separate +lists (e.g. one list for command monitoring events and another for SDAM events)? + + +Should failPoint support a read preference? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The `failPoint`_ operation currently uses a primary read preference. Should it +solicit a ``readPreference`` argument to target nodes other than the primary? +To date, no spec needs this behavior and this change could easily be deferred to +a future minor version of the test format. + + +GridFS Tests +------------ + + +Error Assertions +~~~~~~~~~~~~~~~~ + +The GridFS tests assert for the following error types: "FileNotFound", +"ChunkIsMissing", "ChunkIsWrongSize", and "RevisionNotFound". These types are +not explicitly defined in the spec and only appear in test files. The spec also +does not specify how messages for these errors should be constructed, so it may +not be feasible to use ``errorContains`` assertions. + +Do we care about differentiating these types of errors? If so, we may need to +add them to ``expectedError.type``. + + +Outcome Assertions +~~~~~~~~~~~~~~~~~~ + +In the GridFS tests, ``assert.data`` is similar to `test.outcome`_ but it +uses ``*result`` and ``*actual`` to match document data with a saved result +(e.g. returned file ID) or an arbitrary value (e.g. date). Presently, +`test.outcome`_ uses an exact match and does **not** follow the rules in +`Evaluating Matches`_ (e.g. allow extra fields in actual documents, process +special operators). Should we change `test.outcome`_ to follow the same +comparison rules as we use for `expectedResult`_? + +If so, `test.outcome`_ and `initialData`_ would no longer share the same +`collectionData`_ structure. This is not problematic, but worth noting. + +Alternatively, we could leave `test.outcome`_ as-is and create a new special +operation that *matches* data within a collection. + + Change Log ========== @@ -1883,6 +1966,12 @@ Note: this will be cleared when publishing version 1.0 of the spec * special operations from sessions spec tests +* clarify interaction between runOn and test.runOn, which are evaluated + independently and must both be met for a test to be executed. Also add a note + that test.runOn requirements should only be more restrictive for readability. + +* create open questions from GridFS spec and internal TODO items + 2020-08-25: * note that drivers with global event listeners will need to associate events From 07bbfb84136c2938681ac4a7baaa44f31b13538b Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 26 Aug 2020 18:48:07 -0400 Subject: [PATCH 17/90] open question for representing options in operation.arguments --- .../unified-test-format.rst | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2fa62e82e9..0a4b6162ff 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1922,6 +1922,42 @@ To date, no spec needs this behavior and this change could easily be deferred to a future minor version of the test format. +Representing options in operation.arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Existing spec tests are inconsistent in how they represent optional +parameters for operations. In some cases, optional parameters appear directly +in `operation.arguments`_ alongside required parameters (e.g. CRUD). In other +cases, they are nested under an ``options`` key (e.g. Sessions). + +Most specs *are* consistent about how API methods are defined in spec documents. +The CRUD spec was on the first documents to define API methods and can be +credited with introducing the ``options`` parameter (e.g. +``options: Optional``); however, this was likely done for +readability of the spec (avoiding very long method declarations and/or embedding +``Optional`` syntax therein). Additionally, documenting options in a separate +type (e.g. UpdateOptions) allows the declarations to be shared and reused across +multiple methods. + +That said, this syntax is in no way prescriptive for language implementations. +While some drivers to model options as a struct/object, others solicit them +alongside required parameters (e.g. Python's keyword/named arguments). + +Should this spec require that optional parameters be nested under an ``options`` +key or solicit them directly in `operation.arguments`_? Both are technically +possible, since test runners handle both forms today. + +Note: the CRUD spec WriteModels (e.g. UpdateOneModel) for ``bulkWrite`` to not +use ``options`` keys in either the spec document or test files. As such, the +resolution of this question would not impact how WriteModels are expressed in +test files (see: `bulkWrite`_). + +Note: this question does not pertain to TransactionOptions in SessionOptions, +which is always nested under ``defaultTransactionOptions``. This is a special +case where all drivers represent TransactionOptions as a separate struct/object, +even if they do not do so for ``startTransaction``. + + GridFS Tests ------------ From d80e9290c61864b228c3d5859aeea4327984548e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 27 Aug 2020 14:14:13 -0400 Subject: [PATCH 18/90] Rename schema and test file to convey they are outdated --- source/unified-test-format/{schema.json => schema-pre-1.json} | 0 .../{example.json => schema-testing-example.json} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename source/unified-test-format/{schema.json => schema-pre-1.json} (100%) rename source/unified-test-format/{example.json => schema-testing-example.json} (100%) diff --git a/source/unified-test-format/schema.json b/source/unified-test-format/schema-pre-1.json similarity index 100% rename from source/unified-test-format/schema.json rename to source/unified-test-format/schema-pre-1.json diff --git a/source/unified-test-format/example.json b/source/unified-test-format/schema-testing-example.json similarity index 100% rename from source/unified-test-format/example.json rename to source/unified-test-format/schema-testing-example.json From 98f911da60bd5b3ebbcf40a4d1d74684aad0cfbc Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 27 Aug 2020 14:26:31 -0400 Subject: [PATCH 19/90] clarify schema version comparisons and compatibility --- .../unified-test-format.rst | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 0a4b6162ff..5ab10d9988 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -58,15 +58,31 @@ This specification and the `Test Format`_ follow `semantic versioning `__. Backwards breaking changes (e.g. removing a field, introducing a required field) will warrant a new major version. Backwards compatible changes (e.g. introducing an optional field) will -warrant a new minor version. Small bug fixes and internal changes (e.g. grammar) -will warrant a new patch version. +warrant a new minor version. Small fixes and internal changes (e.g. grammar, +adding clarifying text to the spec) will warrant a new patch version; however, +patch versions SHOULD NOT alter the structure of the test format and thus SHOULD +NOT be relevant to test files. Each test file defines a `schemaVersion`_, which test runners will use to determine compatibility (i.e. whether and how the test file will be -interpreted). Test runners MAY support multiple versions of the test format. -Test runners MUST NOT process incompatible files but are otherwise free to -determine how to handle such files (e.g. skip and log a notice, fail and raise -an error). +interpreted). Test files are considered compatible with a test runner if their +`schemaVersion`_ is less than or equal to a supported version in the test +runner, given the same major version component. For example: + +- A test runner supporting version 1.5.1 could execute test files with versions + 1.0 and 1.5 but *not* 1.6 and 2.0. +- A test runner supporting version 2.1 it could execute test files with versions + 2.0 and 2.1 but *not* 1.0 and 1.5. +- A test runner supporting *both* versions 1.5.1 and 2.0 could execute test + files with versions 1.4, 1.5, and 2.0, but *not* 1.6, 2.1, or 3.0. +- A test runner supporting version 2.0.1 could execute test files with versions + 2.0 and 2.0.1 but *not* 2.0.2 or 2.1. This example is provided for + completeness, but test files SHOULD NOT need to refer to patch versions (as + previously mentioned). + +Test runners MUST NOT process incompatible files but MAY determine how to handle +such files (e.g. skip and log a notice, fail and raise an error). Test runners +MAY support multiple schema versions (as demonstrated in the example above). Each major version of this specification will have a corresponding JSON schema (e.g. `schema-1.json `__), which may be used to programmatically @@ -140,7 +156,9 @@ The top-level fields of a test file are as follows: - ``schemaVersion``: Required string. Version of this specification to which the test file complies. Test runners will use this to determine compatibility (i.e. whether and how the test file will be interpreted). The format of this - string is defined in `Version String`_. + string is defined in `Version String`_; however, test files SHOULD NOT need to + refer to specific patch versions since patch-level changes SHOULD NOT alter + the structure of the test format (as previously noted in `Schema Version`_). .. _runOn: @@ -1998,6 +2016,11 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-08-27: + +* clarify rules for comparing schema versions and note that test files should + not need to refer to patch versions. + 2020-08-26: * special operations from sessions spec tests From 9c1a906cb084156a68b8b33a38de346bb0286725 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 27 Aug 2020 14:45:12 -0400 Subject: [PATCH 20/90] lsid assertions require observed commands with lsid fields --- .../unified-test-format/unified-test-format.rst | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 5ab10d9988..ab29083c90 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1085,6 +1085,10 @@ The following arguments are supported: The client entity SHOULD include "commandStartedEvent" in `observeEvents `_. +The test runner MUST fail this assertion if fewer than two CommandStartedEvents +have been observed on the client or if either command does not include an +``lsid`` field. + An example of this operation follows:: - name: assertDifferentLsidOnLastTwoCommands @@ -1108,6 +1112,10 @@ The following arguments are supported: The client entity SHOULD include "commandStartedEvent" in `observeEvents `_. +The test runner MUST fail this assertion if fewer than two CommandStartedEvents +have been observed on the client or if either command does not include an +``lsid`` field. + An example of this operation follows:: - name: assertSameLsidOnLastTwoCommands @@ -1593,8 +1601,7 @@ in `Executing an Operation`_. If an unexpected error is encountered or an assertion fails, the test runner MUST consider this test to have failed. If any event listeners were enabled on any client entities, the test runner MUST -now disable those event listeners and, for each client, collect the ordered list -of events observed thus far. +now disable those event listeners. If any fail points were configured, the test runner MUST now disable those fail points (on the same server) to avoid spurious failures in subsequent tests. For @@ -2021,6 +2028,11 @@ Note: this will be cleared when publishing version 1.0 of the spec * clarify rules for comparing schema versions and note that test files should not need to refer to patch versions. +* note that lsid assertions require actually having observed events and lsid + fields to compare. also remove note about "collecting observed events" after + executing operations, since events will already need to be accessible while + running operations in order to evaluate some assertions. + 2020-08-26: * special operations from sessions spec tests From 51ccdbf88445368bc85442641d1bea9588927d24 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 27 Aug 2020 16:02:38 -0400 Subject: [PATCH 21/90] Rename topologies link related spec tickets --- .../unified-test-format.rst | 95 ++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index ab29083c90..20bbff8a00 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -218,7 +218,7 @@ The structure of this document is as follows: should be assumed that there is no upper bound on the required server version. The format of this string is defined in `Version String`_. -- ``topology``: Optional string or array of strings. One or more of server +- ``topologies``: Optional string or array of strings. One or more of server topologies against which the tests can be run successfully. Valid topologies are "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded cluster backed by replica sets). If this field is omitted, it should be @@ -1972,6 +1972,11 @@ Should this spec require that optional parameters be nested under an ``options`` key or solicit them directly in `operation.arguments`_? Both are technically possible, since test runners handle both forms today. +With regard to CRUD, this issue dates back to +`SPEC-1158 `__. Therein, Jeremy +originally argued in favor of nesting under an ``options`` key because it was +most consistent the text in existing spec documents. + Note: the CRUD spec WriteModels (e.g. UpdateOneModel) for ``bulkWrite`` to not use ``options`` keys in either the spec document or test files. As such, the resolution of this question would not impact how WriteModels are expressed in @@ -2018,6 +2023,84 @@ Alternatively, we could leave `test.outcome`_ as-is and create a new special operation that *matches* data within a collection. +Human-readable binary data +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In `SPEC-1216 `__, Robert recommended +against changing the GridFS tests' ``$hex`` syntax to ``$binary`` Extended JSON, +as the base64-encoded strings in the latter format would not be human-readable. +This spec could introduce a custom operator in +`Special Operators for Matching Assertions`_ to compare an expected hexidecimal +string with an actual binary value, but we would still need to use ``$binary`` +syntax in `collectionData`_ for both `initialData`_ and `test.outcome`_. + +Another possible work-around is for this spec to discuss or link to a utility +that can easily convert between hex and base64. For example, we can include a +Javascript or Python script in the spec repository that can convert between the +two and assist those editing GridFS test files. + + +Related Issues +============== + +Note: this section should be removed before publishing version 1.0 of the spec. + +The following SPEC tickets are associated with +`DRIVERS-709 `__. This section will +record whether or not each issue is addressed by this spec. + +The following tickets are addressed by the test format: + +* `SPEC-1158 `__: Spec tests should use a consistent format for CRUD options + + Open question: `Representing options in operation.arguments`_ + +* `SPEC-1215 `__: Introduce spec test syntax for meta assertions (e.g. any value, not exists) + + See: `Special Operators for Matching Assertions`_ + +* `SPEC-1216 `__: Update GridFS YAML tests to use newer format + + Open question: `Human-readable binary data`_ + +* `SPEC-1229 `__: Standardize spec-test syntax for topology assertions + + See `runOnRequirement`_, which is used by both `runOn`_ and `test.runOn`_. + +* `SPEC-1254 `__: Rename topology field in spec tests to topologies + + See `runOnRequirement`_. + +* `SPEC-1439 `__: Inconsistent error checking in spec tests + + This may still require test format syntax to assert that an error occurred + without caring about any assertions on the error itself. + +* `SPEC-1713 `__: Allow runOn to be defined per-test in addition to per-file + + See `runOn`_ and `test.runOn`_ and related open question: + `Interaction between top-level and test-level runOn`_ + +* `SPEC-1723 `__: Introduce test file syntax to disable dropping of collection under test + + Only collections in `initialData`_ are dropped, so this can be achieved by + omitting the collection from `initialData`_. Additionally, the format supports + creating an empty collection without inserting any documents (needed by + transaction tests). + +The following tickets can be resolved after the unified test format is approved +and/or other specs begin porting their tests to the format: + +* `SPEC-1102 `__: Add "object: collection" to command monitoring tests +* `SPEC-1133 `__: Use APM to assert outgoing commands in CRUD spec tests +* `SPEC-1144 `__: CRUD tests improvements +* `SPEC-1193 `__: Convert change stream spec tests to runOn format +* `SPEC-1230 `__: Rewrite APM spec tests to use common test format +* `SPEC-1238 `__: Convert retryable write spec tests to multi-operation format +* `SPEC-1261 `__: Use runOn syntax to specify APM test requirements +* `SPEC-1375 `__: changeStream spec tests should be run on sharded clusters + + Change Log ========== @@ -2025,6 +2108,8 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-08-27: +* rename runOn.topology to topologies (SPEC-1254) + * clarify rules for comparing schema versions and note that test files should not need to refer to patch versions. @@ -2033,6 +2118,14 @@ Note: this will be cleared when publishing version 1.0 of the spec executing operations, since events will already need to be accessible while running operations in order to evaluate some assertions. +* open question about human-readable binary data for GridFS + +* rename runOn.topology to topologies + +* create section for related SPEC tickets and explain which are addressed by + this spec or suitable to be completed after the format is approved (e.g. those + that pertain to porting over other spec tests to the new format). + 2020-08-26: * special operations from sessions spec tests From bc267ee550b7a36aee2e0dd02b86554b9575cbde Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 27 Aug 2020 16:52:06 -0400 Subject: [PATCH 22/90] top-level description field --- source/unified-test-format/tests/example-insertOne.yml | 2 ++ source/unified-test-format/unified-test-format.rst | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/source/unified-test-format/tests/example-insertOne.yml b/source/unified-test-format/tests/example-insertOne.yml index 0a902b39bb..84bebe806c 100644 --- a/source/unified-test-format/tests/example-insertOne.yml +++ b/source/unified-test-format/tests/example-insertOne.yml @@ -1,3 +1,5 @@ +description: "example-insertOne" + schemaVersion: "1.0" runOn: diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 20bbff8a00..6082b38584 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -151,6 +151,11 @@ Top-level Fields The top-level fields of a test file are as follows: +- ``description``: Required string. The name of the test file. + + This SHOULD describe the common purpose of tests in this file and MAY refer to + the filename (e.g. "updateOne-hint"). + .. _schemaVersion: - ``schemaVersion``: Required string. Version of this specification to which the @@ -404,6 +409,8 @@ The structure of each document is as follows: - ``description``: Required string. The name of the test. + This SHOULD describe the purpose of this test (e.g. "insertOne is retried"). + .. _test_runOn: - ``runOn``: Optional array of documents. List of server version and/or topology @@ -2126,6 +2133,9 @@ Note: this will be cleared when publishing version 1.0 of the spec this spec or suitable to be completed after the format is approved (e.g. those that pertain to porting over other spec tests to the new format). +* add required top-level description field, which test runners may use for + debugging or log messages + 2020-08-26: * special operations from sessions spec tests From 11cb30808744a19dfcfbcc0fcc58c8a6ed123c2f Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 28 Aug 2020 15:22:54 -0400 Subject: [PATCH 23/90] change stream iteration, open questions, and errorCode assertion --- .../unified-test-format.rst | 154 ++++++++++++++++-- 1 file changed, 137 insertions(+), 17 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 6082b38584..02357e1e1a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -535,9 +535,8 @@ The structure of this document is as follows: This field is mutually exclusive with `expectedError `_. - **TODO**: This is primarily used for change streams. Once an operation for - iterating a change stream is added, it should link to ``saveResultAsEntity`` - as this will be the only way to add a change stream object to the entity map. + This is primarily used for creating a `changeStream`_ entity from the result + of a ``watch`` operation. expectedError @@ -559,12 +558,21 @@ The structure of this document is as follows: - ``server``: server-generated error (e.g. error derived from a server response). -- ``errorContains``: Optional string. A substring of the expected error message. - The test runner MUST assert that the error message contains this string using - a case-insensitive match. +- ``errorContains``: Optional string. A substring of the expected error message + (e.g. "errmsg" field in a server error document). The test runner MUST assert + that the error message contains this string using a case-insensitive match. See `bulkWrite`_ for special considerations for BulkWriteExceptions. +- ``errorCode``: Optional integer. The expected "code" field in the + server-generated error response. The test runner MUST assert that the error + includes a server-generated response whose "code" field equals this value. + In the interest of readability, YAML files SHOULD use a comment to note the + corresponding code name (e.g. ``errorCode: 26 # NamespaceNotFound``). + + Server error codes are defined in + `error_codes.yml `__. + - ``errorCodeName``: Optional string. The expected "codeName" field in the server-generated error response. The test runner MUST assert that the error includes a server-generated response whose "codeName" field equals this value @@ -572,6 +580,9 @@ The structure of this document is as follows: See `bulkWrite`_ for special considerations for BulkWriteExceptions. + Server error codes are defined in + `error_codes.yml `__. + - ``errorLabelsContain``: Optional array of strings. A list of error label strings that the error is expected to have. The test runner MUST assert that the error contains all of the specified labels (e.g. using the @@ -889,7 +900,7 @@ and write concern error messages into the BulkWriteException message MAY optimize the check for ``errorContains`` by examining the concatenated message. Drivers that expose ``code`` but not ``codeName`` through BulkWriteException MAY translate the expected code name to a number (see: -`error_codes.yml `_) +`error_codes.yml `__) and compare with ``code`` instead, but MUST raise an error if the comparison cannot be attempted (e.g. ``code`` is also not available, translation fails). @@ -925,6 +936,38 @@ specifications: - `GridFS <../gridfs/gridfs-spec.rst>`__ +changeStream +~~~~~~~~~~~~ + +Change streams entities are special in that they are not defined in +`createEntities`_ but are instead created by using +`operation.saveResultAsEntity `_ for a ``watch`` +operation on a client, database, or collection entity. + +The `Change Streams <../change-streams/change-streams.rst>`__ spec does not +define a consistent API for the ChangeStream class, since the mechanisms for +iteration and capturing a resume token may differ between synchronous and +asynchronous drivers. To account for this, this section explicitly defines the +supported operations for change stream entities. + + +iterateUntilDocumentOrError +``````````````````````````` + +Iterates the change stream until either a single document is returned or an +error is raised. + +If `expectedResult `_ is specified, it SHOULD be a +single document. + +`Iterating the Change Stream <../change-streams/tests#iterating-the-change-stream>`__ +in the change stream spec cautions drivers that implement a blocking mode of +iteration (e.g. asynchronous drivers) not to iterate the change stream +unnecessarily, as doing so could cause the test runner to block indefinitely. +This should not be a concern for ``iterateUntilDocumentOrError`` as iteration +only continues until either a document or error is encountered. + + Special Test Operations ----------------------- @@ -1836,13 +1879,13 @@ The ``failCommand`` fail point may be configured like so:: configureFailPoint: "failCommand", mode: , data: { - failCommands: ["commandName", "commandName2"], - closeConnection: , - errorCode: , + failCommands: [, ...], + closeConnection: , + errorCode: , writeConcernError: , appName: , - blockConnection: , - blockTimeMS: , + blockConnection: , + blockTimeMS: , } }); @@ -1858,13 +1901,16 @@ if desired: returned as a command error. * ``appName``: Optional string, which is unset by default. If set, the fail point will only apply to connections for MongoClients created with this - ``appname``. New in server 4.4.0-rc2 (`SERVER-47195 `_). + ``appname``. New in server 4.4.0-rc2 + (`SERVER-47195 `_). * ``blockConnection``: Optional boolean, which defaults to ``false``. If ``true``, the server should block the affected commands for ``blockTimeMS``. - New in server 4.3.4 (`SERVER-41070 `_). -* ``blockTimeMS``: Integer, which is required when ``blockConnection`` is - ``true``. The number of milliseconds for which the affected commands should be - blocked. New in server 4.3.4 (`SERVER-41070 `_). + New in server 4.3.4 + (`SERVER-41070 `_). +* ``blockTimeMS``: Optional integer, which is required when ``blockConnection`` + is ``true``. The number of milliseconds for which the affected commands should + be blocked. New in server 4.3.4 + (`SERVER-41070 `_). Determining if a Sharded Cluster Uses Replica Sets @@ -1995,6 +2041,72 @@ case where all drivers represent TransactionOptions as a separate struct/object, even if they do not do so for ``startTransaction``. +Change Stream Tests +------------------- + + +Error Assertions +~~~~~~~~~~~~~~~~ + +`Iterating the Change Stream <../change-streams/tests#iterating-the-change-stream>`__ +in the change stream spec test documentation states: + + If the test expects an error and one was not thrown by either creating the + change stream or executing the test's operations, iterating the change stream + once allows for an error to be thrown by a getMore command. + +It's not clear if this is necessary because the existing change stream spec test +format does not differentiate between errors raised by creating or iterating the +change stream. For instance, an invalid pipeline operator (error code 40324) +should be raised by the initial ``aggregate`` command. + +Since the unified format would allow tests to differentiate between the initial +``watch`` operation and `iterateUntilDocumentOrError`_ operation, and each +can expect its own error, this may not be a concern; however, if the purpose of +the quoted text above to to account for drivers that may not even execute the +``aggregate`` command at the time ``watch`` is called (instead deferring it to +the first iteration of ChangeStream), we may still need to account for that in +the unified test format. + +If so, one possible solution may be to document that ``watch`` operations +expecting an error must also iterate the returned change stream once if the +operation itself did not initially raise an error. Of course, we should confirm +that this (or any) solution is sufficient for all drivers. + + +Command Monitoring Assertions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The change stream spec is also unique among other specs in that its event +assertions only consider as many events as were expected and ignore both extra, +observed events (e.g. getMore) and an optional killCursors (which SHOULD be +issued when resuming). The issue of extra getMore events may pertain to either +sharding or have something to do with variation among drivers in how ``watch`` +is implemented (e.g. eagerly calling ``aggregate`` vs. deferring until the first +iteration). + +A possible solution to address this need may be to extend +`test.expectedEvents `_ to allow a list of command names +to be specified, for which any command monitoring events will be ignored (as we +currently do for security events and ``configureFailPoint``). If that addresses +*both* needs, we may not need anything more; however, there are some change +stream tests that *do* expect getMore, so in such a case we may still need a +separate `test.expectedEvents `_ option to instruct the +test runner to only assert as many observed events are expected and ignore any +additional events. + + +getResumeToken +~~~~~~~~~~~~~~ + +Change stream spec tests do not interact with the resume token so we have not +introduced a ``getResumeToken`` operation for change stream entities. Existing +spec tests do not even check for ``startAfter`` and ``resumeAfter`` fields in +outgoing commands, which is something we could start doing with `$$type`_ +assertions. That decision could be done at any time after existing spec tests +are ported and would not require any change to the unified format. + + GridFS Tests ------------ @@ -2113,6 +2225,14 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-08-28: + +* iterateUntilDocumentOrError operation for change stream entities + +* open questions for change streams + +* errorCode assertion for expectedError + 2020-08-27: * rename runOn.topology to topologies (SPEC-1254) From ace34913d83a66c639733daaacaf9417e7e583d8 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 1 Sep 2020 16:47:15 -0400 Subject: [PATCH 24/90] Future Work, createchangeStream operation, and review fixes --- .../unified-test-format.rst | 167 ++++++++++++++---- 1 file changed, 129 insertions(+), 38 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 02357e1e1a..c411b9d90a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -180,6 +180,10 @@ The top-level fields of a test file are as follows: client, collection, session objects) that should be created before each test case is executed. The structure of each document is defined in `entity`_. + Test files SHOULD define entities in dependency order, such that all + referenced entities (e.g. client) are defined before any of their dependent + entities (e.g. database, session). + .. _initialData: - ``initialData``: Optional array of documents. Data that should exist in @@ -187,15 +191,11 @@ The top-level fields of a test file are as follows: If set, the array should contain at least one document. The structure of each document is defined in `collectionData`_. Before each test and for each - `collectionData`_, the test runner MUST drop and the collection and insert the + `collectionData`_, the test runner MUST drop the collection and insert the specified documents (if any) using a "majority" write concern. If no documents are specified, the test runner MUST create the collection with a "majority" write concern. - The behavior to explicitly create a collection when no documents are specified - is primarily used for testing transactions, since collections cannot be - created within transactions. - .. _tests: - ``tests``: Required array of documents. List of test cases to be executed @@ -290,8 +290,16 @@ The structure of this document is as follows: - ``observeEvents``: Optional string or array of strings. One or more types of events that can be observed for this client. Unspecified event types MUST be ignored by this client's event listeners and SHOULD NOT be included in - `test.expectedEvents `_ for this client. Supported - types correspond to those documented in `expectedEvent`_ and are as follows: + `test.expectedEvents `_ for this client. + + Test files SHOULD NOT observe multiple event types (e.g. command monitoring + *and* SDAM events) for a single client, as this spec presently does not + define a process for collating and asserting mixed events. See + `Mixing event types in observeEvents and expectedEvents`_ for more + information. + + Supported types correspond to those documented in `expectedEvent`_ and are + as follows: - `commandStartedEvent `_ @@ -451,6 +459,12 @@ The structure of each document is as follows: clients), the test runner SHOULD associate each observed event with a client in order to perform these assertions. + Test files SHOULD NOT expect multiple event types (e.g. command monitoring + *and* SDAM events) for a single client, as this spec presently does not define + a process for collating and asserting mixed events. See + `Mixing event types in observeEvents and expectedEvents`_ for more + information. + The array should contain at least one document. The structure of each document is as follows: @@ -529,14 +543,15 @@ The structure of this document is as follows: returned by the operation (if any) will be saved with this name in the `Entity Map`_. The test runner MUST raise an error if the name is already in use. If the operation does not return a value (e.g. void method), the test - runner MAY choose to store an empty value (e.g. ``null``) or do nothing and - leave the entity name undefined. + runner MUST store an empty value (e.g. ``null``) for the entity such that the + name will still be defined in the entity map. This field is mutually exclusive with `expectedError `_. This is primarily used for creating a `changeStream`_ entity from the result - of a ``watch`` operation. + of a `client_createChangeStream`_, `database_createChangeStream`_, or + `collection_createChangeStream`_ operation. expectedError @@ -797,6 +812,25 @@ specifications: - `Change Streams <../change-streams/change-streams.rst>`__ - `Enumerating Databases <../enumerate-databases.rst>`__ +Client operations that require special handling or are not documented by an +existing specification are described below. + + +.. _client_createChangeStream: + +createChangeStream +`````````````````` + +Creates a cluster-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the client's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the client's ``watch`` +operation directly for reasons discussed in `changeStream`_. Test runners MUST +ensure that the server-side cursor is created (i.e. ``aggregate`` is executed) +as part of this operation and before the resulting change stream might be saved +with `operation.saveResultAsEntity `_. + database ~~~~~~~~ @@ -808,7 +842,24 @@ specifications: - `CRUD <../crud/crud.rst>`__ - `Enumerating Collections <../enumerate-collections.rst>`__ -Other database operations not documented by an existing specification follow. +Database operations that require special handling or are not documented by an +existing specification are described below. + + +.. _database_createChangeStream: + +createChangeStream +`````````````````` + +Creates a database-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the database's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the database's ``watch`` +operation directly for reasons discussed in `changeStream`_. Test runners MUST +ensure that the server-side cursor is created (i.e. ``aggregate`` is executed) +as part of this operation and before the resulting change stream might be saved +with `operation.saveResultAsEntity `_. runCommand @@ -850,7 +901,24 @@ specifications: - `Enumerating Indexes <../enumerate-indexes.rst>`__ - `Index Management <../index-management.rst>`__ -Collection operations that require special handling are described below. +Collection operations that require special handling or are not documented by an +existing specification are described below. + + +.. _collection_createChangeStream: + +createChangeStream +`````````````````` + +Creates a collection-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the collection's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the collection's ``watch`` +operation directly for reasons discussed in `changeStream`_. Test runners MUST +ensure that the server-side cursor is created (i.e. ``aggregate`` is executed) +as part of this operation and before the resulting change stream might be saved +with `operation.saveResultAsEntity `_. bulkWrite @@ -941,8 +1009,15 @@ changeStream Change streams entities are special in that they are not defined in `createEntities`_ but are instead created by using -`operation.saveResultAsEntity `_ for a ``watch`` -operation on a client, database, or collection entity. +`operation.saveResultAsEntity `_ with a +`client_createChangeStream`_, `database_createChangeStream`_, or +`collection_createChangeStream`_ operation. + +Test files SHOULD NOT use a ``watch`` operation to create a change stream, as +the implementation of that method may vary among drivers. For example, some +implementations of ``watch`` immediately execute ``aggregate`` and construct the +server-side cursor, while others may defer ``aggregate`` until the change stream +object is iterated. The `Change Streams <../change-streams/change-streams.rst>`__ spec does not define a consistent API for the ChangeStream class, since the mechanisms for @@ -1436,7 +1511,7 @@ Syntax:: { field: { $$exists: } } This operator can be used anywhere the value for a key might be specified in an -expected dcoument. If true, the test runner MUST assert that the key exists in +expected document. If true, the test runner MUST assert that the key exists in the actual document, irrespective of its value (e.g. a key with a ``null`` value would match). If false, the test runner MUST assert that the key does not exist in the actual document. This operator is modeled after the @@ -1977,29 +2052,6 @@ While this is technically feasible, it would add considerable complexity to a test runner to make such comparisons. -Mixing event types in expectedEvent -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Can different event types (e.g. command monitoring, SDAM) appear within the -``events`` array within `test.expectedEvents`_? To date, no specs (including -SDAM) expect events multiple types of events in the same test. `observeEvents`_ -does allow tests to filter events per client, so a test could conceivably expect -mixed types (e.g. CommandStartedEvent and ServerDescriptionChangedEvent). - -Perhaps the underlying question is: when collecting observed events on a client, -should the test runner keep them in one ordered list or group them into separate -lists (e.g. one list for command monitoring events and another for SDAM events)? - - -Should failPoint support a read preference? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The `failPoint`_ operation currently uses a primary read preference. Should it -solicit a ``readPreference`` argument to target nodes other than the primary? -To date, no spec needs this behavior and this change could easily be deferred to -a future minor version of the test format. - - Representing options in operation.arguments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2220,11 +2272,50 @@ and/or other specs begin porting their tests to the format: * `SPEC-1375 `__: changeStream spec tests should be run on sharded clusters +Future Work +=========== + + +Mixing event types in observeEvents and expectedEvents +------------------------------------------------------ + +The test format advises against mixing multiple types of events (e.g. command +monitoring *and* SDAM) in `observeEvents`_ and `test.expectedEvents`_. While +registering event listeners is trivial, determining how to collate events of +multiple types can be a challenge, particularly when some events may not be +predictable (e.g. ServerHeartbeatStartedEvent, CommandStartedEvent for +``getMore`` issued during change stream iteration). If the need arises to expect +multiple types of events in the same test, a future version of this spec can +define an approach for doing so. + + +Target failPoint by read preference +----------------------------------- + +The `failPoint`_ operation currently uses a primary read preference. To date, no +spec has needed behavior to configure a fail point on a non-primary node. If the +need does arise, `failPoint`_ can be enhanced to support a ``readPreference`` +argument. + + Change Log ========== Note: this will be cleared when publishing version 1.0 of the spec +2020-09-01: + +* clarify that saveResultAsEntity should always define the name in the map, even + if the result is empty. + +* test files should order createEntities + +* createChangeStream operation to proxy watch and enforce consistency among + drivers + +* Future Work section to note failPoint readPreference support and mixing + different event types + 2020-08-28: * iterateUntilDocumentOrError operation for change stream entities From b37beea5657c8e336183066fa053beddaf5cc5ce Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 1 Sep 2020 16:49:15 -0400 Subject: [PATCH 25/90] Update last modified and advisors --- source/unified-test-format/unified-test-format.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index c411b9d90a..d67f7a1770 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -5,11 +5,11 @@ Unified Test Format :Spec Title: Unified Test Format :Spec Version: 1.0.0 :Author: Jeremy Mikola -:Advisors: Prashant Mital, Isabel Atkinson +:Advisors: Prashant Mital, Isabel Atkinson, Thomas Reggi :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-08-25 +:Last Modified: 2020-09-01 .. contents:: From b779a14a76ec60e1bbe1bfe367026e071d3c5b7d Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 2 Sep 2020 22:12:18 -0400 Subject: [PATCH 26/90] Fix typos --- source/unified-test-format/tests/example-insertOne.json | 2 +- source/unified-test-format/tests/example-insertOne.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/tests/example-insertOne.json b/source/unified-test-format/tests/example-insertOne.json index d11e1ddce1..a991146cca 100644 --- a/source/unified-test-format/tests/example-insertOne.json +++ b/source/unified-test-format/tests/example-insertOne.json @@ -5,7 +5,7 @@ "minServerVersion": "2.6" } ], - "createEntites": [ + "createEntities": [ { "client": { "id": "client0", diff --git a/source/unified-test-format/tests/example-insertOne.yml b/source/unified-test-format/tests/example-insertOne.yml index 84bebe806c..a11b899412 100644 --- a/source/unified-test-format/tests/example-insertOne.yml +++ b/source/unified-test-format/tests/example-insertOne.yml @@ -5,7 +5,7 @@ schemaVersion: "1.0" runOn: - minServerVersion: "2.6" -createEntites: +createEntities: - client: id: &client0 client0 observeEvents: From b505fdf7f3059f1193eaec70175f361926fcafe1 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 3 Sep 2020 00:03:17 -0400 Subject: [PATCH 27/90] Changes for events, GridFS, and error assertions Close existing Open Questions and add new Future Work sections. --- .../unified-test-format.rst | 505 +++++++++--------- 1 file changed, 266 insertions(+), 239 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index d67f7a1770..2d55a278d2 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -292,9 +292,8 @@ The structure of this document is as follows: be ignored by this client's event listeners and SHOULD NOT be included in `test.expectedEvents `_ for this client. - Test files SHOULD NOT observe multiple event types (e.g. command monitoring - *and* SDAM events) for a single client, as this spec presently does not - define a process for collating and asserting mixed events. See + Test files SHOULD NOT observe events from multiple specs (e.g. command + monitoring *and* SDAM events) for a single client. See `Mixing event types in observeEvents and expectedEvents`_ for more information. @@ -307,6 +306,19 @@ The structure of this document is as follows: - `commandFailedEvent `_ + .. _entity_client_ignoreCommandMonitoringEvents: + + - ``ignoreCommandMonitoringEvents``: Optional string or array of strings. One + or more command names for which the test runner MUST ignore any observed + command monitoring events. The command(s) will be ignored in addition to + ``configureFailPoint`` and any commands containing sensitive information + (per the + `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ + spec). + + Test files SHOULD NOT use this option unless one or more command monitoring + events are specified in `observeEvents `_. + .. _entity_database: - ``database``: Optional document. Corresponds with a Database object. @@ -373,7 +385,8 @@ The structure of this document is as follows: transaction options MUST remain nested under ``defaultTransactionOptions`` and MUST NOT be flattened into ``sessionOptions``. -- ``bucket``: Optional document. Corresponds with a GridFS Bucket object. +- ``bucket``: Optional document. Corresponds with a Bucket object, as defined in + the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. The structure of this document is as follows: @@ -390,6 +403,26 @@ The structure of this document is as follows: specification. The ``readConcern``, ``readPreference``, and ``writeConcern`` options use the same structure as defined in `Common Options`_. +.. _entity_stream: + +- ``stream``: Optional document. Corresponds with a stream as defined in the + `GridFS <../gridfs/gridfs-spec.rst>`__ spec. Test runners MUST ensure that + stream is both readable *and* writable. + + This entity is primarily used with `uploadFromStream`_ and + `uploadFromStreamWithId`_ operations for `bucket`_ entities. + + The structure of this document is as follows: + + - ``id``: Required string. Unique name for this entity. The YAML file SHOULD + define a `node anchor`_ for this field (e.g. ``id: &stream0 stream0``). + + - ``hexBytes``: Required string. The string MUST contain an even number of + hexademical characters (case-insensitive) and MAY be empty. The test runner + MUST raise an error if the string is malformed. The test runner MUST convert + the string to a byte sequence denoting the stream's readable data (if any). + For example, "12ab" would denote a stream with two bytes: "0x12" and "0xab". + collectionData ~~~~~~~~~~~~~~ @@ -459,9 +492,8 @@ The structure of each document is as follows: clients), the test runner SHOULD associate each observed event with a client in order to perform these assertions. - Test files SHOULD NOT expect multiple event types (e.g. command monitoring - *and* SDAM events) for a single client, as this spec presently does not define - a process for collating and asserting mixed events. See + Test files SHOULD NOT expect events from multiple specs (e.g. command + monitoring *and* SDAM events) for a single client. See `Mixing event types in observeEvents and expectedEvents`_ for more information. @@ -474,7 +506,7 @@ The structure of each document is as follows: - ``events``: Required array of documents. List of events, which are expected to be observed (in this order) on the corresponding client while executing `operations`_. If the array is empty, the test runner MUST assert that no - events were observed on the client. + events were observed on the client (excluding ignored events). The structure of each document is defined in `expectedEvent`_. @@ -562,16 +594,19 @@ an executed operation. At least one key is required in this document. The structure of this document is as follows: -- ``type``: Optional string or array of strings. One or more classifications of - errors, at least one of which should apply to the expected error. +- ``isError``: Optional boolean. If true, the test runner MUST assert that an + error was raised. This is primarily used when no other error assertions apply + but the test still needs to assert an expected error. Test files MUST NOT + specify false, as `expectedError`_ is only applicable when an operation is + expected to raise an error. - Valid types are as follows: +- ``isClientError``: Optional boolean. If true, the test runner MUST assert that + the error originates from the client (i.e. it is not derived from a server + response). If false, the test runner MUST assert that the error does not + originate from the client. - - ``client``: client-generated error (e.g. parameter validation error before - a command is sent to the server). - - - ``server``: server-generated error (e.g. error derived from a server - response). + Client errors include, but are not limited to: parameter validation errors + before a command is sent to the server; network errors. - ``errorContains``: Optional string. A substring of the expected error message (e.g. "errmsg" field in a server error document). The test runner MUST assert @@ -588,6 +623,9 @@ The structure of this document is as follows: Server error codes are defined in `error_codes.yml `__. + Test files SHOULD NOT assert error codes for client errors, as specifications + do not define standardized codes for client errors. + - ``errorCodeName``: Optional string. The expected "codeName" field in the server-generated error response. The test runner MUST assert that the error includes a server-generated response whose "codeName" field equals this value @@ -598,6 +636,9 @@ The structure of this document is as follows: Server error codes are defined in `error_codes.yml `__. + Test files SHOULD NOT assert error codes for client errors, as specifications + do not define standardized codes for client errors. + - ``errorLabelsContain``: Optional array of strings. A list of error label strings that the error is expected to have. The test runner MUST assert that the error contains all of the specified labels (e.g. using the @@ -781,16 +822,17 @@ an API method on that class. Some specifications group optional parameters for API methods under an ``options`` parameter (e.g. ``options: Optional`` in the CRUD -spec). While this improves readability of the spec document(s) by allowing -option documentation to be expanded and reused, it would add unnecessary -verbosity to test files. Therefore, test files SHALL declare all required and -optional parameters for an API method directly within -`operation.arguments `_ (e.g. ``upsert`` for ``updateOne`` -is *not* nested under an ``options`` key), unless otherwise stated below. +spec); however, driver APIs vary in how they accept options (e.g. Python's +keyword/named arguments, ``session`` as either an option or required parameter +depending on whether a language supports method overloading). Therefore, test +files SHALL declare all required and optional parameters for an API method +directly within `operation.arguments `_ (e.g. ``upsert`` +for ``updateOne`` is *not* nested under an ``options`` key). If ``session`` is specified in `operation.arguments`_, it is defined according to `commonOptions_session`_. Test runners MUST resolve the ``session`` argument -to session entity *before* passing it as a parameter to any API method. +to `session `_ entity *before* passing it as a parameter to any +API method. If ``readConcern``, ``readPreference``, or ``writeConcern`` are specified in `operation.arguments`_, test runners MUST interpret them according to the @@ -802,6 +844,9 @@ that may appear in a test; however, the following sections discuss all supported entities and their operations in some level of detail. Special handling for certain operations is also discussed below. +Test files SHALL use camelCase when referring to API methods and parameters, +even if the defining specifications use other forms (e.g. snake_case in GridFS). + client ~~~~~~ @@ -982,15 +1027,16 @@ specifications: - `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ - `Driver Sessions <../sessions/driver-sessions.rst>`__ -Session operations that require special handling are described below. +Session operations that require special handling or are not documented by an +existing specification are described below. withTransaction ``````````````` -The ``withTransaction`` operation is unique in that its ``callback`` parameter -is a function and not easily expressed in YAML/JSON. For ease of testing, this -parameter is defined as an array of `operation`_ documents (analogous to +The ``withTransaction`` operation's ``callback`` parameter is a function and not +easily expressed in YAML/JSON. For ease of testing, this parameter is expressed +as an array of `operation`_ documents (analogous to `test.operations `_). Test runners MUST evaluate error and result assertions when executing these operations in the callback. @@ -1003,11 +1049,62 @@ specifications: - `GridFS <../gridfs/gridfs-spec.rst>`__ +Bucket operations that require special handling or are not documented by an +existing specification are described below. + + +.. _downloadToStream: +.. _downloadToStreamByName: + +downloadToStream and downloadToStreamByName +``````````````````````````````````````````` + +These operations SHOULD NOT be used in test files. See +`IO operations for GridFS streams`_ in `Future Work`_. + + +.. _openDownloadStream: +.. _openDownloadStreamByName: + +openDownloadStream and openDownloadStreamByName +``````````````````````````````````````````````` + +The ``openDownloadStream`` and ``openDownloadStreamByName`` operations SHOULD +use `$$matchesHexBytes`_ in `expectedResult `_ to +match the contents of the returned stream. These operations MAY use +`saveResultAsEntity `_ to save the stream for use +with a subsequent operation (e.g. `uploadFromStream`_ ). + + +.. _openUploadStream: +.. _openUploadStreamWithId: + +openUploadStream and openUploadStreamWithId +``````````````````````````````````````````` + +These operations SHOULD NOT be used in test files. See +`IO operations for GridFS streams`_ in `Future Work`_. + + +.. _uploadFromStream: +.. _uploadFromStreamWithId: + +uploadFromStream and uploadFromStreamWithId +``````````````````````````````````````````` + +The ``uploadFromStream`` and ``uploadFromStreamWithId`` operations' ``stream`` +parameter is a stream as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ +spec and not easily expressed in YAML/JSON. This parameter is expressed as an +entity name, which the test runner MUST resolve to a `stream `_ +entity *before* passing it as a parameter to the method. The YAML file SHOULD +use an `alias node`_ for a stream entity's ``id`` field +(e.g. ``stream: *stream0``). + changeStream ~~~~~~~~~~~~ -Change streams entities are special in that they are not defined in +Change stream entities are special in that they are not defined in `createEntities`_ but are instead created by using `operation.saveResultAsEntity `_ with a `client_createChangeStream`_, `database_createChangeStream`_, or @@ -1562,6 +1659,59 @@ admittedly inconsistent with the behavior of the query operator, but there is presently no need for this behavior in tests. +$$matchesEntity +``````````````` + +Syntax, where ``entityName`` is a string:: + + { $$matchesEntity: } + +This operator can be used anywhere a matched value is expected (including an +`expectedResult `_). If the entity name is defined in +the current test's `Entity Map`_, the test runner MUST fetch that entity and +assert that the actual value matches the entity using the standard rules in +`Evaluating Matches`_; otherwise, the test runner MUST raise an error for an +undefined entity. The YAML file SHOULD use an `alias node`_ for the entity name. + +This operator is primarily used to assert identifiers for uploaded GridFS files. + +An example of this operator follows:: + + operations: + - + object: *bucket0 + name: uploadFromStream + arguments: + filename: "filename" + source: *stream0 + expectedResult: { $$type: "objectId" } + saveResultAsEntity: &objectid0 "objectid0" + - object: *filesCollection + name: findOne + arguments: + sort: { uploadDate: -1 } + expectedResult: + _id: { $$matchesEntity: *objectid0 } + + +$$matchesHexBytes +````````````````` + +Syntax, where ``hexBytes`` is an even number of hexademical characters +(case-insensitive):: + + { $$matchesHexBytes: } + +This operator can be used anywhere a matched value is expected (including an +`expectedResult `_) and the actual value is a stream +as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. The test runner +MUST convert the string to a byte sequence and compare it with the full contents +of the stream. The test runner MUST raise an error if the string is malformed. + +This operator is primarily used to assert the contents of stream returned by +`openDownloadStream`_ and `openDownloadStreamByName`_. + + $$unsetOrMatches ```````````````` @@ -1712,14 +1862,18 @@ event types specified in `observeEvents `_. Test runners MAY leave event listeners disabled for tests and/or clients that do not assert expected events. -Test runners MUST ensure that ``configureFailPoint`` commands executed for -`failPoint`_ and `targetedFailPoint`_ operations are excluded from the list of -observed command monitoring events (if applicable). This may require manually -filtering out ``configureFailPoint`` command monitoring events from the list of -observed events. Test runners MUST also ensure that any commands containing -sensitive information are excluded (per the -`Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ -spec). +For each client with command monitoring enabled, the test runner MUST ignore +events for the following: + +- Any command(s) specified in + `ignoreCommandMonitoringEvents `_. + +- Any ``configureFailPoint`` commands executed for `failPoint`_ and + `targetedFailPoint`_ operations. + +- Any commands containing sensitive information (per the + `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ + spec). For each element in `test.operations `_, follow the process in `Executing an Operation`_. If an unexpected error is encountered or an @@ -1781,7 +1935,9 @@ If `operation.object`_ is not "testRunner", this is an entity operation. If runner MUST fetch that entity and note its type; otherwise, the test runner MUST raise an error for an undefined entity. If `operation.name`_ does not correspond to an operation for the entity type (per `Entity Test Operations`_), -the test runner MUST raise an error for an undefined operation. +the test runner MUST raise an error for an undefined operation. Test runners MAY +skip tests that include operations that are intentionally unimplemented (e.g. +``listCollectionNames``). Proceed with preparing the operation's arguments. If ``session`` is specified in `operation.arguments `_, the test runner MUST resolve it @@ -1872,8 +2028,8 @@ points using the special `failPoint`_ or `targetedFailPoint`_ opertions. This internal command is not documented in the MongoDB manual (pending `DOCS-10784`_); however, there is scattered documentation available on the -server wiki (`The "failCommand" Fail Point `_) and employee blogs -(e.g. `Intro to Fail Points `_, +server wiki (`The "failCommand" Fail Point `_) and employee +blogs (e.g. `Intro to Fail Points `_, `Testing Network Errors with MongoDB `_). Documentation can also be gleaned from JIRA tickets (e.g. `SERVER-35004`_, `SERVER-35083`_). This specification does not aim to provide comprehensive documentation for all fail @@ -2016,201 +2172,6 @@ should be described here in detail for historical reference, in addition to any shorter description that may be added to the `Change Log`_. -Open Questions -============== - -Note: these questions should be resolved and this section removed before -publishing version 1.0 of the spec. - - -General -------- - - -Interaction between top-level and test-level runOn -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are at least two ways to handle runOn requirements defined at both the -test and file level: - -* If `test.runOn`_ is specified, those requirements are checked in addition to - any top-level `runOn`_. This behavior is the most straightforward as test - runners will not have to consider inheritance or overriding anything (similar - to reasons top-level ``databaseName`` and ``collectionName`` fields were - ultimately removed). If the top-level `runOn`_ fails, the test runner can skip - the entire file outright (i.e. short-circuit the logical-and of both checks). - -* If `test.runOn`_ is specified, those requirements are checked *instead* of any - top-level `runOn`_. This would mean that a test runner cannot immediately skip - an entire file based on evaluation of the top-level `runOn`_; however, it also - means that humans editing a test file can no longer trust that the top-level - `runOn`_ applies to all tests in the file. - -During review, it was asked if a test runner could enforce that `test.runOn`_ is -only more restrictive (and never more permissive) than the top-level `runOn`_. -While this is technically feasible, it would add considerable complexity to a -test runner to make such comparisons. - - -Representing options in operation.arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Existing spec tests are inconsistent in how they represent optional -parameters for operations. In some cases, optional parameters appear directly -in `operation.arguments`_ alongside required parameters (e.g. CRUD). In other -cases, they are nested under an ``options`` key (e.g. Sessions). - -Most specs *are* consistent about how API methods are defined in spec documents. -The CRUD spec was on the first documents to define API methods and can be -credited with introducing the ``options`` parameter (e.g. -``options: Optional``); however, this was likely done for -readability of the spec (avoiding very long method declarations and/or embedding -``Optional`` syntax therein). Additionally, documenting options in a separate -type (e.g. UpdateOptions) allows the declarations to be shared and reused across -multiple methods. - -That said, this syntax is in no way prescriptive for language implementations. -While some drivers to model options as a struct/object, others solicit them -alongside required parameters (e.g. Python's keyword/named arguments). - -Should this spec require that optional parameters be nested under an ``options`` -key or solicit them directly in `operation.arguments`_? Both are technically -possible, since test runners handle both forms today. - -With regard to CRUD, this issue dates back to -`SPEC-1158 `__. Therein, Jeremy -originally argued in favor of nesting under an ``options`` key because it was -most consistent the text in existing spec documents. - -Note: the CRUD spec WriteModels (e.g. UpdateOneModel) for ``bulkWrite`` to not -use ``options`` keys in either the spec document or test files. As such, the -resolution of this question would not impact how WriteModels are expressed in -test files (see: `bulkWrite`_). - -Note: this question does not pertain to TransactionOptions in SessionOptions, -which is always nested under ``defaultTransactionOptions``. This is a special -case where all drivers represent TransactionOptions as a separate struct/object, -even if they do not do so for ``startTransaction``. - - -Change Stream Tests -------------------- - - -Error Assertions -~~~~~~~~~~~~~~~~ - -`Iterating the Change Stream <../change-streams/tests#iterating-the-change-stream>`__ -in the change stream spec test documentation states: - - If the test expects an error and one was not thrown by either creating the - change stream or executing the test's operations, iterating the change stream - once allows for an error to be thrown by a getMore command. - -It's not clear if this is necessary because the existing change stream spec test -format does not differentiate between errors raised by creating or iterating the -change stream. For instance, an invalid pipeline operator (error code 40324) -should be raised by the initial ``aggregate`` command. - -Since the unified format would allow tests to differentiate between the initial -``watch`` operation and `iterateUntilDocumentOrError`_ operation, and each -can expect its own error, this may not be a concern; however, if the purpose of -the quoted text above to to account for drivers that may not even execute the -``aggregate`` command at the time ``watch`` is called (instead deferring it to -the first iteration of ChangeStream), we may still need to account for that in -the unified test format. - -If so, one possible solution may be to document that ``watch`` operations -expecting an error must also iterate the returned change stream once if the -operation itself did not initially raise an error. Of course, we should confirm -that this (or any) solution is sufficient for all drivers. - - -Command Monitoring Assertions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The change stream spec is also unique among other specs in that its event -assertions only consider as many events as were expected and ignore both extra, -observed events (e.g. getMore) and an optional killCursors (which SHOULD be -issued when resuming). The issue of extra getMore events may pertain to either -sharding or have something to do with variation among drivers in how ``watch`` -is implemented (e.g. eagerly calling ``aggregate`` vs. deferring until the first -iteration). - -A possible solution to address this need may be to extend -`test.expectedEvents `_ to allow a list of command names -to be specified, for which any command monitoring events will be ignored (as we -currently do for security events and ``configureFailPoint``). If that addresses -*both* needs, we may not need anything more; however, there are some change -stream tests that *do* expect getMore, so in such a case we may still need a -separate `test.expectedEvents `_ option to instruct the -test runner to only assert as many observed events are expected and ignore any -additional events. - - -getResumeToken -~~~~~~~~~~~~~~ - -Change stream spec tests do not interact with the resume token so we have not -introduced a ``getResumeToken`` operation for change stream entities. Existing -spec tests do not even check for ``startAfter`` and ``resumeAfter`` fields in -outgoing commands, which is something we could start doing with `$$type`_ -assertions. That decision could be done at any time after existing spec tests -are ported and would not require any change to the unified format. - - -GridFS Tests ------------- - - -Error Assertions -~~~~~~~~~~~~~~~~ - -The GridFS tests assert for the following error types: "FileNotFound", -"ChunkIsMissing", "ChunkIsWrongSize", and "RevisionNotFound". These types are -not explicitly defined in the spec and only appear in test files. The spec also -does not specify how messages for these errors should be constructed, so it may -not be feasible to use ``errorContains`` assertions. - -Do we care about differentiating these types of errors? If so, we may need to -add them to ``expectedError.type``. - - -Outcome Assertions -~~~~~~~~~~~~~~~~~~ - -In the GridFS tests, ``assert.data`` is similar to `test.outcome`_ but it -uses ``*result`` and ``*actual`` to match document data with a saved result -(e.g. returned file ID) or an arbitrary value (e.g. date). Presently, -`test.outcome`_ uses an exact match and does **not** follow the rules in -`Evaluating Matches`_ (e.g. allow extra fields in actual documents, process -special operators). Should we change `test.outcome`_ to follow the same -comparison rules as we use for `expectedResult`_? - -If so, `test.outcome`_ and `initialData`_ would no longer share the same -`collectionData`_ structure. This is not problematic, but worth noting. - -Alternatively, we could leave `test.outcome`_ as-is and create a new special -operation that *matches* data within a collection. - - -Human-readable binary data -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In `SPEC-1216 `__, Robert recommended -against changing the GridFS tests' ``$hex`` syntax to ``$binary`` Extended JSON, -as the base64-encoded strings in the latter format would not be human-readable. -This spec could introduce a custom operator in -`Special Operators for Matching Assertions`_ to compare an expected hexidecimal -string with an actual binary value, but we would still need to use ``$binary`` -syntax in `collectionData`_ for both `initialData`_ and `test.outcome`_. - -Another possible work-around is for this spec to discuss or link to a utility -that can easily convert between hex and base64. For example, we can include a -Javascript or Python script in the spec repository that can convert between the -two and assist those editing GridFS test files. - - Related Issues ============== @@ -2224,7 +2185,8 @@ The following tickets are addressed by the test format: * `SPEC-1158 `__: Spec tests should use a consistent format for CRUD options - Open question: `Representing options in operation.arguments`_ + ``options`` structs are now flattened into `operation.arguments`_. See: + `Entity Test Operations`_. * `SPEC-1215 `__: Introduce spec test syntax for meta assertions (e.g. any value, not exists) @@ -2232,7 +2194,7 @@ The following tickets are addressed by the test format: * `SPEC-1216 `__: Update GridFS YAML tests to use newer format - Open question: `Human-readable binary data`_ + See: `stream `_ entity and `bucket`_ operations * `SPEC-1229 `__: Standardize spec-test syntax for topology assertions @@ -2249,8 +2211,7 @@ The following tickets are addressed by the test format: * `SPEC-1713 `__: Allow runOn to be defined per-test in addition to per-file - See `runOn`_ and `test.runOn`_ and related open question: - `Interaction between top-level and test-level runOn`_ + See `runOn`_ and `test.runOn`_. * `SPEC-1723 `__: Introduce test file syntax to disable dropping of collection under test @@ -2279,7 +2240,7 @@ Future Work Mixing event types in observeEvents and expectedEvents ------------------------------------------------------ -The test format advises against mixing multiple types of events (e.g. command +The test format advises against mixing events from different specs (e.g. command monitoring *and* SDAM) in `observeEvents`_ and `test.expectedEvents`_. While registering event listeners is trivial, determining how to collate events of multiple types can be a challenge, particularly when some events may not be @@ -2289,6 +2250,29 @@ multiple types of events in the same test, a future version of this spec can define an approach for doing so. +Support events types beyond command monitoring +---------------------------------------------- + +The spec currently only supports command monitoring events in `observeEvents`_ +and `test.expectedEvents`_, as those are the only kind of events used in tests +for specifications that will initially adopt the unified test format. New event +types (e.g. SDAM) can be added in future versions of the spec as needed, which +will also require `Mixing event types in observeEvents and expectedEvents`_ to +be addressed. + + +Allow extra observed events to be ignored +----------------------------------------- + +While command monitoring events for specific commands can be ignored (e.g. +killCursors for change streams), the sequence of observed events must otherwise +match the sequence of expected events (including length). The present design +would not support expecting an event for a command while also ignoring extra +events for the same command (e.g. change stream iteration on a sharded cluster +where multiple getMore commands may be issued). No spec tests currently require +this functionality, but that may change in the future. + + Target failPoint by read preference ----------------------------------- @@ -2298,11 +2282,54 @@ need does arise, `failPoint`_ can be enhanced to support a ``readPreference`` argument. +IO operations for GridFS streams +-------------------------------- + +Existing GridFS spec tests refer to "upload", "download", and "download_by_name" +methods, which allow the tests to abstract stream IO and either upload a byte +sequence or assert a downloaded byte sequence. In order to support methods such +as ``downloadToStream``, ``openUploadStream``, and ``openUploadStreamWithId``, +test runners would need to support IO operations to directly read from and write +to a stream entity. + +Currently, `uploadFromStream`_ and `uploadFromStreamWithId`_ are sufficient to +test uploads using a defined `stream `_ entity and +`openDownloadStream`_ and `openDownloadStreamByName`_ are sufficient to test +downloads using `$$matchesHexBytes`_. + + Change Log ========== Note: this will be cleared when publishing version 1.0 of the spec +2020-09-02: + +* Future Work for supporting event types beyond command monitoring (e.g. SDAM) + +* ignoreCommandMonitoringEvents option for client entities, which is needed to + to ignore killCursors for change stream tests. + +* Future Work for ignoring extra, but not all, events for a command (e.g. + multiple getMores for change stream iteration on a sharded cluster) + +* stream entity in createEntities, which is created from a hexadecimal string + and used as an argument for the uploadFromStream bucket operation + +* $$matchesEntity and $$matchesHexBytes match operators, for GridFS tests + +* describe GridFS bucket operations and note unsupported operations, which link + to Future Work for IO operations on GridFS streams + +* isError and isClientError assertions for expectedError + +* note that errorCode and errorCodeName should not be used for client errors + +* Remove Open Questions, which should all be resolved + +* Test runners may skip tests with intentionally unimplemented methods (e.g. + listCollectionNames) + 2020-09-01: * clarify that saveResultAsEntity should always define the name in the map, even From 0dd4ad9a0cc3f0f9e345d9172d51968f9d111b3a Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 3 Sep 2020 17:15:51 -0400 Subject: [PATCH 28/90] runOnRequirements, expectedEventsForClient, refer to YAML objects --- .../tests/example-insertOne.json | 2 +- .../tests/example-insertOne.yml | 2 +- .../unified-test-format.rst | 294 +++++++++--------- 3 files changed, 150 insertions(+), 148 deletions(-) diff --git a/source/unified-test-format/tests/example-insertOne.json b/source/unified-test-format/tests/example-insertOne.json index a991146cca..f2077e5a96 100644 --- a/source/unified-test-format/tests/example-insertOne.json +++ b/source/unified-test-format/tests/example-insertOne.json @@ -1,6 +1,6 @@ { "schemaVersion": "1.0", - "runOn": [ + "runOnRequirements": [ { "minServerVersion": "2.6" } diff --git a/source/unified-test-format/tests/example-insertOne.yml b/source/unified-test-format/tests/example-insertOne.yml index a11b899412..e004080089 100644 --- a/source/unified-test-format/tests/example-insertOne.yml +++ b/source/unified-test-format/tests/example-insertOne.yml @@ -2,7 +2,7 @@ description: "example-insertOne" schemaVersion: "1.0" -runOn: +runOnRequirements: - minServerVersion: "2.6" createEntities: diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2d55a278d2..96a6dd72d2 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -137,8 +137,8 @@ Test Format Each specification test file can define one or more tests, which inherit some top-level configuration (e.g. namespace, initial data). YAML and JSON test files -are parsed as a document by the test runner. This section defines the top-level -keys for that document and links to various sub-sections for definitions of +are parsed as an object by the test runner. This section defines the top-level +keys for that object and links to various sub-sections for definitions of nested structures (e.g. individual `test`_, `operation`_). Although test runners are free to process YAML or JSON files, YAML is the @@ -165,20 +165,18 @@ The top-level fields of a test file are as follows: refer to specific patch versions since patch-level changes SHOULD NOT alter the structure of the test format (as previously noted in `Schema Version`_). -.. _runOn: +.. _runOnRequirements: -- ``runOn``: Optional array of documents. List of server version and/or topology - requirements for which the tests in this file can be run. If no requirements - are met, the test runner MUST skip this test file. - - If set, the array should contain at least one document. The structure of each - document is defined in `runOnRequirement`_. +- ``runOnRequirements``: Optional array of one or more `runOnRequirement`_ + objects. List of server version and/or topology requirements for which the + tests in this file can be run. If no requirements are met, the test runner + MUST skip this test file. .. _createEntities: -- ``createEntities``: Optional array of documents. List of entities (e.g. - client, collection, session objects) that should be created before each test - case is executed. The structure of each document is defined in `entity`_. +- ``createEntities``: Optional array of `entity`_ objects. List of entities + (e.g. client, collection, session objects) that should be created before each + test case is executed. Test files SHOULD define entities in dependency order, such that all referenced entities (e.g. client) are defined before any of their dependent @@ -186,23 +184,18 @@ The top-level fields of a test file are as follows: .. _initialData: -- ``initialData``: Optional array of documents. Data that should exist in - collections before each test case is executed. +- ``initialData``: Optional array of one or more `collectionData`_ objects. Data + that should exist in collections before each test case is executed. - If set, the array should contain at least one document. The structure of each - document is defined in `collectionData`_. Before each test and for each - `collectionData`_, the test runner MUST drop the collection and insert the - specified documents (if any) using a "majority" write concern. If no documents - are specified, the test runner MUST create the collection with a "majority" - write concern. + Before each test and for each `collectionData`_, the test runner MUST drop the + collection and insert the specified documents (if any) using a "majority" + write concern. If no documents are specified, the test runner MUST create the + collection with a "majority" write concern. .. _tests: -- ``tests``: Required array of documents. List of test cases to be executed - independently of each other. - - The array should contain at least one document. The structure of each - document is defined in `test`_. +- ``tests``: Required array of one or more `test`_ objects. List of test cases + to be executed independently of each other. runOnRequirement @@ -211,7 +204,7 @@ runOnRequirement A combination of server version and/or topology requirements for running the test(s). -The structure of this document is as follows: +The structure of this object is as follows: - ``minServerVersion``: Optional string. The minimum server version (inclusive) required to successfully run the tests. If this field is omitted, it should be @@ -240,13 +233,13 @@ entity An entity (e.g. client, collection, session object) that will be created in the `Entity Map`_ before each test is executed. -This document MUST contain **exactly one** top-level key that identifies the -entity type and maps to a nested document, which specifies a unique name for the +This object MUST contain **exactly one** top-level key that identifies the +entity type and maps to a nested object, which specifies a unique name for the entity (``id`` key) and any other parameters necessary for its construction. Tests SHOULD use sequential names based on the entity type (e.g. "session0", "session1"). -When defining an entity document in YAML, a `node anchor`_ SHOULD be created on +When defining an entity object in YAML, a `node anchor`_ SHOULD be created on the entity's ``id`` key. This anchor will allow the unique name to be referenced with an `alias node`_ later in the file (e.g. from another entity or `operation`_ document) and also leverage YAML's parser for reference validation. @@ -254,26 +247,26 @@ with an `alias node`_ later in the file (e.g. from another entity or .. _node anchor: https://yaml.org/spec/1.2/spec.html#id2785586 .. _alias node: https://yaml.org/spec/1.2/spec.html#id2786196 -The structure of this document is as follows: +The structure of this object is as follows: .. _entity_client: -- ``client``: Optional document. Corresponds with a MongoClient object. +- ``client``: Optional object. Defines a MongoClient object. - The structure of this document is as follows: + The structure of this object is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &client0 client0``). - - ``uriOptions``: Optional document. Additional URI options to apply to the + - ``uriOptions``: Optional object. Additional URI options to apply to the test suite's connection string that is used to create this client. Any keys - in this document MUST override conflicting keys in the connection string. + in this object MUST override conflicting keys in the connection string. Documentation for supported options may be found in the `URI Options <../uri-options/uri-options.rst>`__ spec, with one notable - exception: if ``readPreferenceTags`` is specified in this document, the key + exception: if ``readPreferenceTags`` is specified in this object, the key will map to an array of strings, each representing a tag set, since it is - not feasible to define multiple ``readPreferenceTags`` keys in the document. + not feasible to define multiple ``readPreferenceTags`` keys in the object. .. _entity_client_useMultipleMongoses: @@ -321,9 +314,9 @@ The structure of this document is as follows: .. _entity_database: -- ``database``: Optional document. Corresponds with a Database object. +- ``database``: Optional object. Defines a Database object. - The structure of this document is as follows: + The structure of this object is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &database0 database0``). @@ -336,13 +329,13 @@ The structure of this document is as follows: define a `node anchor`_ for this field (e.g. ``databaseName: &database0Name foo``). - - ``databaseOptions``: Optional document. See `collectionOrDatabaseOptions`_. + - ``databaseOptions``: Optional `collectionOrDatabaseOptions`_ object. .. _entity_collection: -- ``collection``: Optional document. Corresponds with a Collection object. +- ``collection``: Optional object. Defines a Collection object. - The structure of this document is as follows: + The structure of this object is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. @@ -356,15 +349,13 @@ The structure of this document is as follows: define a `node anchor`_ for this field (e.g. ``collectionName: &collection0Name foo``). - - ``collectionOptions``: Optional document. See - `collectionOrDatabaseOptions`_. + - ``collectionOptions``: Optional `collectionOrDatabaseOptions`_ object. .. _entity_session: -- ``session``: Optional document. Corresponds with an explicit ClientSession - object. +- ``session``: Optional object. Defines an explicit ClientSession object. - The structure of this document is as follows: + The structure of this object is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &session0 session0``). @@ -373,7 +364,7 @@ The structure of this document is as follows: created. The YAML file SHOULD use an `alias node`_ for a client entity's ``id`` field (e.g. ``client: *client0``). - - ``sessionOptions``: Optional document. Map of parameters to pass to + - ``sessionOptions``: Optional object. Map of parameters to pass to `MongoClient.startSession <../source/sessions/driver-sessions.rst#startsession>`__ when creating the session. Supported options are defined in the following specifications: @@ -385,10 +376,10 @@ The structure of this document is as follows: transaction options MUST remain nested under ``defaultTransactionOptions`` and MUST NOT be flattened into ``sessionOptions``. -- ``bucket``: Optional document. Corresponds with a Bucket object, as defined in - the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. +- ``bucket``: Optional object. Defines a Bucket object, as defined in the + `GridFS <../gridfs/gridfs-spec.rst>`__ spec. - The structure of this document is as follows: + The structure of this object is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &bucket0 bucket0``). @@ -397,7 +388,7 @@ The structure of this document is as follows: be created. The YAML file SHOULD use an `alias node`_ for a database entity's ``id`` field (e.g. ``database: *database0``). - - ``bucketOptions``: Optional document. Additional options used to construct + - ``bucketOptions``: Optional object. Additional options used to construct the bucket object. Supported options are defined in the `GridFS <../source/gridfs/gridfs-spec.rst#configurable-gridfsbucket-class>`__ specification. The ``readConcern``, ``readPreference``, and ``writeConcern`` @@ -405,14 +396,14 @@ The structure of this document is as follows: .. _entity_stream: -- ``stream``: Optional document. Corresponds with a stream as defined in the +- ``stream``: Optional object. Defines a stream, as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. Test runners MUST ensure that stream is both readable *and* writable. This entity is primarily used with `uploadFromStream`_ and `uploadFromStreamWithId`_ operations for `bucket`_ entities. - The structure of this document is as follows: + The structure of this object is as follows: - ``id``: Required string. Unique name for this entity. The YAML file SHOULD define a `node anchor`_ for this field (e.g. ``id: &stream0 stream0``). @@ -431,13 +422,13 @@ List of documents that should correspond to the contents of a collection. This structure is used by both `initialData`_ and `test.outcome `_, which insert and read documents, respectively. -The structure of this document is as follows: +The structure of this object is as follows: - ``collectionName``: Required string. See `commonOptions_collectionName`_. - ``databaseName``: Required string. See `commonOptions_databaseName`_. -- ``documents``: Required array of documents. List of documents corresponding to +- ``documents``: Required array of objects. List of documents corresponding to the contents of the collection. This list may be empty. @@ -446,29 +437,26 @@ test Test case consisting of a sequence of operations to be executed. -The structure of each document is as follows: +The structure of this object is as follows: - ``description``: Required string. The name of the test. This SHOULD describe the purpose of this test (e.g. "insertOne is retried"). -.. _test_runOn: +.. _test_runOnRequirements: -- ``runOn``: Optional array of documents. List of server version and/or topology - requirements for which the tests in this file can be run. If specified, these - requirements are evaluated independently and in addition to any top-level - `runOn`_ requirements. If no requirements in this array are met, the test - runner MUST skip this test. +- ``runOnRequirements``: Optional array of on or more `runOnRequirement`_ + objects. List of server version and/or topology requirements for which this + test can be run. If specified, these requirements are evaluated independently + and in addition to any top-level `runOnRequirements`_. If no requirements in + this array are met, the test runner MUST skip this test. These requirements SHOULD be more restrictive than those specified in the - top-level `runOn`_ requirements (if any). They SHOULD NOT be more permissive. + top-level `runOnRequirements`_ (if any). They SHOULD NOT be more permissive. This is advised because both sets of requirements MUST be satisified in order for a test to be executed and more permissive requirements at the test-level could be taken out of context on their own. - If set, the array should contain at least one document. The structure of each - document is defined in `runOnRequirement`_. - .. _test_skipReason: - ``skipReason``: Optional string. If set, the test will be skipped. The string @@ -476,20 +464,17 @@ The structure of each document is as follows: .. _test_operations: -- ``operations``: Required array of documents. List of operations to be executed - for the test case. - - The array should contain at least one document. The structure of each - document is defined in `operation`_. +- ``operations``: Required array of one or more `operation`_ objects. List of + operations to be executed for the test case. .. _test_expectedEvents: -- ``expectedEvents``: Optional array of documents. Each document will specify a - client entity and a list of events that are expected to be observed (in that - order) on that client while executing `operations `_. +- ``expectedEvents``: Optional array of one or more `expectedEventsForClient`_ + objects. For one or more clients, a list of events that are expected to be + observed in a particular order. If a driver only supports configuring event listeners globally (for all - clients), the test runner SHOULD associate each observed event with a client + clients), the test runner SHOULD associate each observed event with a client in in order to perform these assertions. Test files SHOULD NOT expect events from multiple specs (e.g. command @@ -497,19 +482,6 @@ The structure of each document is as follows: `Mixing event types in observeEvents and expectedEvents`_ for more information. - The array should contain at least one document. The structure of each document - is as follows: - - - ``client``: Required string. Client entity on which the events are expected - to be observed. See `commonOptions_client`_. - - - ``events``: Required array of documents. List of events, which are expected - to be observed (in this order) on the corresponding client while executing - `operations`_. If the array is empty, the test runner MUST assert that no - events were observed on the client (excluding ignored events). - - The structure of each document is defined in `expectedEvent`_. - .. _test_outcome: - ``outcome``: Optional array of documents. Each document will specify expected @@ -526,7 +498,7 @@ operation An operation to be executed as part of the test. -The structure of this document is as follows: +The structure of this object is as follows: .. _operation_name: @@ -543,17 +515,16 @@ The structure of this document is as follows: .. _operation_arguments: -- ``arguments``: Optional document. Map of parameter names and values for the - operation. The structure of this document will vary based on the operation. +- ``arguments``: Optional object. Map of parameter names and values for the + operation. The structure of this object will vary based on the operation. See `Entity Test Operations`_ and `Special Test Operations`_. The ``session`` parameter is handled specially (see `commonOptions_session`_). .. _operation_expectedError: -- ``expectedError``: Optional document. One or more assertions for an expected - error raised by the operation. The structure of this document is - defined in `expectedError`_. +- ``expectedError``: Optional `expectedError`_ object. One or more assertions + for an expected error raised by the operation. This field is mutually exclusive with `expectedResult `_ and @@ -590,9 +561,9 @@ expectedError ~~~~~~~~~~~~~ One or more assertions for an error/exception, which is expected to be raised by -an executed operation. At least one key is required in this document. +an executed operation. At least one key is required in this object. -The structure of this document is as follows: +The structure of this object is as follows: - ``isError``: Optional boolean. If true, the test runner MUST assert that an error was raised. This is primarily used when no other error assertions apply @@ -658,25 +629,42 @@ The structure of this document is as follows: this value. +expectedEventsForClient +~~~~~~~~~~~~~~~~~~~~~~~ + +A list of events that are expected to be observed (in that order) for a client +while executing `operations `_. + +The structure of each object is as follows: + +- ``client``: Required string. Client entity on which the events are expected + to be observed. See `commonOptions_client`_. + +- ``events``: Required array of `expectedEvent`_ objects. List of events, which + are expected to be observed (in this order) on the corresponding client while + executing `operations`_. If the array is empty, the test runner MUST assert + that no events were observed on the client (excluding ignored events). + + expectedEvent ~~~~~~~~~~~~~ An event (e.g. APM), which is expected to be observed while executing the test's operations. -This document MUST contain **exactly one** top-level key that identifies the -event type and maps to a nested document, which contains one or more assertions +This object MUST contain **exactly one** top-level key that identifies the +event type and maps to a nested object, which contains one or more assertions for the event's properties. -The structure of this document is as follows: +The structure of this object is as follows: .. _expectedEvent_commandStartedEvent: -- ``commandStartedEvent``: Optional document. Assertions for a one or more +- ``commandStartedEvent``: Optional object. Assertions for a one or more `CommandStartedEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. - The structure of this document is as follows: + The structure of this object is as follows: - ``command``: Optional document. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. @@ -691,11 +679,11 @@ The structure of this document is as follows: .. _expectedEvent_commandSucceededEvent: -- ``commandSucceededEvent``: Optional document. Assertions for a one or more +- ``commandSucceededEvent``: Optional object. Assertions for a one or more `CommandSucceededEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. - The structure of this document is as follows: + The structure of this object is as follows: - ``reply``: Optional document. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. @@ -705,11 +693,11 @@ The structure of this document is as follows: .. _expectedEvent_commandFailedEvent: -- ``commandFailedEvent``: Optional document. Assertions for a one or more +- ``commandFailedEvent``: Optional object. Assertions for a one or more `CommandFailedEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. - The structure of this document is as follows: + The structure of this object is as follows: - ``commandName``: Optional string. Test runners MUST assert that the actual command name matches this value using a case-insensitive comparison. @@ -720,13 +708,13 @@ collectionOrDatabaseOptions Map of parameters used to construct a collection or database object. -The structure of this document is as follows: +The structure of this object is as follows: - - ``readConcern``: Optional document. See `commonOptions_readConcern`_. + - ``readConcern``: Optional object. See `commonOptions_readConcern`_. - - ``readPreference``: Optional document. See `commonOptions_readPreference`_. + - ``readPreference``: Optional object. See `commonOptions_readPreference`_. - - ``writeConcern``: Optional document. See `commonOptions_writeConcern`_. + - ``writeConcern``: Optional object. See `commonOptions_writeConcern`_. Common Options @@ -755,26 +743,26 @@ The structure of these common options is as follows: .. _commonOptions_readConcern: -- ``readConcern``: Document. Map of parameters to construct a read concern. +- ``readConcern``: Object. Map of parameters to construct a read concern. - The structure of this document is as follows: + The structure of this object is as follows: - ``level``: Required string. .. _commonOptions_readPreference: -- ``readPreference``: Document. Map of parameters to construct a read +- ``readPreference``: Object. Map of parameters to construct a read preference. - The structure of this document is as follows: + The structure of this object is as follows: - ``mode``: Required string. - - ``tagSets``: Optional array of documents. + - ``tagSets``: Optional array of objects. - ``maxStalenessSeconds``: Optional integer. - - ``hedge``: Optional document. + - ``hedge``: Optional object. .. _commonOptions_client: @@ -790,9 +778,9 @@ The structure of these common options is as follows: .. _commonOptions_writeConcern: -- ``writeConcern``: Document. Map of parameters to construct a write concern. +- ``writeConcern``: Object. Map of parameters to construct a write concern. - The structure of this document is as follows: + The structure of this object is as follows: - ``journal``: Optional boolean. @@ -804,8 +792,9 @@ The structure of these common options is as follows: Version String -------------- -Version strings, which are used for `schemaVersion`_ and `runOn`_, MUST conform -to one of the following formats, where each component is an integer: +Version strings, which are used for `schemaVersion`_ and `runOnRequirement`_, +MUST conform to one of the following formats, where each component is an +integer: - ``..`` - ``.`` (```` is assumed to be zero) @@ -926,13 +915,13 @@ The following arguments are supported: by languages that are unable preserve the order of keys in the ``command`` argument when parsing YAML/JSON. -- ``readConcern``: Optional document. See `commonOptions_readConcern`_. +- ``readConcern``: Optional object. See `commonOptions_readConcern`_. -- ``readPreference``: Optional document. See `commonOptions_readPreference`_. +- ``readPreference``: Optional object. See `commonOptions_readPreference`_. - ``session``: Optional string. See `commonOptions_session`_. -- ``writeConcern``: Optional document. See `commonOptions_writeConcern`_. +- ``writeConcern``: Optional object. See `commonOptions_writeConcern`_. collection @@ -973,7 +962,7 @@ The ``requests`` parameter for ``bulkWrite`` is documented as a list of WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily expressed in YAML and JSON. To account for this, test files MUST nest -each WriteModel document in a single-key document, where the key identifies the +each WriteModel object in a single-key object, where the key identifies the request type (e.g. "insertOne"), as in the following example:: arguments: @@ -1036,7 +1025,7 @@ withTransaction The ``withTransaction`` operation's ``callback`` parameter is a function and not easily expressed in YAML/JSON. For ease of testing, this parameter is expressed -as an array of `operation`_ documents (analogous to +as an array of `operation`_ objects (analogous to `test.operations `_). Test runners MUST evaluate error and result assertions when executing these operations in the callback. @@ -1165,10 +1154,11 @@ The following arguments are supported: The client entity SHOULD specify false for `useMultipleMongoses `_ if this operation - could be executed on a sharded topology (according to `runOn`_ or - `test.runOn `_). This is advised because server selection rules - for mongos could lead to unpredictable behavior if different servers were - selected for configuring the fail point and executing subsequent operations. + could be executed on a sharded topology (according to `runOnRequirements`_ or + `test.runOnRequirements `_). This is advised because + server selection rules for mongos could lead to unpredictable behavior if + different servers were selected for configuring the fail point and executing + subsequent operations. When executing this operation, the test runner MUST keep a record of the fail point so that it can be disabled after the test. The test runner MUST also @@ -1589,14 +1579,14 @@ sufficient. For instance, a test file cannot anticipate what a session ID will be at runtime, but may still want to analyze the contents of an ``lsid`` field in a command document. To address this need, special operators can be used. -These operators are documents with a single key identifying the operator. Such +These operators are objects with a single key identifying the operator. Such keys are prefixed with ``$$`` to ease in detecting an operator (test runners -need only inspect the first key of each document) and differentiate the document +need only inspect the first key of each object) and differentiate the object from MongoDB query operators, which use a single `$` prefix. The key will map to some value that influences the operator's behavior (if applicable). When examining the structure of an expected value during a comparison, test -runners MUST examine the first key of any document for a ``$$`` prefix and, if +runners MUST examine the first key of any object for a ``$$`` prefix and, if present, defer to the special logic defined in this section. @@ -1814,8 +1804,8 @@ determine if the test file can be processed further. Test runners MAY support multiple versions and MUST NOT process incompatible files (as discussed in `Schema Version`_). -If `runOn`_ is specified, the test runner MUST skip the test file unless one or -more `runOnRequirement`_ documents are satisfied. +If `runOnRequirements`_ is specified, the test runner MUST skip the test file +unless one or more `runOnRequirement`_ objects are satisfied. For each element in `tests`_, follow the process in `Executing a Test`_. @@ -1837,8 +1827,9 @@ forgoing any additional assertions. If `test.skipReason `_ is specified, the test runner MUST skip this test and MAY use the string value to log a message. -If `test.runOn `_ is specified, the test runner MUST skip the test -unless one or more `runOnRequirement`_ documents are satisfied. +If `test.runOnRequirementss `_ is specified, the test +runner MUST skip the test unless one or more `runOnRequirement`_ objects are +satisfied. If `initialData`_ is specified, for each `collectionData`_ therein the test runner MUST drop the collection and insert the specified documents (if any) @@ -1888,7 +1879,7 @@ any fail points configured using `targetedFailPoint`_, the test runner MUST disable the fail point on the same mongos server on which it was originally configured. See `Disabling Fail Points`_ for more information. -If `test.expectedEvents `_ is specified, for each document +If `test.expectedEvents `_ is specified, for each object therein the test runner MUST assert that the number and sequence of expected events match the number and sequence of actual events observed on the specified client. If the list of expected events is empty, the test runner MUST assert @@ -2052,24 +2043,24 @@ and has the following structure:: db.adminCommand({ configureFailPoint: , - mode: , - data: + mode: , + data: }); The value of ``configureFailPoint`` is a string denoting the fail point to be configured (e.g. "failCommand"). The ``mode`` option is a generic fail point option and may be assigned a string -or document value. The string values "alwaysOn" and "off" may be used to -enable or disable the fail point, respectively. A document may be used to -specify either ``times`` or ``skip``, which are mutually exclusive: +or object value. The string values "alwaysOn" and "off" may be used to enable or +disable the fail point, respectively. An object may be used to specify either +``times`` or ``skip``, which are mutually exclusive: - ``{ times: }`` may be used to limit the number of times the fail point may trigger before transitioning to "off". - ``{ skip: }`` may be used to defer the first trigger of a fail point, after which it will transition to "alwaysOn". -The ``data`` option is a document that may be used to specify any options that +The ``data`` option is an object that may be used to specify any options that control the particular fail point's behavior. In order to use ``configureFailPoint``, the undocumented ``enableTestCommands`` @@ -2108,12 +2099,12 @@ The ``failCommand`` fail point may be configured like so:: db.adminCommand({ configureFailPoint: "failCommand", - mode: , + mode: , data: { failCommands: [, ...], closeConnection: , errorCode: , - writeConcernError: , + writeConcernError: , appName: , blockConnection: , blockTimeMS: , @@ -2198,7 +2189,8 @@ The following tickets are addressed by the test format: * `SPEC-1229 `__: Standardize spec-test syntax for topology assertions - See `runOnRequirement`_, which is used by both `runOn`_ and `test.runOn`_. + See `runOnRequirement`_, which is used by both `runOnRequirements`_ and + `test.runOnRequirements`_. * `SPEC-1254 `__: Rename topology field in spec tests to topologies @@ -2211,7 +2203,7 @@ The following tickets are addressed by the test format: * `SPEC-1713 `__: Allow runOn to be defined per-test in addition to per-file - See `runOn`_ and `test.runOn`_. + See `runOnRequirements`_ and `test.runOnRequirements`_. * `SPEC-1723 `__: Introduce test file syntax to disable dropping of collection under test @@ -2303,6 +2295,14 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-03: + +* Rename top-level and test-level runOn to runOnRequirements + +* Define expectedEventsForClient, between expectedEvents and expectedEvent + +* Replace "document" with "object" unless referring to a MongoDB document + 2020-09-02: * Future Work for supporting event types beyond command monitoring (e.g. SDAM) @@ -2330,6 +2330,8 @@ Note: this will be cleared when publishing version 1.0 of the spec * Test runners may skip tests with intentionally unimplemented methods (e.g. listCollectionNames) +* use camelCase instead of snake_case for API methods and parameters + 2020-09-01: * clarify that saveResultAsEntity should always define the name in the map, even From 3b70475e6ae56dc4de04d4014e6806bf3ac9b11c Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 8 Sep 2020 12:53:04 -0400 Subject: [PATCH 29/90] Sync JSON file with YAML to add description field --- source/unified-test-format/tests/example-insertOne.json | 1 + 1 file changed, 1 insertion(+) diff --git a/source/unified-test-format/tests/example-insertOne.json b/source/unified-test-format/tests/example-insertOne.json index f2077e5a96..68f2ccf8de 100644 --- a/source/unified-test-format/tests/example-insertOne.json +++ b/source/unified-test-format/tests/example-insertOne.json @@ -1,4 +1,5 @@ { + "description": "example-insertOne", "schemaVersion": "1.0", "runOnRequirements": [ { From 8038b4bd105140172f83d9ee75c32f05352e8f41 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 8 Sep 2020 16:07:43 -0400 Subject: [PATCH 30/90] Require arrays, rename "expect" prefix, clarify "sharded" requirement --- .../tests/example-insertOne.json | 4 +- .../tests/example-insertOne.yml | 4 +- .../unified-test-format.rst | 147 ++++++++++-------- 3 files changed, 83 insertions(+), 72 deletions(-) diff --git a/source/unified-test-format/tests/example-insertOne.json b/source/unified-test-format/tests/example-insertOne.json index 68f2ccf8de..be41f9eacb 100644 --- a/source/unified-test-format/tests/example-insertOne.json +++ b/source/unified-test-format/tests/example-insertOne.json @@ -53,14 +53,14 @@ "_id": 2 } }, - "expectedResult": { + "expectResult": { "insertedId": { "$$unsetOrMatches": 2 } } } ], - "expectedEvents": [ + "expectEvents": [ { "client": "client0", "events": [ diff --git a/source/unified-test-format/tests/example-insertOne.yml b/source/unified-test-format/tests/example-insertOne.yml index e004080089..22b27bb2f8 100644 --- a/source/unified-test-format/tests/example-insertOne.yml +++ b/source/unified-test-format/tests/example-insertOne.yml @@ -33,9 +33,9 @@ tests: name: insertOne arguments: document: { _id: 2 } - expectedResult: + expectResult: insertedId: { $$unsetOrMatches: 2 } - expectedEvents: + expectEvents: - client: *client0 events: - commandStartedEvent: diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 96a6dd72d2..82148df0fe 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -216,15 +216,17 @@ The structure of this object is as follows: should be assumed that there is no upper bound on the required server version. The format of this string is defined in `Version String`_. -- ``topologies``: Optional string or array of strings. One or more of server - topologies against which the tests can be run successfully. Valid topologies - are "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded +- ``topologies``: Optional array of strings. One or more of server topologies + against which the tests can be run successfully. Valid topologies are + "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded cluster backed by replica sets). If this field is omitted, it should be assumed that there is no topology requirement for the test. When matching a "sharded-replicaset" topology, test runners MUST ensure that all shards are backed by a replica set. The process for doing so is described - in `Determining if a Sharded Cluster Uses Replica Sets`_. + in `Determining if a Sharded Cluster Uses Replica Sets`_. When matching a + "sharded" topology, test runners MUST accept any type of sharded cluster (i.e. + "sharded" implies "sharded-replicaset", but not vice versa). entity @@ -280,14 +282,14 @@ The structure of this object is as follows: .. _entity_client_observeEvents: - - ``observeEvents``: Optional string or array of strings. One or more types of - events that can be observed for this client. Unspecified event types MUST - be ignored by this client's event listeners and SHOULD NOT be included in - `test.expectedEvents `_ for this client. + - ``observeEvents``: Optional array of strings. One or more types of events + that can be observed for this client. Unspecified event types MUST be + ignored by this client's event listeners and SHOULD NOT be included in + `test.expectEvents `_ for this client. Test files SHOULD NOT observe events from multiple specs (e.g. command monitoring *and* SDAM events) for a single client. See - `Mixing event types in observeEvents and expectedEvents`_ for more + `Mixing event types in observeEvents and expectEvents`_ for more information. Supported types correspond to those documented in `expectedEvent`_ and are @@ -301,9 +303,9 @@ The structure of this object is as follows: .. _entity_client_ignoreCommandMonitoringEvents: - - ``ignoreCommandMonitoringEvents``: Optional string or array of strings. One - or more command names for which the test runner MUST ignore any observed - command monitoring events. The command(s) will be ignored in addition to + - ``ignoreCommandMonitoringEvents``: Optional array of strings. One or more + command names for which the test runner MUST ignore any observed command + monitoring events. The command(s) will be ignored in addition to ``configureFailPoint`` and any commands containing sensitive information (per the `Command Monitoring <../command-monitoring/command-monitoring.rst#security>`__ @@ -467,9 +469,9 @@ The structure of this object is as follows: - ``operations``: Required array of one or more `operation`_ objects. List of operations to be executed for the test case. -.. _test_expectedEvents: +.. _test_expectEvents: -- ``expectedEvents``: Optional array of one or more `expectedEventsForClient`_ +- ``expectEvents``: Optional array of one or more `expectedEventsForClient`_ objects. For one or more clients, a list of events that are expected to be observed in a particular order. @@ -479,7 +481,7 @@ The structure of this object is as follows: Test files SHOULD NOT expect events from multiple specs (e.g. command monitoring *and* SDAM events) for a single client. See - `Mixing event types in observeEvents and expectedEvents`_ for more + `Mixing event types in observeEvents and expectEvents`_ for more information. .. _test_outcome: @@ -521,24 +523,23 @@ The structure of this object is as follows: The ``session`` parameter is handled specially (see `commonOptions_session`_). -.. _operation_expectedError: +.. _operation_expectError: -- ``expectedError``: Optional `expectedError`_ object. One or more assertions - for an expected error raised by the operation. +- ``expectError``: Optional `expectedError`_ object. One or more assertions for + an error expected to be raised by the operation. This field is mutually exclusive with - `expectedResult `_ and + `expectResult `_ and `saveResultAsEntity `_. -.. _operation_expectedResult: +.. _operation_expectResult: -- ``expectedResult``: Optional mixed type. A value corresponding to the expected +- ``expectResult``: Optional mixed type. A value corresponding to the expected result of the operation. This field may be a scalar value, a single document, or an array of documents in the case of a multi-document read. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. - This field is mutually exclusive with - `expectedError `_. + This field is mutually exclusive with `expectError `_. .. _operation_saveResultAsEntity: @@ -549,8 +550,7 @@ The structure of this object is as follows: runner MUST store an empty value (e.g. ``null``) for the entity such that the name will still be defined in the entity map. - This field is mutually exclusive with - `expectedError `_. + This field is mutually exclusive with `expectError `_. This is primarily used for creating a `changeStream`_ entity from the result of a `client_createChangeStream`_, `database_createChangeStream`_, or @@ -620,13 +620,13 @@ The structure of this object is as follows: the error does not contain any of the specified labels (e.g. using the ``hasErrorLabel`` method). -.. _expectedError_expectedResult: +.. _expectedError_expectResult: -- ``expectedResult``: Optional mixed type. This field follows the same rules as - `operation.expectedResult `_ and is only used in - cases where the error includes a result (e.g. `bulkWrite`_). If specified, the - test runner MUST assert that the error includes a result and that it matches - this value. +- ``expectResult``: Optional mixed type. This field follows the same rules as + `operation.expectResult `_ and is only used in cases + where the error includes a result (e.g. `bulkWrite`_). If specified, the test + runner MUST assert that the error includes a result and that it matches this + value. expectedEventsForClient @@ -989,7 +989,7 @@ request type (e.g. "insertOne"), as in the following example:: While operations typically raise an error *or* return a result, the ``bulkWrite`` operation is unique in that it may report both via the ``writeResult`` property of a BulkWriteException. In this case, the intermediary -write result may be matched with `expectedError_expectedResult`_. Because +write result may be matched with `expectedError_expectResult`_. Because ``writeResult`` is optional for drivers to implement, such assertions should utilize the `$$unsetOrMatches`_ operator. @@ -1059,8 +1059,8 @@ openDownloadStream and openDownloadStreamByName ``````````````````````````````````````````````` The ``openDownloadStream`` and ``openDownloadStreamByName`` operations SHOULD -use `$$matchesHexBytes`_ in `expectedResult `_ to -match the contents of the returned stream. These operations MAY use +use `$$matchesHexBytes`_ in `expectResult `_ to match +the contents of the returned stream. These operations MAY use `saveResultAsEntity `_ to save the stream for use with a subsequent operation (e.g. `uploadFromStream`_ ). @@ -1118,8 +1118,8 @@ iterateUntilDocumentOrError Iterates the change stream until either a single document is returned or an error is raised. -If `expectedResult `_ is specified, it SHOULD be a -single document. +If `expectResult `_ is specified, it SHOULD be a single +document. `Iterating the Change Stream <../change-streams/tests#iterating-the-change-stream>`__ in the change stream spec cautions drivers that implement a blocking mode of @@ -1482,7 +1482,7 @@ Evaluating Matches ------------------ Expected values in tests (e.g. -`operation.expectedResult `_) are expressed as either +`operation.expectResult `_) are expressed as either relaxed or canonical `Extended JSON <../extended-json.rst>`_. The algorithm for matching expected and actual values is specified with the @@ -1629,10 +1629,10 @@ Syntax, where ``bsonType`` is a string or integer:: { $$type: } { $$type: [ , , ... ] } -This operator can be used anywhere a matched value is expected (including an -`expectedResult `_). The test runner MUST assert that -the actual value exists and matches one of the expected types, which correspond -to the documented types for the +This operator can be used anywhere a matched value is expected (including +`expectResult `_). The test runner MUST assert that the +actual value exists and matches one of the expected types, which correspond to +the documented types for the `$type `__ query operator. @@ -1656,10 +1656,10 @@ Syntax, where ``entityName`` is a string:: { $$matchesEntity: } -This operator can be used anywhere a matched value is expected (including an -`expectedResult `_). If the entity name is defined in -the current test's `Entity Map`_, the test runner MUST fetch that entity and -assert that the actual value matches the entity using the standard rules in +This operator can be used anywhere a matched value is expected (including +`expectResult `_). If the entity name is defined in the +current test's `Entity Map`_, the test runner MUST fetch that entity and assert +that the actual value matches the entity using the standard rules in `Evaluating Matches`_; otherwise, the test runner MUST raise an error for an undefined entity. The YAML file SHOULD use an `alias node`_ for the entity name. @@ -1674,13 +1674,13 @@ An example of this operator follows:: arguments: filename: "filename" source: *stream0 - expectedResult: { $$type: "objectId" } + expectResult: { $$type: "objectId" } saveResultAsEntity: &objectid0 "objectid0" - object: *filesCollection name: findOne arguments: sort: { uploadDate: -1 } - expectedResult: + expectResult: _id: { $$matchesEntity: *objectid0 } @@ -1692,11 +1692,11 @@ Syntax, where ``hexBytes`` is an even number of hexademical characters { $$matchesHexBytes: } -This operator can be used anywhere a matched value is expected (including an -`expectedResult `_) and the actual value is a stream -as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. The test runner -MUST convert the string to a byte sequence and compare it with the full contents -of the stream. The test runner MUST raise an error if the string is malformed. +This operator can be used anywhere a matched value is expected (including +`expectResult `_) and the actual value is a stream as +defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. The test runner MUST +convert the string to a byte sequence and compare it with the full contents of +the stream. The test runner MUST raise an error if the string is malformed. This operator is primarily used to assert the contents of stream returned by `openDownloadStream`_ and `openDownloadStreamByName`_. @@ -1709,8 +1709,8 @@ Syntax:: { $$unsetOrMatches: } -This operator can be used anywhere a matched value is expected (including an -`expectedResult `_). The test runner MUST assert that +This operator can be used anywhere a matched value is expected (including +`expectResult `_). The test runner MUST assert that actual value either does not exist or matches the expected value. Matching the expected value should use the standard rules in `Evaluating Matches`_, which means that it may contain special operators. @@ -1721,13 +1721,13 @@ BulkWriteException). An example of this operator used for a result's field follows:: - expectedResult: + expectResult: insertedId: { $$unsetOrMatches: 2 } An example of this operator used for an entire result follows:: - expectedError: - expectedResult: + expectError: + expectResult: $$unsetOrMatches: deletedCount: 0 insertedCount: 2 @@ -1847,7 +1847,7 @@ for each target collection the test runner SHOULD execute a non-transactional ``distinct`` command on each mongos server using the internal MongoClient. See `StaleDbVersion Errors on Sharded Clusters`_ for more information. -If `test.expectedEvents `_ is specified, for each client +If `test.expectEvents `_ is specified, for each client entity the test runner MUST enable all event listeners necessary to collect the event types specified in `observeEvents `_. Test runners MAY leave event listeners disabled for tests and/or clients that do not @@ -1879,7 +1879,7 @@ any fail points configured using `targetedFailPoint`_, the test runner MUST disable the fail point on the same mongos server on which it was originally configured. See `Disabling Fail Points`_ for more information. -If `test.expectedEvents `_ is specified, for each object +If `test.expectEvents `_ is specified, for each object therein the test runner MUST assert that the number and sequence of expected events match the number and sequence of actual events observed on the specified client. If the list of expected events is empty, the test runner MUST assert @@ -1939,13 +1939,13 @@ Before executing the operation, the test runner MUST be prepared to catch a potential error from the operation (e.g. enter a ``try`` block). Proceed with executing the operation and capture its result or error. -If `operation.expectedError `_ is specified, the test +If `operation.expectError `_ is specified, the test runner MUST assert that the operation yielded an error; otherwise, the test runner MUST assert that the operation did not yield an error. If an error was expected, the test runner MUST evaluate any assertions in `expectedError`_ accordingly. -If `operation.expectedResult `_ is specified, the test +If `operation.expectResult `_ is specified, the test MUST assert that it matches the actual result of the operation according to the rules outlined in `Evaluating Matches`_. @@ -2229,11 +2229,11 @@ Future Work =========== -Mixing event types in observeEvents and expectedEvents ------------------------------------------------------- +Mixing event types in observeEvents and expectEvents +---------------------------------------------------- The test format advises against mixing events from different specs (e.g. command -monitoring *and* SDAM) in `observeEvents`_ and `test.expectedEvents`_. While +monitoring *and* SDAM) in `observeEvents`_ and `test.expectEvents`_. While registering event listeners is trivial, determining how to collate events of multiple types can be a challenge, particularly when some events may not be predictable (e.g. ServerHeartbeatStartedEvent, CommandStartedEvent for @@ -2246,11 +2246,11 @@ Support events types beyond command monitoring ---------------------------------------------- The spec currently only supports command monitoring events in `observeEvents`_ -and `test.expectedEvents`_, as those are the only kind of events used in tests -for specifications that will initially adopt the unified test format. New event +and `test.expectEvents`_, as those are the only kind of events used in tests for +specifications that will initially adopt the unified test format. New event types (e.g. SDAM) can be added in future versions of the spec as needed, which -will also require `Mixing event types in observeEvents and expectedEvents`_ to -be addressed. +will also require `Mixing event types in observeEvents and expectEvents`_ to be +addressed. Allow extra observed events to be ignored @@ -2295,6 +2295,17 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-08: + +* Replace " or array of " with "array of " for + ``topologies``, ``observeEvents``, and ``ignoreCommandMonitoringEvents``. + +* Rename ``expected`` prefix to ``expect`` in test field names. Applies to + ``expectEvents``, ``expectedError``, and ``expectResult``. Structures such as + ` `expectedEvent`` were not renamed. + +* Clarify that "sharded" implies "sharded-replicaset". + 2020-09-03: * Rename top-level and test-level runOn to runOnRequirements From d960cbefa2c4fa66608542f869a42ff7a6373c6f Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 8 Sep 2020 16:52:39 -0400 Subject: [PATCH 31/90] Remove note about clearing state between test files --- source/unified-test-format/unified-test-format.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 82148df0fe..597e36439a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-01 +:Last Modified: 2020-09-08 .. contents:: @@ -1790,9 +1790,7 @@ Executing a Test File ~~~~~~~~~~~~~~~~~~~~~ The instructions in this section apply for each test file loaded by the test -runner. After processing a test file, test runners SHOULD reset any internal -state that resulted from doing so. For example, an internal MongoClient created -for one test file SHOULD NOT be shared with another. +runner. Test files, which may be YAML or JSON files, MUST be interpreted using an `Extended JSON`_ parser. The parser MUST accept relaxed and canonical Extended @@ -2306,6 +2304,9 @@ Note: this will be cleared when publishing version 1.0 of the spec * Clarify that "sharded" implies "sharded-replicaset". +* Remove note about clearing state between test files. Clearing state is mainly + relevant for tests, and is already discussed. + 2020-09-03: * Rename top-level and test-level runOn to runOnRequirements From 87dc9d70106bebad920344f6e44a097f25a19615 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 8 Sep 2020 18:46:05 -0400 Subject: [PATCH 32/90] Note that server versions are compared numerically --- source/unified-test-format/unified-test-format.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 597e36439a..3b37c6f9eb 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -71,7 +71,7 @@ runner, given the same major version component. For example: - A test runner supporting version 1.5.1 could execute test files with versions 1.0 and 1.5 but *not* 1.6 and 2.0. -- A test runner supporting version 2.1 it could execute test files with versions +- A test runner supporting version 2.1 could execute test files with versions 2.0 and 2.1 but *not* 1.0 and 1.5. - A test runner supporting *both* versions 1.5.1 and 2.0 could execute test files with versions 1.4, 1.5, and 2.0, but *not* 1.6, 2.1, or 3.0. @@ -204,6 +204,9 @@ runOnRequirement A combination of server version and/or topology requirements for running the test(s). +Server versions SHALL be compared numerically and do not follow the comparison +rules discussed in `Schema Version`_. + The structure of this object is as follows: - ``minServerVersion``: Optional string. The minimum server version (inclusive) @@ -2307,6 +2310,8 @@ Note: this will be cleared when publishing version 1.0 of the spec * Remove note about clearing state between test files. Clearing state is mainly relevant for tests, and is already discussed. +* Note that server versions are compared numerically. + 2020-09-03: * Rename top-level and test-level runOn to runOnRequirements From acbec780c59efe8e74ec78e73ed39acc1ca1c0bf Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 8 Sep 2020 18:48:24 -0400 Subject: [PATCH 33/90] Fix typo --- source/unified-test-format/unified-test-format.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 3b37c6f9eb..98baf463c0 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1828,7 +1828,7 @@ forgoing any additional assertions. If `test.skipReason `_ is specified, the test runner MUST skip this test and MAY use the string value to log a message. -If `test.runOnRequirementss `_ is specified, the test +If `test.runOnRequirements `_ is specified, the test runner MUST skip the test unless one or more `runOnRequirement`_ objects are satisfied. From 9997cd60b8afd6c9779c980769291be03f4d6b79 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 9 Sep 2020 12:50:47 -0400 Subject: [PATCH 34/90] Clarify type of observeEvents --- source/unified-test-format/unified-test-format.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 98baf463c0..d6a27d3637 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -295,8 +295,8 @@ The structure of this object is as follows: `Mixing event types in observeEvents and expectEvents`_ for more information. - Supported types correspond to those documented in `expectedEvent`_ and are - as follows: + Supported types correspond to the top-level keys (strings) documented in + `expectedEvent`_ and are as follows: - `commandStartedEvent `_ From 5480885e819e8affe154b38954e3ae09f36caa5e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 11 Sep 2020 16:59:16 -0400 Subject: [PATCH 35/90] Require BSON types be expressed as strings for $$type operator --- .../unified-test-format.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index d6a27d3637..53b2147a92 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-08 +:Last Modified: 2020-09-11 .. contents:: @@ -1627,23 +1627,23 @@ An example of this operator checking for a field's absence follows:: $$type `````` -Syntax, where ``bsonType`` is a string or integer:: +Syntax:: - { $$type: } - { $$type: [ , , ... ] } + { $$type: } + { $$type: [ , , ... ] } This operator can be used anywhere a matched value is expected (including `expectResult `_). The test runner MUST assert that the actual value exists and matches one of the expected types, which correspond to -the documented types for the +the documented string types for the `$type `__ query operator. An example of this operator follows:: command: - getMore: { $$type: [ int, long ] } - collection: { $$type: 2 } # string + getMore: { $$type: [ "int", "long" ] } + collection: { $$type: "string" } When the actual value is an array, test runners MUST NOT examine types of the array's elements. Only the type of actual field should be checked. This is @@ -2296,6 +2296,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-11: + +* Require BSON types be expressed as strings for $$type operator + 2020-09-08: * Replace " or array of " with "array of " for From 96e67132751638bb766d59f81b28c05c79ad3b60 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 15 Sep 2020 17:36:50 -0400 Subject: [PATCH 36/90] Clarify test.outcome and comparison rules --- .../unified-test-format.rst | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 53b2147a92..de1b4f05a0 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-11 +:Last Modified: 2020-09-15 .. contents:: @@ -489,13 +489,12 @@ The structure of this object is as follows: .. _test_outcome: -- ``outcome``: Optional array of documents. Each document will specify expected - contents of a collection after all operations have been executed. The list of - documents therein SHOULD be sorted ascendingly by the ``_id`` field to allow - for deterministic comparisons. +- ``outcome``: Optional array of one or more `collectionData`_ objects. Data + that should exist in collections after each test case is executed. - If set, the array should contain at least one document. The structure of each - document is defined in `collectionData`_. + The list of documents herein SHOULD be sorted ascendingly by the ``_id`` field + to allow for deterministic comparisons. The procedure for asserting collection + contents is discussed in `Executing a Test`_. operation @@ -1889,10 +1888,14 @@ described in `expectedEvent`_. If `test.outcome `_ is specified, for each `collectionData`_ therein the test runner MUST assert that the collection contains exactly the -expected data. The test runner MUST query each collection using an ascending -sort order on the ``_id`` field (i.e. ``{ _id: 1 }``), a ``primary`` read -preference, a ``local`` read concern, and the internal MongoClient. If the list -of documents is empty, the test runner MUST assert that the collection is empty. +expected data. The test runner MUST query each collection using the internal +MongoClient, an ascending sort order on the ``_id`` field (i.e. ``{ _id: 1 }``), +a "primary" read preference, a "local" read concern. The documents must match +expected data exactly (`Evaluating Matches`_ does not apply); however, languages +that are unable to preserve the order of keys in documents MAY permit variations +in field order or otherwise normalize actual and expected documents before +comparison. If the list of documents is empty, the test runner MUST assert that +the collection is empty. Clear the entity map for this test. For each ClientSession in the entity map, the test runner MUST end the session (e.g. call @@ -2296,6 +2299,11 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-15: + +* Revise definition for test.outcome, clarify comparison rules for outcome + documents, and permit flexibility for field order during comparisons. + 2020-09-11: * Require BSON types be expressed as strings for $$type operator From a44d3a23c7d13aee80069904e9edf924feb0fbcc Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 15 Sep 2020 17:37:00 -0400 Subject: [PATCH 37/90] Make formatting of "primary" RP consistent with WC and RC --- source/unified-test-format/unified-test-format.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index de1b4f05a0..de4e41613f 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1145,7 +1145,7 @@ failPoint ~~~~~~~~~ The ``failPoint`` operation instructs the test runner to configure a fail point -using a ``primary`` read preference using the specified client. +using a "primary" read preference using the specified client. The following arguments are supported: @@ -2272,10 +2272,10 @@ this functionality, but that may change in the future. Target failPoint by read preference ----------------------------------- -The `failPoint`_ operation currently uses a primary read preference. To date, no -spec has needed behavior to configure a fail point on a non-primary node. If the -need does arise, `failPoint`_ can be enhanced to support a ``readPreference`` -argument. +The `failPoint`_ operation currently uses a "primary" read preference. To date, +no spec has needed behavior to configure a fail point on a non-primary node. If +the need does arise, `failPoint`_ can be enhanced to support a +``readPreference`` argument. IO operations for GridFS streams From 26f7855fa22cbaeafb57635d45c26bc203826f65 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 17 Sep 2020 14:41:51 -0400 Subject: [PATCH 38/90] Add Goals, clarify schemaVersion, and Future Work entries --- .../unified-test-format.rst | 83 +++++++++++++++++-- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index de4e41613f..218e1c174a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -24,6 +24,7 @@ tests to a single schema, drivers can implement a single test runner to execute acceptance tests for multiple specifications, thereby reducing maintenance of existing specs and implementation time for new specifications. + META ==== @@ -31,6 +32,26 @@ The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in `RFC 2119 `__. + +Goals +===== + +This test format can be used to define tests for the following specifications: + +- `Change Streams <../change-streams/change-streams.rst>`__ +- `Command Monitoring <../command-monitoring/command-monitoring.rst>`__ +- `CRUD <../crud/crud.rst>`__ +- `GridFS <../gridfs/gridfs-spec.rst>`__ +- `Retryable Reads <../retryable-reads/retryable-reads.rst>`__ +- `Retryable Writes <../retryable-writes/retryable-writes.rst>`__ +- `Sessions <../sessions/driver-sessions.rst>`__ +- `Transactions <../transactions/transactions.rst>`__ +- `Convenient API for Transactions <../transactions-convenient-api/transactions-convenient-api.rst>`__ + +This is not an exhaustive list. Specifications that are known to not be +supported by this format may be discussed under `Future Work`_. + + Specification ============= @@ -159,11 +180,17 @@ The top-level fields of a test file are as follows: .. _schemaVersion: - ``schemaVersion``: Required string. Version of this specification to which the - test file complies. Test runners will use this to determine compatibility - (i.e. whether and how the test file will be interpreted). The format of this - string is defined in `Version String`_; however, test files SHOULD NOT need to - refer to specific patch versions since patch-level changes SHOULD NOT alter - the structure of the test format (as previously noted in `Schema Version`_). + test file complies. + + Test files SHOULD be conservative when specifying a schema version. For + example, if the latest schema version is 1.1 but the test file complies with + schema version 1.0, the test file should specify 1.0. + + Test runners will use this to determine compatibility (i.e. whether and how + the test file will be interpreted). The format of this string is defined in + `Version String`_; however, test files SHOULD NOT need to refer to specific + patch versions since patch-level changes SHOULD NOT alter the structure of the + test format (as previously noted in `Schema Version`_). .. _runOnRequirements: @@ -2269,6 +2296,16 @@ where multiple getMore commands may be issued). No spec tests currently require this functionality, but that may change in the future. +Assert expected log messages +---------------------------- + +When drivers support standardized logging, the test format may need to support +assertions for messages expected to be logged while executing operations. Since +log messages are strings, this may require an operator to match regex patterns +within strings. Additionally, the test runner may need to support ignoring extra +log output, similar to `Allow extra observed events to be ignored`_. + + Target failPoint by read preference ----------------------------------- @@ -2294,11 +2331,47 @@ test uploads using a defined `stream `_ entity and downloads using `$$matchesHexBytes`_. +Support Client-side Encryption integration tests +------------------------------------------------ + +Supporting client-side encryption spec tests will require the following changes +to the test format: + +- ``json_schema`` will need to be specified when creating a collection, via + either the collection entity definition or `initialData`_. +- ``key_vault_data`` can be expressed via `initialData`_ +- ``autoEncryptOpts`` will need to be specified when defining a client entity. + Preparation of this field may require reading AWS credentials from environment + variables. + +The process for executing tests should not require significant changes, but test +files will need to express a dependency on mongocryptd. + + +Support SDAM integration tests +------------------------------ + +SDAM integration tests should not require test format changes, but will +introduce several new special test operations for the "testRunner" object. While +the tests themselves only define expectations for command monitoring events, +some special operations may require observing additional event types. There are +also special operations for defining threads and executing operations within +threads, which may warrant introducing a new "thread" entity type. + + Change Log ========== Note: this will be cleared when publishing version 1.0 of the spec +2020-09-17: + +* Future work for logging assertions, FLE tests, and SDAM tests + +* Test files should specify schema version conservatively. + +* Add Goals section and note out-of-scope specs in Future Work. + 2020-09-15: * Revise definition for test.outcome, clarify comparison rules for outcome From 2fd02b5e655d32795efc622d142c68170d96dede Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 17 Sep 2020 15:45:30 -0400 Subject: [PATCH 39/90] Elaborate on schema version --- .../unified-test-format.rst | 75 +++++++++++++++---- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 218e1c174a..33d2ffc805 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -76,13 +76,39 @@ Schema Version -------------- This specification and the `Test Format`_ follow -`semantic versioning `__. Backwards breaking changes (e.g. -removing a field, introducing a required field) will warrant a new major -version. Backwards compatible changes (e.g. introducing an optional field) will -warrant a new minor version. Small fixes and internal changes (e.g. grammar, -adding clarifying text to the spec) will warrant a new patch version; however, -patch versions SHOULD NOT alter the structure of the test format and thus SHOULD -NOT be relevant to test files. +`semantic versioning `__. The version is primarily used to +validate test files with a `JSON schema `__ and also +allow test runners to determine whether a particular test file is supported. + +New tests files SHOULD always be written using the latest major version of this +specification; however, test files SHOULD be conservative in the minor version +they specify (as noted in `schemaVersion`_). + + +JSON Schema Validation +~~~~~~~~~~~~~~~~~~~~~~ + +Each major version of this specification will have a corresponding JSON schema +for its most recent minor version (e.g. ``schema-1.1.json``). A JSON schema for +a particular minor version MUST be capable of validating any and all test files +in that major version series up to and including the minor version. For example, +``schema-2.1.json`` should validate test files with `schemaVersion`_ "2.0" and +"2.1", but would not be expected to validate "1.0", "2.2", or "3.0". + +The JSON schema MUST remain consistent with the `Test Format`_ section. If and +when a new major version is introduced, the `Breaking Changes`_ section MUST be +updated and any JSON schema(s) for a previous major version(s) MUST remain +available so that older test files can still be validated. + +`Ajv `__ MAY be used to programmatically validate both YAML +and JSON files using the JSON schema. The JSON schema MUST NOT use syntax that +is unsupported by this tool, which bears mentioning because there are multiple +versions of the +`JSON schema specification `__. + + +Test Runner Support +~~~~~~~~~~~~~~~~~~~ Each test file defines a `schemaVersion`_, which test runners will use to determine compatibility (i.e. whether and how the test file will be @@ -105,15 +131,32 @@ Test runners MUST NOT process incompatible files but MAY determine how to handle such files (e.g. skip and log a notice, fail and raise an error). Test runners MAY support multiple schema versions (as demonstrated in the example above). -Each major version of this specification will have a corresponding JSON schema -(e.g. `schema-1.json `__), which may be used to programmatically -validate YAML and JSON files using a tool such as `Ajv `__. -The latest JSON schema MUST remain consistent with the `Test Format`_ section. -If and when a new major version is introduced, the `Breaking Changes`_ section -must be updated and JSON schema(s) for any previous major version(s) MUST remain -available so that older test files can still be validated. New tests files -SHOULD always be written using the latest version of this specification. +Impact of Spec Changes on Schema Version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Backwards-breaking changes SHALL warrant a new major version. These changes +include, but are not limited to: + +- Subtractive changes, such as removing a field, operation, or type of supported + entity or event +- Changing an existing field from optional to required +- Introducing a new, required field in the test format +- Significant changes to test file execution (not BC) + +Backwards-compatible changes SHALL warrant a new minor version. These changes +include, but are not limited to: + +- Additive changes, such as a introducing a new `Special Test Operation`_ or + type of supported entity or event +- Changing an existing field from required to optional +- Introducing a new, optional field in the test format +- Minor changes to test file execution (BC) + +Small fixes and internal spec changes (e.g. grammar, adding clarifying text to +the spec) MAY warrant a new patch version; however, patch versions SHOULD NOT +alter the structure of the test format and thus SHOULD NOT be relevant to test +files (as noted in `schemaVersion`_). Entity Map @@ -2372,6 +2415,8 @@ Note: this will be cleared when publishing version 1.0 of the spec * Add Goals section and note out-of-scope specs in Future Work. +* Elaborate on schema version. + 2020-09-15: * Revise definition for test.outcome, clarify comparison rules for outcome From 81c2e74304bc16690b0bef1a67904fba5e4a3eb8 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 17 Sep 2020 23:08:45 -0400 Subject: [PATCH 40/90] Clarify (un)supported entity types --- .../unified-test-format.rst | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 33d2ffc805..5d21399d9e 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -63,8 +63,9 @@ Entity Any object or value that is indexed by a unique name and stored in the `Entity Map`_. This will typically be a driver object (e.g. client, session) defined in `createEntities`_ but may also be a - `saved operation result `_. Entities are - referenced throughout the test file (e.g. `Entity Test Operations`_). + `saved operation result `_. A exhaustive list + of supported types is presented in `Supported Entity Types`_. Entities are + referenced by name throughout the test file (e.g. `Entity Test Operations`_). Internal MongoClient A MongoClient created specifically for use with internal test operations, such @@ -196,6 +197,28 @@ Consider the following examples:: - session: { id: session1, client: session0 } +Supported Entity Types +~~~~~~~~~~~~~~~~~~~~~~ + +Test runners MUST support the following types of entities: + +- MongoClient. See `entity_client`_ and `client`_. +- Database. See `entity_database`_ and `database`_. +- Collection. See `entity_collection`_ and `collection`_ +- ClientSession. See `entity_session`_ and `session`_. +- GridFS Bucket. See `entity_bucket`_ and `bucket`_. +- GridFS Stream. See `entity_stream`_. +- ChangeStream. See `changeStream`_. +- The following BSON types: + + - 0x01 - 0x13 + - 0xFF + - 0x7F + +This is an exhaustive list of permissible types for the entity map. Test runners +MUST raise an error if another type would be added to the entity map. + + Test Format ----------- @@ -451,6 +474,8 @@ The structure of this object is as follows: transaction options MUST remain nested under ``defaultTransactionOptions`` and MUST NOT be flattened into ``sessionOptions``. +.. _entity_bucket: + - ``bucket``: Optional object. Defines a Bucket object, as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. @@ -618,16 +643,10 @@ The structure of this object is as follows: - ``saveResultAsEntity``: Optional string. If specified, the actual result returned by the operation (if any) will be saved with this name in the `Entity Map`_. The test runner MUST raise an error if the name is already in - use. If the operation does not return a value (e.g. void method), the test - runner MUST store an empty value (e.g. ``null``) for the entity such that the - name will still be defined in the entity map. + use or if the result is not a `Supported Entity Type`_. This field is mutually exclusive with `expectError `_. - This is primarily used for creating a `changeStream`_ entity from the result - of a `client_createChangeStream`_, `database_createChangeStream`_, or - `collection_createChangeStream`_ operation. - expectedError ~~~~~~~~~~~~~ @@ -2024,10 +2043,9 @@ MUST assert that it matches the actual result of the operation according to the rules outlined in `Evaluating Matches`_. If `operation.saveResultAsEntity `_ is specified, -the test runner MUST store the result (if any) in the current test's entity map -using the specified name. If the operation did not return a result (e.g. -``void`` method), the test runner MAY decide to store an empty value (e.g. -``null``) or do nothing and leave the entity name undefined. +the test runner MUST store the result in the current test's entity map using the +specified name. If the operation did not return a result or the result is not a +`Supported Entity Type`_ then the test runner MUST raise an error. After asserting the operation's error and/or result and optionally saving the result, proceed to the subsequent operation. @@ -2417,6 +2435,9 @@ Note: this will be cleared when publishing version 1.0 of the spec * Elaborate on schema version. +* Exhaustively list supported entity types and error when attempting to add an + unsupported type to the entity map + 2020-09-15: * Revise definition for test.outcome, clarify comparison rules for outcome From 3b7448231c907d5252829156ceeb7d662dea4440 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Mon, 21 Sep 2020 14:30:05 -0400 Subject: [PATCH 41/90] Allow equivalent language types for BSON types in entity map --- source/unified-test-format/unified-test-format.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 5d21399d9e..0cf86a6dc6 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-15 +:Last Modified: 2020-09-21 .. contents:: @@ -209,7 +209,7 @@ Test runners MUST support the following types of entities: - GridFS Bucket. See `entity_bucket`_ and `bucket`_. - GridFS Stream. See `entity_stream`_. - ChangeStream. See `changeStream`_. -- The following BSON types: +- The following BSON types and/or equivalent language types: - 0x01 - 0x13 - 0xFF @@ -2425,6 +2425,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-21: + +* Entity map can store equivalent language types for supported BSON types. + 2020-09-17: * Future work for logging assertions, FLE tests, and SDAM tests From 4b39a33005b45973ef8c1772816eb8f96db212a6 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 22 Sep 2020 13:38:02 -0400 Subject: [PATCH 42/90] Raise errors for unsupported operations and arguments --- .../unified-test-format/unified-test-format.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 0cf86a6dc6..dd9a624d8d 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-21 +:Last Modified: 2020-09-22 .. contents:: @@ -2018,15 +2018,18 @@ If `operation.object`_ is not "testRunner", this is an entity operation. If `operation.object`_ is defined in the current test's `Entity Map`_, the test runner MUST fetch that entity and note its type; otherwise, the test runner MUST raise an error for an undefined entity. If `operation.name`_ does not -correspond to an operation for the entity type (per `Entity Test Operations`_), -the test runner MUST raise an error for an undefined operation. Test runners MAY -skip tests that include operations that are intentionally unimplemented (e.g. +correspond to a known operation for the entity type (per +`Entity Test Operations`_), the test runner MUST raise an error for an +unsupported operation. Test runners MAY skip tests that include operations that +are intentionally unimplemented (e.g. ``listCollectionNames``). Proceed with preparing the operation's arguments. If ``session`` is specified in `operation.arguments `_, the test runner MUST resolve it to a session entity and MUST raise an error if the name is undefined or maps to -an unexpected type. +an unexpected type. If a key in `operation.arguments`_ does not correspond to a +known parameter/option for the operation, the test runner MUST raise an error +for an unsupported argument. Before executing the operation, the test runner MUST be prepared to catch a potential error from the operation (e.g. enter a ``try`` block). Proceed with @@ -2425,6 +2428,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-22 + +* Test runners must raise errors for unsupported operations and arguments + 2020-09-21: * Entity map can store equivalent language types for supported BSON types. From ffc577348021993fe0a244e631e98988d64a1e23 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 22 Sep 2020 15:18:32 -0400 Subject: [PATCH 43/90] Fix link syntax --- source/unified-test-format/unified-test-format.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index dd9a624d8d..2aeddccdda 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -148,7 +148,7 @@ include, but are not limited to: Backwards-compatible changes SHALL warrant a new minor version. These changes include, but are not limited to: -- Additive changes, such as a introducing a new `Special Test Operation`_ or +- Additive changes, such as a introducing a new `Special Test Operations`_ or type of supported entity or event - Changing an existing field from required to optional - Introducing a new, optional field in the test format From 30160fa9e070a187d6bf9bdae228decfddb6fd04 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 22 Sep 2020 16:00:29 -0400 Subject: [PATCH 44/90] Feedback from Oleg --- .../unified-test-format.rst | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2aeddccdda..92c67cfc1e 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -209,14 +209,15 @@ Test runners MUST support the following types of entities: - GridFS Bucket. See `entity_bucket`_ and `bucket`_. - GridFS Stream. See `entity_stream`_. - ChangeStream. See `changeStream`_. -- The following BSON types and/or equivalent language types: +- Values of the following BSON types and/or equivalent language types: - 0x01 - 0x13 - 0xFF - 0x7F -This is an exhaustive list of permissible types for the entity map. Test runners -MUST raise an error if another type would be added to the entity map. +This is an exhaustive list of supported types for the entity map. Test runners +MUST raise an error if an attempt is made to store an unsupported type in the +entity map. Test Format @@ -245,8 +246,8 @@ The top-level fields of a test file are as follows: .. _schemaVersion: -- ``schemaVersion``: Required string. Version of this specification to which the - test file complies. +- ``schemaVersion``: Required string. Version of this specification with which + the test file complies. Test files SHOULD be conservative when specifying a schema version. For example, if the latest schema version is 1.1 but the test file complies with @@ -373,8 +374,14 @@ The structure of this object is as follows: to multiple mongos hosts (e.g. by inspecting the connection string). If false and the topology is a sharded cluster, the test runner MUST ensure that this MongoClient connects to only a single mongos host (e.g. by - modifying the connection string). This option has no effect for non-sharded - topologies. + modifying the connection string). + + If this option is not specified and the topology is a sharded cluster, the + test runner MUST NOT enforce any limit on the number of mongos hosts in the + connection string and any tests using this client SHOULD NOT depend on a + particular number of mongos hosts. + + This option has no effect for non-sharded topologies. .. _entity_client_observeEvents: @@ -518,9 +525,9 @@ The structure of this object is as follows: collectionData ~~~~~~~~~~~~~~ -List of documents that should correspond to the contents of a collection. This -structure is used by both `initialData`_ and `test.outcome `_, -which insert and read documents, respectively. +List of documents corresponding to the contents of a collection. This structure +is used by both `initialData`_ and `test.outcome `_, which insert +and read documents, respectively. The structure of this object is as follows: @@ -2432,6 +2439,8 @@ Note: this will be cleared when publishing version 1.0 of the spec * Test runners must raise errors for unsupported operations and arguments +* Clarify behavior when useMultipleMongoses is unspecified + 2020-09-21: * Entity map can store equivalent language types for supported BSON types. From ecee77ebe120434f665f20e29cec724664d056ac Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 23 Sep 2020 14:10:32 -0400 Subject: [PATCH 45/90] $$sessionLsid error handling and heartbeatFrequencyMS tweaks --- .../unified-test-format.rst | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 92c67cfc1e..540818507a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1808,7 +1808,7 @@ Syntax:: { $$unsetOrMatches: } This operator can be used anywhere a matched value is expected (including -`expectResult `_). The test runner MUST assert that +`expectResult `_). The test runner MUST assert that the actual value either does not exist or matches the expected value. Matching the expected value should use the standard rules in `Evaluating Matches`_, which means that it may contain special operators. @@ -1840,12 +1840,15 @@ $$sessionLsid Syntax:: - { $$sessionLsid: } + { $$sessionLsid: } -This operation is used for matching any value with the logical session ID of a -`session entity `_. The value will refer to a unique name of a -session entity. The YAML file SHOULD use an `alias node`_ for a session entity's -``id`` field (e.g. ``session: *session0``). +This operator can be used anywhere a matched value is expected (including +`expectResult `_). If the +`session entity `_session entity is defined in the current +test's `Entity Map`_, the test runner MUST assert that the actual value equals +its logical session ID; otherwise, the test runner MUST raise an error for an +undefined or mistyped entity. The YAML file SHOULD use an `alias node`_ for a +session entity's ``id`` field (e.g. ``session: *session0``). An example of this operator follows:: @@ -1943,6 +1946,12 @@ for each target collection the test runner SHOULD execute a non-transactional ``distinct`` command on each mongos server using the internal MongoClient. See `StaleDbVersion Errors on Sharded Clusters`_ for more information. +If the test might execute a ``configureFailPoint`` command, for each target +client the test runner MAY specify a reduced value for ``heartbeatFrequencyMS`` +(and ``minHeartbeatFrequencyMS`` if possible) to speed up SDAM recovery time and +server selection after a failure; however, test runners MUST NOT do so for any +client that specifies ``heartbeatFrequencyMS`` in its ``uriOptions``. + If `test.expectEvents `_ is specified, for each client entity the test runner MUST enable all event listeners necessary to collect the event types specified in `observeEvents `_. Test @@ -2435,7 +2444,14 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec -2020-09-22 +2020-09-23: + +* Clarify error handling for $$sessionLsid + +* Suggest lowering heartbeatFrequencyMS and minHeartbeatFrquencyMS for clients + using fail points + +2020-09-22: * Test runners must raise errors for unsupported operations and arguments From ba6f5976b49f3a72607243b97dd94cde291c0487 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 23 Sep 2020 14:29:05 -0400 Subject: [PATCH 46/90] Clarify BSON type support and advise against using deprecated types --- .../unified-test-format.rst | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 540818507a..dfff15480a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-22 +:Last Modified: 2020-09-23 .. contents:: @@ -209,16 +209,23 @@ Test runners MUST support the following types of entities: - GridFS Bucket. See `entity_bucket`_ and `bucket`_. - GridFS Stream. See `entity_stream`_. - ChangeStream. See `changeStream`_. -- Values of the following BSON types and/or equivalent language types: +- All known BSON types and/or equivalent language types for the target driver. + For the present version of the spec, the following BSON types are known: + 0x01-0x13, 0x7F, 0xFF. - - 0x01 - 0x13 - - 0xFF - - 0x7F + Tests SHOULD NOT utilize deprecated types (e.g. 0x0E: Symbol), since they may + not be supported by all drivers and could yield runtime errors (e.g. while + loading a test file with an Extended JSON parser). This is an exhaustive list of supported types for the entity map. Test runners MUST raise an error if an attempt is made to store an unsupported type in the entity map. +Adding new entity types (including known BSON types) to this list will require +a minor version bump to the spec and schema version. Removing entity types will +require a major version bump. See `Impact of Spec Changes on Schema Version`_ +for more information. + Test Format ----------- @@ -2446,6 +2453,8 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-09-23: +* Clarify BSON type support and advise against using deprecated types + * Clarify error handling for $$sessionLsid * Suggest lowering heartbeatFrequencyMS and minHeartbeatFrquencyMS for clients From f8bb778720afc8ae539198d78f1b4c15bc92b123 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 23 Sep 2020 14:42:53 -0400 Subject: [PATCH 47/90] Clarify server version string comparison rules --- source/unified-test-format/unified-test-format.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index dfff15480a..7a7ea3c180 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -305,8 +305,12 @@ runOnRequirement A combination of server version and/or topology requirements for running the test(s). -Server versions SHALL be compared numerically and do not follow the comparison -rules discussed in `Schema Version`_. +Server version strings consist of three components: major, minor, and patch. +Each component is a non-negative integer and components are delimited by a dot +character (i.e. "."). Version strings MUST include at least a major component. +If a component is omitted, its value is zero. Each component of a version +string SHALL be compared numerically. For example, "4.0.10" is greater than +"4.0.9" and "3.6" and less than "4.2.0". The structure of this object is as follows: @@ -2453,6 +2457,8 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-09-23: +* Clarify server version string comparison rules + * Clarify BSON type support and advise against using deprecated types * Clarify error handling for $$sessionLsid From 955d59febf387220ae89d1201ab1def97b1304b8 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 24 Sep 2020 14:24:46 -0400 Subject: [PATCH 48/90] Revise version docs, MUST reset state, explain SHOULD in Design Rationale --- .../unified-test-format.rst | 92 ++++++++++++------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 7a7ea3c180..377f10735d 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-23 +:Last Modified: 2020-09-24 .. contents:: @@ -32,6 +32,10 @@ The keywords "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in `RFC 2119 `__. +This document tends to use "SHOULD" more frequently than other specifications, +but mainly in the context of providing guidance on writing test files. This is +discussed in more detail in `Design Rationale`_. + Goals ===== @@ -276,7 +280,7 @@ The top-level fields of a test file are as follows: .. _createEntities: - ``createEntities``: Optional array of `entity`_ objects. List of entities - (e.g. client, collection, session objects) that should be created before each + (e.g. client, collection, session objects) that SHALL be created before each test case is executed. Test files SHOULD define entities in dependency order, such that all @@ -286,7 +290,7 @@ The top-level fields of a test file are as follows: .. _initialData: - ``initialData``: Optional array of one or more `collectionData`_ objects. Data - that should exist in collections before each test case is executed. + that will exist in collections before each test case is executed. Before each test and for each `collectionData`_, the test runner MUST drop the collection and insert the specified documents (if any) using a "majority" @@ -305,30 +309,27 @@ runOnRequirement A combination of server version and/or topology requirements for running the test(s). -Server version strings consist of three components: major, minor, and patch. -Each component is a non-negative integer and components are delimited by a dot -character (i.e. "."). Version strings MUST include at least a major component. -If a component is omitted, its value is zero. Each component of a version -string SHALL be compared numerically. For example, "4.0.10" is greater than -"4.0.9" and "3.6" and less than "4.2.0". +The format of server version strings is defined in `Version String`_. When +comparing server version strings, each component SHALL be compared numerically. +For example, "4.0.10" is greater than "4.0.9" and "3.6" and less than "4.2.0". The structure of this object is as follows: - ``minServerVersion``: Optional string. The minimum server version (inclusive) - required to successfully run the tests. If this field is omitted, it should be - assumed that there is no lower bound on the required server version. The - format of this string is defined in `Version String`_. + required to successfully run the tests. If this field is omitted, there is no + lower bound on the required server version. The format of this string is + defined in `Version String`_. - ``maxServerVersion``: Optional string. The maximum server version (inclusive) - against which the tests can be run successfully. If this field is omitted, it - should be assumed that there is no upper bound on the required server version. - The format of this string is defined in `Version String`_. + against which the tests can be run successfully. If this field is omitted, + there is no upper bound on the required server version. The format of this + string is defined in `Version String`_. - ``topologies``: Optional array of strings. One or more of server topologies against which the tests can be run successfully. Valid topologies are "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded - cluster backed by replica sets). If this field is omitted, it should be - assumed that there is no topology requirement for the test. + cluster backed by replica sets). If this field is omitted, there is no + topology requirement for the test. When matching a "sharded-replicaset" topology, test runners MUST ensure that all shards are backed by a replica set. The process for doing so is described @@ -570,7 +571,7 @@ The structure of this object is as follows: this array are met, the test runner MUST skip this test. These requirements SHOULD be more restrictive than those specified in the - top-level `runOnRequirements`_ (if any). They SHOULD NOT be more permissive. + top-level `runOnRequirements`_ (if any) and SHOULD NOT be more permissive. This is advised because both sets of requirements MUST be satisified in order for a test to be executed and more permissive requirements at the test-level could be taken out of context on their own. @@ -592,7 +593,7 @@ The structure of this object is as follows: observed in a particular order. If a driver only supports configuring event listeners globally (for all - clients), the test runner SHOULD associate each observed event with a client in + clients), the test runner SHOULD associate each observed event with a client in order to perform these assertions. Test files SHOULD NOT expect events from multiple specs (e.g. command @@ -603,7 +604,7 @@ The structure of this object is as follows: .. _test_outcome: - ``outcome``: Optional array of one or more `collectionData`_ objects. Data - that should exist in collections after each test case is executed. + that is expected to exist in collections after each test case is executed. The list of documents herein SHOULD be sorted ascendingly by the ``_id`` field to allow for deterministic comparisons. The procedure for asserting collection @@ -625,7 +626,7 @@ The structure of this object is as follows: .. _operation_object: - ``object``: Required string. Name of the object on which to perform the - operation. This should correspond to either an `entity`_ name (for + operation. This SHOULD correspond to either an `entity`_ name (for `Entity Test Operations`_) or "testRunner" (for `Special Test Operations`_). If the object is an entity, The YAML file SHOULD use an `alias node`_ for its ``id`` field (e.g. ``object: *collection0``). @@ -902,8 +903,8 @@ Version String -------------- Version strings, which are used for `schemaVersion`_ and `runOnRequirement`_, -MUST conform to one of the following formats, where each component is an -integer: +MUST conform to one of the following formats, where each component is a +non-negative integer: - ``..`` - ``.`` (```` is assumed to be zero) @@ -1099,7 +1100,7 @@ While operations typically raise an error *or* return a result, the ``bulkWrite`` operation is unique in that it may report both via the ``writeResult`` property of a BulkWriteException. In this case, the intermediary write result may be matched with `expectedError_expectResult`_. Because -``writeResult`` is optional for drivers to implement, such assertions should +``writeResult`` is optional for drivers to implement, such assertions SHOULD utilize the `$$unsetOrMatches`_ operator. Additionally, BulkWriteException is unique in that it aggregates one or more @@ -1195,8 +1196,8 @@ parameter is a stream as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec and not easily expressed in YAML/JSON. This parameter is expressed as an entity name, which the test runner MUST resolve to a `stream `_ entity *before* passing it as a parameter to the method. The YAML file SHOULD -use an `alias node`_ for a stream entity's ``id`` field -(e.g. ``stream: *stream0``). +use an `alias node`_ for a stream entity's ``id`` field (e.g. +``stream: *stream0``). changeStream @@ -1752,7 +1753,7 @@ An example of this operator follows:: collection: { $$type: "string" } When the actual value is an array, test runners MUST NOT examine types of the -array's elements. Only the type of actual field should be checked. This is +array's elements. Only the type of actual field SHALL be checked. This is admittedly inconsistent with the behavior of the `$type `__ query operator, but there is presently no need for this behavior in tests. @@ -1821,7 +1822,7 @@ Syntax:: This operator can be used anywhere a matched value is expected (including `expectResult `_). The test runner MUST assert that the actual value either does not exist or matches the expected value. Matching the -expected value should use the standard rules in `Evaluating Matches`_, which +expected value MUST use the standard rules in `Evaluating Matches`_, which means that it may contain special operators. This operator is primarily used to assert driver-optional fields from the CRUD @@ -1873,7 +1874,7 @@ Test Runner Implementation The sections below describe instructions for instantiating the test runner, loading each test file, and executing each test within a test file. Test runners -SHOULD NOT share state created by processing a test file with the processing of +MUST NOT share state created by processing a test file with the processing of subsequent test files, and likewise for tests within a test file. @@ -1924,9 +1925,9 @@ Executing a Test ~~~~~~~~~~~~~~~~ The instructions in this section apply for each `test`_ occuring in a test file -loaded by the test runner. After processing a test, test runners SHOULD reset +loaded by the test runner. After processing a test, test runners MUST reset any internal state that resulted from doing so. For example, the `Entity Map`_ -created for one test SHOULD NOT be shared with another. +created for one test MUST NOT be shared with another. If at any point while executing this test an unexpected error is encountered or an assertion fails, the test runner MUST consider this test to have failed and @@ -2160,8 +2161,8 @@ points available for driver testing, but some fail points are documented in Configuring Fail Points ~~~~~~~~~~~~~~~~~~~~~~~ -The ``configureFailPoint`` command should be executed on the ``admin`` database -and has the following structure:: +The ``configureFailPoint`` command is executed on the ``admin`` database and has +the following structure:: db.adminCommand({ configureFailPoint: , @@ -2276,19 +2277,29 @@ This specification was primarily derived from the test formats used by the `Transactions <../transactions/transactions.rst>`__ and `CRUD <../crud/crud.rst>`__ specs, which have served models or other specs. +This specification commonly uses "SHOULD" when providing guidance on writing +test files. While this may appear contradictory to the driver mantra preferring +"MUST", it is intentional. Some of this guidance addresses style (e.g. adding +comments, using YAML anchors) and cannot be enforced with a JSON schema. Other +guidance needs to be purposefully ignored in order to test the test runner +implementation (e.g. defining entities out of order to trigger runtime errors). +The specification does prefer "MUST" in other contexts, such as discussing parts +of the test file format that *are* enforceable by the JSON schema or the test +runner implementation. + Breaking Changes ================ This section is reserved for future use. Any breaking changes to the test format -should be described here in detail for historical reference, in addition to any +SHOULD be described here in detail for historical reference, in addition to any shorter description that may be added to the `Change Log`_. Related Issues ============== -Note: this section should be removed before publishing version 1.0 of the spec. +Note: this section will be removed before publishing version 1.0 of the spec. The following SPEC tickets are associated with `DRIVERS-709 `__. This section will @@ -2455,6 +2466,17 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-24: + +* Explain in "Design Rationale" why "SHOULD" is used more commonly in this + document. + +* Simplify paragraph about comparing server versions and refer to Version String + section for format info. + +* Change "SHOULD" to "MUST" to prohibit test runners from sharing state between + test files and test cases (e.g. entity map must be reset between tests). + 2020-09-23: * Clarify server version string comparison rules From ef1d60a9517a89b04b0d13eca465e2133f57be0e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 24 Sep 2020 20:43:43 -0400 Subject: [PATCH 49/90] Improve Evaluating Matches docs --- .../unified-test-format.rst | 120 ++++++++++++------ 1 file changed, 82 insertions(+), 38 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 377f10735d..619addfbad 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -652,7 +652,7 @@ The structure of this object is as follows: - ``expectResult``: Optional mixed type. A value corresponding to the expected result of the operation. This field may be a scalar value, a single document, - or an array of documents in the case of a multi-document read. Test runners + or an array of documents in the case of a multi-document read. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. This field is mutually exclusive with `expectError `_. @@ -662,7 +662,7 @@ The structure of this object is as follows: - ``saveResultAsEntity``: Optional string. If specified, the actual result returned by the operation (if any) will be saved with this name in the `Entity Map`_. The test runner MUST raise an error if the name is already in - use or if the result is not a `Supported Entity Type`_. + use or if the result does not comply with `Supported Entity Types`_. This field is mutually exclusive with `expectError `_. @@ -1600,16 +1600,25 @@ following pseudo-code:: function match (expected, actual): if expected is a document: - if first key of expected starts with "$$": - assert that the special operator (identified by key) matches + // handle special operators (e.g. $$type) + if first and only key of expected starts with "$$": + execute any assertion(s) for the special operator return assert that actual is a document for every key/value in expected: + // handle key-based operators (e.g. $$exists, $$unsetOrMatches) + if value is a document and its first and only key starts with "$$": + execute any assertion(s) for the special operator + continue to the next iteration unless actual value must be matched + assert that actual[key] exists assert that actual[key] matches value + if expected is not the root document: + assert that actual does not contain additional keys + return if expected is an array: @@ -1622,7 +1631,7 @@ following pseudo-code:: return // expected is neither a document nor array - assert that actual and expected are the same type + assert that actual and expected are the same type, noting flexible numerics assert that actual and expected are equal The rules for comparing documents and arrays are discussed in more detail in @@ -1641,46 +1650,79 @@ numerically equivalent. For example, an expected value of ``1`` would match an actual value of ``1.0`` (e.g. ``ok`` field in a server response) but would not match ``1.5``. -When comparing types that may contain documents (e.g. CodeWScope), test runners -MUST follow standard document matching rules when comparing those properties. +When comparing types that contain documents as internal properties (e.g. +CodeWScope), the rules in `Evaluating Matches`_ do not apply and the documents +MUST match exactly; however, test runners MUST permit variation in document key +order or otherwise normalize the documents before comparison. + +Allowing Extra Fields in Root-level Documents +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Extra Fields in Actual Documents -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When matching root-level documents, test runners MUST permit the actual document +to contain additional fields not present in the expected document. Examples of +root-level documents include, but are not limited to: -When matching expected and actual *documents*, test runners MUST permit the -actual documents to contain additional fields not present in the expected -document. For example, the following documents match:: +- ``command`` for `CommandStartedEvent `_ +- ``reply`` for `CommandSucceededEvent `_ +- `expectResult`_ for a `collection`_ ``findOneAndUpdate`` operation +- `expectResult`_ for a `changeStream`_ `iterateUntilDocumentOrError`_ operation +- each array element in `expectResult`_ for a `collection`_ ``find`` or + ``aggregate`` operation + +For example, the following documents match:: expected: { x: 1 } actual: { x: 1, y: 1 } -The inverse is not true. For example, the following documents would not match:: +The inverse is not true. For example, the following documents do not match:: expected: { x: 1, y: 1 } actual: { x: 1 } +Test runners MUST NOT permit additional fields in nested documents. For example, +the following documents do not match:: + + expected: { x: { y: 1 } } + actual: { x: { y: 1, z: 1 } } + It may be helpful to think of expected documents as a form of query criteria. The intention behind this rule is that it is not always feasible or relevant for a test to exhaustively specify all fields in an expected document (e.g. cluster -time in a `CommandStartedEvent `_ command). +time in ``command`` for `CommandStartedEvent`_). -Note that this rule for allowing extra fields in actual values only applies when -matching documents. When comparing arrays, expected and actual values MUST -contain the same number of elements. For example, the following arrays -corresponding to a ``distinct`` operation result would not match:: - - expected: [ 1, 2, 3 ] - actual: [ 1, 2, 3, 4 ] - -That said, any individual documents *within* an array or list (e.g. result of a -``find`` operation) MAY be matched according to the rules in this section. For -example, the following arrays would match:: +When the expected value is an array, test runners MUST differentiate between +an array of values, which may be documents, (e.g. ``distinct``) and an array of +root-level documents (e.g. ``find``, ``aggregate``). For example, the following +array of documents would not match if returned by ``distinct``, but would match +if returned via ``find`` (after iterating the cursor): expected: [ { x: 1 }, { x: 2 } ] actual: [ { x: 1, y: 1 }, { x: 2, y: 2 } ] +Document Key Order Variation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When matching documents, test runners MUST NOT require keys in the expected and +actual document to appear in the same order. For example, the following +documents would match: + + expected: { x: 1, y: 1 } + actual: { y: 1, x: 1 } + + +Arrays Must Contain the Same Number of Elements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When comparing arrays, expected and actual values MUST contain the same number +of elements. For example, the following arrays corresponding to a ``distinct`` +operation result would not match:: + + expected: [ 1, 2, 3 ] + actual: [ 1, 2, 3, 4 ] + + Special Operators for Matching Assertions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1696,8 +1738,8 @@ from MongoDB query operators, which use a single `$` prefix. The key will map to some value that influences the operator's behavior (if applicable). When examining the structure of an expected value during a comparison, test -runners MUST examine the first key of any object for a ``$$`` prefix and, if -present, defer to the special logic defined in this section. +runners MUST check if the value is an object whose first and only key starts +with ``$$`` and, if so, defer to the special logic defined in this section. $$exists @@ -1856,9 +1898,9 @@ Syntax:: This operator can be used anywhere a matched value is expected (including `expectResult `_). If the -`session entity `_session entity is defined in the current -test's `Entity Map`_, the test runner MUST assert that the actual value equals -its logical session ID; otherwise, the test runner MUST raise an error for an +`session entity `_ is defined in the current test's +`Entity Map`_, the test runner MUST assert that the actual value equals its +logical session ID; otherwise, the test runner MUST raise an error for an undefined or mistyped entity. The YAML file SHOULD use an `alias node`_ for a session entity's ``id`` field (e.g. ``session: *session0``). @@ -2007,12 +2049,11 @@ If `test.outcome `_ is specified, for each `collectionData`_ therein the test runner MUST assert that the collection contains exactly the expected data. The test runner MUST query each collection using the internal MongoClient, an ascending sort order on the ``_id`` field (i.e. ``{ _id: 1 }``), -a "primary" read preference, a "local" read concern. The documents must match -expected data exactly (`Evaluating Matches`_ does not apply); however, languages -that are unable to preserve the order of keys in documents MAY permit variations -in field order or otherwise normalize actual and expected documents before -comparison. If the list of documents is empty, the test runner MUST assert that -the collection is empty. +a "primary" read preference, a "local" read concern. When comparing collection +data, the rules in `Evaluating Matches`_ do not apply and the documents MUST +match exactly; however, test runners MUST permit variations in document key +order or otherwise normalize the documents before comparison. If the list of +documents is empty, the test runner MUST assert that the collection is empty. Clear the entity map for this test. For each ClientSession in the entity map, the test runner MUST end the session (e.g. call @@ -2075,8 +2116,8 @@ rules outlined in `Evaluating Matches`_. If `operation.saveResultAsEntity `_ is specified, the test runner MUST store the result in the current test's entity map using the -specified name. If the operation did not return a result or the result is not a -`Supported Entity Type`_ then the test runner MUST raise an error. +specified name. If the operation did not return a result or the result does not +comply with `Supported Entity Types`_ then the test runner MUST raise an error. After asserting the operation's error and/or result and optionally saving the result, proceed to the subsequent operation. @@ -2468,6 +2509,9 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-09-24: +* Improve docs for Evaluating Matches. Extra keys are only permitted in + root-level documents. Always permit key order variation. + * Explain in "Design Rationale" why "SHOULD" is used more commonly in this document. From 0912d2adc156a1832fa63b6177d26564de074e3f Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 24 Sep 2020 21:51:51 -0400 Subject: [PATCH 50/90] Test runners MUST raise an error for incompatible files --- source/unified-test-format/unified-test-format.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 619addfbad..841536004c 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -132,9 +132,10 @@ runner, given the same major version component. For example: completeness, but test files SHOULD NOT need to refer to patch versions (as previously mentioned). -Test runners MUST NOT process incompatible files but MAY determine how to handle -such files (e.g. skip and log a notice, fail and raise an error). Test runners -MAY support multiple schema versions (as demonstrated in the example above). +Test runners MUST NOT process incompatible files and MUST raise an error if they +encounter an incompatible file (as discussed in `Executing a Test File`_). Test +runners MAY support multiple schema versions (as demonstrated in the example +above). Impact of Spec Changes on Schema Version @@ -1955,7 +1956,10 @@ files may use either. Upon loading a file, the test runner MUST read the `schemaVersion`_ field and determine if the test file can be processed further. Test runners MAY support multiple versions and MUST NOT process incompatible files (as discussed in -`Schema Version`_). +`Test Runner Support`_). If a test file is incompatible, test runners MUST raise +an error and MAY do so by reporting a test failure. Test runners MAY make an +effort to infer the number of tests (and their descriptions) from an +incompatible file and report a failure for each test. If `runOnRequirements`_ is specified, the test runner MUST skip the test file unless one or more `runOnRequirement`_ objects are satisfied. @@ -2509,6 +2513,8 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-09-24: +* Test runners MUST raise an error for incompatible files + * Improve docs for Evaluating Matches. Extra keys are only permitted in root-level documents. Always permit key order variation. From 0abb9e8d0e27bf878bde8cd1b8898f99032aa90f Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 24 Sep 2020 21:52:10 -0400 Subject: [PATCH 51/90] Future work for operations/arguments and schema version --- .../unified-test-format.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 841536004c..736dfcb343 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -2506,6 +2506,26 @@ also special operations for defining threads and executing operations within threads, which may warrant introducing a new "thread" entity type. +Incorporate referenced entity operations into the schema version +---------------------------------------------------------------- + +The `Schema Version`_ is not impacted by changes to operations defined in other +specs and referenced in `Entity Test Operations` (e.g. ``find`` for CRUD). The +`operation.name `_ and +`operation.arguments `_ fields are loosely defined in the +JSON schema as string and object types, respectively. + +Ideally, all operations (and their arguments) would be enforced by the JSON +schema *and* any changes to operations would affect the schema version +accordingly. For example, a new ``find`` option would warrant a minor version +bump both for the CRUD spec and this spec and its schema. + +As discussed in `Executing an Operation`_, test runners MUST raise errors for +unsupported operations and arguments. This is a concession until such time that +better processes can be established for versioning other specs *and* collating +spec changes developed in parallel or during the same release cycle. + + Change Log ========== @@ -2513,6 +2533,8 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-09-24: +* Future work for incorporating referenced entity operations into schema version + * Test runners MUST raise an error for incompatible files * Improve docs for Evaluating Matches. Extra keys are only permitted in From dc7383b5208e23d7e26f494d97bbc0b5ba2587be Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Mon, 28 Sep 2020 13:10:56 +0800 Subject: [PATCH 52/90] Prohibit $$unsetOrMatches for array elements --- .../unified-test-format/unified-test-format.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 736dfcb343..3ce3fc5b94 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-24 +:Last Modified: 2020-09-28 .. contents:: @@ -1863,10 +1863,11 @@ Syntax:: { $$unsetOrMatches: } This operator can be used anywhere a matched value is expected (including -`expectResult `_). The test runner MUST assert that the -actual value either does not exist or matches the expected value. Matching the -expected value MUST use the standard rules in `Evaluating Matches`_, which -means that it may contain special operators. +`expectResult `_), excluding an array element because +`Arrays Must Contain the Same Number of Elements`_. The test runner MUST assert +that the actual value either does not exist or matches the expected value. +Matching the expected value MUST use the standard rules in +`Evaluating Matches`_, which means that it may contain special operators. This operator is primarily used to assert driver-optional fields from the CRUD spec (e.g. ``insertedId`` for InsertOneResult, ``writeResult`` for @@ -2531,6 +2532,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-28: + +* Prohibit $$unsetOrMatches for array elements + 2020-09-24: * Future work for incorporating referenced entity operations into schema version From 85dcf2ccb50352271d1a25f65c3a5de2cb6488c6 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 29 Sep 2020 15:24:01 +0800 Subject: [PATCH 53/90] Optional arrays must be non-empty, revise expectedEvent and $$matchesHexBytes --- .../unified-test-format.rst | 83 +++++++++++-------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 3ce3fc5b94..61006c7818 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-28 +:Last Modified: 2020-09-29 .. contents:: @@ -280,9 +280,9 @@ The top-level fields of a test file are as follows: .. _createEntities: -- ``createEntities``: Optional array of `entity`_ objects. List of entities - (e.g. client, collection, session objects) that SHALL be created before each - test case is executed. +- ``createEntities``: Optional array of one or more `entity`_ objects. List of + entities (e.g. client, collection, session objects) that SHALL be created + before each test case is executed. Test files SHOULD define entities in dependency order, such that all referenced entities (e.g. client) are defined before any of their dependent @@ -326,7 +326,7 @@ The structure of this object is as follows: there is no upper bound on the required server version. The format of this string is defined in `Version String`_. -- ``topologies``: Optional array of strings. One or more of server topologies +- ``topologies``: Optional array of one or more strings. Server topologies against which the tests can be run successfully. Valid topologies are "single", "replicaset", "sharded", and "sharded-replicaset" (i.e. sharded cluster backed by replica sets). If this field is omitted, there is no @@ -398,7 +398,7 @@ The structure of this object is as follows: .. _entity_client_observeEvents: - - ``observeEvents``: Optional array of strings. One or more types of events + - ``observeEvents``: Optional array of one or more strings. Types of events that can be observed for this client. Unspecified event types MUST be ignored by this client's event listeners and SHOULD NOT be included in `test.expectEvents `_ for this client. @@ -419,8 +419,8 @@ The structure of this object is as follows: .. _entity_client_ignoreCommandMonitoringEvents: - - ``ignoreCommandMonitoringEvents``: Optional array of strings. One or more - command names for which the test runner MUST ignore any observed command + - ``ignoreCommandMonitoringEvents``: Optional array of one or more strings. + Command names for which the test runner MUST ignore any observed command monitoring events. The command(s) will be ignored in addition to ``configureFailPoint`` and any commands containing sensitive information (per the @@ -565,7 +565,7 @@ The structure of this object is as follows: .. _test_runOnRequirements: -- ``runOnRequirements``: Optional array of on or more `runOnRequirement`_ +- ``runOnRequirements``: Optional array of one or more `runOnRequirement`_ objects. List of server version and/or topology requirements for which this test can be run. If specified, these requirements are evaluated independently and in addition to any top-level `runOnRequirements`_. If no requirements in @@ -721,15 +721,15 @@ The structure of this object is as follows: Test files SHOULD NOT assert error codes for client errors, as specifications do not define standardized codes for client errors. -- ``errorLabelsContain``: Optional array of strings. A list of error label - strings that the error is expected to have. The test runner MUST assert that - the error contains all of the specified labels (e.g. using the +- ``errorLabelsContain``: Optional array of one or more strings. A list of error + label strings that the error is expected to have. The test runner MUST assert + that the error contains all of the specified labels (e.g. using the ``hasErrorLabel`` method). -- ``errorLabelsOmit``: Optional array of strings. A list of error label strings - that the error is expected not to have. The test runner MUST assert that - the error does not contain any of the specified labels (e.g. using the - ``hasErrorLabel`` method). +- ``errorLabelsOmit``: Optional array of one or more strings. A list of error + label strings that the error is expected not to have. The test runner MUST + assert that the error does not contain any of the specified labels (e.g. using + the ``hasErrorLabel`` method). .. _expectedError_expectResult: @@ -767,51 +767,57 @@ This object MUST contain **exactly one** top-level key that identifies the event type and maps to a nested object, which contains one or more assertions for the event's properties. +Some event properties are omitted in the following structures because they +cannot be reliably tested. Taking command monitoring events as an example, +``requestId`` and ``operationId`` are nondeterministic and types for +``connectionId`` and ``failure`` can vary by implementation. + The structure of this object is as follows: .. _expectedEvent_commandStartedEvent: -- ``commandStartedEvent``: Optional object. Assertions for a one or more +- ``commandStartedEvent``: Optional object. Assertions for one or more `CommandStartedEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. The structure of this object is as follows: - - ``command``: Optional document. Test runners MUST follow the rules in + - ``command``: Optional document. A value corresponding to the expected + command document. Test runners MUST follow the rules in `Evaluating Matches`_ when processing this assertion. - - ``commandName``: Optional string. Test runners MUST assert that the actual - command name matches this value using a case-insensitive comparison. + - ``commandName``: Optional string. Test runners MUST assert that the command + name matches this value. - - ``databaseName``: Optional string. Test runners MUST assert that the actual - command name matches this value using a case-insensitive comparison. THe - YAML file SHOULD use an `alias node`_ for this value (e.g. - ``databaseName: *database0Name``). + - ``databaseName``: Optional string. Test runners MUST assert that the + database name matches this value. The YAML file SHOULD use an `alias node`_ + for this value (e.g. ``databaseName: *database0Name``). .. _expectedEvent_commandSucceededEvent: -- ``commandSucceededEvent``: Optional object. Assertions for a one or more +- ``commandSucceededEvent``: Optional object. Assertions for one or more `CommandSucceededEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. The structure of this object is as follows: - - ``reply``: Optional document. Test runners MUST follow the rules in - `Evaluating Matches`_ when processing this assertion. + - ``reply``: Optional document. A value corresponding to the expected + reply document. Test runners MUST follow the rules in `Evaluating Matches`_ + when processing this assertion. - - ``commandName``: Optional string. Test runners MUST assert that the actual - command name matches this value using a case-insensitive comparison. + - ``commandName``: Optional string. Test runners MUST assert that the command + name matches this value. .. _expectedEvent_commandFailedEvent: -- ``commandFailedEvent``: Optional object. Assertions for a one or more +- ``commandFailedEvent``: Optional object. Assertions for one or more `CommandFailedEvent <../command-monitoring/command-monitoring.rst#api>`__ fields. The structure of this object is as follows: - - ``commandName``: Optional string. Test runners MUST assert that the actual - command name matches this value using a case-insensitive comparison. + - ``commandName``: Optional string. Test runners MUST assert that the command + name matches this value. collectionOrDatabaseOptions @@ -1841,7 +1847,7 @@ $$matchesHexBytes ````````````````` Syntax, where ``hexBytes`` is an even number of hexademical characters -(case-insensitive):: +(case-insensitive) and MAY be empty:: { $$matchesHexBytes: } @@ -2532,6 +2538,17 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-09-29: + +* Clarify that some optional arrays must contain one or more elements, since the + fields could otherwise be omitted. + +* $$matchesHexBytes may be an empty string, like hexBytes stream entity option + +* Note why some event properties are omitted in expectedEvent assertions + +* Do not use case-insensitive comparisons for commandName + 2020-09-28: * Prohibit $$unsetOrMatches for array elements From 0e00466b6a241dc3f3b203a833a524cbaf97a559 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 29 Sep 2020 16:25:11 +0800 Subject: [PATCH 54/90] Update schema for current spec syntax --- source/unified-test-format/schema-1.0.json | 282 ++++++++++++++++++ source/unified-test-format/schema-pre-1.json | 285 ------------------- 2 files changed, 282 insertions(+), 285 deletions(-) create mode 100644 source/unified-test-format/schema-1.0.json delete mode 100644 source/unified-test-format/schema-pre-1.json diff --git a/source/unified-test-format/schema-1.0.json b/source/unified-test-format/schema-1.0.json new file mode 100644 index 0000000000..5ebe359dde --- /dev/null +++ b/source/unified-test-format/schema-1.0.json @@ -0,0 +1,282 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": ["description", "schemaVersion", "tests"], + "properties": { + "description": { "type": "string" }, + "schemaVersion": { "$ref": "#/definitions/version" }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/entity" } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/test" } + } + }, + + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { "$ref": "#/definitions/version" }, + "minServerVersion": { "$ref": "#/definitions/version" }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": ["single", "replicaset", "sharded", "sharded-replicaset"] + } + } + } + }, + + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": { "type": "string" }, + "uriOptions": { "type": "object" }, + "useMultipleMongoses": { "type": "boolean" }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": ["commandStartedEvent", "commandSucceededEvent", "commandFailedEvent"] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": ["id", "client", "databaseName"], + "properties": { + "id": { "type": "string" }, + "client": { "type": "string" }, + "databaseName": { "type": "string" }, + "databaseOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": ["id", "database", "collectionName"], + "properties": { + "id": { "type": "string" }, + "database": { "type": "string" }, + "collectionName": { "type": "string" }, + "collectionOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": ["id", "client"], + "properties": { + "id": { "type": "string" }, + "client": { "type": "string" }, + "sessionOptions": { "type": "object" } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": ["id", "database"], + "properties": { + "id": { "type": "string" }, + "database": { "type": "string" }, + "bucketOptions": { "type": "object" } + } + }, + "stream": { + "type": "object", + "additionalProperties": false, + "required": ["id", "hexBytes"], + "properties": { + "id": { "type": "string" }, + "hexBytes": { + "type": "string", + "pattern": "^([0-9A-Fa-f]{2})*$" + } + } + } + } + }, + + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": ["collectionName", "databaseName", "documents"], + "properties": { + "collectionName": { "type": "string" }, + "databaseName": { "type": "string" }, + "documents": { + "type": "array", + "items": { "type": "object" } + } + } + }, + + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": ["client", "events"], + "properties": { + "client": { "type": "string" }, + "events": { + "type": "array", + "items": { "$ref": "#/definitions/expectedEvent" } + } + } + }, + + "expectedEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { "type": "object" }, + "commandName": { "type": "string" }, + "databaseName": { "type": "string" } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { "type": "object" }, + "commandName": { "type": "string" } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { "type": "string" } + } + } + } + }, + + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { "type": "object" }, + "readPreference": { "type": "object" }, + "writeConcern": { "type": "object" } + } + }, + + "operation": { + "type": "object", + "additionalProperties": false, + "required": ["name", "object"], + "properties": { + "name": { "type": "string" }, + "object": { "type": "string" }, + "arguments": { "type": "object" }, + "expectError": { "$ref": "#/definitions/expectedError" }, + "expectResult": {}, + "saveResultAsEntity": { "type": "string" } + }, + "allOf": [ + { "not": { "required": ["expectError", "expectResult"] } }, + { "not": { "required": ["expectError", "saveResultAsEntity"] } } + ] + }, + + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { "type": "boolean" }, + "isClientError": { "type": "boolean" }, + "errorContains": { "type": "string" }, + "errorCode": { "type": "integer" }, + "errorCodeName": { "type": "string" }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { "type": "string" } + }, + "expectResult": {} + } + }, + + "test": { + "type": "object", + "required": ["description", "operations"], + "properties": { + "description": { "type": "string" }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/runOnRequirement" } + }, + "skipReason": { "type": "string" }, + "operations": { + "type": "array", + "items": { "$ref": "#/definitions/operation" } + }, + "expectEvents": { + "type": "array", + "items": { "$ref": "#/definitions/expectedEventsForClient" } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/collectionData" } + } + } + } + } +} diff --git a/source/unified-test-format/schema-pre-1.json b/source/unified-test-format/schema-pre-1.json deleted file mode 100644 index b94bdc7b67..0000000000 --- a/source/unified-test-format/schema-pre-1.json +++ /dev/null @@ -1,285 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - - "type": "object", - "title": "Unified Test Format", - "description": "Schema definition for unified test format.", - "additionalProperties": false, - "required": ["tests"], - "properties": { - "runOn": { - "type": "array", - "description": "List of server version and/or topology requirements for which the tests can be run. If the test environment satisfies one or more of these requirements, the tests may be executed; otherwise, this file should be skipped. If this field is omitted, the tests can be assumed to have no particular requirements and should be executed.", - "minItems": 1, - "items": { "$ref": "#/definitions/runOnRequirement" } - }, - - "databaseName": { - "type": "string", - "description": "Name of database under test. This is primarily useful when the database name must be referenced in an assertion." - }, - - "collectionName": { - "type": "string", - "description": "Name of collection under test. This is primarily useful when the collection name must be referenced in an assertion." - }, - - "initialData": { - "type": "array", - "description": "Data that should exist in collections before each test case is executed.", - "minItems": 1, - "items": { "$ref": "#/definitions/collectionData" } - }, - - "tests": { - "type": "array", - "description": "Test cases.", - "minItems": 1, - "items": { "$ref": "#/definitions/test" } - } - }, - - "definitions": { - "runOnRequirement": { - "type": "object", - "description": "A combination of server version and/or topology requirements for running the tests.", - "additionalProperties": false, - "minProperties": 1, - "properties": { - "maxServerVersion": { - "type": "string", - "description": "The maximum server version (inclusive) against which the tests can be run successfully. If this field is omitted, it should be assumed that there is no upper bound on the required server version.", - "pattern": "^[0-9]+(\\.[0-9]+){1,2}$", - "examples": ["4.0", "4.2.0"] - }, - "minServerVersion": { - "type": "string", - "description": "The minimum server version (inclusive) required to successfully run the tests. If this field is omitted, it should be assumed that there is no lower bound on the required server version.", - "pattern": "^[0-9]+(\\.[0-9]+){1,2}$", - "examples": ["4.0", "4.2.0"] - }, - "topology": { - "type": "array", - "description": "List of server topologies against which the tests can be run successfully. Valid topologies are \"single\", \"replicaset\", and \"sharded\". If this field is omitted, the default is all topologies.", - "default": ["single", "replicaset", "sharded"], - "minItems": 1, - "items": { - "type": "string", - "enum": ["single", "replicaset", "sharded"] - } - } - } - }, - - "collectionData": { - "type": "object", - "description": "List of documents that should correspond to the contents of a collection.", - "additionalProperties": false, - "required": ["documents"], - "properties": { - "collection": { - "type": "string", - "description": "Collection name. If omitted, this defaults to the collection under test." - }, - "database": { - "type": "string", - "description": "Database name. If omitted, this defaults to the database under test." - }, - "documents": { - "type": "array", - "description": "List of documents corresponding to the contents of the collection. This list may be empty.", - "items": { "type": "object" } - } - } - }, - - "event": { - "type": "object", - "description": "An event (e.g. APM, SDAM) to be observed while running the test operations.", - "additionalProperties": false, - "minProperties": 1, - "maxProperties": 1, - "properties": { - "commandStartedEvent": { - "type": "object", - "additionalProperties": false, - "minProperties": 1, - "properties": { - "command": { "type": "object" }, - "commandName": { "type": "string" }, - "databaseName": { "type": "string" } - } - } - } - }, - - "collectionOrDatabaseOptions": { - "type": "object", - "description": "Map of parameters to pass when creating a collection or database.", - "additionalProperties": false, - "properties": { - "readConcern": { "type": "object" }, - "readPreference": { "type": "object" }, - "writeConcern": { "type": "object" } - } - }, - - "operation": { - "type": "object", - "description": "An operation to be executed as part of the test.", - "additionalProperties": false, - "required": ["name"], - "properties": { - "name": { - "type": "string", - "description": "Name of the operation (e.g. method) to perform on the object." - }, - "object": { - "type": "string", - "description": "Name of the object on which to perform the operation.", - "default": "collection", - "enum": ["collection", "database", "session0", "session1", "testRunner"] - }, - "collectionOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" }, - "databaseOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" }, - "command_name": { - "type": "string", - "description": "Required only when name is \"runCommand\". The name of the command to run. This may be used by languages that are unable preserve the order of keys in the command argument when parsing YAML or JSON." - }, - "arguments": { - "type": "object", - "description": "Map of parameter names and values for the operation." - }, - "error": { - "type": "boolean", - "description": "If true, the test should expect the operation to raise an error/exception. This could be either a server-generated or a driver-generated error.", - "default": false - }, - "result": { - "description": "The return value from the operation, if any. This field may be a scalar value, a single document, or an array of documents in the case of a multi-document read. If the operation is expected to return an error, the result is a single document containing several \"error\" fields defined in a conditional schema." - } - }, - "allOf": [ - { - "$comment": "Require command_name for runCommand operations", - "oneOf": [ - { - "properties": { "name": { "const": "runCommand" }}, - "required": ["command_name"] - }, - { - "properties": { "name": { "not": { "const": "runCommand" }}} - } - ] - }, - { - "$comment": "Conditional result schema when error is true", - "anyOf": [ - { "properties": { "error": false }}, - { "properties": { "error": { "const": false }}}, - { - "properties": { - "error": { "const": true }, - "result": { - "type": "object", - "additionalProperties": false, - "minProperties": 1, - "properties": { - "errorContains": { - "type": "string", - "description": "A substring of the expected error message." - }, - "errorCodeName": { - "type": "string", - "description": "The expected \"codeName\" field in the server error response." - }, - "errorLabelsContain": { - "type": "array", - "description": "A list of error label strings that the error is expected to have.", - "items": { "type": "string" } - }, - "errorLabelsOmit": { - "type": "array", - "description": "A list of error label strings that the error is expected not to have.", - "items": { "type": "string" } - } - } - } - } - } - ] - } - ] - }, - - "test": { - "type": "object", - "description": "Test case consisting of a sequence of operations to be executed. The test may optionally include configuration directives and event/outcome assertions.", - "required": ["description", "operations"], - "properties": { - "description": { - "type": "string", - "description": "The name of the test." - }, - "skipReason": { - "type": "string", - "description": "If set, the test will be skipped. The string should explain the reason for skipping the test (e.g. JIRA ticket)." - }, - "useMultipleMongoses": { - "type": "boolean", - "description": "If true, the MongoClient for this test should be initialized with multiple mongos seed addresses. If false or omitted, only a single mongos address should be specified. This field has no effect for non-sharded topologies." - }, - "clientOptions": { - "$comment": "TODO: this schema is not sufficient for expressing multiple readPreferenceTags, since keys would repeat", - "type": "object", - "description": "Additional connection string options to pass to the MongoClient constructor.", - "additionalProperties": { - "type": ["number", "string"] - } - }, - "failPoint": { - "type": "object", - "description": "A server failpoint to enable expressed as a complete configureFailPoint command to run on the admin database. This option and useMultipleMongoses:true are mutually exclusive.", - "additionalProperties": false, - "required": ["configureFailPoint", "mode"], - "properties": { - "configureFailPoint": { "type": "string" }, - "mode": { "type": ["object", "string"] }, - "data": { "type": "object" } - } - }, - "sessionOptions": { - "type": "object", - "description": " Map of session names (e.g. \"session0\") to documents, each of which denotes parameters to pass to MongoClient.startSession() when creating that session.", - "additionalProperties": false, - "patternProperties": { - "^session[0-1]$": { "type": "object" } - } - }, - "operations": { - "type": "array", - "description": "List of operations to be executed for the test case.", - "items": { "$ref": "#/definitions/operation" } - }, - "expectedEvents": { - "type": "array", - "description": "List of events, which are expected to be observed in this order by running the operations.", - "items": { "$ref": "#/definitions/event" } - }, - "outcome": { - "type": "array", - "description": "Data that should exist in collections after all operations have been executed. The list of documents should be sorted ascendingly by the \"_id\" field to allow for deterministic comparisons.", - "minItems": 1, - "items": { "$ref": "#/definitions/collectionData" } - } - }, - "allOf": [ - { - "$comment": "useMultipleMongoses:true and failPoint are mutually exclusive", - "if": { "properties": { "useMultipleMongoses": { "const": true }}}, - "then": { "properties": { "failPoint": { "not": { "type": "object" }}}} - } - ] - } - } -} From fddd2b7545170e7d542c699f97631f0009afdf71 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 29 Sep 2020 16:26:25 +0800 Subject: [PATCH 55/90] Update Makefile for new schema --- source/Makefile | 6 ------ source/unified-test-format/Makefile | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 source/unified-test-format/Makefile diff --git a/source/Makefile b/source/Makefile index 74f89ba425..eaa584eb6b 100644 --- a/source/Makefile +++ b/source/Makefile @@ -17,9 +17,3 @@ HAS_JSYAML: echo 'Error: need "npm install -g js-yaml"' 1>&2; \ exit 1; \ fi - -HAS_AJV: - @if ! command -v ajv > /dev/null; then \ - echo 'Error: need "npm install -g ajv"' 1>&2; \ - exit 1; \ - fi diff --git a/source/unified-test-format/Makefile b/source/unified-test-format/Makefile new file mode 100644 index 0000000000..b499144390 --- /dev/null +++ b/source/unified-test-format/Makefile @@ -0,0 +1,17 @@ +YAML_FILES=$(shell find tests/ -iname '*.yml') +JSON_FILES=$(shell find tests/ -iname '*.json') + +.PHONY: all validate HAS_AJV + +all: validate + +validate: HAS_AJV + @for f in $(YAML_FILES) $(JSON_FILES); do \ + ajv validate -s schema-1.0.json -d $$f; \ + done + +HAS_AJV: + @if ! command -v ajv > /dev/null; then \ + echo 'Error: need "npm install -g ajv"' 1>&2; \ + exit 1; \ + fi From 8b205175b035e791ada417c40dca8132fb0ea846 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 30 Sep 2020 15:36:25 +0800 Subject: [PATCH 56/90] expectedError.isError can only be true --- source/unified-test-format/schema-1.0.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/schema-1.0.json b/source/unified-test-format/schema-1.0.json index 5ebe359dde..38997847db 100644 --- a/source/unified-test-format/schema-1.0.json +++ b/source/unified-test-format/schema-1.0.json @@ -49,7 +49,7 @@ "items": { "type": "string", "enum": ["single", "replicaset", "sharded", "sharded-replicaset"] - } + } } } }, @@ -233,7 +233,10 @@ "additionalProperties": false, "minProperties": 1, "properties": { - "isError": { "type": "boolean" }, + "isError": { + "type": "boolean", + "const": true + }, "isClientError": { "type": "boolean" }, "errorContains": { "type": "string" }, "errorCode": { "type": "integer" }, From 731b8b8db1f4a5dc67b9404a6357d6f7c610c98b Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 30 Sep 2020 16:42:01 +0800 Subject: [PATCH 57/90] Add invalid test files for testing the schema itself --- source/unified-test-format/Makefile | 17 -------- source/unified-test-format/tests/Makefile | 18 +++++++++ source/unified-test-format/tests/README.rst | 39 +++++++++++++++++++ .../collectionData-additionalProperties.yml | 26 +++++++++++++ ...collectionData-collectionName-required.yml | 24 ++++++++++++ .../collectionData-collectionName-type.yml | 25 ++++++++++++ .../collectionData-databaseName-required.yml | 24 ++++++++++++ .../collectionData-databaseName-type.yml | 25 ++++++++++++ .../collectionData-documents-items.yml | 25 ++++++++++++ .../collectionData-documents-required.yml | 24 ++++++++++++ .../invalid/collectionData-documents-type.yml | 25 ++++++++++++ ...OrDatabaseOptions-additionalProperties.yml | 17 ++++++++ ...tionOrDatabaseOptions-readConcern-type.yml | 17 ++++++++ ...nOrDatabaseOptions-readPreference-type.yml | 17 ++++++++ ...ionOrDatabaseOptions-writeConcern-type.yml | 17 ++++++++ .../tests/invalid/createEntities-items.yml | 9 +++++ .../tests/invalid/createEntities-minItems.yml | 9 +++++ .../tests/invalid/createEntities-type.yml | 9 +++++ .../tests/invalid/description-required.yml | 5 +++ .../invalid/entity-additionalProperties.yml | 10 +++++ .../entity-bucket-additionalProperties.yml | 19 +++++++++ .../entity-bucket-bucketOptions-type.yml | 19 +++++++++ .../entity-bucket-database-required.yml | 17 ++++++++ .../invalid/entity-bucket-database-type.yml | 18 +++++++++ .../invalid/entity-bucket-id-required.yml | 17 ++++++++ .../tests/invalid/entity-bucket-id-type.yml | 18 +++++++++ .../entity-client-additionalProperties.yml | 12 ++++++ .../invalid/entity-client-id-required.yml | 10 +++++ .../tests/invalid/entity-client-id-type.yml | 11 ++++++ ...nt-ignoreCommandMonitoringEvents-items.yml | 12 ++++++ ...ignoreCommandMonitoringEvents-minItems.yml | 12 ++++++ ...ent-ignoreCommandMonitoringEvents-type.yml | 12 ++++++ .../entity-client-observeEvents-enum.yml | 12 ++++++ .../entity-client-observeEvents-items.yml | 12 ++++++ .../entity-client-observeEvents-minItems.yml | 12 ++++++ .../entity-client-observeEvents-type.yml | 12 ++++++ .../invalid/entity-client-uriOptions-type.yml | 12 ++++++ ...entity-client-useMultipleMongoses-type.yml | 12 ++++++ ...entity-collection-additionalProperties.yml | 20 ++++++++++ ...ity-collection-collectionName-required.yml | 18 +++++++++ .../entity-collection-collectionName-type.yml | 19 +++++++++ ...tity-collection-collectionOptions-type.yml | 20 ++++++++++ .../entity-collection-database-required.yml | 18 +++++++++ .../entity-collection-database-type.yml | 19 +++++++++ .../invalid/entity-collection-id-required.yml | 18 +++++++++ .../invalid/entity-collection-id-type.yml | 19 +++++++++ .../entity-database-additionalProperties.yml | 16 ++++++++ .../entity-database-client-required.yml | 14 +++++++ .../invalid/entity-database-client-type.yml | 15 +++++++ .../entity-database-databaseName-required.yml | 14 +++++++ .../entity-database-databaseName-type.yml | 15 +++++++ .../entity-database-databaseOptions-type.yml | 16 ++++++++ .../invalid/entity-database-id-required.yml | 14 +++++++ .../tests/invalid/entity-database-id-type.yml | 15 +++++++ .../tests/invalid/entity-maxProperties.yml | 15 +++++++ .../tests/invalid/entity-minProperties.yml | 10 +++++ .../entity-session-additionalProperties.yml | 15 +++++++ .../entity-session-client-required.yml | 13 +++++++ .../invalid/entity-session-client-type.yml | 14 +++++++ .../invalid/entity-session-id-required.yml | 13 +++++++ .../tests/invalid/entity-session-id-type.yml | 14 +++++++ .../entity-session-sessionOptions-type.yml | 15 +++++++ .../entity-stream-additionalProperties.yml | 13 +++++++ .../entity-stream-hexBytes-pattern.yml | 12 ++++++ .../entity-stream-hexBytes-required.yml | 11 ++++++ .../invalid/entity-stream-hexBytes-type.yml | 12 ++++++ .../invalid/entity-stream-id-required.yml | 11 ++++++ .../tests/invalid/entity-stream-id-type.yml | 12 ++++++ .../expectedError-additionalProperties.yml | 15 +++++++ .../invalid/expectedError-errorCode-type.yml | 15 +++++++ .../expectedError-errorCodeName-type.yml | 15 +++++++ .../expectedError-errorContains-type.yml | 15 +++++++ ...expectedError-errorLabelsContain-items.yml | 15 +++++++ ...ectedError-errorLabelsContain-minItems.yml | 15 +++++++ .../expectedError-errorLabelsContain-type.yml | 15 +++++++ .../expectedError-errorLabelsOmit-items.yml | 15 +++++++ ...expectedError-errorLabelsOmit-minItems.yml | 15 +++++++ .../expectedError-errorLabelsOmit-type.yml | 15 +++++++ .../expectedError-isClientError-type.yml | 15 +++++++ .../invalid/expectedError-isError-const.yml | 15 +++++++ .../invalid/expectedError-isError-type.yml | 15 +++++++ .../invalid/expectedError-minProperties.yml | 14 +++++++ .../expectedEvent-additionalProperties.yml | 17 ++++++++ ...nt-commandFailedEvent-commandName-type.yml | 18 +++++++++ ...mmandStartedEvent-additionalProperties.yml | 18 +++++++++ ...Event-commandStartedEvent-command-type.yml | 18 +++++++++ ...t-commandStartedEvent-commandName-type.yml | 18 +++++++++ ...-commandStartedEvent-databaseName-type.yml | 18 +++++++++ ...commandSucceededEvent-commandName-type.yml | 18 +++++++++ ...Event-commandSucceededEvent-reply-type.yml | 18 +++++++++ .../invalid/expectedEvent-maxProperties.yml | 18 +++++++++ .../invalid/expectedEvent-minProperties.yml | 17 ++++++++ ...edEventsForClient-additionalProperties.yml | 17 ++++++++ ...xpectedEventsForClient-client-required.yml | 15 +++++++ .../expectedEventsForClient-client-type.yml | 16 ++++++++ .../expectedEventsForClient-events-items.yml | 16 ++++++++ ...xpectedEventsForClient-events-required.yml | 15 +++++++ .../expectedEventsForClient-events-type.yml | 16 ++++++++ .../tests/invalid/initialData-items.yml | 9 +++++ .../tests/invalid/initialData-minItems.yml | 9 +++++ .../tests/invalid/initialData-type.yml | 9 +++++ .../operation-additionalProperties.yml | 14 +++++++ .../invalid/operation-arguments-type.yml | 14 +++++++ ...xpectError-conflicts_with_expectResult.yml | 16 ++++++++ ...rror-conflicts_with_saveResultAsEntity.yml | 16 ++++++++ .../invalid/operation-expectError-type.yml | 14 +++++++ .../invalid/operation-expectEvents-type.yml | 14 +++++++ .../tests/invalid/operation-name-required.yml | 12 ++++++ .../tests/invalid/operation-name-type.yml | 13 +++++++ .../invalid/operation-object-required.yml | 12 ++++++ .../tests/invalid/operation-object-type.yml | 13 +++++++ .../operation-saveResultAsEntity-type.yml | 14 +++++++ .../runOnRequirement-additionalProperties.yml | 11 ++++++ ...OnRequirement-maxServerVersion-pattern.yml | 10 +++++ ...runOnRequirement-maxServerVersion-type.yml | 10 +++++ .../runOnRequirement-minProperties.yml | 10 +++++ ...OnRequirement-minServerVersion-pattern.yml | 10 +++++ ...runOnRequirement-minServerVersion-type.yml | 10 +++++ .../runOnRequirement-topologies-enum.yml | 10 +++++ .../runOnRequirement-topologies-items.yml | 10 +++++ .../runOnRequirement-topologies-minItems.yml | 10 +++++ .../runOnRequirement-topologies-type.yml | 10 +++++ .../tests/invalid/runOnRequirements-items.yml | 9 +++++ .../invalid/runOnRequirements-minItems.yml | 9 +++++ .../tests/invalid/runOnRequirements-type.yml | 9 +++++ .../tests/invalid/schemaVersion-pattern.yml | 7 ++++ .../tests/invalid/schemaVersion-required.yml | 5 +++ .../tests/invalid/schemaVersion-type.yml | 7 ++++ .../invalid/test-additionalProperties.yml | 8 ++++ .../invalid/test-description-required.yml | 6 +++ .../tests/invalid/test-description-type.yml | 7 ++++ .../tests/invalid/test-expectEvents-items.yml | 8 ++++ .../tests/invalid/test-expectEvents-type.yml | 8 ++++ .../tests/invalid/test-operations-items.yml | 7 ++++ .../invalid/test-operations-required.yml | 6 +++ .../tests/invalid/test-operations-type.yml | 7 ++++ .../tests/invalid/test-outcome-items.yml | 8 ++++ .../tests/invalid/test-outcome-minItems.yml | 8 ++++ .../tests/invalid/test-outcome-type.yml | 8 ++++ .../invalid/test-runOnRequirements-items.yml | 8 ++++ .../test-runOnRequirements-minItems.yml | 8 ++++ .../invalid/test-runOnRequirements-type.yml | 8 ++++ .../tests/invalid/test-skipReason-type.yml | 8 ++++ .../tests/invalid/tests-items.yml | 5 +++ .../tests/invalid/tests-minItems.yml | 5 +++ .../tests/invalid/tests-required.yml | 3 ++ .../tests/invalid/tests-type.yml | 5 +++ 147 files changed, 2003 insertions(+), 17 deletions(-) delete mode 100644 source/unified-test-format/Makefile create mode 100644 source/unified-test-format/tests/Makefile create mode 100644 source/unified-test-format/tests/README.rst create mode 100644 source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-documents-items.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-documents-required.yml create mode 100644 source/unified-test-format/tests/invalid/collectionData-documents-type.yml create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml create mode 100644 source/unified-test-format/tests/invalid/createEntities-items.yml create mode 100644 source/unified-test-format/tests/invalid/createEntities-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/createEntities-type.yml create mode 100644 source/unified-test-format/tests/invalid/description-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-database-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-database-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-id-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-id-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-id-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-id-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-database-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-database-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-id-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-collection-id-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-client-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-client-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-id-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-database-id-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-maxProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-minProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-session-client-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-session-client-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-session-id-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-session-id-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml create mode 100644 source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml create mode 100644 source/unified-test-format/tests/invalid/entity-stream-id-required.yml create mode 100644 source/unified-test-format/tests/invalid/entity-stream-id-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-isError-const.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-isError-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedError-minProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml create mode 100644 source/unified-test-format/tests/invalid/initialData-items.yml create mode 100644 source/unified-test-format/tests/invalid/initialData-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/initialData-type.yml create mode 100644 source/unified-test-format/tests/invalid/operation-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/operation-arguments-type.yml create mode 100644 source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml create mode 100644 source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml create mode 100644 source/unified-test-format/tests/invalid/operation-expectError-type.yml create mode 100644 source/unified-test-format/tests/invalid/operation-expectEvents-type.yml create mode 100644 source/unified-test-format/tests/invalid/operation-name-required.yml create mode 100644 source/unified-test-format/tests/invalid/operation-name-type.yml create mode 100644 source/unified-test-format/tests/invalid/operation-object-required.yml create mode 100644 source/unified-test-format/tests/invalid/operation-object-type.yml create mode 100644 source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirements-items.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/runOnRequirements-type.yml create mode 100644 source/unified-test-format/tests/invalid/schemaVersion-pattern.yml create mode 100644 source/unified-test-format/tests/invalid/schemaVersion-required.yml create mode 100644 source/unified-test-format/tests/invalid/schemaVersion-type.yml create mode 100644 source/unified-test-format/tests/invalid/test-additionalProperties.yml create mode 100644 source/unified-test-format/tests/invalid/test-description-required.yml create mode 100644 source/unified-test-format/tests/invalid/test-description-type.yml create mode 100644 source/unified-test-format/tests/invalid/test-expectEvents-items.yml create mode 100644 source/unified-test-format/tests/invalid/test-expectEvents-type.yml create mode 100644 source/unified-test-format/tests/invalid/test-operations-items.yml create mode 100644 source/unified-test-format/tests/invalid/test-operations-required.yml create mode 100644 source/unified-test-format/tests/invalid/test-operations-type.yml create mode 100644 source/unified-test-format/tests/invalid/test-outcome-items.yml create mode 100644 source/unified-test-format/tests/invalid/test-outcome-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/test-outcome-type.yml create mode 100644 source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml create mode 100644 source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml create mode 100644 source/unified-test-format/tests/invalid/test-skipReason-type.yml create mode 100644 source/unified-test-format/tests/invalid/tests-items.yml create mode 100644 source/unified-test-format/tests/invalid/tests-minItems.yml create mode 100644 source/unified-test-format/tests/invalid/tests-required.yml create mode 100644 source/unified-test-format/tests/invalid/tests-type.yml diff --git a/source/unified-test-format/Makefile b/source/unified-test-format/Makefile deleted file mode 100644 index b499144390..0000000000 --- a/source/unified-test-format/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -YAML_FILES=$(shell find tests/ -iname '*.yml') -JSON_FILES=$(shell find tests/ -iname '*.json') - -.PHONY: all validate HAS_AJV - -all: validate - -validate: HAS_AJV - @for f in $(YAML_FILES) $(JSON_FILES); do \ - ajv validate -s schema-1.0.json -d $$f; \ - done - -HAS_AJV: - @if ! command -v ajv > /dev/null; then \ - echo 'Error: need "npm install -g ajv"' 1>&2; \ - exit 1; \ - fi diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile new file mode 100644 index 0000000000..966360cd5c --- /dev/null +++ b/source/unified-test-format/tests/Makefile @@ -0,0 +1,18 @@ +SCHEMA=../schema-1.0.json + +.PHONY: all invalid HAS_AJV + +all: invalid + +invalid: + @if ajv test -s $(SCHEMA) -d "invalid/*" --invalid > /dev/null; then \ + echo "PASSED"; \ + else \ + echo "FAILED"; \ + fi + +HAS_AJV: + @if ! command -v ajv > /dev/null; then \ + echo 'Error: need "npm install -g ajv-cli"' 1>&2; \ + exit 1; \ + fi diff --git a/source/unified-test-format/tests/README.rst b/source/unified-test-format/tests/README.rst new file mode 100644 index 0000000000..dd422f7f02 --- /dev/null +++ b/source/unified-test-format/tests/README.rst @@ -0,0 +1,39 @@ +========================= +Unified Test Format Tests +========================= + +.. contents:: + +---- + +Introduction +============ + +This directory contains tests for the Unified Test Format's schema and test +runner implementation(s). Tests are organized in the following directories: + +- ``invalid``: These files do not validate against the schema and are used to + test the schema itself. + +- ``valid-pass``: These files validate against the schema and should pass when + executed with a test runner. + +- ``valid-fail``: These files validate against the schema but should produce + runtime errors or failures when executed with a test runner. Some do so by + violating the "SHOULD" and "SHOULD NOT" guidance in the spec (e.g. referencing + an undefined entity). + +Validating Test Files +===================== + +JSON and YAML test files can be validated using `Ajv `__ +and a schema from the parent directory (e.g. ``schema-1.0.json``). + +Test files can be validated individually like so:: + + ajv -s ../schema-1.0.json -d path/to/test.yml + +Ajv can also be used to assert the validity of test files:: + + ajv test -s ../schema-1.0.json -d "invalid/*.yml" --invalid + ajv test -s ../schema-1.0.json -d "valid-*/*.yml" --valid diff --git a/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml new file mode 100644 index 0000000000..796f65be56 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml @@ -0,0 +1,26 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml new file mode 100644 index 0000000000..c0fbfe98f6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml @@ -0,0 +1,24 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - databaseName: *database0Name + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml new file mode 100644 index 0000000000..23e6d59861 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml @@ -0,0 +1,25 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: 0 + databaseName: *database0Name + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml new file mode 100644 index 0000000000..20c03c8065 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml @@ -0,0 +1,24 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml new file mode 100644 index 0000000000..9dcf8d17c7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml @@ -0,0 +1,25 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: 0 + documents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-items.yml b/source/unified-test-format/tests/invalid/collectionData-documents-items.yml new file mode 100644 index 0000000000..22e4179b26 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-items.yml @@ -0,0 +1,25 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-required.yml b/source/unified-test-format/tests/invalid/collectionData-documents-required.yml new file mode 100644 index 0000000000..aa318176b8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-required.yml @@ -0,0 +1,24 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-type.yml b/source/unified-test-format/tests/invalid/collectionData-documents-type.yml new file mode 100644 index 0000000000..d8404a2dfb --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-type.yml @@ -0,0 +1,25 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: &database0Name "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: &collection0Name "foo" + foo: 0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml new file mode 100644 index 0000000000..f7620923cb --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml new file mode 100644 index 0000000000..59dbf2e26e --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + readConcern: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml new file mode 100644 index 0000000000..8cafbad02c --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + readPreference: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml new file mode 100644 index 0000000000..8cbd4516bd --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: + writeConcern: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/createEntities-items.yml b/source/unified-test-format/tests/invalid/createEntities-items.yml new file mode 100644 index 0000000000..e444490701 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-items.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/createEntities-minItems.yml b/source/unified-test-format/tests/invalid/createEntities-minItems.yml new file mode 100644 index 0000000000..502d6e76b9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-minItems.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/createEntities-type.yml b/source/unified-test-format/tests/invalid/createEntities-type.yml new file mode 100644 index 0000000000..ce23e6dc4c --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-type.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/description-required.yml b/source/unified-test-format/tests/invalid/description-required.yml new file mode 100644 index 0000000000..73a69a8b6d --- /dev/null +++ b/source/unified-test-format/tests/invalid/description-required.yml @@ -0,0 +1,5 @@ +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-additionalProperties.yml new file mode 100644 index 0000000000..743bd7c1d2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-additionalProperties.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml new file mode 100644 index 0000000000..f905e2b58e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml @@ -0,0 +1,19 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + database: *database0 + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml new file mode 100644 index 0000000000..94f8535cac --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml @@ -0,0 +1,19 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + database: *database0 + bucketOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml b/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml new file mode 100644 index 0000000000..569df8c330 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml new file mode 100644 index 0000000000..21e3c68fc9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: &bucket0 "bucket0" + database: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml b/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml new file mode 100644 index 0000000000..ecae379127 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + database: *database0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml new file mode 100644 index 0000000000..ddb6eb96b9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - bucket: + id: 0 + database: *database0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml new file mode 100644 index 0000000000..a772fde01e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-id-required.yml b/source/unified-test-format/tests/invalid/entity-client-id-required.yml new file mode 100644 index 0000000000..3ead68edd5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-required.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: {} + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-id-type.yml b/source/unified-test-format/tests/invalid/entity-client-id-type.yml new file mode 100644 index 0000000000..ae0ee681df --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-type.yml @@ -0,0 +1,11 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml new file mode 100644 index 0000000000..2cfbe0fdd4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + ignoreCommandMonitoringEvents: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml new file mode 100644 index 0000000000..03a0128c80 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + ignoreCommandMonitoringEvents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml new file mode 100644 index 0000000000..a0bee34058 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + ignoreCommandMonitoringEvents: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml new file mode 100644 index 0000000000..7c28bab216 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: ["foo"] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml new file mode 100644 index 0000000000..8b75e6e204 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml new file mode 100644 index 0000000000..99ce389971 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml new file mode 100644 index 0000000000..2344c8bd2b --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + observeEvents: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml new file mode 100644 index 0000000000..d91dc9866b --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + uriOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml new file mode 100644 index 0000000000..035089fee6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + useMultipleMongoses: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml new file mode 100644 index 0000000000..025b689338 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml @@ -0,0 +1,20 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: "foo" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml new file mode 100644 index 0000000000..f3ad71560c --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml new file mode 100644 index 0000000000..5e455d5a75 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml @@ -0,0 +1,19 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml new file mode 100644 index 0000000000..318f230ebe --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml @@ -0,0 +1,20 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: *database0 + collectionName: "foo" + collectionOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-required.yml b/source/unified-test-format/tests/invalid/entity-collection-database-required.yml new file mode 100644 index 0000000000..f8d754d450 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-required.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-type.yml b/source/unified-test-format/tests/invalid/entity-collection-database-type.yml new file mode 100644 index 0000000000..e54a80386a --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-type.yml @@ -0,0 +1,19 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: &collection0 "collection0" + database: 0 + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-required.yml b/source/unified-test-format/tests/invalid/entity-collection-id-required.yml new file mode 100644 index 0000000000..f9c6889f8e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-required.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + database: *database0 + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-type.yml b/source/unified-test-format/tests/invalid/entity-collection-id-type.yml new file mode 100644 index 0000000000..6b8e31560a --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-type.yml @@ -0,0 +1,19 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + - collection: + id: 0 + database: *database0 + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml new file mode 100644 index 0000000000..f6e0279c6a --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-client-required.yml b/source/unified-test-format/tests/invalid/entity-database-client-required.yml new file mode 100644 index 0000000000..d232744292 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-required.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-client-type.yml b/source/unified-test-format/tests/invalid/entity-database-client-type.yml new file mode 100644 index 0000000000..32a4cad8e6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: 0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml new file mode 100644 index 0000000000..9d328b9609 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml new file mode 100644 index 0000000000..7ce9e631ad --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml new file mode 100644 index 0000000000..a57e9fe647 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + databaseOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-id-required.yml b/source/unified-test-format/tests/invalid/entity-database-id-required.yml new file mode 100644 index 0000000000..2520e6476b --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-required.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + client: *client0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-database-id-type.yml b/source/unified-test-format/tests/invalid/entity-database-id-type.yml new file mode 100644 index 0000000000..b31e38e082 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - database: + id: 0 + client: *client0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-maxProperties.yml b/source/unified-test-format/tests/invalid/entity-maxProperties.yml new file mode 100644 index 0000000000..97329a61ff --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-maxProperties.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + database: + id: &database0 "database0" + client: *client0 + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-minProperties.yml b/source/unified-test-format/tests/invalid/entity-minProperties.yml new file mode 100644 index 0000000000..1ec37d1c75 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-minProperties.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - {} + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml new file mode 100644 index 0000000000..5a2db0c197 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + client: *client0 + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-client-required.yml b/source/unified-test-format/tests/invalid/entity-session-client-required.yml new file mode 100644 index 0000000000..9ea824f3ec --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-required.yml @@ -0,0 +1,13 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-client-type.yml b/source/unified-test-format/tests/invalid/entity-session-client-type.yml new file mode 100644 index 0000000000..fb615208a8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-type.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + client: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-id-required.yml b/source/unified-test-format/tests/invalid/entity-session-id-required.yml new file mode 100644 index 0000000000..bf8321dd9f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-required.yml @@ -0,0 +1,13 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + client: *client0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-id-type.yml b/source/unified-test-format/tests/invalid/entity-session-id-type.yml new file mode 100644 index 0000000000..6726e1a8b6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-type.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: 0 + client: *client0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml new file mode 100644 index 0000000000..e2deb6fd78 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + - session: + id: &session0 "session0" + client: *client0 + sessionOptions: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml new file mode 100644 index 0000000000..312ecd5bda --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml @@ -0,0 +1,13 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + hexBytes: "FF" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml new file mode 100644 index 0000000000..bd8481e883 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + hexBytes: "FFF" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml new file mode 100644 index 0000000000..ea47f2f7ac --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml @@ -0,0 +1,11 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml new file mode 100644 index 0000000000..51fc9f3d83 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: &stream0 "stream0" + hexBytes: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-required.yml b/source/unified-test-format/tests/invalid/entity-stream-id-required.yml new file mode 100644 index 0000000000..66da9c1539 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-required.yml @@ -0,0 +1,11 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - stream: + hexBytes: "FF" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-type.yml b/source/unified-test-format/tests/invalid/entity-stream-id-type.yml new file mode 100644 index 0000000000..10b70d82fe --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-type.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - stream: + id: 0 + hexBytes: "FF" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml new file mode 100644 index 0000000000..322c6d4b2f --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml new file mode 100644 index 0000000000..a3cea1177b --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorCode: "foo" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml new file mode 100644 index 0000000000..af0157b04a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorCodeName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml new file mode 100644 index 0000000000..19cbbe0922 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorContains: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml new file mode 100644 index 0000000000..e6a046db12 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsContain: [0] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml new file mode 100644 index 0000000000..4c240e5222 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsContain: [] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml new file mode 100644 index 0000000000..3d298600dc --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsContain: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml new file mode 100644 index 0000000000..c738a7a186 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsOmit: [0] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml new file mode 100644 index 0000000000..8dc24a72c3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsOmit: [] diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml new file mode 100644 index 0000000000..0fe6c06c78 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + errorLabelsOmit: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml new file mode 100644 index 0000000000..ad0d89cec9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isClientError: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-const.yml b/source/unified-test-format/tests/invalid/expectedError-isError-const.yml new file mode 100644 index 0000000000..babbb291d6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-const.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: false diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-type.yml b/source/unified-test-format/tests/invalid/expectedError-isError-type.yml new file mode 100644 index 0000000000..16bad39796 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-type.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: 0 diff --git a/source/unified-test-format/tests/invalid/expectedError-minProperties.yml b/source/unified-test-format/tests/invalid/expectedError-minProperties.yml new file mode 100644 index 0000000000..54c1fe49f8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-minProperties.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: {} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml new file mode 100644 index 0000000000..08ab186f2a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml new file mode 100644 index 0000000000..cf6094189d --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandFailedEvent: + commandName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml new file mode 100644 index 0000000000..175e36d19e --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml new file mode 100644 index 0000000000..025ce7df0d --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml new file mode 100644 index 0000000000..3fb4ed05b1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + commandName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml new file mode 100644 index 0000000000..5afd053df3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + databaseName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml new file mode 100644 index 0000000000..a817033ad4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandSucceededEvent: + commandName: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml new file mode 100644 index 0000000000..fe0e09ce56 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandSucceededEvent: + reply: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml new file mode 100644 index 0000000000..b3e061a215 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml @@ -0,0 +1,18 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: {} + commandSucceededEvent: {} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml new file mode 100644 index 0000000000..6739dc77fd --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: + - {} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml new file mode 100644 index 0000000000..59e8f1d767 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml @@ -0,0 +1,17 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: [] + foo: 0 diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml new file mode 100644 index 0000000000..122eb97963 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - events: [] diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml new file mode 100644 index 0000000000..69016b8dc8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: 0 + events: [] diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml new file mode 100644 index 0000000000..9f5ba41a37 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: [0] diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml new file mode 100644 index 0000000000..226e37f127 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml @@ -0,0 +1,15 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml new file mode 100644 index 0000000000..a2a059194c --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: + - client: *client0 + events: 0 diff --git a/source/unified-test-format/tests/invalid/initialData-items.yml b/source/unified-test-format/tests/invalid/initialData-items.yml new file mode 100644 index 0000000000..ddd84cf60a --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-items.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +initialData: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/initialData-minItems.yml b/source/unified-test-format/tests/invalid/initialData-minItems.yml new file mode 100644 index 0000000000..97dbd3ff13 --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-minItems.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +initialData: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/initialData-type.yml b/source/unified-test-format/tests/invalid/initialData-type.yml new file mode 100644 index 0000000000..543e7a91e8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-type.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +initialData: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/operation-additionalProperties.yml b/source/unified-test-format/tests/invalid/operation-additionalProperties.yml new file mode 100644 index 0000000000..5d7b3adc77 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-additionalProperties.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + foo: 0 diff --git a/source/unified-test-format/tests/invalid/operation-arguments-type.yml b/source/unified-test-format/tests/invalid/operation-arguments-type.yml new file mode 100644 index 0000000000..aa26607b47 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-arguments-type.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + arguments: 0 diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml new file mode 100644 index 0000000000..9ae43992b6 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: true + expectResult: {} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml new file mode 100644 index 0000000000..a7b6fee1ae --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml @@ -0,0 +1,16 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: + isError: true + saveResultAsEntity: "foo" diff --git a/source/unified-test-format/tests/invalid/operation-expectError-type.yml b/source/unified-test-format/tests/invalid/operation-expectError-type.yml new file mode 100644 index 0000000000..8336fca1d8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-type.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectError: 0 diff --git a/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml b/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml new file mode 100644 index 0000000000..31945f2eb5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + expectEvents: 0 diff --git a/source/unified-test-format/tests/invalid/operation-name-required.yml b/source/unified-test-format/tests/invalid/operation-name-required.yml new file mode 100644 index 0000000000..fd62b19420 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-required.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - object: *client0 diff --git a/source/unified-test-format/tests/invalid/operation-name-type.yml b/source/unified-test-format/tests/invalid/operation-name-type.yml new file mode 100644 index 0000000000..a7a47d2f05 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-type.yml @@ -0,0 +1,13 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: 0 + object: *client0 diff --git a/source/unified-test-format/tests/invalid/operation-object-required.yml b/source/unified-test-format/tests/invalid/operation-object-required.yml new file mode 100644 index 0000000000..c00461d742 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-required.yml @@ -0,0 +1,12 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" diff --git a/source/unified-test-format/tests/invalid/operation-object-type.yml b/source/unified-test-format/tests/invalid/operation-object-type.yml new file mode 100644 index 0000000000..9f4930f3c1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-type.yml @@ -0,0 +1,13 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: 0 diff --git a/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml new file mode 100644 index 0000000000..4e8aad3890 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml @@ -0,0 +1,14 @@ +description: "foo" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 "client0" + +tests: + - description: "foo" + operations: + - name: "foo" + object: *client0 + saveResultAsEntity: 0 diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml new file mode 100644 index 0000000000..c30a559ce0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml @@ -0,0 +1,11 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + foo: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml new file mode 100644 index 0000000000..c07bfc20a5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - maxServerVersion: "1.2.3.4" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml new file mode 100644 index 0000000000..677d72c3b4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - maxServerVersion: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml new file mode 100644 index 0000000000..b03fb50441 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - {} + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml new file mode 100644 index 0000000000..bd6644a597 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "1.2.3.4" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml new file mode 100644 index 0000000000..f8c53e11ef --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml new file mode 100644 index 0000000000..3c5ec191de --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: ["foo"] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml new file mode 100644 index 0000000000..dfe39f4181 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml new file mode 100644 index 0000000000..1f3431b469 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml new file mode 100644 index 0000000000..935af7889b --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml @@ -0,0 +1,10 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: + - topologies: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-items.yml b/source/unified-test-format/tests/invalid/runOnRequirements-items.yml new file mode 100644 index 0000000000..d620af55a0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-items.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: [0] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml new file mode 100644 index 0000000000..4c1045e0dd --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: [] + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-type.yml b/source/unified-test-format/tests/invalid/runOnRequirements-type.yml new file mode 100644 index 0000000000..c159787826 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-type.yml @@ -0,0 +1,9 @@ +description: "foo" + +schemaVersion: "1.0" + +runOnRequirements: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml b/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml new file mode 100644 index 0000000000..dfd284905e --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml @@ -0,0 +1,7 @@ +description: "foo" + +schemaVersion: "1.2.3.4" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/schemaVersion-required.yml b/source/unified-test-format/tests/invalid/schemaVersion-required.yml new file mode 100644 index 0000000000..85303d0fac --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-required.yml @@ -0,0 +1,5 @@ +description: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/schemaVersion-type.yml b/source/unified-test-format/tests/invalid/schemaVersion-type.yml new file mode 100644 index 0000000000..3bd03eda0d --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-type.yml @@ -0,0 +1,7 @@ +description: "foo" + +schemaVersion: 0 + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/invalid/test-additionalProperties.yml b/source/unified-test-format/tests/invalid/test-additionalProperties.yml new file mode 100644 index 0000000000..a27a845d9d --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-additionalProperties.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + foo: 0 diff --git a/source/unified-test-format/tests/invalid/test-description-required.yml b/source/unified-test-format/tests/invalid/test-description-required.yml new file mode 100644 index 0000000000..0855d6dcc5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-required.yml @@ -0,0 +1,6 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - operation: [] diff --git a/source/unified-test-format/tests/invalid/test-description-type.yml b/source/unified-test-format/tests/invalid/test-description-type.yml new file mode 100644 index 0000000000..ea599f2978 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-type.yml @@ -0,0 +1,7 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: 0 + operation: [] diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-items.yml b/source/unified-test-format/tests/invalid/test-expectEvents-items.yml new file mode 100644 index 0000000000..cecee32f99 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-items.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + expectEvents: [0] diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-type.yml b/source/unified-test-format/tests/invalid/test-expectEvents-type.yml new file mode 100644 index 0000000000..b70f57c2d5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-type.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + expectEvents: 0 diff --git a/source/unified-test-format/tests/invalid/test-operations-items.yml b/source/unified-test-format/tests/invalid/test-operations-items.yml new file mode 100644 index 0000000000..d6573b1a06 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-items.yml @@ -0,0 +1,7 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [0] diff --git a/source/unified-test-format/tests/invalid/test-operations-required.yml b/source/unified-test-format/tests/invalid/test-operations-required.yml new file mode 100644 index 0000000000..8828ad1b08 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-required.yml @@ -0,0 +1,6 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" diff --git a/source/unified-test-format/tests/invalid/test-operations-type.yml b/source/unified-test-format/tests/invalid/test-operations-type.yml new file mode 100644 index 0000000000..ffaa4ad390 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-type.yml @@ -0,0 +1,7 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: 0 diff --git a/source/unified-test-format/tests/invalid/test-outcome-items.yml b/source/unified-test-format/tests/invalid/test-outcome-items.yml new file mode 100644 index 0000000000..c685a717a1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-items.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + outcome: [0] diff --git a/source/unified-test-format/tests/invalid/test-outcome-minItems.yml b/source/unified-test-format/tests/invalid/test-outcome-minItems.yml new file mode 100644 index 0000000000..96fc63e5df --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-minItems.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + outcome: [] diff --git a/source/unified-test-format/tests/invalid/test-outcome-type.yml b/source/unified-test-format/tests/invalid/test-outcome-type.yml new file mode 100644 index 0000000000..0c1a14e650 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-type.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + outcome: 0 diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml new file mode 100644 index 0000000000..540609b813 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + runOnRequirements: [0] diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml new file mode 100644 index 0000000000..30501e5fae --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + runOnRequirements: [] diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml new file mode 100644 index 0000000000..e1d7e34b2a --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + runOnRequirements: 0 diff --git a/source/unified-test-format/tests/invalid/test-skipReason-type.yml b/source/unified-test-format/tests/invalid/test-skipReason-type.yml new file mode 100644 index 0000000000..d77ab4e913 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-skipReason-type.yml @@ -0,0 +1,8 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: + - description: "foo" + operations: [] + skipReason: 0 diff --git a/source/unified-test-format/tests/invalid/tests-items.yml b/source/unified-test-format/tests/invalid/tests-items.yml new file mode 100644 index 0000000000..328f72cde7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-items.yml @@ -0,0 +1,5 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: [0] diff --git a/source/unified-test-format/tests/invalid/tests-minItems.yml b/source/unified-test-format/tests/invalid/tests-minItems.yml new file mode 100644 index 0000000000..c7aefc882d --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-minItems.yml @@ -0,0 +1,5 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: [] diff --git a/source/unified-test-format/tests/invalid/tests-required.yml b/source/unified-test-format/tests/invalid/tests-required.yml new file mode 100644 index 0000000000..13bed7011d --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-required.yml @@ -0,0 +1,3 @@ +description: "foo" + +schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/tests-type.yml b/source/unified-test-format/tests/invalid/tests-type.yml new file mode 100644 index 0000000000..46e16d3ca5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-type.yml @@ -0,0 +1,5 @@ +description: "foo" + +schemaVersion: "1.0" + +tests: 0 From 2e76cb381bae2677723fe2ac6c3b1278b69de1c1 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 30 Sep 2020 16:47:15 +0800 Subject: [PATCH 58/90] Prohibit additionalProperties in test objects --- source/unified-test-format/schema-1.0.json | 1 + 1 file changed, 1 insertion(+) diff --git a/source/unified-test-format/schema-1.0.json b/source/unified-test-format/schema-1.0.json index 38997847db..3ce636c952 100644 --- a/source/unified-test-format/schema-1.0.json +++ b/source/unified-test-format/schema-1.0.json @@ -257,6 +257,7 @@ "test": { "type": "object", + "additionalProperties": false, "required": ["description", "operations"], "properties": { "description": { "type": "string" }, From eb7fb4bbfaf7dd5278586f46b8b235ccbc1688f1 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 2 Oct 2020 13:31:31 +0800 Subject: [PATCH 59/90] Requirements for model objects and iterables and clarify failPoint Also restructures Entity Test Operations sections --- .../unified-test-format.rst | 222 ++++++++++++------ 1 file changed, 154 insertions(+), 68 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 61006c7818..2d0578eaad 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-09-29 +:Last Modified: 2020-10-02 .. contents:: @@ -73,8 +73,14 @@ Entity Internal MongoClient A MongoClient created specifically for use with internal test operations, such - as inserting collection data before a test, configuring fail points during a - test, or asserting collection data after a test. + as inserting collection data before a test, performing special assertions + during a test, or asserting collection data after a test. + +Iterable + This term is used by various specifications as the return type for operations + that return a sequence of items, which may be iterated. For example, the CRUD + spec uses this as the return value for ``find`` and permit API flexibility + rather than stipulate that a cursor object be returned directly. Schema Version @@ -207,13 +213,13 @@ Supported Entity Types Test runners MUST support the following types of entities: -- MongoClient. See `entity_client`_ and `client`_. -- Database. See `entity_database`_ and `database`_. -- Collection. See `entity_collection`_ and `collection`_ -- ClientSession. See `entity_session`_ and `session`_. -- GridFS Bucket. See `entity_bucket`_ and `bucket`_. +- MongoClient. See `entity_client`_ and `Client Operations`_. +- Database. See `entity_database`_ and `Database Operations`_. +- Collection. See `entity_collection`_ and `Collection Operations`_ +- ClientSession. See `entity_session`_ and `Session Operations`_. +- GridFS Bucket. See `entity_bucket`_ and `Bucket Operations`_. - GridFS Stream. See `entity_stream`_. -- ChangeStream. See `changeStream`_. +- ChangeStream. See `ChangeStream Operations`_. - All known BSON types and/or equivalent language types for the target driver. For the present version of the spec, the following BSON types are known: 0x01-0x13, 0x7F, 0xFF. @@ -521,7 +527,7 @@ The structure of this object is as follows: stream is both readable *and* writable. This entity is primarily used with `uploadFromStream`_ and - `uploadFromStreamWithId`_ operations for `bucket`_ entities. + `uploadFromStreamWithId`_ `Bucket Operations`_. The structure of this object is as follows: @@ -653,8 +659,8 @@ The structure of this object is as follows: - ``expectResult``: Optional mixed type. A value corresponding to the expected result of the operation. This field may be a scalar value, a single document, - or an array of documents in the case of a multi-document read. Test runners - MUST follow the rules in `Evaluating Matches`_ when processing this assertion. + or an array of values. Test runners MUST follow the rules in + `Evaluating Matches`_ when processing this assertion. This field is mutually exclusive with `expectError `_. @@ -926,6 +932,18 @@ Entity operations correspond to an API method on a driver object. If "collection0") then `operation.name `_ is expected to reference an API method on that class. +Test files SHALL use camelCase when referring to API methods and parameters, +even if the defining specifications use other forms (e.g. snake_case in GridFS). + +This spec does not provide exhaustive documentation for all possible API methods +that may appear in a test; however, the following sections discuss all supported +entities and their operations in some level of detail. Special handling for +certain operations is also discussed as needed. + + +Expressing Required and Optional Parameters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Some specifications group optional parameters for API methods under an ``options`` parameter (e.g. ``options: Optional`` in the CRUD spec); however, driver APIs vary in how they accept options (e.g. Python's @@ -935,6 +953,10 @@ files SHALL declare all required and optional parameters for an API method directly within `operation.arguments `_ (e.g. ``upsert`` for ``updateOne`` is *not* nested under an ``options`` key). + +Special Handling for Arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + If ``session`` is specified in `operation.arguments`_, it is defined according to `commonOptions_session`_. Test runners MUST resolve the ``session`` argument to `session `_ entity *before* passing it as a parameter to any @@ -945,17 +967,31 @@ If ``readConcern``, ``readPreference``, or ``writeConcern`` are specified in corresponding definition in `Common Options`_ and MUST convert the value into the appropriate object *before* passing it as a parameter to any API method. -This spec does not provide exhaustive documentation for all possible API methods -that may appear in a test; however, the following sections discuss all supported -entities and their operations in some level of detail. Special handling for -certain operations is also discussed below. -Test files SHALL use camelCase when referring to API methods and parameters, -even if the defining specifications use other forms (e.g. snake_case in GridFS). +Converting Returned Model Objects to Documents +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +For operations that return a model object (e.g. ``BulkWriteResult`` for +``bulkWrite``), the test runner MUST convert the model object to a document when +evaluating `expectResult `_ or +`saveResultAsEntity `_. Similarly, for operations +that may return iterables of model objects (e.g. ``DatabaseInfo`` for +``listDatabases``), the test runner MUST convert the iterable to an array of +documents when evaluating `expectResult`_ or `saveResultAsEntity`_. + + +Iterating Returned Iterables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unless otherwise stated, test runners MUST fully iterate any iterable returned +by an operation as part of that operation's execution. This is necessary to +ensure consistent behavior among drivers, as discussed in `aggregate`_ and +`find`_, and also ensures that error and event assertions can be evaluated +consistently. -client -~~~~~~ + +Client Operations +----------------- These operations and their arguments may be documented in the following specifications: @@ -970,21 +1006,26 @@ existing specification are described below. .. _client_createChangeStream: createChangeStream -`````````````````` +~~~~~~~~~~~~~~~~~~ Creates a cluster-level change stream and ensures that the server-side cursor has been created. This operation proxies the client's ``watch`` method and supports the same arguments and options. Test files SHOULD NOT use the client's ``watch`` -operation directly for reasons discussed in `changeStream`_. Test runners MUST -ensure that the server-side cursor is created (i.e. ``aggregate`` is executed) -as part of this operation and before the resulting change stream might be saved -with `operation.saveResultAsEntity `_. +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. -database -~~~~~~~~ + +Database Operations +------------------- These operations and their arguments may be documented in the following specifications: @@ -1000,21 +1041,26 @@ existing specification are described below. .. _database_createChangeStream: createChangeStream -`````````````````` +~~~~~~~~~~~~~~~~~~ Creates a database-level change stream and ensures that the server-side cursor has been created. This operation proxies the database's ``watch`` method and supports the same arguments and options. Test files SHOULD NOT use the database's ``watch`` -operation directly for reasons discussed in `changeStream`_. Test runners MUST -ensure that the server-side cursor is created (i.e. ``aggregate`` is executed) -as part of this operation and before the resulting change stream might be saved -with `operation.saveResultAsEntity `_. +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. + +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. runCommand -`````````` +~~~~~~~~~~ Generic command runner. @@ -1041,8 +1087,8 @@ The following arguments are supported: - ``writeConcern``: Optional object. See `commonOptions_writeConcern`_. -collection -~~~~~~~~~~ +Collection Operations +--------------------- These operations and their arguments may be documented in the following specifications: @@ -1056,24 +1102,16 @@ Collection operations that require special handling or are not documented by an existing specification are described below. -.. _collection_createChangeStream: - -createChangeStream -`````````````````` - -Creates a collection-level change stream and ensures that the server-side cursor -has been created. +aggregate +~~~~~~~~~ -This operation proxies the collection's ``watch`` method and supports the same -arguments and options. Test files SHOULD NOT use the collection's ``watch`` -operation directly for reasons discussed in `changeStream`_. Test runners MUST -ensure that the server-side cursor is created (i.e. ``aggregate`` is executed) -as part of this operation and before the resulting change stream might be saved -with `operation.saveResultAsEntity `_. +When executing an ``aggregate`` operation, the test runner MUST fully iterate +the result. This will ensure consistent behavior between drivers that eagerly +create a server-side cursor and those that do so lazily when iteration begins. bulkWrite -````````` +~~~~~~~~~ The ``requests`` parameter for ``bulkWrite`` is documented as a list of WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) @@ -1124,8 +1162,37 @@ and compare with ``code`` instead, but MUST raise an error if the comparison cannot be attempted (e.g. ``code`` is also not available, translation fails). -session -~~~~~~~ +.. _collection_createChangeStream: + +createChangeStream +~~~~~~~~~~~~~~~~~~ + +Creates a collection-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the collection's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the collection's ``watch`` +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. + +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. + + +find +~~~~ + +When executing a ``find`` operation, the test runner MUST fully iterate the +result. This will ensure consistent behavior between drivers that eagerly create +a server-side cursor and those that do so lazily when iteration begins. + + +Session Operations +------------------ These operations and their arguments may be documented in the following specifications: @@ -1138,7 +1205,7 @@ existing specification are described below. withTransaction -``````````````` +~~~~~~~~~~~~~~~ The ``withTransaction`` operation's ``callback`` parameter is a function and not easily expressed in YAML/JSON. For ease of testing, this parameter is expressed @@ -1147,8 +1214,8 @@ as an array of `operation`_ objects (analogous to result assertions when executing these operations in the callback. -bucket -~~~~~~ +Bucket Operations +----------------- These operations and their arguments may be documented in the following specifications: @@ -1163,7 +1230,7 @@ existing specification are described below. .. _downloadToStreamByName: downloadToStream and downloadToStreamByName -``````````````````````````````````````````` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ These operations SHOULD NOT be used in test files. See `IO operations for GridFS streams`_ in `Future Work`_. @@ -1173,7 +1240,7 @@ These operations SHOULD NOT be used in test files. See .. _openDownloadStreamByName: openDownloadStream and openDownloadStreamByName -``````````````````````````````````````````````` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``openDownloadStream`` and ``openDownloadStreamByName`` operations SHOULD use `$$matchesHexBytes`_ in `expectResult `_ to match @@ -1186,7 +1253,7 @@ with a subsequent operation (e.g. `uploadFromStream`_ ). .. _openUploadStreamWithId: openUploadStream and openUploadStreamWithId -``````````````````````````````````````````` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ These operations SHOULD NOT be used in test files. See `IO operations for GridFS streams`_ in `Future Work`_. @@ -1196,7 +1263,7 @@ These operations SHOULD NOT be used in test files. See .. _uploadFromStreamWithId: uploadFromStream and uploadFromStreamWithId -``````````````````````````````````````````` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``uploadFromStream`` and ``uploadFromStreamWithId`` operations' ``stream`` parameter is a stream as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ @@ -1207,8 +1274,8 @@ use an `alias node`_ for a stream entity's ``id`` field (e.g. ``stream: *stream0``). -changeStream -~~~~~~~~~~~~ +ChangeStream Operations +----------------------- Change stream entities are special in that they are not defined in `createEntities`_ but are instead created by using @@ -1230,7 +1297,7 @@ supported operations for change stream entities. iterateUntilDocumentOrError -``````````````````````````` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ Iterates the change stream until either a single document is returned or an error is raised. @@ -1260,7 +1327,8 @@ failPoint ~~~~~~~~~ The ``failPoint`` operation instructs the test runner to configure a fail point -using a "primary" read preference using the specified client. +using a "primary" read preference using the specified client entity (fail points +are not configured using the internal MongoClient). The following arguments are supported: @@ -1301,7 +1369,8 @@ targetedFailPoint ~~~~~~~~~~~~~~~~~ The ``targetedFailPoint`` operation instructs the test runner to configure a -fail point on a specific mongos. +fail point on a specific mongos using the client entity associated with a +pinned session. The following arguments are supported: @@ -1672,10 +1741,10 @@ root-level documents include, but are not limited to: - ``command`` for `CommandStartedEvent `_ - ``reply`` for `CommandSucceededEvent `_ -- `expectResult`_ for a `collection`_ ``findOneAndUpdate`` operation -- `expectResult`_ for a `changeStream`_ `iterateUntilDocumentOrError`_ operation -- each array element in `expectResult`_ for a `collection`_ ``find`` or - ``aggregate`` operation +- `expectResult`_ for a ``findOneAndUpdate`` `Collection Operation`_ +- `expectResult`_ for a `iterateUntilDocumentOrError`_ `ChangeStream Operation`_ +- each array element in `expectResult`_ for a `find`_ or `aggregate`_ + `Collection Operation`_ For example, the following documents match:: @@ -2115,6 +2184,11 @@ Before executing the operation, the test runner MUST be prepared to catch a potential error from the operation (e.g. enter a ``try`` block). Proceed with executing the operation and capture its result or error. +Note that some operations require special handling, as discussed in +`Entity Test Operations`_. For example, model objects may need to be converted +to documents (before matching or saving in the entity map) and returned +iterables may need to be fully iterated. + If `operation.expectError `_ is specified, the test runner MUST assert that the operation yielded an error; otherwise, the test runner MUST assert that the operation did not yield an error. If an error was @@ -2370,7 +2444,7 @@ The following tickets are addressed by the test format: * `SPEC-1216 `__: Update GridFS YAML tests to use newer format - See: `stream `_ entity and `bucket`_ operations + See: `stream `_ entity and `Bucket Operations`_ * `SPEC-1229 `__: Standardize spec-test syntax for topology assertions @@ -2538,6 +2612,18 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-10-02: + +* Restructure Entity Test Operations sections. + +* Clarify that fail points are not configured using the internal MongoClient. + +* Require model objects returned by operations be converted to documents before + being matched or saved in the entity map. + +* Require iterables returned by operations be fully iterated when executing an + operation, unless otherwise stated. + 2020-09-29: * Clarify that some optional arrays must contain one or more elements, since the From 92d4368f8113801473e561ce9d1e8954b58af0c1 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 2 Oct 2020 13:35:19 +0800 Subject: [PATCH 60/90] Fix RST syntax and typos --- source/unified-test-format/unified-test-format.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2d0578eaad..989a8fbc6a 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1741,10 +1741,10 @@ root-level documents include, but are not limited to: - ``command`` for `CommandStartedEvent `_ - ``reply`` for `CommandSucceededEvent `_ -- `expectResult`_ for a ``findOneAndUpdate`` `Collection Operation`_ -- `expectResult`_ for a `iterateUntilDocumentOrError`_ `ChangeStream Operation`_ -- each array element in `expectResult`_ for a `find`_ or `aggregate`_ - `Collection Operation`_ +- `expectResult`_ for ``findOneAndUpdate`` `Collection Operations`_ +- `expectResult`_ for `iterateUntilDocumentOrError`_ `ChangeStream Operations`_ +- each array element in `expectResult`_ for `find`_ or `aggregate`_ + `Collection Operations`_ For example, the following documents match:: @@ -2707,7 +2707,7 @@ Note: this will be cleared when publishing version 1.0 of the spec * Rename ``expected`` prefix to ``expect`` in test field names. Applies to ``expectEvents``, ``expectedError``, and ``expectResult``. Structures such as - ` `expectedEvent`` were not renamed. + ``expectedEvent`` were not renamed. * Clarify that "sharded" implies "sharded-replicaset". From 13eabc27798b96e8a6429d89d5b8f64e1aef9b9e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 2 Oct 2020 14:27:46 +0800 Subject: [PATCH 61/90] Specify descriptions for invalid schema tests --- .../tests/invalid/collectionData-additionalProperties.yml | 2 +- .../tests/invalid/collectionData-collectionName-required.yml | 2 +- .../tests/invalid/collectionData-collectionName-type.yml | 2 +- .../tests/invalid/collectionData-databaseName-required.yml | 2 +- .../tests/invalid/collectionData-databaseName-type.yml | 2 +- .../tests/invalid/collectionData-documents-items.yml | 2 +- .../tests/invalid/collectionData-documents-required.yml | 2 +- .../tests/invalid/collectionData-documents-type.yml | 2 +- .../collectionOrDatabaseOptions-additionalProperties.yml | 2 +- .../invalid/collectionOrDatabaseOptions-readConcern-type.yml | 2 +- .../invalid/collectionOrDatabaseOptions-readPreference-type.yml | 2 +- .../invalid/collectionOrDatabaseOptions-writeConcern-type.yml | 2 +- .../unified-test-format/tests/invalid/createEntities-items.yml | 2 +- .../tests/invalid/createEntities-minItems.yml | 2 +- .../unified-test-format/tests/invalid/createEntities-type.yml | 2 +- .../unified-test-format/tests/invalid/description-required.yml | 2 ++ .../tests/invalid/entity-additionalProperties.yml | 2 +- .../tests/invalid/entity-bucket-additionalProperties.yml | 2 +- .../tests/invalid/entity-bucket-bucketOptions-type.yml | 2 +- .../tests/invalid/entity-bucket-database-required.yml | 2 +- .../tests/invalid/entity-bucket-database-type.yml | 2 +- .../tests/invalid/entity-bucket-id-required.yml | 2 +- .../unified-test-format/tests/invalid/entity-bucket-id-type.yml | 2 +- .../tests/invalid/entity-client-additionalProperties.yml | 2 +- .../tests/invalid/entity-client-id-required.yml | 2 +- .../unified-test-format/tests/invalid/entity-client-id-type.yml | 2 +- .../entity-client-ignoreCommandMonitoringEvents-items.yml | 2 +- .../entity-client-ignoreCommandMonitoringEvents-minItems.yml | 2 +- .../entity-client-ignoreCommandMonitoringEvents-type.yml | 2 +- .../tests/invalid/entity-client-observeEvents-enum.yml | 2 +- .../tests/invalid/entity-client-observeEvents-items.yml | 2 +- .../tests/invalid/entity-client-observeEvents-minItems.yml | 2 +- .../tests/invalid/entity-client-observeEvents-type.yml | 2 +- .../tests/invalid/entity-client-uriOptions-type.yml | 2 +- .../tests/invalid/entity-client-useMultipleMongoses-type.yml | 2 +- .../tests/invalid/entity-collection-additionalProperties.yml | 2 +- .../tests/invalid/entity-collection-collectionName-required.yml | 2 +- .../tests/invalid/entity-collection-collectionName-type.yml | 2 +- .../tests/invalid/entity-collection-collectionOptions-type.yml | 2 +- .../tests/invalid/entity-collection-database-required.yml | 2 +- .../tests/invalid/entity-collection-database-type.yml | 2 +- .../tests/invalid/entity-collection-id-required.yml | 2 +- .../tests/invalid/entity-collection-id-type.yml | 2 +- .../tests/invalid/entity-database-additionalProperties.yml | 2 +- .../tests/invalid/entity-database-client-required.yml | 2 +- .../tests/invalid/entity-database-client-type.yml | 2 +- .../tests/invalid/entity-database-databaseName-required.yml | 2 +- .../tests/invalid/entity-database-databaseName-type.yml | 2 +- .../tests/invalid/entity-database-databaseOptions-type.yml | 2 +- .../tests/invalid/entity-database-id-required.yml | 2 +- .../tests/invalid/entity-database-id-type.yml | 2 +- .../unified-test-format/tests/invalid/entity-maxProperties.yml | 2 +- .../unified-test-format/tests/invalid/entity-minProperties.yml | 2 +- .../tests/invalid/entity-session-additionalProperties.yml | 2 +- .../tests/invalid/entity-session-client-required.yml | 2 +- .../tests/invalid/entity-session-client-type.yml | 2 +- .../tests/invalid/entity-session-id-required.yml | 2 +- .../tests/invalid/entity-session-id-type.yml | 2 +- .../tests/invalid/entity-session-sessionOptions-type.yml | 2 +- .../tests/invalid/entity-stream-additionalProperties.yml | 2 +- .../tests/invalid/entity-stream-hexBytes-pattern.yml | 2 +- .../tests/invalid/entity-stream-hexBytes-required.yml | 2 +- .../tests/invalid/entity-stream-hexBytes-type.yml | 2 +- .../tests/invalid/entity-stream-id-required.yml | 2 +- .../unified-test-format/tests/invalid/entity-stream-id-type.yml | 2 +- .../tests/invalid/expectedError-additionalProperties.yml | 2 +- .../tests/invalid/expectedError-errorCode-type.yml | 2 +- .../tests/invalid/expectedError-errorCodeName-type.yml | 2 +- .../tests/invalid/expectedError-errorContains-type.yml | 2 +- .../tests/invalid/expectedError-errorLabelsContain-items.yml | 2 +- .../tests/invalid/expectedError-errorLabelsContain-minItems.yml | 2 +- .../tests/invalid/expectedError-errorLabelsContain-type.yml | 2 +- .../tests/invalid/expectedError-errorLabelsOmit-items.yml | 2 +- .../tests/invalid/expectedError-errorLabelsOmit-minItems.yml | 2 +- .../tests/invalid/expectedError-errorLabelsOmit-type.yml | 2 +- .../tests/invalid/expectedError-isClientError-type.yml | 2 +- .../tests/invalid/expectedError-isError-const.yml | 2 +- .../tests/invalid/expectedError-isError-type.yml | 2 +- .../tests/invalid/expectedError-minProperties.yml | 2 +- .../tests/invalid/expectedEvent-additionalProperties.yml | 2 +- .../expectedEvent-commandFailedEvent-commandName-type.yml | 2 +- .../expectedEvent-commandStartedEvent-additionalProperties.yml | 2 +- .../invalid/expectedEvent-commandStartedEvent-command-type.yml | 2 +- .../expectedEvent-commandStartedEvent-commandName-type.yml | 2 +- .../expectedEvent-commandStartedEvent-databaseName-type.yml | 2 +- .../expectedEvent-commandSucceededEvent-commandName-type.yml | 2 +- .../invalid/expectedEvent-commandSucceededEvent-reply-type.yml | 2 +- .../tests/invalid/expectedEvent-maxProperties.yml | 2 +- .../tests/invalid/expectedEvent-minProperties.yml | 2 +- .../invalid/expectedEventsForClient-additionalProperties.yml | 2 +- .../tests/invalid/expectedEventsForClient-client-required.yml | 2 +- .../tests/invalid/expectedEventsForClient-client-type.yml | 2 +- .../tests/invalid/expectedEventsForClient-events-items.yml | 2 +- .../tests/invalid/expectedEventsForClient-events-required.yml | 2 +- .../tests/invalid/expectedEventsForClient-events-type.yml | 2 +- source/unified-test-format/tests/invalid/initialData-items.yml | 2 +- .../unified-test-format/tests/invalid/initialData-minItems.yml | 2 +- source/unified-test-format/tests/invalid/initialData-type.yml | 2 +- .../tests/invalid/operation-additionalProperties.yml | 2 +- .../tests/invalid/operation-arguments-type.yml | 2 +- .../operation-expectError-conflicts_with_expectResult.yml | 2 +- .../operation-expectError-conflicts_with_saveResultAsEntity.yml | 2 +- .../tests/invalid/operation-expectError-type.yml | 2 +- .../tests/invalid/operation-expectEvents-type.yml | 2 +- .../tests/invalid/operation-name-required.yml | 2 +- .../unified-test-format/tests/invalid/operation-name-type.yml | 2 +- .../tests/invalid/operation-object-required.yml | 2 +- .../unified-test-format/tests/invalid/operation-object-type.yml | 2 +- .../tests/invalid/operation-saveResultAsEntity-type.yml | 2 +- .../tests/invalid/runOnRequirement-additionalProperties.yml | 2 +- .../tests/invalid/runOnRequirement-maxServerVersion-pattern.yml | 2 +- .../tests/invalid/runOnRequirement-maxServerVersion-type.yml | 2 +- .../tests/invalid/runOnRequirement-minProperties.yml | 2 +- .../tests/invalid/runOnRequirement-minServerVersion-pattern.yml | 2 +- .../tests/invalid/runOnRequirement-minServerVersion-type.yml | 2 +- .../tests/invalid/runOnRequirement-topologies-enum.yml | 2 +- .../tests/invalid/runOnRequirement-topologies-items.yml | 2 +- .../tests/invalid/runOnRequirement-topologies-minItems.yml | 2 +- .../tests/invalid/runOnRequirement-topologies-type.yml | 2 +- .../tests/invalid/runOnRequirements-items.yml | 2 +- .../tests/invalid/runOnRequirements-minItems.yml | 2 +- .../tests/invalid/runOnRequirements-type.yml | 2 +- .../unified-test-format/tests/invalid/schemaVersion-pattern.yml | 2 +- .../tests/invalid/schemaVersion-required.yml | 2 +- source/unified-test-format/tests/invalid/schemaVersion-type.yml | 2 +- .../tests/invalid/test-additionalProperties.yml | 2 +- .../tests/invalid/test-description-required.yml | 2 +- .../unified-test-format/tests/invalid/test-description-type.yml | 2 +- .../tests/invalid/test-expectEvents-items.yml | 2 +- .../tests/invalid/test-expectEvents-type.yml | 2 +- .../unified-test-format/tests/invalid/test-operations-items.yml | 2 +- .../tests/invalid/test-operations-required.yml | 2 +- .../unified-test-format/tests/invalid/test-operations-type.yml | 2 +- source/unified-test-format/tests/invalid/test-outcome-items.yml | 2 +- .../unified-test-format/tests/invalid/test-outcome-minItems.yml | 2 +- source/unified-test-format/tests/invalid/test-outcome-type.yml | 2 +- .../tests/invalid/test-runOnRequirements-items.yml | 2 +- .../tests/invalid/test-runOnRequirements-minItems.yml | 2 +- .../tests/invalid/test-runOnRequirements-type.yml | 2 +- .../unified-test-format/tests/invalid/test-skipReason-type.yml | 2 +- source/unified-test-format/tests/invalid/tests-items.yml | 2 +- source/unified-test-format/tests/invalid/tests-minItems.yml | 2 +- source/unified-test-format/tests/invalid/tests-required.yml | 2 +- source/unified-test-format/tests/invalid/tests-type.yml | 2 +- 144 files changed, 145 insertions(+), 143 deletions(-) diff --git a/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml index 796f65be56..6a2256a639 100644 --- a/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml index c0fbfe98f6..2f20805581 100644 --- a/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-collectionName-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml index 23e6d59861..a9da0e0129 100644 --- a/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-collectionName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml index 20c03c8065..d8d6e4d571 100644 --- a/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-databaseName-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml index 9dcf8d17c7..27dfaafee1 100644 --- a/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-databaseName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-items.yml b/source/unified-test-format/tests/invalid/collectionData-documents-items.yml index 22e4179b26..6e860e896f 100644 --- a/source/unified-test-format/tests/invalid/collectionData-documents-items.yml +++ b/source/unified-test-format/tests/invalid/collectionData-documents-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-documents-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-required.yml b/source/unified-test-format/tests/invalid/collectionData-documents-required.yml index aa318176b8..0452842bc7 100644 --- a/source/unified-test-format/tests/invalid/collectionData-documents-required.yml +++ b/source/unified-test-format/tests/invalid/collectionData-documents-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-documents-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-type.yml b/source/unified-test-format/tests/invalid/collectionData-documents-type.yml index d8404a2dfb..db6d8b417a 100644 --- a/source/unified-test-format/tests/invalid/collectionData-documents-type.yml +++ b/source/unified-test-format/tests/invalid/collectionData-documents-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionData-documents-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml index f7620923cb..e5b92562a2 100644 --- a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionOrDatabaseOptions-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml index 59dbf2e26e..671a5b0241 100644 --- a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionOrDatabaseOptions-readConcern-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml index 8cafbad02c..84b328fc46 100644 --- a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionOrDatabaseOptions-readPreference-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml index 8cbd4516bd..5dc3f8b63c 100644 --- a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "collectionOrDatabaseOptions-writeConcern-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/createEntities-items.yml b/source/unified-test-format/tests/invalid/createEntities-items.yml index e444490701..f0d00e33b2 100644 --- a/source/unified-test-format/tests/invalid/createEntities-items.yml +++ b/source/unified-test-format/tests/invalid/createEntities-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "createEntities-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/createEntities-minItems.yml b/source/unified-test-format/tests/invalid/createEntities-minItems.yml index 502d6e76b9..11641ea7d8 100644 --- a/source/unified-test-format/tests/invalid/createEntities-minItems.yml +++ b/source/unified-test-format/tests/invalid/createEntities-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "createEntities-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/createEntities-type.yml b/source/unified-test-format/tests/invalid/createEntities-type.yml index ce23e6dc4c..c181f31922 100644 --- a/source/unified-test-format/tests/invalid/createEntities-type.yml +++ b/source/unified-test-format/tests/invalid/createEntities-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "createEntities-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/description-required.yml b/source/unified-test-format/tests/invalid/description-required.yml index 73a69a8b6d..5bc6ecff5e 100644 --- a/source/unified-test-format/tests/invalid/description-required.yml +++ b/source/unified-test-format/tests/invalid/description-required.yml @@ -1,3 +1,5 @@ +# description: "description-required" + schemaVersion: "1.0" tests: diff --git a/source/unified-test-format/tests/invalid/entity-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-additionalProperties.yml index 743bd7c1d2..4978518c5e 100644 --- a/source/unified-test-format/tests/invalid/entity-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml index f905e2b58e..e864aab955 100644 --- a/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-bucket-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml index 94f8535cac..2d80f5ca3f 100644 --- a/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml +++ b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-bucket-bucketOptions-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml b/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml index 569df8c330..df7f8d9457 100644 --- a/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-bucket-database-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml index 21e3c68fc9..e735d16e16 100644 --- a/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-bucket-database-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml b/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml index ecae379127..b47435870f 100644 --- a/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-bucket-id-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml b/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml index ddb6eb96b9..33522fea32 100644 --- a/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-bucket-id-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml index a772fde01e..b93e51d08a 100644 --- a/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-id-required.yml b/source/unified-test-format/tests/invalid/entity-client-id-required.yml index 3ead68edd5..794a025f23 100644 --- a/source/unified-test-format/tests/invalid/entity-client-id-required.yml +++ b/source/unified-test-format/tests/invalid/entity-client-id-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-id-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-id-type.yml b/source/unified-test-format/tests/invalid/entity-client-id-type.yml index ae0ee681df..fc872b2b22 100644 --- a/source/unified-test-format/tests/invalid/entity-client-id-type.yml +++ b/source/unified-test-format/tests/invalid/entity-client-id-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-id-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml index 2cfbe0fdd4..8f63e4d7e6 100644 --- a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-ignoreCommandMonitoringEvents-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml index 03a0128c80..128282c6f3 100644 --- a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-ignoreCommandMonitoringEvents-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml index a0bee34058..9fa8bfa323 100644 --- a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-ignoreCommandMonitoringEvents-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml index 7c28bab216..5deaeb6212 100644 --- a/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-observeEvents-enum" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml index 8b75e6e204..ad73f22699 100644 --- a/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-observeEvents-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml index 99ce389971..bd16e78c94 100644 --- a/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-observeEvents-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml index 2344c8bd2b..b75e97faea 100644 --- a/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-observeEvents-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml index d91dc9866b..09bc9f25d3 100644 --- a/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml +++ b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-uriOptions-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml index 035089fee6..ef5e4baae9 100644 --- a/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml +++ b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-client-useMultipleMongoses-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml index 025b689338..2010de017a 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml index f3ad71560c..97e1616207 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-collectionName-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml index 5e455d5a75..46bcb473f8 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-collectionName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml index 318f230ebe..58d101981d 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-collectionOptions-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-required.yml b/source/unified-test-format/tests/invalid/entity-collection-database-required.yml index f8d754d450..6a584cd9b6 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-database-required.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-database-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-database-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-type.yml b/source/unified-test-format/tests/invalid/entity-collection-database-type.yml index e54a80386a..8ed3c743e9 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-database-type.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-database-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-database-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-required.yml b/source/unified-test-format/tests/invalid/entity-collection-id-required.yml index f9c6889f8e..83de062263 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-id-required.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-id-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-id-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-type.yml b/source/unified-test-format/tests/invalid/entity-collection-id-type.yml index 6b8e31560a..c7bfa502ee 100644 --- a/source/unified-test-format/tests/invalid/entity-collection-id-type.yml +++ b/source/unified-test-format/tests/invalid/entity-collection-id-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-collection-id-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml index f6e0279c6a..dcfdc048dc 100644 --- a/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-client-required.yml b/source/unified-test-format/tests/invalid/entity-database-client-required.yml index d232744292..6e271b1859 100644 --- a/source/unified-test-format/tests/invalid/entity-database-client-required.yml +++ b/source/unified-test-format/tests/invalid/entity-database-client-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-client-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-client-type.yml b/source/unified-test-format/tests/invalid/entity-database-client-type.yml index 32a4cad8e6..fe04610863 100644 --- a/source/unified-test-format/tests/invalid/entity-database-client-type.yml +++ b/source/unified-test-format/tests/invalid/entity-database-client-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-client-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml index 9d328b9609..cd2c6e77a0 100644 --- a/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-databaseName-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml index 7ce9e631ad..34ec075b64 100644 --- a/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-databaseName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml index a57e9fe647..77828e5390 100644 --- a/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml +++ b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-databaseOptions-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-id-required.yml b/source/unified-test-format/tests/invalid/entity-database-id-required.yml index 2520e6476b..51e63b4ab1 100644 --- a/source/unified-test-format/tests/invalid/entity-database-id-required.yml +++ b/source/unified-test-format/tests/invalid/entity-database-id-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-id-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-database-id-type.yml b/source/unified-test-format/tests/invalid/entity-database-id-type.yml index b31e38e082..1174c36ca4 100644 --- a/source/unified-test-format/tests/invalid/entity-database-id-type.yml +++ b/source/unified-test-format/tests/invalid/entity-database-id-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-database-id-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-maxProperties.yml b/source/unified-test-format/tests/invalid/entity-maxProperties.yml index 97329a61ff..c6f8bcffc5 100644 --- a/source/unified-test-format/tests/invalid/entity-maxProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-maxProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-maxProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-minProperties.yml b/source/unified-test-format/tests/invalid/entity-minProperties.yml index 1ec37d1c75..d758bd5745 100644 --- a/source/unified-test-format/tests/invalid/entity-minProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-minProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-minProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml index 5a2db0c197..1bb2233043 100644 --- a/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-session-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-session-client-required.yml b/source/unified-test-format/tests/invalid/entity-session-client-required.yml index 9ea824f3ec..3c1fe4e3c0 100644 --- a/source/unified-test-format/tests/invalid/entity-session-client-required.yml +++ b/source/unified-test-format/tests/invalid/entity-session-client-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-session-client-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-session-client-type.yml b/source/unified-test-format/tests/invalid/entity-session-client-type.yml index fb615208a8..60353c08db 100644 --- a/source/unified-test-format/tests/invalid/entity-session-client-type.yml +++ b/source/unified-test-format/tests/invalid/entity-session-client-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-session-client-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-session-id-required.yml b/source/unified-test-format/tests/invalid/entity-session-id-required.yml index bf8321dd9f..2aa4269d16 100644 --- a/source/unified-test-format/tests/invalid/entity-session-id-required.yml +++ b/source/unified-test-format/tests/invalid/entity-session-id-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-session-id-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-session-id-type.yml b/source/unified-test-format/tests/invalid/entity-session-id-type.yml index 6726e1a8b6..058dcdb1b7 100644 --- a/source/unified-test-format/tests/invalid/entity-session-id-type.yml +++ b/source/unified-test-format/tests/invalid/entity-session-id-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-session-id-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml index e2deb6fd78..2d651f690b 100644 --- a/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml +++ b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-session-sessionOptions-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml index 312ecd5bda..e2f4f8e93d 100644 --- a/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-stream-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml index bd8481e883..3e1aa8e45d 100644 --- a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-stream-hexBytes-pattern" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml index ea47f2f7ac..b1a4b98561 100644 --- a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-stream-hexBytes-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml index 51fc9f3d83..79b8188c5b 100644 --- a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-stream-hexBytes-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-required.yml b/source/unified-test-format/tests/invalid/entity-stream-id-required.yml index 66da9c1539..0ca388aade 100644 --- a/source/unified-test-format/tests/invalid/entity-stream-id-required.yml +++ b/source/unified-test-format/tests/invalid/entity-stream-id-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-stream-id-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-type.yml b/source/unified-test-format/tests/invalid/entity-stream-id-type.yml index 10b70d82fe..d7b17952cb 100644 --- a/source/unified-test-format/tests/invalid/entity-stream-id-type.yml +++ b/source/unified-test-format/tests/invalid/entity-stream-id-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "entity-stream-id-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml index 322c6d4b2f..6146367962 100644 --- a/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml index a3cea1177b..2110e9c6ad 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorCode-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml index af0157b04a..c35ec3ed4a 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorCodeName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml index 19cbbe0922..f4989b61e3 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorContains-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml index e6a046db12..4382fa645a 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorLabelsContain-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml index 4c240e5222..ebd650e11a 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorLabelsContain-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml index 3d298600dc..537cfb9cfe 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorLabelsContain-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml index c738a7a186..d80711ef0f 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorLabelsOmit-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml index 8dc24a72c3..205162cf67 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorLabelsOmit-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml index 0fe6c06c78..6c1947b043 100644 --- a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-errorLabelsOmit-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml index ad0d89cec9..8b8afc1061 100644 --- a/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-isClientError-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-const.yml b/source/unified-test-format/tests/invalid/expectedError-isError-const.yml index babbb291d6..347ae2965a 100644 --- a/source/unified-test-format/tests/invalid/expectedError-isError-const.yml +++ b/source/unified-test-format/tests/invalid/expectedError-isError-const.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-isError-const" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-type.yml b/source/unified-test-format/tests/invalid/expectedError-isError-type.yml index 16bad39796..3e53f4af90 100644 --- a/source/unified-test-format/tests/invalid/expectedError-isError-type.yml +++ b/source/unified-test-format/tests/invalid/expectedError-isError-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-isError-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedError-minProperties.yml b/source/unified-test-format/tests/invalid/expectedError-minProperties.yml index 54c1fe49f8..5292d85008 100644 --- a/source/unified-test-format/tests/invalid/expectedError-minProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedError-minProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedError-minProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml index 08ab186f2a..c2c1cb5a7c 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml index cf6094189d..57504b78f8 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandFailedEvent-commandName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml index 175e36d19e..6ea724ddfb 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandStartedEvent-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml index 025ce7df0d..91ba129869 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandStartedEvent-command-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml index 3fb4ed05b1..07c968cdd4 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandStartedEvent-commandName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml index 5afd053df3..355d90d6f7 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandStartedEvent-databaseName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml index a817033ad4..740b377fa2 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandSucceededEvent-commandName-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml index fe0e09ce56..5a4b35e272 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-commandSucceededEvent-reply-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml index b3e061a215..d349133ea1 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-maxProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml index 6739dc77fd..88de63a898 100644 --- a/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEvent-minProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml index 59e8f1d767..a15835d1ca 100644 --- a/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEventsForClient-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml index 122eb97963..57db7b07e2 100644 --- a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEventsForClient-client-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml index 69016b8dc8..015fd7849b 100644 --- a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEventsForClient-client-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml index 9f5ba41a37..e5a6f4606b 100644 --- a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEventsForClient-events-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml index 226e37f127..dd14eb2b8e 100644 --- a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEventsForClient-events-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml index a2a059194c..b51730f4ff 100644 --- a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "expectedEventsForClient-events-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/initialData-items.yml b/source/unified-test-format/tests/invalid/initialData-items.yml index ddd84cf60a..87fa3e562b 100644 --- a/source/unified-test-format/tests/invalid/initialData-items.yml +++ b/source/unified-test-format/tests/invalid/initialData-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "initialData-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/initialData-minItems.yml b/source/unified-test-format/tests/invalid/initialData-minItems.yml index 97dbd3ff13..dc1afde97f 100644 --- a/source/unified-test-format/tests/invalid/initialData-minItems.yml +++ b/source/unified-test-format/tests/invalid/initialData-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "initialData-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/initialData-type.yml b/source/unified-test-format/tests/invalid/initialData-type.yml index 543e7a91e8..946c0e4364 100644 --- a/source/unified-test-format/tests/invalid/initialData-type.yml +++ b/source/unified-test-format/tests/invalid/initialData-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "initialData-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-additionalProperties.yml b/source/unified-test-format/tests/invalid/operation-additionalProperties.yml index 5d7b3adc77..d5dbef8e8f 100644 --- a/source/unified-test-format/tests/invalid/operation-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/operation-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-arguments-type.yml b/source/unified-test-format/tests/invalid/operation-arguments-type.yml index aa26607b47..1564f21601 100644 --- a/source/unified-test-format/tests/invalid/operation-arguments-type.yml +++ b/source/unified-test-format/tests/invalid/operation-arguments-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-arguments-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml index 9ae43992b6..aa9430fdb1 100644 --- a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-expectError-conflicts_with_expectResult" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml index a7b6fee1ae..6c369598fe 100644 --- a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-expectError-conflicts_with_saveResultAsEntity" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-expectError-type.yml b/source/unified-test-format/tests/invalid/operation-expectError-type.yml index 8336fca1d8..55588d4359 100644 --- a/source/unified-test-format/tests/invalid/operation-expectError-type.yml +++ b/source/unified-test-format/tests/invalid/operation-expectError-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-expectError-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml b/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml index 31945f2eb5..2c9936ed28 100644 --- a/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml +++ b/source/unified-test-format/tests/invalid/operation-expectEvents-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-expectEvents-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-name-required.yml b/source/unified-test-format/tests/invalid/operation-name-required.yml index fd62b19420..253cb579b9 100644 --- a/source/unified-test-format/tests/invalid/operation-name-required.yml +++ b/source/unified-test-format/tests/invalid/operation-name-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-name-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-name-type.yml b/source/unified-test-format/tests/invalid/operation-name-type.yml index a7a47d2f05..f9323050b5 100644 --- a/source/unified-test-format/tests/invalid/operation-name-type.yml +++ b/source/unified-test-format/tests/invalid/operation-name-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-name-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-object-required.yml b/source/unified-test-format/tests/invalid/operation-object-required.yml index c00461d742..eec6101bbe 100644 --- a/source/unified-test-format/tests/invalid/operation-object-required.yml +++ b/source/unified-test-format/tests/invalid/operation-object-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-object-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-object-type.yml b/source/unified-test-format/tests/invalid/operation-object-type.yml index 9f4930f3c1..fb8e10267b 100644 --- a/source/unified-test-format/tests/invalid/operation-object-type.yml +++ b/source/unified-test-format/tests/invalid/operation-object-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-object-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml index 4e8aad3890..7d20938f32 100644 --- a/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml +++ b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "operation-saveResultAsEntity-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml index c30a559ce0..044b803d29 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml index c07bfc20a5..ce5af27df5 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-maxServerVersion-pattern" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml index 677d72c3b4..a63df28232 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-maxServerVersion-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml index b03fb50441..e8b91fd4f5 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-minProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml index bd6644a597..69d4986665 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-minServerVersion-pattern" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml index f8c53e11ef..e2727c5800 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-minServerVersion-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml index 3c5ec191de..924a67515d 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-topologies-enum" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml index dfe39f4181..7b2265fb26 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-topologies-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml index 1f3431b469..3e6d11ab8f 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-topologies-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml index 935af7889b..5a1652d8d9 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirement-topologies-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-items.yml b/source/unified-test-format/tests/invalid/runOnRequirements-items.yml index d620af55a0..a86bbb1318 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirements-items.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirements-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirements-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml index 4c1045e0dd..034786a220 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirements-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-type.yml b/source/unified-test-format/tests/invalid/runOnRequirements-type.yml index c159787826..42afc21ced 100644 --- a/source/unified-test-format/tests/invalid/runOnRequirements-type.yml +++ b/source/unified-test-format/tests/invalid/runOnRequirements-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "runOnRequirements-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml b/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml index dfd284905e..3d36ee64b5 100644 --- a/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml +++ b/source/unified-test-format/tests/invalid/schemaVersion-pattern.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "schemaVersion-pattern" schemaVersion: "1.2.3.4" diff --git a/source/unified-test-format/tests/invalid/schemaVersion-required.yml b/source/unified-test-format/tests/invalid/schemaVersion-required.yml index 85303d0fac..14a13872d8 100644 --- a/source/unified-test-format/tests/invalid/schemaVersion-required.yml +++ b/source/unified-test-format/tests/invalid/schemaVersion-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "schemaVersion-required" tests: - description: "foo" diff --git a/source/unified-test-format/tests/invalid/schemaVersion-type.yml b/source/unified-test-format/tests/invalid/schemaVersion-type.yml index 3bd03eda0d..9dfeed1e78 100644 --- a/source/unified-test-format/tests/invalid/schemaVersion-type.yml +++ b/source/unified-test-format/tests/invalid/schemaVersion-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "schemaVersion-type" schemaVersion: 0 diff --git a/source/unified-test-format/tests/invalid/test-additionalProperties.yml b/source/unified-test-format/tests/invalid/test-additionalProperties.yml index a27a845d9d..65a2e2aa64 100644 --- a/source/unified-test-format/tests/invalid/test-additionalProperties.yml +++ b/source/unified-test-format/tests/invalid/test-additionalProperties.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-additionalProperties" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-description-required.yml b/source/unified-test-format/tests/invalid/test-description-required.yml index 0855d6dcc5..d1c725b69a 100644 --- a/source/unified-test-format/tests/invalid/test-description-required.yml +++ b/source/unified-test-format/tests/invalid/test-description-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-description-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-description-type.yml b/source/unified-test-format/tests/invalid/test-description-type.yml index ea599f2978..891db9bbc1 100644 --- a/source/unified-test-format/tests/invalid/test-description-type.yml +++ b/source/unified-test-format/tests/invalid/test-description-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-description-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-items.yml b/source/unified-test-format/tests/invalid/test-expectEvents-items.yml index cecee32f99..1a889b1b4f 100644 --- a/source/unified-test-format/tests/invalid/test-expectEvents-items.yml +++ b/source/unified-test-format/tests/invalid/test-expectEvents-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-expectEvents-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-type.yml b/source/unified-test-format/tests/invalid/test-expectEvents-type.yml index b70f57c2d5..0a1e9d97d5 100644 --- a/source/unified-test-format/tests/invalid/test-expectEvents-type.yml +++ b/source/unified-test-format/tests/invalid/test-expectEvents-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-expectEvents-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-operations-items.yml b/source/unified-test-format/tests/invalid/test-operations-items.yml index d6573b1a06..6bb097fa3c 100644 --- a/source/unified-test-format/tests/invalid/test-operations-items.yml +++ b/source/unified-test-format/tests/invalid/test-operations-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-operations-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-operations-required.yml b/source/unified-test-format/tests/invalid/test-operations-required.yml index 8828ad1b08..80af1a900a 100644 --- a/source/unified-test-format/tests/invalid/test-operations-required.yml +++ b/source/unified-test-format/tests/invalid/test-operations-required.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-operations-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-operations-type.yml b/source/unified-test-format/tests/invalid/test-operations-type.yml index ffaa4ad390..713c7ff0ec 100644 --- a/source/unified-test-format/tests/invalid/test-operations-type.yml +++ b/source/unified-test-format/tests/invalid/test-operations-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-operations-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-outcome-items.yml b/source/unified-test-format/tests/invalid/test-outcome-items.yml index c685a717a1..bd5565e4fb 100644 --- a/source/unified-test-format/tests/invalid/test-outcome-items.yml +++ b/source/unified-test-format/tests/invalid/test-outcome-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-outcome-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-outcome-minItems.yml b/source/unified-test-format/tests/invalid/test-outcome-minItems.yml index 96fc63e5df..f8d87d1f56 100644 --- a/source/unified-test-format/tests/invalid/test-outcome-minItems.yml +++ b/source/unified-test-format/tests/invalid/test-outcome-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-outcome-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-outcome-type.yml b/source/unified-test-format/tests/invalid/test-outcome-type.yml index 0c1a14e650..cfc44183a5 100644 --- a/source/unified-test-format/tests/invalid/test-outcome-type.yml +++ b/source/unified-test-format/tests/invalid/test-outcome-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-outcome-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml index 540609b813..2324fa8045 100644 --- a/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-runOnRequirements-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml index 30501e5fae..2f3e0391ad 100644 --- a/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-runOnRequirements-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml index e1d7e34b2a..12bd90e859 100644 --- a/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-runOnRequirements-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/test-skipReason-type.yml b/source/unified-test-format/tests/invalid/test-skipReason-type.yml index d77ab4e913..7921aefc79 100644 --- a/source/unified-test-format/tests/invalid/test-skipReason-type.yml +++ b/source/unified-test-format/tests/invalid/test-skipReason-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "test-skipReason-type" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/tests-items.yml b/source/unified-test-format/tests/invalid/tests-items.yml index 328f72cde7..8c9829634c 100644 --- a/source/unified-test-format/tests/invalid/tests-items.yml +++ b/source/unified-test-format/tests/invalid/tests-items.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "tests-items" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/tests-minItems.yml b/source/unified-test-format/tests/invalid/tests-minItems.yml index c7aefc882d..834120f835 100644 --- a/source/unified-test-format/tests/invalid/tests-minItems.yml +++ b/source/unified-test-format/tests/invalid/tests-minItems.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "tests-minItems" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/tests-required.yml b/source/unified-test-format/tests/invalid/tests-required.yml index 13bed7011d..41c5072acf 100644 --- a/source/unified-test-format/tests/invalid/tests-required.yml +++ b/source/unified-test-format/tests/invalid/tests-required.yml @@ -1,3 +1,3 @@ -description: "foo" +description: "tests-required" schemaVersion: "1.0" diff --git a/source/unified-test-format/tests/invalid/tests-type.yml b/source/unified-test-format/tests/invalid/tests-type.yml index 46e16d3ca5..c16781a727 100644 --- a/source/unified-test-format/tests/invalid/tests-type.yml +++ b/source/unified-test-format/tests/invalid/tests-type.yml @@ -1,4 +1,4 @@ -description: "foo" +description: "tests-type" schemaVersion: "1.0" From ae29867cf65214f2131941ce4513fa92c1e519a8 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 2 Oct 2020 21:11:26 +0800 Subject: [PATCH 62/90] Initial valid-fail and valid-pass tests --- source/unified-test-format/tests/Makefile | 19 +- .../entity-bucket-database-undefined.yml | 12 ++ .../entity-collection-database-undefined.yml | 13 ++ .../entity-database-client-undefined.yml | 13 ++ .../entity-session-client-undefined.yml | 12 ++ .../valid-fail/schemaVersion-unsupported.yml | 7 + .../tests/valid-pass/poc-change-streams.yml | 172 ++++++++++++++++++ .../valid-pass/poc-command-monitoring.yml | 102 +++++++++++ 8 files changed, 342 insertions(+), 8 deletions(-) create mode 100644 source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml create mode 100644 source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml create mode 100644 source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml create mode 100644 source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml create mode 100644 source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-change-streams.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile index 966360cd5c..2df2261eb5 100644 --- a/source/unified-test-format/tests/Makefile +++ b/source/unified-test-format/tests/Makefile @@ -1,15 +1,18 @@ SCHEMA=../schema-1.0.json -.PHONY: all invalid HAS_AJV +.PHONY: all invalid valid-fail valid-pass HAS_AJV -all: invalid +all: invalid valid-fail valid-pass -invalid: - @if ajv test -s $(SCHEMA) -d "invalid/*" --invalid > /dev/null; then \ - echo "PASSED"; \ - else \ - echo "FAILED"; \ - fi +invalid: HAS_AJV + @# Redirect stdout to hide expected validation errors + @ajv test -s $(SCHEMA) -d "invalid/*.yml" --invalid > /dev/null && echo "invalid/*.yml passed test" + +valid-fail: HAS_AJV + @ajv test -s $(SCHEMA) -d "valid-fail/*.yml" --valid + +valid-pass: HAS_AJV + @ajv test -s $(SCHEMA) -d "valid-pass/*.yml" --valid HAS_AJV: @if ! command -v ajv > /dev/null; then \ diff --git a/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml new file mode 100644 index 0000000000..7aeda8e1ac --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.yml @@ -0,0 +1,12 @@ +description: "entity-bucket-database-undefined" + +schemaVersion: "1.0" + +createEntities: + - bucket: + id: &bucket0 "bucket0" + database: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml new file mode 100644 index 0000000000..a66b6ed292 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.yml @@ -0,0 +1,13 @@ +description: "entity-collection-database-undefined" + +schemaVersion: "1.0" + +createEntities: + - collection: + id: &collection0 "collection0" + database: "foo" + collectionName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml new file mode 100644 index 0000000000..34b58cece5 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.yml @@ -0,0 +1,13 @@ +description: "entity-database-client-undefined" + +schemaVersion: "1.0" + +createEntities: + - database: + id: &database0 "database0" + client: "foo" + databaseName: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml new file mode 100644 index 0000000000..16eaea46b7 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.yml @@ -0,0 +1,12 @@ +description: "entity-session-client-undefined" + +schemaVersion: "1.0" + +createEntities: + - session: + id: &session0 "session0" + client: "foo" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml new file mode 100644 index 0000000000..0cb6994d24 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.yml @@ -0,0 +1,7 @@ +description: "schemaVersion-unsupported" + +schemaVersion: "0.1" + +tests: + - description: "foo" + operations: [] diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml new file mode 100644 index 0000000000..a0e918f601 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -0,0 +1,172 @@ +description: "poc-change-streams" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + # Original tests in change-streams.yml did not observe getMore commands + ignoreCommandMonitoringEvents: [ getMore ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name change-stream-tests + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name change-stream-tests-2 + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name test2 + - collection: + id: &collection2 collection2 + database: *database1 + collectionName: &collection2Name test + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + - collectionName: *collection1Name + databaseName: *database0Name + documents: [] + - collectionName: *collection2Name + databaseName: *database1Name + documents: [] + +tests: + - description: "Executing a watch helper on a MongoClient results in notifications for changes to all collections in all databases in the cluster." + runOnRequirements: + - minServerVersion: "3.8.0" + topologies: [ replicaset ] + operations: + - name: createChangeStream + object: *client0 + saveResultAsEntity: &changeStream0 changeStream0 + - name: insertOne + object: *collection1 + arguments: { x: 1 } + - name: insertOne + object: *collection2 + arguments: { y: 1 } + - name: insertOne + object: *collection0 + arguments: { z: 1 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database0Name + coll: *collection1Name + fullDocument: { x: 1 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database1Name + coll: *collection2Name + fullDocument: { y: 1 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database0Name + coll: *collection0Name + fullDocument: { z: 1 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: 1 + cursor: {} + pipeline: + - $changeStream: { allChangesForCluster: true } + commandName: aggregate + databaseName: admin + + - description: "Test consecutive resume" + runOnRequirements: + - minServerVersion: "4.1.7" + topologies: [ replicaset ] + operations: + - name: failPoint + object: testRunner + arguments: + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ getMore ] + closeConnection: true + - name: createChangeStream + object: *collection0 + arguments: + batchSize: 1 + saveResultAsEntity: *changeStream0 + - name: insertOne + object: *collection0 + arguments: { x: 1 } + - name: insertOne + object: *collection0 + arguments: { x: 2 } + - name: insertOne + object: *collection0 + arguments: { x: 3 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database0Name + coll: *collection0Name + fullDocument: { x: 1 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database0Name + coll: *collection0Name + fullDocument: { x: 2 } + - name: iterateUntilDocumentOrError + object: *changeStream0 + expectResult: + operationType: insert + ns: + db: *database0Name + coll: *collection0Name + fullDocument: { x: 3 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection0Name + cursor: { batchSize: 1 } + pipeline: + - $changeStream: {} + commandName: aggregate + databaseName: *database0Name + # The original test only asserted the first command, since expected + # events were only an ordered subset. This file does ignore getMore + # commands but we must expect the second aggregate command. While + # doing so we can also assert that it includes a resume token. + - commandStartedEvent: + command: + aggregate: *collection0Name + cursor: { batchSize: 1 } + pipeline: + - $changeStream: + resumeAfter: { $exists: true } + commandName: aggregate + databaseName: *database0Name diff --git a/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml new file mode 100644 index 0000000000..39d923185a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml @@ -0,0 +1,102 @@ +description: "poc-command-monitoring" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + - commandFailedEvent + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name command-monitoring-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + - { _id: 5, x: 55 } + +tests: + - description: "A successful find event with a getmore and the server kills the cursor" + runOnRequirements: + - minServerVersion: "3.1" + topologies: [ single, replicaset ] + operations: + - name: find + object: *collection0 + arguments: + filter: { _id: { $gte: 1 }} + sort: { _id: 1 } + batchSize: 3 + limit: 4 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: { $gte : 1 } } + sort: { _id: 1 } + batchSize: 3 + limit: 4 + commandName: find + databaseName: *database0Name + - commandSucceededEvent: + reply: + ok: 1 + cursor: + id: { $$type: [ int, long ] } + ns: &namespace command-monitoring-tests.test + firstBatch: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + commandName: find + - commandStartedEvent: + command: + getMore: { $$type: [ int, long ] } + collection: *collection0Name + batchSize: 1 + commandName: getMore + databaseName: *database0Name + - commandSucceededEvent: + reply: + ok: 1 + cursor: + id: 0 + ns: *namespace + nextBatch: + - { _id: 4, x: 44 } + commandName: getMore + + - description: "A failed find event" + operations: + - name: find + object: *collection0 + arguments: + filter: { $or: true } + expectError: + isError: true + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collection0Name + filter: { $or: true } + commandName: find + databaseName: *database0Name + - commandFailedEvent: + commandName: find From 198aa4cd904d56d7291720b7c86b624a92f5f3c8 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 2 Oct 2020 21:47:39 +0800 Subject: [PATCH 63/90] Clarify iteration rules --- source/unified-test-format/unified-test-format.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 989a8fbc6a..38680ebe93 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -983,11 +983,11 @@ documents when evaluating `expectResult`_ or `saveResultAsEntity`_. Iterating Returned Iterables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Unless otherwise stated, test runners MUST fully iterate any iterable returned -by an operation as part of that operation's execution. This is necessary to -ensure consistent behavior among drivers, as discussed in `aggregate`_ and -`find`_, and also ensures that error and event assertions can be evaluated -consistently. +Unless otherwise stated by an operation below, test runners MUST fully iterate +any iterable returned by an operation as part of that operation's execution. +This is necessary to ensure consistent behavior among drivers, as discussed in +`aggregate`_ and `find`_, and also ensures that error and event assertions can +be evaluated consistently. Client Operations From f947095d249341e85216b77b92dd0a70a46ddb2f Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 3 Oct 2020 17:45:12 +0800 Subject: [PATCH 64/90] Don't mix testRunner ops with error/result expectations --- .../unified-test-format.rst | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 38680ebe93..341b7ed361 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -655,6 +655,9 @@ The structure of this object is as follows: `expectResult `_ and `saveResultAsEntity `_. + This field SHOULD NOT be used for `Special Test Operations`_ (i.e. + ``object: testRunner``). + .. _operation_expectResult: - ``expectResult``: Optional mixed type. A value corresponding to the expected @@ -664,6 +667,9 @@ The structure of this object is as follows: This field is mutually exclusive with `expectError `_. + This field SHOULD NOT be used for `Special Test Operations`_ (i.e. + ``object: testRunner``). + .. _operation_saveResultAsEntity: - ``saveResultAsEntity``: Optional string. If specified, the actual result @@ -673,6 +679,9 @@ The structure of this object is as follows: This field is mutually exclusive with `expectError `_. + This field SHOULD NOT be used for `Special Test Operations`_ (i.e. + ``object: testRunner``). + expectedError ~~~~~~~~~~~~~ @@ -1322,6 +1331,12 @@ special test operations (e.g. assertions). These operations are distinguished by `operation.name `_ field will correspond to an operation defined below. +Special test operations return no result and are always expected to succeed. +These operations SHOULD NOT be combined with +`expectError `_, +`expectResult `_, or +`saveResultAsEntity `_. + failPoint ~~~~~~~~~ @@ -2612,6 +2627,11 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-10-03: + +* Note that special test operations should not be combined with expectError, + expectResult, or saveResultAsEntity. + 2020-10-02: * Restructure Entity Test Operations sections. From 3790681222fc27fec38ebb486e76a9a5dc6a14ff Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sun, 4 Oct 2020 15:39:38 +0800 Subject: [PATCH 65/90] Additional POC tests --- .../tests/valid-pass/poc-change-streams.yml | 1 + .../valid-pass/poc-command-monitoring.yml | 3 +- .../tests/valid-pass/poc-crud.yml | 183 +++++++++++++++ .../tests/valid-pass/poc-gridfs.yml | 156 +++++++++++++ .../tests/valid-pass/poc-retryable-reads.yml | 161 +++++++++++++ .../tests/valid-pass/poc-retryable-writes.yml | 212 ++++++++++++++++++ 6 files changed, 714 insertions(+), 2 deletions(-) create mode 100644 source/unified-test-format/tests/valid-pass/poc-crud.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-gridfs.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml index a0e918f601..1645b7fbcb 100644 --- a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -5,6 +5,7 @@ schemaVersion: "1.0" createEntities: - client: id: &client0 client0 + useMultipleMongoses: false observeEvents: [ commandStartedEvent ] # Original tests in change-streams.yml did not observe getMore commands ignoreCommandMonitoringEvents: [ getMore ] diff --git a/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml index 39d923185a..19a282327c 100644 --- a/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml +++ b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.yml @@ -87,8 +87,7 @@ tests: object: *collection0 arguments: filter: { $or: true } - expectError: - isError: true + expectError: { isError: true } expectEvents: - client: *client0 events: diff --git a/source/unified-test-format/tests/valid-pass/poc-crud.yml b/source/unified-test-format/tests/valid-pass/poc-crud.yml new file mode 100644 index 0000000000..7d101a077a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-crud.yml @@ -0,0 +1,183 @@ +description: "poc-crud" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name admin + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name coll1 + - collection: + id: &collection2 collection2 + database: *database0 + collectionName: &collection2Name coll2 + collectionOptions: + readConcern: { level: majority } + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - collectionName: *collection1Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - collectionName: *collection2Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - collectionName: &out aggregate_out + databaseName: *database0Name + documents: [] + +tests: + - description: "BulkWrite with mixed ordered operations" + operations: + - name: bulkWrite + object: *collection0 + arguments: + requests: + - insertOne: + document: { _id: 3, x: 33 } + - updateOne: + filter: { _id: 2 } + update: { $inc: { x: 1 } } + - updateMany: + filter: { _id: { $gt: 1 } } + update: { $inc: { x: 1 } } + - insertOne: + document: { _id: 4, x: 44 } + - deleteMany: + filter: { x: { $nin: [ 24, 34 ] } } + - replaceOne: + filter: { _id: 4 } + replacement: { _id: 4, x: 44 } + upsert: true + ordered: true + expectResult: + deletedCount: 2 + insertedCount: 2 + insertedIds: { $$unsetOrMatches: { 0: 3, 3: 4 } } + matchedCount: 3 + modifiedCount: 3 + upsertedCount: 1 + upsertedIds: { 5: 4 } + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - {_id: 2, x: 24 } + - {_id: 3, x: 34 } + - {_id: 4, x: 44 } + + - description: "InsertMany continue-on-error behavior with unordered (duplicate key in requests)" + operations: + - name: insertMany + object: *collection1 + arguments: + documents: + - { _id: 2, x: 22 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + ordered: false + expectError: + expectResult: + deletedCount: 0 + insertedCount: 2 + # Since the map of insertedIds is generated before execution it + # could indicate inserts that did not actually succeed. We omit + # this field rather than expect drivers to provide an accurate + # map filtered by write errors. + matchedCount: 0 + modifiedCount: 0 + upsertedCount: 0 + upsertedIds: { } + outcome: + - collectionName: *collection1Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + + - description: "ReplaceOne prohibits atomic modifiers" + operations: + - name: replaceOne + object: *collection1 + arguments: + filter: { _id: 1 } + replacement: { $set: { x: 22 }} + expectError: + isClientError: true + expectEvents: + - client: *client0 + events: [] + outcome: + - collectionName: *collection1Name + databaseName: *database0Name + documents: + - { _id: 1, x: 11 } + + - description: "readConcern majority with out stage" + runOnRequirements: + - minServerVersion: "4.1.0" + topologies: [ replicaset, sharded-replicaset ] + operations: + - name: aggregate + object: *collection2 + arguments: + pipeline: &pipeline + - $sort: { x : 1 } + - $match: { _id: { $gt: 1 } } + - $out: *out + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collection2Name + pipeline: *pipeline + readConcern: { level: majority } + # The following two assertions were not in the original test + commandName: aggregate + databaseName: *database0Name + outcome: + - collectionName: *out + databaseName: *database0Name + documents: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + + - description: "Aggregate with $listLocalSessions" + runOnRequirements: + - minServerVersion: "3.6.0" + operations: + - name: aggregate + object: *database1 + arguments: + pipeline: + - $listLocalSessions: { } + - $limit: 1 + - $addFields: { dummy: "dummy field"} + - $project: { _id: 0, dummy: 1} + expectResult: + - { dummy: "dummy field" } diff --git a/source/unified-test-format/tests/valid-pass/poc-gridfs.yml b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml new file mode 100644 index 0000000000..b2dc52df62 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml @@ -0,0 +1,156 @@ +description: "poc-gridfs" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name gridfs-tests + - bucket: + id: &bucket0 bucket0 + database: *database0 + - collection: + id: &bucket0_files_collection bucket0_files_collection + database: *database0 + collectionName: &bucket0_files_collectionName fs.files + - collection: + id: &bucket0_chunks_collection bucket0_chunks_collection + database: *database0 + collectionName: &bucket0_chunks_collectionName fs.chunks + - stream: + id: &stream0 stream0 + hexBytes: "1122334455" + +initialData: + - collectionName: *bucket0_files_collectionName + databaseName: *database0Name + documents: + - _id: { $oid: "000000000000000000000005" } + length: 10 + chunkSize: 4 + uploadDate: { $date: "1970-01-01T00:00:00.000Z" } + md5: "57d83cd477bfb1ccd975ab33d827a92b" + filename: "length-10" + contentType: "application/octet-stream" + aliases: [] + metadata: {} + - collectionName: *bucket0_chunks_collectionName + databaseName: *database0Name + documents: + - _id: { $oid: "000000000000000000000005" } + files_id: { $oid: "000000000000000000000005" } + n: 0 + data: { $binary: { base64: "ESIzRA==", subType: "00" } } # hex: 11223344 + - _id: { $oid: "000000000000000000000006" } + files_id: { $oid: "000000000000000000000005" } + n: 1 + data: { $binary: { base64: "VWZ3iA==", subType: "00" } } # hex: 55667788 + - _id: { $oid: "000000000000000000000007" } + files_id: { $oid: "000000000000000000000005" } + n: 2 + data: { $binary: { base64: "mao=", subType: "00" } } # hex: 99aa + +tests: + # Changed from original test ("length is 8") to operate on same initialData + - description: "Delete when length is 10" + operations: + - name: delete + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000005" } + # Original test uses "assert.data" syntax to modify outcome collection for + # comparison. This can be accomplished using "outcome" directly. + outcome: + - collectionName: *bucket0_files_collectionName + databaseName: *database0Name + documents: [] + - collectionName: *bucket0_chunks_collectionName + databaseName: *database0Name + documents: [] + + - description: "Download when there are three chunks" + operations: + # Original test uses "download" operation. We use an explicit operation + # that returns a stream and then assert the contents of that stream. + - name: openDownloadStream + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000005" } + expectResult: { $$matchesHexbytes: "112233445566778899aa" } + + - description: "Download when files entry does not exist" + operations: + - name: openDownloadStream + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000000" } + # Original test expects "FileNotFound" error, which isn't specified + expectError: { isError: true } + + - description: "Download when an intermediate chunk is missing" + operations: + # Original test uses "arrange" syntax to modify initialData. This can be + # accomplished as a delete operation on the chunks collection. + - name: deleteOne + object: *bucket0_chunks_collection + arguments: + filter: + files_id: { $oid: "000000000000000000000005" } + n: 1 + expectResult: + deletedCount: 1 + - name: openDownloadStream + object: *bucket0 + arguments: + id: { $oid: "000000000000000000000005" } + # Original test expects "ChunkIsMissing" error, which isn't specified + expectError: { isError: true } + + - description: "Upload when length is 5" + operations: + # Original test uses "upload" operation. We use an explicit operation + # that takes a stream, which has been created from the expected hex bytes. + - name: uploadFromStream + object: *bucket0 + arguments: + filename: filename + source: *stream0 + chunkSizeBytes: 4 + # Original test references the result directly in "assert.data". Here, + # we need to save the result as an entity, which we can later reference. + expectResult: { $$type: objectId } + saveResultAsEntity: &oid0 oid0 + # "outcome" does not allow operators, but we can perform the assertions + # with separate find operations. + - name: find + object: *bucket0_files_collection + arguments: + filter: {} + sort: { uploadDate: -1 } + expectResult: + - _id: { $$matchesEntity: *oid0 } + length: 5 + chunkSize: 4 + uploadDate: { $$type: date } + md5: "283d4fea5dded59cf837d3047328f5af" + filename: filename + - name: find + object: *bucket0_chunks_collection + arguments: + filter: {} + # We cannot use the saved ObjectId when querying, but sort and limit + # can be used to return the expected chunks in order + sort: { _id: -1, n: 1 } + limit: 2 + expectResult: + - _id: { $$type: "objectId" } + files_id: { $$matchesEntity: *oid0 } + n: 0 + data: { $binary: { base64: "ESIzRA==", subType: "00" } } # hex 11223344 + - _id: { $$type: "objectId" } + files_id: { $$matchesEntity: *oid0 } + n: 1 + data: { $binary: { base64: "VQ==", subType: "00" } } # hex 55 diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml new file mode 100644 index 0000000000..0d4323e78f --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml @@ -0,0 +1,161 @@ +description: "poc-retryable-reads" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + topologies: [ single, replicaset ] + - minServerVersion: "4.1.7" + topologies: [ sharded ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + uriOptions: { retryReads: false } + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName retryable-reads-tests + - database: + id: &database1 database1 + client: *client1 + databaseName: *databaseName + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName coll + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: *collectionName + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - {_id: 1, x: 11} + - {_id: 2, x: 22} + - {_id: 3, x: 33} + +tests: + - description: "Aggregate succeeds after InterruptedAtShutdown" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ aggregate ] + errorCode: 11600 # InterruptedAtShutdown + - name: aggregate + object: *collection0 + arguments: + pipeline: &pipeline + - $match: { _id: { $gt: 1 } } + - $sort: { x: 1 } + expectResult: + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + aggregate: *collectionName + pipeline: *pipeline + databaseName: *databaseName + - commandStartedEvent: + command: + aggregate: *collectionName + pipeline: *pipeline + databaseName: *databaseName + + - description: "Find fails on first attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client1 # uses retryReads=false + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ find ] + closeConnection: true + - name: find + object: collection1 + arguments: + filter: {} + # Other arguments in the original test are not relevant + expectError: { isError: true } + expectEvents: + - client: *client1 + events: + - commandStartedEvent: + command: + find: *collectionName + filter: {} + databaseName: *databaseName + + - description: "Find fails on second attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ find ] + closeConnection: true + - name: find + object: collection1 + arguments: + filter: {} + # Other arguments in the original test are not relevant + expectError: { isError: true } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + find: *collectionName + filter: {} + databaseName: *databaseName + - commandStartedEvent: + command: + find: *collectionName + filter: {} + databaseName: *databaseName + + - description: "ListDatabases succeeds on second attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ listDatabases ] + closeConnection: true + - name: listDatabases + object: *client0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: { listDatabases: 1 } + - commandStartedEvent: + command: { listDatabases: 1 } diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml new file mode 100644 index 0000000000..84c5c8e7bd --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml @@ -0,0 +1,212 @@ +description: "poc-retryable-writes" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "3.6" + topologies: [ replicaset ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + uriOptions: { retryWrites: false } + useMultipleMongoses: false + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName retryable-writes-tests + - database: + id: &database1 database1 + client: *client1 + databaseName: *databaseName + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName coll + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: *collectionName + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + +tests: + - description: "FindOneAndUpdate is committed on first attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 1 } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 } } + returnDocument: Before + expectResult: { _id: 1, x: 11 } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + + - description: "FindOneAndUpdate is not committed on first attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 1 } + data: { failBeforeCommitExceptionCode: 1 } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 } } + returnDocument: Before + expectResult: { _id: 1, x: 11 } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 12 } + - { _id: 2, x: 22 } + + - description: "FindOneAndUpdate is never committed" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: onPrimaryTransactionalWrite + mode: { times: 2 } + data: { failBeforeCommitExceptionCode: 1 } + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 } } + returnDocument: Before + expectError: { isError: true } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + + - description: "InsertMany succeeds after PrimarySteppedDown" + runOnRequirements: &failCommand_requirements + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.7" + # Original test uses "sharded", but retryable writes requires a sharded + # cluster backed by replica sets + topologies: [ sharded-replicaset ] + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: 189 # PrimarySteppedDown + errorLabels: [ RetryableWriteError ] + - name: insertMany + object: *collection0 + arguments: + documents: + # Documents are modified from original test for "initialData" + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + ordered: true + expectResult: + # insertMany returns a BulkWriteResult, but there is no reason to + # assert other fields + insertedCount: 2 + insertedIds: { $$unsetOrMatches: { 0: 3, 1: 4 } } + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } + - { _id: 4, x: 44 } + + - description: "InsertOne fails after connection failure when retryWrites option is false" + runOnRequirements: *failCommand_requirements + operations: + - name: failPoint + object: testRunner + arguments: + client: *client1 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - name: insertOne + object: *collection1 + arguments: + document: { _id: 3, x: 33 } + expectError: + # If retryWrites is false, the driver should not add the + # RetryableWriteError label to the error. + errorLabelsOmit: [ RetryableWriteError ] + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + + - description: "InsertOne fails after multiple retryable writeConcernErrors" + runOnRequirements: *failCommand_requirements + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 2 } + data: + failCommands: [ insert ] + writeConcernError: + code: 91 # ShutdownInProgress + errmsg: "Replication is being shut down" + - name: insertOne + object: *collection1 + arguments: + document: { _id: 3, x: 33 } + expectError: + errorLabelsContain: [ RetryableWriteError ] + outcome: + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + - { _id: 3, x: 33 } # The write was still applied From e80d6c6856b2e19d039dc2e7bdbafe1ac6873bd4 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sun, 4 Oct 2020 15:48:06 +0800 Subject: [PATCH 66/90] Clarifications for CRUD operations --- .../returnDocument-enum-invalid.yaml | 34 +++++++++++++++++++ .../unified-test-format.rst | 33 ++++++++++++++++-- 2 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml diff --git a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml new file mode 100644 index 0000000000..4a28a6379b --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml @@ -0,0 +1,34 @@ +description: "returnDocument-enum-invalid" + +schemaVersion: "1.0" + +createEntities: + - client: + id: &client0 client0 + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName test + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName coll + +tests: + - description: "FindOneAndReplace returnDocument invalid enum value" + operations: + - name: findOneAndReplace + object: *collection0 + arguments: + filter: { _id: 1 } + replacement: { _id: 1, x: 111 } + returnDocument: invalid + + - description: "FindOneAndUpdate returnDocument invalid enum value" + operations: + - name: findOneAndUpdate + object: *collection0 + arguments: + filter: { _id: 1 } + update: { $inc: { x : 1 }} + returnDocument: invalid diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 341b7ed361..515239a8c5 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-10-02 +:Last Modified: 2020-10-04 .. contents:: @@ -1127,7 +1127,8 @@ WriteModel interfaces. Each WriteModel implementation (e.g. InsertOneModel) provides important context to the method, but that type information is not easily expressed in YAML and JSON. To account for this, test files MUST nest each WriteModel object in a single-key object, where the key identifies the -request type (e.g. "insertOne"), as in the following example:: +request type (e.g. "insertOne") and its value is an object expressing the +parameters, as in the following example:: arguments: requests: @@ -1150,6 +1151,10 @@ request type (e.g. "insertOne"), as in the following example:: filter: { x: { $gt: 2 } } ordered: true +Because the ``insertedIds`` field of BulkWriteResult is optional for drivers to +implement, assertions for that field SHOULD utilize the `$$unsetOrMatches`_ +operator. + While operations typically raise an error *or* return a result, the ``bulkWrite`` operation is unique in that it may report both via the ``writeResult`` property of a BulkWriteException. In this case, the intermediary @@ -1171,6 +1176,24 @@ and compare with ``code`` instead, but MUST raise an error if the comparison cannot be attempted (e.g. ``code`` is also not available, translation fails). +findOneAndReplace and findOneAndUpdate +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``returnDocument`` option for ``findOneAndReplace`` and ``findOneAndUpdate`` +is documented as an enum with possible values "Before" and "After". Test files +SHOULD express ``returnDocument`` a string and test runners MUST raise an error +if its value does not case-insensitively match either enum value. + + +insertMany +~~~~~~~~~~ + +The CRUD spec documents ``insertMany`` as returning a BulkWriteResult. Because +the ``insertedIds`` field of BulkWriteResult is optional for drivers to +implement, assertions for that field SHOULD utilize the `$$unsetOrMatches`_ +operator. + + .. _collection_createChangeStream: createChangeStream @@ -2627,6 +2650,12 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-10-04: + +* Clarifications for matching insertedIds in BulkWriteResult + +* Test runners must error if returnDocument enum is invalid + 2020-10-03: * Note that special test operations should not be combined with expectError, From 99ea7f15ef65d616dc37710c98b459fef80ce50c Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sun, 4 Oct 2020 17:22:59 +0800 Subject: [PATCH 67/90] Advise using $$unsetOrMatches for InsertOneResult --- .../unified-test-format.rst | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 515239a8c5..fc1fb7c753 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1194,6 +1194,25 @@ implement, assertions for that field SHOULD utilize the `$$unsetOrMatches`_ operator. +insertOne +~~~~~~~~~ + +The CRUD spec documents ``insertOne`` as returning an InsertOneResult; however, +because all fields InsertOneResult are optional drivers are permitted to forgo +it entirely and have ``insertOne`` return nothing (i.e. void method). Tests +asserting InsertOneResult SHOULD utilize the `$$unsetOrMatches`_ operator for +*both* the result object and any optional fields within, as in the following +examples:: + + - name: insertOne + object: *collection0 + arguments: + document: { _id: 2 } + expectResult: + $$unsetOrMatches: + insertedId: { $$unsetOrMatches: 2 } + + .. _collection_createChangeStream: createChangeStream @@ -2652,6 +2671,8 @@ Note: this will be cleared when publishing version 1.0 of the spec 2020-10-04: +* Advise using $$unsetOrMatches for InsertOneResult + * Clarifications for matching insertedIds in BulkWriteResult * Test runners must error if returnDocument enum is invalid From 602833d99c736557f2746c561aad7efe7f9a09d2 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sun, 4 Oct 2020 17:23:53 +0800 Subject: [PATCH 68/90] POC for session tests --- .../tests/valid-pass/poc-sessions.yml | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 source/unified-test-format/tests/valid-pass/poc-sessions.yml diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.yml b/source/unified-test-format/tests/valid-pass/poc-sessions.yml new file mode 100644 index 0000000000..382fe44f01 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.yml @@ -0,0 +1,206 @@ +description: "poc-sessions" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "3.6.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name session-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - session: + id: &session0 session0 + client: *client0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + +tests: + - description: "Server supports explicit sessions" + operations: + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 2 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 2 } } } + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + - name: endSession + object: *session0 + - &find_with_implicit_session + name: find + object: *collection0 + arguments: + filter: { _id: -1 } + expectResult: [] + - name: assertSameLsidOnLastTwoCommands + object: testRunner + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collection0Name + documents: [ { _id: 2 } ] + ordered: true + lsid: { $$sessionLsid: *session0 } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: -1 } + lsid: { $$sessionLsid: *session0 } + commandName: find + databaseName: *database0Name + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + + - description: "Server supports implicit sessions" + operations: + - name: insertOne + object: *collection0 + arguments: + document: { _id: 2 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 2 } } } + - *find_with_implicit_session + - name: assertSameLsidOnLastTwoCommands + object: testRunner + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collection0Name + documents: + - { _id: 2 } + ordered: true + # Original test did not include any assertion, but we can use + # $$type to expect an arbitrary lsid document + lsid: { $$type: object } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: -1 } + lsid: { $$type: object } + commandName: find + databaseName: *database0Name + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + + - description: "Dirty explicit session is discarded" + runOnRequirements: + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.8" + topologies: [ sharded ] + # Original test specified retryWrites=true, but that is now the default + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - name: assertSessionNotDirty + object: testRunner + arguments: + session: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 2 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 2 } } } + - name: assertSessionDirty + object: testRunner + arguments: + session: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 3 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 3 } } } + - name: assertSessionDirty + object: testRunner + arguments: + session: *session0 + - name: endSession + object: *session0 + - *find_with_implicit_session + - name: assertDifferentLsidOnLastTwoCommands + object: testRunner + expectEvents: + - client: *client0 + events: + # ajv's YAML parser is unable to handle anchors on array elements, so + # we define an anchor on the commandStartedEvent object instead + - commandStartedEvent: &insert_attempt + command: + insert: *collection0Name + documents: + - { _id: 2 } + ordered: true + lsid: { $sessionLsid: *session0 } + txnNumber: 1 + commandName: insert + databaseName: *database0Name + - commandStartedEvent: *insert_attempt + - commandStartedEvent: + command: + insert: *collection0Name + documents: + - { _id: 3 } + ordered: true + lsid: { $sessionLsid: *session0 } + txnNumber: 2 + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + find: *collection0Name + filter: { _id: -1 } + lsid: { $$type: object } + commandName: find + databaseName: *database0Name + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } From a42a31c1032a91dfc6a39b2f022d76c34f105ca2 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sun, 4 Oct 2020 22:10:50 +0800 Subject: [PATCH 69/90] POC for transaction tests --- .../poc-transactions-convenient-api.yml | 239 ++++++++++++++++++ .../poc-transactions-mongos-pin-auto.yml | 170 +++++++++++++ .../tests/valid-pass/poc-transactions.yml | 170 +++++++++++++ 3 files changed, 579 insertions(+) create mode 100644 source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml create mode 100644 source/unified-test-format/tests/valid-pass/poc-transactions.yml diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml new file mode 100644 index 0000000000..021f5b5611 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml @@ -0,0 +1,239 @@ +description: "poc-transactions-convenient-api" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + uriOptions: + readConcernLevel: local + w: 1 + useMultipleMongoses: true + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName transaction-tests + - database: + id: &database1 database1 + client: *client1 + databaseName: *databaseName + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collectionName test + - collection: + id: &collection1 collection1 + database: *database1 + collectionName: *collectionName + - session: + id: &session0 session0 + client: *client0 + - session: + id: &session1 session1 + client: *client1 + - session: + id: &session2 session2 + client: *client0 + sessionOptions: + defaultTransactionOptions: + readConcern: { level: majority } + writeConcern: { w: 1 } + +initialData: + - collectionName: *collectionName + databaseName: *databaseName + documents: [] + +tests: + - description: "withTransaction and no transaction options set" + operations: + - name: withTransaction + object: *session0 + arguments: + callback: + operations: + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + # omitted fields + readConcern: { $$exists: false } + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + autocommit: false + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + writeConcern: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: &outcome + - collectionName: *collectionName + databaseName: *databaseName + documents: + - { _id: 1 } + + - description: "withTransaction inherits transaction options from client" + operations: + - name: withTransaction + object: *session1 + arguments: + callback: + operations: + - name: insertOne + object: *collection1 + arguments: + session: *session1 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + expectEvents: + - client: *client1 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session1 } + txnNumber: 1 + startTransaction: true + autocommit: false + readConcern: { level: local } + # omitted fields + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session1 } + txnNumber: 1 + autocommit: false + writeConcern: { w: 1 } + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: *outcome + + - description: "withTransaction inherits transaction options from defaultTransactionOptions" + operations: + - name: withTransaction + object: *session2 + arguments: + callback: + operations: + - name: insertOne + object: *collection0 + arguments: + session: *session2 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + expectEvents: + - client: *client1 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session2 } + txnNumber: 1 + startTransaction: true + autocommit: false + readConcern: { level: majority } + # omitted fields + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session2 } + txnNumber: 1 + autocommit: false + writeConcern: { w: 1 } + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: *outcome + + - description: "withTransaction explicit transaction options" + operations: + - name: withTransaction + object: *session0 + arguments: + callback: + operations: + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + readConcern: { level: majority } + writeConcern: { w: 1 } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + insert: *collectionName + documents: [ { _id: 1 } ] + ordered: true + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + readConcern: { level: majority } + # omitted fields + writeConcern: { $$exists: false } + commandName: insert + databaseName: *databaseName + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + autocommit: false + writeConcern: { w: 1 } + # omitted fields + readConcern: { $$exists: false } + startTransaction: { $$exists: false } + commandName: commitTransaction + databaseName: admin + outcome: *outcome diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml new file mode 100644 index 0000000000..d39a406d96 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml @@ -0,0 +1,170 @@ +description: "poc-transactions-mongos-pin-auto" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + +createEntities: + - client: + id: &client0 client0 + useMultipleMongoses: true + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name transaction-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - session: + id: &session0 session0 + client: *client0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + +tests: + - description: "remain pinned after non-transient Interrupted error on insertOne" + operations: + - &startTransaction + name: startTransaction + object: *session0 + - &firstInsert + name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 3 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 3 } } } + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + errorCode: 11601 # Interrupted + - name: insertOne + object: collection + arguments: + session: *session0 + document: { _id: 4 } + expectError: + errorLabelsOmit: [ TransientTransactionError, UnknownTransactionCommitResult ] + errorCodeName: Interrupted + - name: assertSessionPinned + object: testRunner + arguments: + session: *session0 + - name: commitTransaction + object: *session0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: &firstInsertEvent + command: + insert: *collection0Name + documents: [ { _id: 3 } ] + ordered: true + readConcern: { $$exists: false } + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: &secondInsertEvent + command: + insert: *collection0Name + documents: [ { _id: 4 } ] + ordered: true + readConcern: { $$exists: false } + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + commandName: insert + databaseName: *database0Name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + # Original test expected any value, but we can assert an object + recoveryToken: { $$type: object } + commandName: commitTransaction + databaseName: admin + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } + + - description: "unpin after transient error within a transaction" + operations: + - *startTransaction + - *firstInsert + - name: targetedFailPoint + object: testRunner + arguments: + session: *session0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ insert ] + closeConnection: true + - name: insertOne + object: collection + arguments: + session: *session0 + document: { _id: 4 } + expectError: + errorLabelsContain: [ TransientTransactionError ] + errorLabelsOmit: [ UnknownTransactionCommitResult ] + - name: assertSessionUnpinned + object: testRunner + arguments: + session: *session0 + - name: abortTransaction + object: *session0 + expectEvents: + - client: *client0 + events: + - commandStartedEvent: *firstInsertEvent + - commandStartedEvent: *secondInsertEvent + - commandStartedEvent: + command: + abortTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + # Original test expected any value, but we can assert an object + recoveryToken: { $$type: object } + commandName: abortTransaction + databaseName: admin + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1 } + - { _id: 2 } + - { _id: 3 } diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions.yml b/source/unified-test-format/tests/valid-pass/poc-transactions.yml new file mode 100644 index 0000000000..b3101fc2c2 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions.yml @@ -0,0 +1,170 @@ +description: "poc-transactions" + +schemaVersion: "1.0" + +runOnRequirements: + - minServerVersion: "4.0" + topologies: [ replicaset ] + - minServerVersion: "4.1.8" + topologies: [ sharded-replicaset ] + +createEntities: + - client: + id: &client0 client0 + observeEvents: [ commandStartedEvent ] + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name transaction-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name test + - session: + id: &session0 session0 + client: *client0 + +initialData: + - collectionName: *collection0Name + databaseName: *database0Name + documents: [] + +tests: + - description: "Client side error in command starting transaction" + operations: + - name: startTransaction + object: *session0 + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: { .: . } } + # Original test only asserted a generic error + expectError: { isClientError: true } + - name: assertSessionTransactionState + object: testRunner + arguments: + session: *session0 + state: starting + + - description: "create index on a non-existing collection" + runOnRequirements: + - minServerVersion: "4.3.4" + topologies: [ replicaset, sharded-replicaset ] + operations: + - name: dropCollection + object: *database0 + arguments: + collection: *collection0Name + - name: startTransaction + object: *session0 + - name: createCollection + object: *database0 + arguments: + session: *session0 + collection: *collection0Name + - name: assertCollectionNotExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + - name: commitTransaction + object: *session0 + - name: assertCollectionExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + drop: *collection0Name + writeConcern: { $$exists: false } + commandName: drop + databaseName: *database0Name + - commandStartedEvent: + command: + create: *collection0Name + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + commandName: create + databaseName: *database0Name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + commandName: commitTransaction + databaseName: admin + + - description: "create index on a non-existing collection" + runOnRequirements: + - minServerVersion: "4.3.4" + topologies: [ replicaset, sharded-replicaset ] + operations: + - name: dropCollection + object: *database0 + arguments: + collection: *collection0Name + - name: startTransaction + object: *session0 + - name: createIndex + object: *collection0 + arguments: + session: *session0 + name: &indexName "x_1" + keys: { x: 1 } + - name: assertIndexNotExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + indexName: *indexName + - name: commitTransaction + object: *session0 + - name: assertIndexExists + object: testRunner + arguments: + databaseName: *database0Name + collectionName: *collection0Name + indexName: *indexName + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + command: + drop: *collection0Name + writeConcern: { $$exists: false } + commandName: drop + databaseName: *database0Name + - commandStartedEvent: + command: + createIndexes: *collection0Name + indexes: + - name: *indexName + key: { x: 1 } + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: true + autocommit: false + writeConcern: { $$exists: false } + commandName: createIndexes + databaseName: *database0Name + - commandStartedEvent: + command: + commitTransaction: 1 + lsid: { $$sessionLsid: *session0 } + txnNumber: 1 + startTransaction: { $$exists: false } + autocommit: false + writeConcern: { $$exists: false } + commandName: commitTransaction + databaseName: admin From c0c1c021b963b1bc765dfefb028abf7573f63e43 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Mon, 5 Oct 2020 23:39:07 +0800 Subject: [PATCH 70/90] Fix insertOne syntax --- .../tests/valid-pass/poc-change-streams.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml index 1645b7fbcb..677d9de7fb 100644 --- a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -52,13 +52,16 @@ tests: saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne object: *collection1 - arguments: { x: 1 } + arguments: + document: { x: 1 } - name: insertOne object: *collection2 - arguments: { y: 1 } + arguments: + document: { y: 1 } - name: insertOne object: *collection0 - arguments: { z: 1 } + arguments: + document: { z: 1 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: @@ -103,6 +106,7 @@ tests: - name: failPoint object: testRunner arguments: + client: *client0 failPoint: configureFailPoint: failCommand mode: { times: 2 } @@ -116,13 +120,16 @@ tests: saveResultAsEntity: *changeStream0 - name: insertOne object: *collection0 - arguments: { x: 1 } + arguments: + document: { x: 1 } - name: insertOne object: *collection0 - arguments: { x: 2 } + arguments: + document: { x: 2 } - name: insertOne object: *collection0 - arguments: { x: 3 } + arguments: + document: { x: 3 } - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: From 5a95b6dccb9ec60e5a3ad5a8d127c8f56f041943 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 6 Oct 2020 18:22:20 +0800 Subject: [PATCH 71/90] Use complete fullDocument assertions in change stream tests --- .../tests/valid-pass/poc-change-streams.yml | 26 ++++++++++++++----- .../tests/valid-pass/poc-gridfs.yml | 4 +-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml index 677d9de7fb..7934fecc8f 100644 --- a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -69,7 +69,9 @@ tests: ns: db: *database0Name coll: *collection1Name - fullDocument: { x: 1 } + fullDocument: + _id: { $$type: objectId } + x: 1 - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: @@ -77,7 +79,11 @@ tests: ns: db: *database1Name coll: *collection2Name - fullDocument: { y: 1 } + fullDocument: + # Original tests did not include _id, but matching now only permits + # extra keys for root-level documents. + _id: { $$type: objectId } + y: 1 - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: @@ -85,7 +91,9 @@ tests: ns: db: *database0Name coll: *collection0Name - fullDocument: { z: 1 } + fullDocument: + _id: { $$type: objectId } + z: 1 expectEvents: - client: *client0 events: @@ -137,7 +145,9 @@ tests: ns: db: *database0Name coll: *collection0Name - fullDocument: { x: 1 } + fullDocument: + _id: { $$type: objectId } + x: 1 - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: @@ -145,7 +155,9 @@ tests: ns: db: *database0Name coll: *collection0Name - fullDocument: { x: 2 } + fullDocument: + _id: { $$type: objectId } + x: 2 - name: iterateUntilDocumentOrError object: *changeStream0 expectResult: @@ -153,7 +165,9 @@ tests: ns: db: *database0Name coll: *collection0Name - fullDocument: { x: 3 } + fullDocument: + _id: { $$type: objectId } + x: 3 expectEvents: - client: *client0 events: diff --git a/source/unified-test-format/tests/valid-pass/poc-gridfs.yml b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml index b2dc52df62..91baaed6fd 100644 --- a/source/unified-test-format/tests/valid-pass/poc-gridfs.yml +++ b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml @@ -146,11 +146,11 @@ tests: sort: { _id: -1, n: 1 } limit: 2 expectResult: - - _id: { $$type: "objectId" } + - _id: { $$type: objectId } files_id: { $$matchesEntity: *oid0 } n: 0 data: { $binary: { base64: "ESIzRA==", subType: "00" } } # hex 11223344 - - _id: { $$type: "objectId" } + - _id: { $$type: objectId } files_id: { $$matchesEntity: *oid0 } n: 1 data: { $binary: { base64: "VQ==", subType: "00" } } # hex 55 From c0f468e8b8aa81d048aab3f305059c4561281f9e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Tue, 6 Oct 2020 18:25:22 +0800 Subject: [PATCH 72/90] Clarify rules for enabling event listeners --- .../unified-test-format/unified-test-format.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index fc1fb7c753..c7ff08adcb 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-10-04 +:Last Modified: 2020-10-06 .. contents:: @@ -2143,11 +2143,12 @@ client the test runner MAY specify a reduced value for ``heartbeatFrequencyMS`` server selection after a failure; however, test runners MUST NOT do so for any client that specifies ``heartbeatFrequencyMS`` in its ``uriOptions``. -If `test.expectEvents `_ is specified, for each client -entity the test runner MUST enable all event listeners necessary to collect the -event types specified in `observeEvents `_. Test -runners MAY leave event listeners disabled for tests and/or clients that do not -assert expected events. +For each client entity where `observeEvents `_ +has been specified, the test runner MUST enable all event listeners necessary to +collect the specified event types. Test runners MAY leave event listeners +disabled for tests that do not assert events (i.e. tests that omit both +`test.expectEvents `_ and special operations such as +`assertSameLsidOnLastTwoCommands`_). For each client with command monitoring enabled, the test runner MUST ignore events for the following: @@ -2669,6 +2670,10 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-10-06: + +* Clarify rules for enabling event listeners + 2020-10-04: * Advise using $$unsetOrMatches for InsertOneResult From cd42a550d0505d7739da65c49faac16513c23c11 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 11:20:56 +0800 Subject: [PATCH 73/90] Fix event assertions for ChangeStream tests --- .../tests/valid-pass/poc-change-streams.yml | 102 +++++++++++------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml index 7934fecc8f..386578815b 100644 --- a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -3,43 +3,56 @@ description: "poc-change-streams" schemaVersion: "1.0" createEntities: + # Entities for creating changeStreams - client: id: &client0 client0 useMultipleMongoses: false observeEvents: [ commandStartedEvent ] - # Original tests in change-streams.yml did not observe getMore commands - ignoreCommandMonitoringEvents: [ getMore ] + # Original tests do not observe getMore commands but only because event + # assertions ignore extra events. killCursors is explicitly ignored. + ignoreCommandMonitoringEvents: [ getMore, killCursors ] - database: id: &database0 database0 client: *client0 databaseName: &database0Name change-stream-tests - - database: - id: &database1 database1 - client: *client0 - databaseName: &database1Name change-stream-tests-2 - collection: id: &collection0 collection0 database: *database0 collectionName: &collection0Name test + # Entities for executing insert operations + - client: + id: &client1 client1 + - database: + id: &database1 database1 + client: *client1 + databaseName: &database1Name change-stream-tests + - database: + id: &database2 database2 + client: *client1 + databaseName: &database2Name change-stream-tests-2 - collection: id: &collection1 collection1 - database: *database0 - collectionName: &collection1Name test2 + database: *database1 + collectionName: &collection1Name test - collection: id: &collection2 collection2 database: *database1 - collectionName: &collection2Name test + collectionName: &collection2Name test2 + - collection: + id: &collection3 collection3 + database: *database2 + collectionName: &collection3Name test initialData: - - collectionName: *collection0Name - databaseName: *database0Name - documents: [] - collectionName: *collection1Name - databaseName: *database0Name + databaseName: *database1Name documents: [] - collectionName: *collection2Name databaseName: *database1Name documents: [] + - collectionName: *collection3Name + databaseName: *database2Name + documents: [] tests: - description: "Executing a watch helper on a MongoClient results in notifications for changes to all collections in all databases in the cluster." @@ -51,15 +64,15 @@ tests: object: *client0 saveResultAsEntity: &changeStream0 changeStream0 - name: insertOne - object: *collection1 + object: *collection2 arguments: document: { x: 1 } - name: insertOne - object: *collection2 + object: *collection3 arguments: document: { y: 1 } - name: insertOne - object: *collection0 + object: *collection1 arguments: document: { z: 1 } - name: iterateUntilDocumentOrError @@ -67,8 +80,8 @@ tests: expectResult: operationType: insert ns: - db: *database0Name - coll: *collection1Name + db: *database1Name + coll: *collection2Name fullDocument: _id: { $$type: objectId } x: 1 @@ -77,8 +90,8 @@ tests: expectResult: operationType: insert ns: - db: *database1Name - coll: *collection2Name + db: *database2Name + coll: *collection3Name fullDocument: # Original tests did not include _id, but matching now only permits # extra keys for root-level documents. @@ -89,8 +102,8 @@ tests: expectResult: operationType: insert ns: - db: *database0Name - coll: *collection0Name + db: *database1Name + coll: *collection1Name fullDocument: _id: { $$type: objectId } z: 1 @@ -102,7 +115,11 @@ tests: aggregate: 1 cursor: {} pipeline: - - $changeStream: { allChangesForCluster: true } + - $changeStream: + allChangesForCluster: true + # Some drivers may send a default value for fullDocument + # or omit it entirely (see: SPEC-1350). + fullDocument: { $$unsetOrMatches: default } commandName: aggregate databaseName: admin @@ -127,15 +144,15 @@ tests: batchSize: 1 saveResultAsEntity: *changeStream0 - name: insertOne - object: *collection0 + object: *collection1 arguments: document: { x: 1 } - name: insertOne - object: *collection0 + object: *collection1 arguments: document: { x: 2 } - name: insertOne - object: *collection0 + object: *collection1 arguments: document: { x: 3 } - name: iterateUntilDocumentOrError @@ -143,8 +160,8 @@ tests: expectResult: operationType: insert ns: - db: *database0Name - coll: *collection0Name + db: *database1Name + coll: *collection1Name fullDocument: _id: { $$type: objectId } x: 1 @@ -153,8 +170,8 @@ tests: expectResult: operationType: insert ns: - db: *database0Name - coll: *collection0Name + db: *database1Name + coll: *collection1Name fullDocument: _id: { $$type: objectId } x: 2 @@ -163,8 +180,8 @@ tests: expectResult: operationType: insert ns: - db: *database0Name - coll: *collection0Name + db: *database1Name + coll: *collection1Name fullDocument: _id: { $$type: objectId } x: 3 @@ -173,22 +190,27 @@ tests: events: - commandStartedEvent: command: - aggregate: *collection0Name + aggregate: *collection1Name cursor: { batchSize: 1 } pipeline: - - $changeStream: {} + - $changeStream: + fullDocument: { $$unsetOrMatches: default } commandName: aggregate - databaseName: *database0Name + databaseName: *database1Name # The original test only asserted the first command, since expected # events were only an ordered subset. This file does ignore getMore - # commands but we must expect the second aggregate command. While - # doing so we can also assert that it includes a resume token. - - commandStartedEvent: + # commands but we must expect the subsequent aggregate commands, since + # each failed getMore will resume. While doing so we can also assert + # that those commands include a resume token. + - &resumingAggregate + commandStartedEvent: command: - aggregate: *collection0Name + aggregate: *collection1Name cursor: { batchSize: 1 } pipeline: - $changeStream: - resumeAfter: { $exists: true } + fullDocument: { $$unsetOrMatches: default } + resumeAfter: { $$exists: true } commandName: aggregate databaseName: *database0Name + - *resumingAggregate From a4f20d0141ef196e405aef4b7f8999cc30390e26 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 11:21:28 +0800 Subject: [PATCH 74/90] Fix syntax of callback argument for withTransaction tests --- .../poc-transactions-convenient-api.yml | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml index 021f5b5611..cc959e2c50 100644 --- a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml @@ -62,13 +62,12 @@ tests: object: *session0 arguments: callback: - operations: - - name: insertOne - object: *collection0 - arguments: - session: *session0 - document: { _id: 1 } - expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } expectEvents: - client: *client0 events: @@ -110,13 +109,12 @@ tests: object: *session1 arguments: callback: - operations: - - name: insertOne - object: *collection1 - arguments: - session: *session1 - document: { _id: 1 } - expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + - name: insertOne + object: *collection1 + arguments: + session: *session1 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } expectEvents: - client: *client1 events: @@ -154,13 +152,12 @@ tests: object: *session2 arguments: callback: - operations: - - name: insertOne - object: *collection0 - arguments: - session: *session2 - document: { _id: 1 } - expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + - name: insertOne + object: *collection0 + arguments: + session: *session2 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } expectEvents: - client: *client1 events: @@ -198,13 +195,12 @@ tests: object: *session0 arguments: callback: - operations: - - name: insertOne - object: *collection0 - arguments: - session: *session0 - document: { _id: 1 } - expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } + - name: insertOne + object: *collection0 + arguments: + session: *session0 + document: { _id: 1 } + expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } readConcern: { level: majority } writeConcern: { w: 1 } expectEvents: From 5944094056e25737364f53de1f25fec9ad29e79e Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 11:23:21 +0800 Subject: [PATCH 75/90] Remove stream entity and add proxy operations for GridFS --- .../tests/valid-pass/poc-gridfs.yml | 26 ++-- .../unified-test-format.rst | 128 +++++++++--------- 2 files changed, 78 insertions(+), 76 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-gridfs.yml b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml index 91baaed6fd..8a0478658b 100644 --- a/source/unified-test-format/tests/valid-pass/poc-gridfs.yml +++ b/source/unified-test-format/tests/valid-pass/poc-gridfs.yml @@ -20,9 +20,6 @@ createEntities: id: &bucket0_chunks_collection bucket0_chunks_collection database: *database0 collectionName: &bucket0_chunks_collectionName fs.chunks - - stream: - id: &stream0 stream0 - hexBytes: "1122334455" initialData: - collectionName: *bucket0_files_collectionName @@ -75,15 +72,15 @@ tests: operations: # Original test uses "download" operation. We use an explicit operation # that returns a stream and then assert the contents of that stream. - - name: openDownloadStream + - name: download object: *bucket0 arguments: id: { $oid: "000000000000000000000005" } - expectResult: { $$matchesHexbytes: "112233445566778899aa" } + expectResult: { $$matchesHexBytes: "112233445566778899aa" } - description: "Download when files entry does not exist" operations: - - name: openDownloadStream + - name: download object: *bucket0 arguments: id: { $oid: "000000000000000000000000" } @@ -102,7 +99,7 @@ tests: n: 1 expectResult: deletedCount: 1 - - name: openDownloadStream + - name: download object: *bucket0 arguments: id: { $oid: "000000000000000000000005" } @@ -113,11 +110,11 @@ tests: operations: # Original test uses "upload" operation. We use an explicit operation # that takes a stream, which has been created from the expected hex bytes. - - name: uploadFromStream + - name: upload object: *bucket0 arguments: filename: filename - source: *stream0 + source: { $$hexBytes: "1122334455" } chunkSizeBytes: 4 # Original test references the result directly in "assert.data". Here, # we need to save the result as an entity, which we can later reference. @@ -130,6 +127,7 @@ tests: arguments: filter: {} sort: { uploadDate: -1 } + limit: 1 expectResult: - _id: { $$matchesEntity: *oid0 } length: 5 @@ -140,11 +138,11 @@ tests: - name: find object: *bucket0_chunks_collection arguments: - filter: {} - # We cannot use the saved ObjectId when querying, but sort and limit - # can be used to return the expected chunks in order - sort: { _id: -1, n: 1 } - limit: 2 + # We cannot use the saved ObjectId when querying, but filtering by a + # non-zero timestamp will exclude initialData and sort can return the + # expected chunks in order. + filter: { _id: { $gt: { $oid: "000000000000000000000007" } } } + sort: { n: 1 } expectResult: - _id: { $$type: objectId } files_id: { $$matchesEntity: *oid0 } diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index c7ff08adcb..2d08fa1ba6 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-10-06 +:Last Modified: 2020-10-07 .. contents:: @@ -218,7 +218,6 @@ Test runners MUST support the following types of entities: - Collection. See `entity_collection`_ and `Collection Operations`_ - ClientSession. See `entity_session`_ and `Session Operations`_. - GridFS Bucket. See `entity_bucket`_ and `Bucket Operations`_. -- GridFS Stream. See `entity_stream`_. - ChangeStream. See `ChangeStream Operations`_. - All known BSON types and/or equivalent language types for the target driver. For the present version of the spec, the following BSON types are known: @@ -520,26 +519,6 @@ The structure of this object is as follows: specification. The ``readConcern``, ``readPreference``, and ``writeConcern`` options use the same structure as defined in `Common Options`_. -.. _entity_stream: - -- ``stream``: Optional object. Defines a stream, as defined in the - `GridFS <../gridfs/gridfs-spec.rst>`__ spec. Test runners MUST ensure that - stream is both readable *and* writable. - - This entity is primarily used with `uploadFromStream`_ and - `uploadFromStreamWithId`_ `Bucket Operations`_. - - The structure of this object is as follows: - - - ``id``: Required string. Unique name for this entity. The YAML file SHOULD - define a `node anchor`_ for this field (e.g. ``id: &stream0 stream0``). - - - ``hexBytes``: Required string. The string MUST contain an even number of - hexademical characters (case-insensitive) and MAY be empty. The test runner - MUST raise an error if the string is malformed. The test runner MUST convert - the string to a byte sequence denoting the stream's readable data (if any). - For example, "12ab" would denote a stream with two bytes: "0x12" and "0xab". - collectionData ~~~~~~~~~~~~~~ @@ -1277,8 +1256,21 @@ Bucket operations that require special handling or are not documented by an existing specification are described below. -.. _downloadToStream: -.. _downloadToStreamByName: +.. _download: +.. _downloadByName: + +download and downloadByName +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +These operations proxy the bucket's ``openDownloadStream`` and +``openDownloadStreamByName`` methods and support the same parameters and +options, but return a string containing the stream's contents instead of the +stream itself. Test runners MUST fully read the stream to yield the returned +string. This is also necessary to ensure that any expected errors are raised +(e.g. missing chunks). Test files SHOULD use `$$matchesHexBytes`_ in +`expectResult `_ to assert the contents of the returned +string. + downloadToStream and downloadToStreamByName ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1287,17 +1279,11 @@ These operations SHOULD NOT be used in test files. See `IO operations for GridFS streams`_ in `Future Work`_. -.. _openDownloadStream: -.. _openDownloadStreamByName: - openDownloadStream and openDownloadStreamByName ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``openDownloadStream`` and ``openDownloadStreamByName`` operations SHOULD -use `$$matchesHexBytes`_ in `expectResult `_ to match -the contents of the returned stream. These operations MAY use -`saveResultAsEntity `_ to save the stream for use -with a subsequent operation (e.g. `uploadFromStream`_ ). +These operations SHOULD NOT be used in test files. See +`download and downloadByName`_. .. _openUploadStream: @@ -1310,19 +1296,33 @@ These operations SHOULD NOT be used in test files. See `IO operations for GridFS streams`_ in `Future Work`_. -.. _uploadFromStream: -.. _uploadFromStreamWithId: +.. _upload: +.. _uploadWithId: + +upload and uploadWithId +~~~~~~~~~~~~~~~~~~~~~~~ + +These operations proxy the bucket's ``uploadFromStream`` and +``uploadFromStreamWithId`` methods and support the same parameters and options +with one exception: the ``source`` parameter is an object specifying hex bytes +from which test runners MUST construct a readable stream for the underlying +methods. The structure of ``source`` is as follows:: + + { $$hexBytes: } + +The string MUST contain an even number of hexademical characters +(case-insensitive) and MAY be empty. The test runner MUST raise an error if the +structure of ``source`` or its string is malformed. The test runner MUST convert +the string to a byte sequence denoting the stream's readable data (if any). For +example, "12ab" would denote a stream with two bytes: "0x12" and "0xab". + + uploadFromStream and uploadFromStreamWithId ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``uploadFromStream`` and ``uploadFromStreamWithId`` operations' ``stream`` -parameter is a stream as defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ -spec and not easily expressed in YAML/JSON. This parameter is expressed as an -entity name, which the test runner MUST resolve to a `stream `_ -entity *before* passing it as a parameter to the method. The YAML file SHOULD -use an `alias node`_ for a stream entity's ``id`` field (e.g. -``stream: *stream0``). +These operations SHOULD NOT be used in test files. See +`upload and uploadWithId`_. ChangeStream Operations @@ -1955,18 +1955,19 @@ An example of this operator follows:: operations: - object: *bucket0 - name: uploadFromStream + name: upload arguments: filename: "filename" - source: *stream0 + source: { $$hexBytes: "12AB" } expectResult: { $$type: "objectId" } saveResultAsEntity: &objectid0 "objectid0" - object: *filesCollection - name: findOne + name: find arguments: sort: { uploadDate: -1 } + limit: 1 expectResult: - _id: { $$matchesEntity: *objectid0 } + - _id: { $$matchesEntity: *objectid0 } $$matchesHexBytes @@ -1978,13 +1979,10 @@ Syntax, where ``hexBytes`` is an even number of hexademical characters { $$matchesHexBytes: } This operator can be used anywhere a matched value is expected (including -`expectResult `_) and the actual value is a stream as -defined in the `GridFS <../gridfs/gridfs-spec.rst>`__ spec. The test runner MUST -convert the string to a byte sequence and compare it with the full contents of -the stream. The test runner MUST raise an error if the string is malformed. - -This operator is primarily used to assert the contents of stream returned by -`openDownloadStream`_ and `openDownloadStreamByName`_. +`expectResult `_) and the actual value is a string. +The test runner MUST raise an error if the ``hexBytes`` string is malformed. +This operator is primarily used to assert the results of `download`_ and +`downloadByName`_, which return stream contents as a string. $$unsetOrMatches @@ -2502,7 +2500,7 @@ The following tickets are addressed by the test format: * `SPEC-1216 `__: Update GridFS YAML tests to use newer format - See: `stream `_ entity and `Bucket Operations`_ + See: `Bucket Operations`_ * `SPEC-1229 `__: Standardize spec-test syntax for topology assertions @@ -2604,17 +2602,17 @@ the need does arise, `failPoint`_ can be enhanced to support a IO operations for GridFS streams -------------------------------- -Existing GridFS spec tests refer to "upload", "download", and "download_by_name" +Original GridFS spec tests refer to "upload", "download", and "download_by_name" methods, which allow the tests to abstract stream IO and either upload a byte -sequence or assert a downloaded byte sequence. In order to support methods such -as ``downloadToStream``, ``openUploadStream``, and ``openUploadStreamWithId``, -test runners would need to support IO operations to directly read from and write -to a stream entity. +sequence or assert a downloaded byte sequence. These operations correspond to +the `download`_, `downloadByName`_, `upload`_, and `uploadWithId`_ +`Bucket Operations`_. -Currently, `uploadFromStream`_ and `uploadFromStreamWithId`_ are sufficient to -test uploads using a defined `stream `_ entity and -`openDownloadStream`_ and `openDownloadStreamByName`_ are sufficient to test -downloads using `$$matchesHexBytes`_. +In order to support methods such as ``downloadToStream``, ``openUploadStream``, +and ``openUploadStreamWithId``, test runners would need to represent streams as +entities and support IO operations to directly read from and write to a stream +entity. This may not be worth the added complexity if the existing operations +provide adequate test coverage for GridFS implementations. Support Client-side Encryption integration tests @@ -2670,6 +2668,12 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-10-07: + +* Removed stream entities. Created download, downloadByName, upload, and + uploadWithId operations for GridFS buckets, which proxy the underlying + methods and convert between streams and hex strings. + 2020-10-06: * Clarify rules for enabling event listeners From 3dbb89545fd05a0dd4ab15c38c1470250afff45b Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 11:24:28 +0800 Subject: [PATCH 76/90] Reorganized entity operation content and advise not using watch --- .../unified-test-format.rst | 91 ++++++++++++------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2d08fa1ba6..9e9543d414 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1012,6 +1012,13 @@ and test files SHOULD NOT specify `operation.expectResult `_ for this operation. +watch +~~~~~ + +This operation SHOULD NOT be used in test files. See +`client_createChangeStream`_. + + Database Operations ------------------- @@ -1075,6 +1082,13 @@ The following arguments are supported: - ``writeConcern``: Optional object. See `commonOptions_writeConcern`_. +watch +~~~~~ + +This operation SHOULD NOT be used in test files. See +`database_createChangeStream`_. + + Collection Operations --------------------- @@ -1155,6 +1169,35 @@ and compare with ``code`` instead, but MUST raise an error if the comparison cannot be attempted (e.g. ``code`` is also not available, translation fails). +.. _collection_createChangeStream: + +createChangeStream +~~~~~~~~~~~~~~~~~~ + +Creates a collection-level change stream and ensures that the server-side cursor +has been created. + +This operation proxies the collection's ``watch`` method and supports the same +arguments and options. Test files SHOULD NOT use the collection's ``watch`` +operation directly for reasons discussed in `ChangeStream Operations`_. Test +runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` +is executed) as part of this operation and before the resulting change stream +might be saved with +`operation.saveResultAsEntity `_. + +Test runners MUST NOT iterate the change stream when executing this operation +and test files SHOULD NOT specify +`operation.expectResult `_ for this operation. + + +find +~~~~ + +When executing a ``find`` operation, the test runner MUST fully iterate the +result. This will ensure consistent behavior between drivers that eagerly create +a server-side cursor and those that do so lazily when iteration begins. + + findOneAndReplace and findOneAndUpdate ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1192,33 +1235,11 @@ examples:: insertedId: { $$unsetOrMatches: 2 } -.. _collection_createChangeStream: - -createChangeStream -~~~~~~~~~~~~~~~~~~ - -Creates a collection-level change stream and ensures that the server-side cursor -has been created. - -This operation proxies the collection's ``watch`` method and supports the same -arguments and options. Test files SHOULD NOT use the collection's ``watch`` -operation directly for reasons discussed in `ChangeStream Operations`_. Test -runners MUST ensure that the server-side cursor is created (i.e. ``aggregate`` -is executed) as part of this operation and before the resulting change stream -might be saved with -`operation.saveResultAsEntity `_. - -Test runners MUST NOT iterate the change stream when executing this operation -and test files SHOULD NOT specify -`operation.expectResult `_ for this operation. - - -find -~~~~ +watch +~~~~~ -When executing a ``find`` operation, the test runner MUST fully iterate the -result. This will ensure consistent behavior between drivers that eagerly create -a server-side cursor and those that do so lazily when iteration begins. +This operation SHOULD NOT be used in test files. See +`collection_createChangeStream`_. Session Operations @@ -1777,17 +1798,21 @@ actual values, as long as they are consistent: - Convert both values to BSON, and compare bytes - Convert both values to native representations, and compare accordingly +When comparing types that contain documents as internal properties (e.g. +CodeWScope), the rules in `Evaluating Matches`_ do not apply and the documents +MUST match exactly; however, test runners MUST permit variation in document key +order or otherwise normalize the documents before comparison. + + +Flexible Numeric Comparisons +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + When comparing numeric types (excluding Decimal128), test runners MUST consider 32-bit, 64-bit, and floating point numbers to be equal if their values are numerically equivalent. For example, an expected value of ``1`` would match an actual value of ``1.0`` (e.g. ``ok`` field in a server response) but would not match ``1.5``. -When comparing types that contain documents as internal properties (e.g. -CodeWScope), the rules in `Evaluating Matches`_ do not apply and the documents -MUST match exactly; however, test runners MUST permit variation in document key -order or otherwise normalize the documents before comparison. - Allowing Extra Fields in Root-level Documents ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2674,6 +2699,10 @@ Note: this will be cleared when publishing version 1.0 of the spec uploadWithId operations for GridFS buckets, which proxy the underlying methods and convert between streams and hex strings. +* Create heading for Flexible Numeric Comparisons + +* Note that watch operations should not be used (point to createChangeStream) + 2020-10-06: * Clarify rules for enabling event listeners From 374924d158785fe92912ec39f10aeb34cc95a5dc Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 12:45:16 +0800 Subject: [PATCH 77/90] Fix retryable reads test and add additional find test --- .../tests/valid-pass/poc-retryable-reads.yml | 50 +++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml index 0d4323e78f..c1ea7ec696 100644 --- a/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.yml @@ -79,12 +79,47 @@ tests: pipeline: *pipeline databaseName: *databaseName + - description: "Find succeeds on second attempt" + operations: + - name: failPoint + object: testRunner + arguments: + client: *client0 + failPoint: + configureFailPoint: failCommand + mode: { times: 1 } + data: + failCommands: [ find ] + closeConnection: true + # Find options and expected result changed to use common initialData + - name: find + object: collection0 + arguments: + filter: {} + sort: { _id: 1 } + limit: 2 + expectResult: + - { _id: 1, x: 11 } + - { _id: 2, x: 22 } + expectEvents: + - client: *client0 + events: + - &findAttempt + commandStartedEvent: + command: + find: *collectionName + filter: {} + sort: { _id: 1 } + limit: 2 + databaseName: *databaseName + - *findAttempt + - description: "Find fails on first attempt" operations: - name: failPoint object: testRunner arguments: - client: *client1 # uses retryReads=false + client: *client0 failPoint: configureFailPoint: failCommand mode: { times: 1 } @@ -92,7 +127,7 @@ tests: failCommands: [ find ] closeConnection: true - name: find - object: collection1 + object: collection1 # client uses retryReads=false arguments: filter: {} # Other arguments in the original test are not relevant @@ -119,7 +154,7 @@ tests: failCommands: [ find ] closeConnection: true - name: find - object: collection1 + object: collection0 arguments: filter: {} # Other arguments in the original test are not relevant @@ -127,16 +162,13 @@ tests: expectEvents: - client: *client0 events: - - commandStartedEvent: - command: - find: *collectionName - filter: {} - databaseName: *databaseName - - commandStartedEvent: + - &findAttempt + commandStartedEvent: command: find: *collectionName filter: {} databaseName: *databaseName + - *findAttempt - description: "ListDatabases succeeds on second attempt" operations: From 29b8aaba3d50f1169b9683b41343e6d56d34519b Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 12:49:51 +0800 Subject: [PATCH 78/90] Fix retryable writes test to use correct collection object --- .../tests/valid-pass/poc-retryable-writes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml index 84c5c8e7bd..0b94fbd803 100644 --- a/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.yml @@ -198,7 +198,7 @@ tests: code: 91 # ShutdownInProgress errmsg: "Replication is being shut down" - name: insertOne - object: *collection1 + object: *collection0 arguments: document: { _id: 3, x: 33 } expectError: From 4de1a936817f753fb3f661aed4f6efbb4b72317c Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 15:54:03 +0800 Subject: [PATCH 79/90] Fix syntax errors in session tests --- .../tests/valid-pass/poc-sessions.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.yml b/source/unified-test-format/tests/valid-pass/poc-sessions.yml index 382fe44f01..b700573d50 100644 --- a/source/unified-test-format/tests/valid-pass/poc-sessions.yml +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.yml @@ -54,6 +54,8 @@ tests: expectResult: [] - name: assertSameLsidOnLastTwoCommands object: testRunner + arguments: + client: *client0 expectEvents: - client: *client0 events: @@ -89,6 +91,8 @@ tests: - *find_with_implicit_session - name: assertSameLsidOnLastTwoCommands object: testRunner + arguments: + client: *client0 expectEvents: - client: *client0 events: @@ -164,6 +168,8 @@ tests: - *find_with_implicit_session - name: assertDifferentLsidOnLastTwoCommands object: testRunner + arguments: + client: *client0 expectEvents: - client: *client0 events: @@ -175,7 +181,7 @@ tests: documents: - { _id: 2 } ordered: true - lsid: { $sessionLsid: *session0 } + lsid: { $$sessionLsid: *session0 } txnNumber: 1 commandName: insert databaseName: *database0Name @@ -186,7 +192,7 @@ tests: documents: - { _id: 3 } ordered: true - lsid: { $sessionLsid: *session0 } + lsid: { $$sessionLsid: *session0 } txnNumber: 2 commandName: insert databaseName: *database0Name From 78291ddf67ed4dd8bb7667427b231578d60f0ef1 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 22:01:04 +0800 Subject: [PATCH 80/90] Fix typos in transaction tests --- .../tests/valid-pass/poc-transactions-convenient-api.yml | 2 +- .../tests/valid-pass/poc-transactions-mongos-pin-auto.yml | 4 ++-- .../unified-test-format/tests/valid-pass/poc-transactions.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml index cc959e2c50..4f981d15dd 100644 --- a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.yml @@ -159,7 +159,7 @@ tests: document: { _id: 1 } expectResult: { $$unsetOrMatches: { insertedId: { $$unsetOrMatches: 1 } } } expectEvents: - - client: *client1 + - client: *client0 events: - commandStartedEvent: command: diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml index d39a406d96..c2545b301f 100644 --- a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml @@ -54,7 +54,7 @@ tests: failCommands: [ insert ] errorCode: 11601 # Interrupted - name: insertOne - object: collection + object: *collection0 arguments: session: *session0 document: { _id: 4 } @@ -131,7 +131,7 @@ tests: failCommands: [ insert ] closeConnection: true - name: insertOne - object: collection + object: *collection0 arguments: session: *session0 document: { _id: 4 } diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions.yml b/source/unified-test-format/tests/valid-pass/poc-transactions.yml index b3101fc2c2..5f229e464c 100644 --- a/source/unified-test-format/tests/valid-pass/poc-transactions.yml +++ b/source/unified-test-format/tests/valid-pass/poc-transactions.yml @@ -47,7 +47,7 @@ tests: session: *session0 state: starting - - description: "create index on a non-existing collection" + - description: "explicitly create collection using create command" runOnRequirements: - minServerVersion: "4.3.4" topologies: [ replicaset, sharded-replicaset ] From f04079fe71d04a2e20b74214da11914e92f8787a Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 22:08:48 +0800 Subject: [PATCH 81/90] Specify useMultipleMongoses:false for failPoint tests --- .../unified-test-format/tests/valid-pass/poc-change-streams.yml | 1 + source/unified-test-format/tests/valid-pass/poc-sessions.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml index 386578815b..bdf83fdc60 100644 --- a/source/unified-test-format/tests/valid-pass/poc-change-streams.yml +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.yml @@ -22,6 +22,7 @@ createEntities: # Entities for executing insert operations - client: id: &client1 client1 + useMultipleMongoses: false - database: id: &database1 database1 client: *client1 diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.yml b/source/unified-test-format/tests/valid-pass/poc-sessions.yml index b700573d50..f27a5fd8aa 100644 --- a/source/unified-test-format/tests/valid-pass/poc-sessions.yml +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.yml @@ -8,6 +8,7 @@ runOnRequirements: createEntities: - client: id: &client0 client0 + useMultipleMongoses: false observeEvents: [ commandStartedEvent ] - database: id: &database0 database0 From db33be3fad5e1d3921c4ffdb243a01e012af0779 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Wed, 7 Oct 2020 22:27:17 +0800 Subject: [PATCH 82/90] Fix outcome assertion in session pinning test --- .../tests/valid-pass/poc-transactions-mongos-pin-auto.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml index c2545b301f..47db7c3188 100644 --- a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.yml @@ -167,4 +167,3 @@ tests: documents: - { _id: 1 } - { _id: 2 } - - { _id: 3 } From 2665c01c44e0470b5208e71b51d8d8ca17455600 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 8 Oct 2020 13:41:56 +0800 Subject: [PATCH 83/90] Add related issue and iterateUntilDocumentOrError clarification --- .../unified-test-format.rst | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 9e9543d414..213d8a9747 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -9,7 +9,7 @@ Unified Test Format :Status: Draft :Type: Standards :Minimum Server Version: N/A -:Last Modified: 2020-10-07 +:Last Modified: 2020-10-08 .. contents:: @@ -1372,9 +1372,8 @@ iterateUntilDocumentOrError ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Iterates the change stream until either a single document is returned or an -error is raised. - -If `expectResult `_ is specified, it SHOULD be a single +error is raised. This operation takes no arguments. If +`expectResult `_ is specified, it SHOULD be a single document. `Iterating the Change Stream <../change-streams/tests#iterating-the-change-stream>`__ @@ -1384,6 +1383,15 @@ unnecessarily, as doing so could cause the test runner to block indefinitely. This should not be a concern for ``iterateUntilDocumentOrError`` as iteration only continues until either a document or error is encountered. +Test runners MUST ensure that this operation will not inadvertently skip the +first document in a change stream. Albeit rare, this could happen if +``iterateUntilDocumentOrError`` were to blindly invoke ``next`` (or equivalent) +on a change stream in a driver where newly created change streams are already +positioned at their first element and the change stream cursor had a non-empty +``firstBatch`` (i.e. ``resumeAfter`` or ``startAfter`` used). Alternatively, +some drivers may use a different iterator method for advancing a change stream +to its first position (e.g. ``rewind`` in PHP). + Special Test Operations ----------------------- @@ -2552,6 +2560,10 @@ The following tickets are addressed by the test format: creating an empty collection without inserting any documents (needed by transaction tests). +* `SPEC-1660 `__: Consolidate failCommand documentation + + See `Server Fail Points`_ and `failCommand`_. + The following tickets can be resolved after the unified test format is approved and/or other specs begin porting their tests to the format: @@ -2693,6 +2705,13 @@ Change Log Note: this will be cleared when publishing version 1.0 of the spec +2020-10-08: + +* Note special consideration to ensure iterateUntilDocumentOrError does not skip + the first result in a change stream + +* Add SPEC-1660 to related issues + 2020-10-07: * Removed stream entities. Created download, downloadByName, upload, and From f3df323700300bbe6bb724dc53cc2874402136f2 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Thu, 8 Oct 2020 23:52:17 +0800 Subject: [PATCH 84/90] Rename returnDocument test and fix syntax error --- ...-enum-invalid.yaml => returnDocument-enum-invalid.yml} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename source/unified-test-format/tests/valid-fail/{returnDocument-enum-invalid.yaml => returnDocument-enum-invalid.yml} (87%) diff --git a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml similarity index 87% rename from source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml rename to source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml index 4a28a6379b..d37d3d701b 100644 --- a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yaml +++ b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml @@ -5,10 +5,10 @@ schemaVersion: "1.0" createEntities: - client: id: &client0 client0 - - database: - id: &database0 database0 - client: *client0 - databaseName: &databaseName test + - database: + id: &database0 database0 + client: *client0 + databaseName: &databaseName test - collection: id: &collection0 collection0 database: *database0 From 4bfda0053e288e7bd1e510c960934f3bff33bc11 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 9 Oct 2020 09:59:44 +0800 Subject: [PATCH 85/90] Require sharded-replicaset for retryable write session test --- source/unified-test-format/tests/valid-pass/poc-sessions.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.yml b/source/unified-test-format/tests/valid-pass/poc-sessions.yml index f27a5fd8aa..cb16657da3 100644 --- a/source/unified-test-format/tests/valid-pass/poc-sessions.yml +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.yml @@ -123,12 +123,13 @@ tests: - { _id: 2 } - description: "Dirty explicit session is discarded" + # Original test specified retryWrites=true, but that is now the default. + # Retryable writes will require a sharded-replicaset, though. runOnRequirements: - minServerVersion: "4.0" topologies: [ replicaset ] - minServerVersion: "4.1.8" - topologies: [ sharded ] - # Original test specified retryWrites=true, but that is now the default + topologies: [ sharded-replicaset ] operations: - name: failPoint object: testRunner From 0732830c26237942dbf687cc141048fe8e8b2e2d Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Fri, 9 Oct 2020 12:52:18 +0800 Subject: [PATCH 86/90] Fix typo --- source/unified-test-format/unified-test-format.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 213d8a9747..2a10047189 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -1203,8 +1203,8 @@ findOneAndReplace and findOneAndUpdate The ``returnDocument`` option for ``findOneAndReplace`` and ``findOneAndUpdate`` is documented as an enum with possible values "Before" and "After". Test files -SHOULD express ``returnDocument`` a string and test runners MUST raise an error -if its value does not case-insensitively match either enum value. +SHOULD express ``returnDocument`` as a string and test runners MUST raise an +error if its value does not case-insensitively match either enum value. insertMany From 40bb54e85fed7172ee0925c445420954742391a9 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 10 Oct 2020 20:59:54 +0800 Subject: [PATCH 87/90] Remove "Related Issues" section and clear "Change Log" --- .../unified-test-format.rst | 427 ------------------ 1 file changed, 427 deletions(-) diff --git a/source/unified-test-format/unified-test-format.rst b/source/unified-test-format/unified-test-format.rst index 2a10047189..b6b1e5e860 100644 --- a/source/unified-test-format/unified-test-format.rst +++ b/source/unified-test-format/unified-test-format.rst @@ -2511,72 +2511,6 @@ SHOULD be described here in detail for historical reference, in addition to any shorter description that may be added to the `Change Log`_. -Related Issues -============== - -Note: this section will be removed before publishing version 1.0 of the spec. - -The following SPEC tickets are associated with -`DRIVERS-709 `__. This section will -record whether or not each issue is addressed by this spec. - -The following tickets are addressed by the test format: - -* `SPEC-1158 `__: Spec tests should use a consistent format for CRUD options - - ``options`` structs are now flattened into `operation.arguments`_. See: - `Entity Test Operations`_. - -* `SPEC-1215 `__: Introduce spec test syntax for meta assertions (e.g. any value, not exists) - - See: `Special Operators for Matching Assertions`_ - -* `SPEC-1216 `__: Update GridFS YAML tests to use newer format - - See: `Bucket Operations`_ - -* `SPEC-1229 `__: Standardize spec-test syntax for topology assertions - - See `runOnRequirement`_, which is used by both `runOnRequirements`_ and - `test.runOnRequirements`_. - -* `SPEC-1254 `__: Rename topology field in spec tests to topologies - - See `runOnRequirement`_. - -* `SPEC-1439 `__: Inconsistent error checking in spec tests - - This may still require test format syntax to assert that an error occurred - without caring about any assertions on the error itself. - -* `SPEC-1713 `__: Allow runOn to be defined per-test in addition to per-file - - See `runOnRequirements`_ and `test.runOnRequirements`_. - -* `SPEC-1723 `__: Introduce test file syntax to disable dropping of collection under test - - Only collections in `initialData`_ are dropped, so this can be achieved by - omitting the collection from `initialData`_. Additionally, the format supports - creating an empty collection without inserting any documents (needed by - transaction tests). - -* `SPEC-1660 `__: Consolidate failCommand documentation - - See `Server Fail Points`_ and `failCommand`_. - -The following tickets can be resolved after the unified test format is approved -and/or other specs begin porting their tests to the format: - -* `SPEC-1102 `__: Add "object: collection" to command monitoring tests -* `SPEC-1133 `__: Use APM to assert outgoing commands in CRUD spec tests -* `SPEC-1144 `__: CRUD tests improvements -* `SPEC-1193 `__: Convert change stream spec tests to runOn format -* `SPEC-1230 `__: Rewrite APM spec tests to use common test format -* `SPEC-1238 `__: Convert retryable write spec tests to multi-operation format -* `SPEC-1261 `__: Use runOn syntax to specify APM test requirements -* `SPEC-1375 `__: changeStream spec tests should be run on sharded clusters - - Future Work =========== @@ -2702,364 +2636,3 @@ spec changes developed in parallel or during the same release cycle. Change Log ========== - -Note: this will be cleared when publishing version 1.0 of the spec - -2020-10-08: - -* Note special consideration to ensure iterateUntilDocumentOrError does not skip - the first result in a change stream - -* Add SPEC-1660 to related issues - -2020-10-07: - -* Removed stream entities. Created download, downloadByName, upload, and - uploadWithId operations for GridFS buckets, which proxy the underlying - methods and convert between streams and hex strings. - -* Create heading for Flexible Numeric Comparisons - -* Note that watch operations should not be used (point to createChangeStream) - -2020-10-06: - -* Clarify rules for enabling event listeners - -2020-10-04: - -* Advise using $$unsetOrMatches for InsertOneResult - -* Clarifications for matching insertedIds in BulkWriteResult - -* Test runners must error if returnDocument enum is invalid - -2020-10-03: - -* Note that special test operations should not be combined with expectError, - expectResult, or saveResultAsEntity. - -2020-10-02: - -* Restructure Entity Test Operations sections. - -* Clarify that fail points are not configured using the internal MongoClient. - -* Require model objects returned by operations be converted to documents before - being matched or saved in the entity map. - -* Require iterables returned by operations be fully iterated when executing an - operation, unless otherwise stated. - -2020-09-29: - -* Clarify that some optional arrays must contain one or more elements, since the - fields could otherwise be omitted. - -* $$matchesHexBytes may be an empty string, like hexBytes stream entity option - -* Note why some event properties are omitted in expectedEvent assertions - -* Do not use case-insensitive comparisons for commandName - -2020-09-28: - -* Prohibit $$unsetOrMatches for array elements - -2020-09-24: - -* Future work for incorporating referenced entity operations into schema version - -* Test runners MUST raise an error for incompatible files - -* Improve docs for Evaluating Matches. Extra keys are only permitted in - root-level documents. Always permit key order variation. - -* Explain in "Design Rationale" why "SHOULD" is used more commonly in this - document. - -* Simplify paragraph about comparing server versions and refer to Version String - section for format info. - -* Change "SHOULD" to "MUST" to prohibit test runners from sharing state between - test files and test cases (e.g. entity map must be reset between tests). - -2020-09-23: - -* Clarify server version string comparison rules - -* Clarify BSON type support and advise against using deprecated types - -* Clarify error handling for $$sessionLsid - -* Suggest lowering heartbeatFrequencyMS and minHeartbeatFrquencyMS for clients - using fail points - -2020-09-22: - -* Test runners must raise errors for unsupported operations and arguments - -* Clarify behavior when useMultipleMongoses is unspecified - -2020-09-21: - -* Entity map can store equivalent language types for supported BSON types. - -2020-09-17: - -* Future work for logging assertions, FLE tests, and SDAM tests - -* Test files should specify schema version conservatively. - -* Add Goals section and note out-of-scope specs in Future Work. - -* Elaborate on schema version. - -* Exhaustively list supported entity types and error when attempting to add an - unsupported type to the entity map - -2020-09-15: - -* Revise definition for test.outcome, clarify comparison rules for outcome - documents, and permit flexibility for field order during comparisons. - -2020-09-11: - -* Require BSON types be expressed as strings for $$type operator - -2020-09-08: - -* Replace " or array of " with "array of " for - ``topologies``, ``observeEvents``, and ``ignoreCommandMonitoringEvents``. - -* Rename ``expected`` prefix to ``expect`` in test field names. Applies to - ``expectEvents``, ``expectedError``, and ``expectResult``. Structures such as - ``expectedEvent`` were not renamed. - -* Clarify that "sharded" implies "sharded-replicaset". - -* Remove note about clearing state between test files. Clearing state is mainly - relevant for tests, and is already discussed. - -* Note that server versions are compared numerically. - -2020-09-03: - -* Rename top-level and test-level runOn to runOnRequirements - -* Define expectedEventsForClient, between expectedEvents and expectedEvent - -* Replace "document" with "object" unless referring to a MongoDB document - -2020-09-02: - -* Future Work for supporting event types beyond command monitoring (e.g. SDAM) - -* ignoreCommandMonitoringEvents option for client entities, which is needed to - to ignore killCursors for change stream tests. - -* Future Work for ignoring extra, but not all, events for a command (e.g. - multiple getMores for change stream iteration on a sharded cluster) - -* stream entity in createEntities, which is created from a hexadecimal string - and used as an argument for the uploadFromStream bucket operation - -* $$matchesEntity and $$matchesHexBytes match operators, for GridFS tests - -* describe GridFS bucket operations and note unsupported operations, which link - to Future Work for IO operations on GridFS streams - -* isError and isClientError assertions for expectedError - -* note that errorCode and errorCodeName should not be used for client errors - -* Remove Open Questions, which should all be resolved - -* Test runners may skip tests with intentionally unimplemented methods (e.g. - listCollectionNames) - -* use camelCase instead of snake_case for API methods and parameters - -2020-09-01: - -* clarify that saveResultAsEntity should always define the name in the map, even - if the result is empty. - -* test files should order createEntities - -* createChangeStream operation to proxy watch and enforce consistency among - drivers - -* Future Work section to note failPoint readPreference support and mixing - different event types - -2020-08-28: - -* iterateUntilDocumentOrError operation for change stream entities - -* open questions for change streams - -* errorCode assertion for expectedError - -2020-08-27: - -* rename runOn.topology to topologies (SPEC-1254) - -* clarify rules for comparing schema versions and note that test files should - not need to refer to patch versions. - -* note that lsid assertions require actually having observed events and lsid - fields to compare. also remove note about "collecting observed events" after - executing operations, since events will already need to be accessible while - running operations in order to evaluate some assertions. - -* open question about human-readable binary data for GridFS - -* rename runOn.topology to topologies - -* create section for related SPEC tickets and explain which are addressed by - this spec or suitable to be completed after the format is approved (e.g. those - that pertain to porting over other spec tests to the new format). - -* add required top-level description field, which test runners may use for - debugging or log messages - -2020-08-26: - -* special operations from sessions spec tests - -* clarify interaction between runOn and test.runOn, which are evaluated - independently and must both be met for a test to be executed. Also add a note - that test.runOn requirements should only be more restrictive for readability. - -* create open questions from GridFS spec and internal TODO items - -2020-08-25: - -* note that drivers with global event listeners will need to associate events - with clients for executing assertions (copied from change streams spec). - -* require databaseName and collectionName when creating database and collection - entities, respectively. also require those options in initialData, outcome, - and special operations (YAML aliases may be used). remove top-level - databaseName and collectionName fields (again) and any language for test - runners generate their own names. - -* remove createOnServer option for collection entities. initialData will now - explicitly create a collection if the list of documents is empty. - -* note handling of read concern, read preference, and write concern options for - entity operations. - -* document arguments for all special operations - -* define observeEvents option for client entities to filter observed events - -* revise docs for configuring event listeners and remove wording that assumed - a test runner might only collect command monitoring events. - -* failPoint now takes a client entity and no longer uses internal MongoClient. - this is related to moving the option for multi-mongos and single-mongos to the - client entity. - -* replace top-level allowMultipleMongoses option, which previously applied to - all clients, with a useMultipleMongoses client entity option. This option can - be used to require multiple mongos hosts (desirable for targetedFailPoint) or - modify a connection string to a single host (desirable for failPoint). - -* construct internal MongoClient when initializing the test runner and allow it - to be shared throughout. This is possible because it no longer cares about the - number of mongos hosts (formerly allowMultipleMongoses). - -2020-08-23: - -* describe how to determine if sharded clusters use replica sets - -* use flexible comparisons for numeric types (to help with ``ok`` matches) - -* copy special procedures from transaction spec - -* document test runner implementation, including loading test files, executing - tests, and executing operations - -* clarify interactions with fail points and APM - -* better describe argument handling for entity operations and document edge - cases for bulkWrite ops - -* document command started, succeeded, and failed events (per APM spec) - -* clarify logic for expectedError assertions - -* restructure expectedEvents to list events per client - -* createOnServer option for collection entities (mainly for transactions) - -2020-08-21: - -* clarify error assertions for BulkWriteException - -* note that YAML is the canonical format and discuss js-yaml - -* note that configureFailPoint must be excluded from APM - -* reformat external links to YAML spec and fail point docs - -* add schemaVersion field and document how the spec will handle versions - -* move client.allowMultipleMongoses to top-level option, since it should apply - to all clients (internal and entities). also note that targetedFailPoint and - failPoint should not be used in the same test file, since the latter requires - allowMultipleMongoses:false and would not provide meaningful test coverage of - mongos pinning for sessions. - -* add terms and define Entity and Internal MongoClient - -* note that failPoint always uses internal MongoClient and targetedFailPoint - uses the client of its session argument - -* start writing steps for test execution - -2020-08-19: - -* added test.runOn and clarified that it can override top-level runOn requirements - -* runOn.topology can be a single string in addition to array of strings - -* added "sharded-replicaset" topology type, which will be relevant for change - streams, transactions, retryable writes. - -* removed top-level collectionName and databaseName fields, since they can be - specified when creating collection and database entities. - -* removed test.clientOptions, since client entities can specify their own options - -* moved operation.failPoint to failPoint special operation - -* operation.object is now required and takes either an entity name (e.g. - "collection0") or "testRunner" - -* operation.commandName moved to an argument of the runCommand database - operation. Since that method is documented entirely in this spec, I didn't - see an issue with consolidating. - -* renamed operation.result to expectedResult and noted that it may co-exist with - error assertions in special cases (e.g. BulkWriteException). - -* remove error assertions from operation.result. These are now specified under - operation.expectedError, which replaces the error boolean and requires at - least one assertion. Added a type assertion (e.g. client, server), which - should be useful for discerning client-side and server-side errors (currently - achieved with APM assertions). - -* added operation.saveResultAsEntity to capture a result in the entity map - (primarily for use with change streams) - -* consolidated documentation for ignoring configureFailPoint commands in APM and - also disabling fail points after a test, which is now referenced from the - failPoint and targetedFailPoint operations - -* removed $$assert nesting in favor of $$, since test runners can - easily check the first document key for a ``$$`` prefix. - -* completed section on evaluating matches and added pseudo-code From 796de8c5c336aed402f0faf13610ec7ce6f34df9 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 10 Oct 2020 21:04:59 +0800 Subject: [PATCH 88/90] Fix schema validation error in returnDocument-enum-invalid.yml --- .../tests/valid-fail/returnDocument-enum-invalid.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml index d37d3d701b..b877f5e965 100644 --- a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml +++ b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.yml @@ -6,9 +6,9 @@ createEntities: - client: id: &client0 client0 - database: - id: &database0 database0 - client: *client0 - databaseName: &databaseName test + id: &database0 database0 + client: *client0 + databaseName: &databaseName test - collection: id: &collection0 collection0 database: *database0 From c8815ad660b701dd10855417eeadea56944ab0dc Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 10 Oct 2020 21:16:24 +0800 Subject: [PATCH 89/90] Generate JSON for YAML tests --- .../collectionData-additionalProperties.json | 40 ++ ...ollectionData-collectionName-required.json | 38 ++ .../collectionData-collectionName-type.json | 39 ++ .../collectionData-databaseName-required.json | 38 ++ .../collectionData-databaseName-type.json | 39 ++ .../collectionData-documents-items.json | 41 ++ .../collectionData-documents-required.json | 38 ++ .../collectionData-documents-type.json | 39 ++ ...rDatabaseOptions-additionalProperties.json | 27 + ...ionOrDatabaseOptions-readConcern-type.json | 27 + ...OrDatabaseOptions-readPreference-type.json | 27 + ...onOrDatabaseOptions-writeConcern-type.json | 27 + .../tests/invalid/createEntities-items.json | 13 + .../invalid/createEntities-minItems.json | 11 + .../tests/invalid/createEntities-type.json | 11 + .../tests/invalid/description-required.json | 9 + .../invalid/entity-additionalProperties.json | 15 + .../entity-bucket-additionalProperties.json | 31 ++ .../entity-bucket-bucketOptions-type.json | 31 ++ .../entity-bucket-database-required.json | 29 + .../invalid/entity-bucket-database-type.json | 30 ++ .../invalid/entity-bucket-id-required.json | 29 + .../tests/invalid/entity-bucket-id-type.json | 30 ++ .../entity-client-additionalProperties.json | 18 + .../invalid/entity-client-id-required.json | 15 + .../tests/invalid/entity-client-id-type.json | 17 + ...t-ignoreCommandMonitoringEvents-items.json | 20 + ...gnoreCommandMonitoringEvents-minItems.json | 18 + ...nt-ignoreCommandMonitoringEvents-type.json | 18 + .../entity-client-observeEvents-enum.json | 20 + .../entity-client-observeEvents-items.json | 20 + .../entity-client-observeEvents-minItems.json | 18 + .../entity-client-observeEvents-type.json | 18 + .../entity-client-uriOptions-type.json | 18 + ...ntity-client-useMultipleMongoses-type.json | 18 + ...ntity-collection-additionalProperties.json | 32 ++ ...ty-collection-collectionName-required.json | 30 ++ ...entity-collection-collectionName-type.json | 31 ++ ...ity-collection-collectionOptions-type.json | 32 ++ .../entity-collection-database-required.json | 30 ++ .../entity-collection-database-type.json | 31 ++ .../entity-collection-id-required.json | 30 ++ .../invalid/entity-collection-id-type.json | 31 ++ .../entity-database-additionalProperties.json | 25 + .../entity-database-client-required.json | 23 + .../invalid/entity-database-client-type.json | 24 + ...entity-database-databaseName-required.json | 23 + .../entity-database-databaseName-type.json | 24 + .../entity-database-databaseOptions-type.json | 25 + .../invalid/entity-database-id-required.json | 23 + .../invalid/entity-database-id-type.json | 24 + .../tests/invalid/entity-maxProperties.json | 22 + .../tests/invalid/entity-minProperties.json | 13 + .../entity-session-additionalProperties.json | 24 + .../entity-session-client-required.json | 22 + .../invalid/entity-session-client-type.json | 23 + .../invalid/entity-session-id-required.json | 22 + .../tests/invalid/entity-session-id-type.json | 23 + .../entity-session-sessionOptions-type.json | 24 + .../entity-stream-additionalProperties.json | 19 + .../entity-stream-hexBytes-pattern.json | 18 + .../entity-stream-hexBytes-required.json | 17 + .../invalid/entity-stream-hexBytes-type.json | 18 + .../invalid/entity-stream-id-required.json | 17 + .../tests/invalid/entity-stream-id-type.json | 18 + .../expectedError-additionalProperties.json | 25 + .../invalid/expectedError-errorCode-type.json | 25 + .../expectedError-errorCodeName-type.json | 25 + .../expectedError-errorContains-type.json | 25 + ...xpectedError-errorLabelsContain-items.json | 27 + ...ctedError-errorLabelsContain-minItems.json | 25 + ...expectedError-errorLabelsContain-type.json | 25 + .../expectedError-errorLabelsOmit-items.json | 27 + ...xpectedError-errorLabelsOmit-minItems.json | 25 + .../expectedError-errorLabelsOmit-type.json | 25 + .../expectedError-isClientError-type.json | 25 + .../invalid/expectedError-isError-const.json | 25 + .../invalid/expectedError-isError-type.json | 25 + .../invalid/expectedError-minProperties.json | 23 + .../expectedEvent-additionalProperties.json | 32 ++ ...t-commandFailedEvent-commandName-type.json | 34 ++ ...mandStartedEvent-additionalProperties.json | 34 ++ ...vent-commandStartedEvent-command-type.json | 34 ++ ...-commandStartedEvent-commandName-type.json | 34 ++ ...commandStartedEvent-databaseName-type.json | 34 ++ ...ommandSucceededEvent-commandName-type.json | 34 ++ ...vent-commandSucceededEvent-reply-type.json | 34 ++ .../invalid/expectedEvent-maxProperties.json | 33 ++ .../invalid/expectedEvent-minProperties.json | 30 ++ ...dEventsForClient-additionalProperties.json | 29 + ...pectedEventsForClient-client-required.json | 27 + .../expectedEventsForClient-client-type.json | 28 + .../expectedEventsForClient-events-items.json | 30 ++ ...pectedEventsForClient-events-required.json | 27 + .../expectedEventsForClient-events-type.json | 28 + .../tests/invalid/initialData-items.json | 13 + .../tests/invalid/initialData-minItems.json | 11 + .../tests/invalid/initialData-type.json | 11 + .../operation-additionalProperties.json | 23 + .../invalid/operation-arguments-type.json | 23 + ...pectError-conflicts_with_expectResult.json | 26 + ...ror-conflicts_with_saveResultAsEntity.json | 26 + .../invalid/operation-expectError-type.json | 23 + .../invalid/operation-expectEvents-type.json | 23 + .../invalid/operation-name-required.json | 21 + .../tests/invalid/operation-name-type.json | 22 + .../invalid/operation-object-required.json | 21 + .../tests/invalid/operation-object-type.json | 22 + .../operation-saveResultAsEntity-type.json | 23 + ...runOnRequirement-additionalProperties.json | 16 + ...nRequirement-maxServerVersion-pattern.json | 15 + ...unOnRequirement-maxServerVersion-type.json | 15 + .../runOnRequirement-minProperties.json | 13 + ...nRequirement-minServerVersion-pattern.json | 15 + ...unOnRequirement-minServerVersion-type.json | 15 + .../runOnRequirement-topologies-enum.json | 17 + .../runOnRequirement-topologies-items.json | 17 + .../runOnRequirement-topologies-minItems.json | 15 + .../runOnRequirement-topologies-type.json | 15 + .../invalid/runOnRequirements-items.json | 13 + .../invalid/runOnRequirements-minItems.json | 11 + .../tests/invalid/runOnRequirements-type.json | 11 + .../tests/invalid/schemaVersion-pattern.json | 10 + .../tests/invalid/schemaVersion-required.json | 9 + .../tests/invalid/schemaVersion-type.json | 10 + .../invalid/test-additionalProperties.json | 11 + .../invalid/test-description-required.json | 9 + .../tests/invalid/test-description-type.json | 10 + .../invalid/test-expectEvents-items.json | 13 + .../tests/invalid/test-expectEvents-type.json | 11 + .../tests/invalid/test-operations-items.json | 12 + .../invalid/test-operations-required.json | 9 + .../tests/invalid/test-operations-type.json | 10 + .../tests/invalid/test-outcome-items.json | 13 + .../tests/invalid/test-outcome-minItems.json | 11 + .../tests/invalid/test-outcome-type.json | 11 + .../invalid/test-runOnRequirements-items.json | 13 + .../test-runOnRequirements-minItems.json | 11 + .../invalid/test-runOnRequirements-type.json | 11 + .../tests/invalid/test-skipReason-type.json | 11 + .../tests/invalid/tests-items.json | 7 + .../tests/invalid/tests-minItems.json | 5 + .../tests/invalid/tests-required.json | 4 + .../tests/invalid/tests-type.json | 5 + .../entity-bucket-database-undefined.json | 18 + .../entity-collection-database-undefined.json | 19 + .../entity-database-client-undefined.json | 19 + .../entity-session-client-undefined.json | 18 + .../returnDocument-enum-invalid.json | 66 +++ .../valid-fail/schemaVersion-unsupported.json | 10 + .../tests/valid-pass/poc-change-streams.json | 410 ++++++++++++++ .../valid-pass/poc-command-monitoring.json | 222 ++++++++ .../tests/valid-pass/poc-crud.json | 446 ++++++++++++++++ .../tests/valid-pass/poc-gridfs.json | 299 +++++++++++ .../tests/valid-pass/poc-retryable-reads.json | 433 +++++++++++++++ .../valid-pass/poc-retryable-writes.json | 481 +++++++++++++++++ .../tests/valid-pass/poc-sessions.json | 466 ++++++++++++++++ .../poc-transactions-convenient-api.json | 505 ++++++++++++++++++ .../poc-transactions-mongos-pin-auto.json | 409 ++++++++++++++ .../tests/valid-pass/poc-transactions.json | 322 +++++++++++ 160 files changed, 7284 insertions(+) create mode 100644 source/unified-test-format/tests/invalid/collectionData-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-collectionName-required.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-collectionName-type.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-databaseName-required.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-databaseName-type.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-documents-items.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-documents-required.json create mode 100644 source/unified-test-format/tests/invalid/collectionData-documents-type.json create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json create mode 100644 source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json create mode 100644 source/unified-test-format/tests/invalid/createEntities-items.json create mode 100644 source/unified-test-format/tests/invalid/createEntities-minItems.json create mode 100644 source/unified-test-format/tests/invalid/createEntities-type.json create mode 100644 source/unified-test-format/tests/invalid/description-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-database-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-database-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-id-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-bucket-id-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-id-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-id-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-database-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-database-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-id-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-collection-id-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-client-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-client-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-databaseName-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-databaseName-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-id-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-database-id-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-maxProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-minProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-session-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-session-client-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-session-client-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-session-id-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-session-id-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json create mode 100644 source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json create mode 100644 source/unified-test-format/tests/invalid/entity-stream-id-required.json create mode 100644 source/unified-test-format/tests/invalid/entity-stream-id-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorCode-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorContains-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-isClientError-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-isError-const.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-isError-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedError-minProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedEvent-minProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json create mode 100644 source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json create mode 100644 source/unified-test-format/tests/invalid/initialData-items.json create mode 100644 source/unified-test-format/tests/invalid/initialData-minItems.json create mode 100644 source/unified-test-format/tests/invalid/initialData-type.json create mode 100644 source/unified-test-format/tests/invalid/operation-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/operation-arguments-type.json create mode 100644 source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json create mode 100644 source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json create mode 100644 source/unified-test-format/tests/invalid/operation-expectError-type.json create mode 100644 source/unified-test-format/tests/invalid/operation-expectEvents-type.json create mode 100644 source/unified-test-format/tests/invalid/operation-name-required.json create mode 100644 source/unified-test-format/tests/invalid/operation-name-type.json create mode 100644 source/unified-test-format/tests/invalid/operation-object-required.json create mode 100644 source/unified-test-format/tests/invalid/operation-object-type.json create mode 100644 source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirements-items.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirements-minItems.json create mode 100644 source/unified-test-format/tests/invalid/runOnRequirements-type.json create mode 100644 source/unified-test-format/tests/invalid/schemaVersion-pattern.json create mode 100644 source/unified-test-format/tests/invalid/schemaVersion-required.json create mode 100644 source/unified-test-format/tests/invalid/schemaVersion-type.json create mode 100644 source/unified-test-format/tests/invalid/test-additionalProperties.json create mode 100644 source/unified-test-format/tests/invalid/test-description-required.json create mode 100644 source/unified-test-format/tests/invalid/test-description-type.json create mode 100644 source/unified-test-format/tests/invalid/test-expectEvents-items.json create mode 100644 source/unified-test-format/tests/invalid/test-expectEvents-type.json create mode 100644 source/unified-test-format/tests/invalid/test-operations-items.json create mode 100644 source/unified-test-format/tests/invalid/test-operations-required.json create mode 100644 source/unified-test-format/tests/invalid/test-operations-type.json create mode 100644 source/unified-test-format/tests/invalid/test-outcome-items.json create mode 100644 source/unified-test-format/tests/invalid/test-outcome-minItems.json create mode 100644 source/unified-test-format/tests/invalid/test-outcome-type.json create mode 100644 source/unified-test-format/tests/invalid/test-runOnRequirements-items.json create mode 100644 source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json create mode 100644 source/unified-test-format/tests/invalid/test-runOnRequirements-type.json create mode 100644 source/unified-test-format/tests/invalid/test-skipReason-type.json create mode 100644 source/unified-test-format/tests/invalid/tests-items.json create mode 100644 source/unified-test-format/tests/invalid/tests-minItems.json create mode 100644 source/unified-test-format/tests/invalid/tests-required.json create mode 100644 source/unified-test-format/tests/invalid/tests-type.json create mode 100644 source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json create mode 100644 source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json create mode 100644 source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json create mode 100644 source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json create mode 100644 source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json create mode 100644 source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-change-streams.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-command-monitoring.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-crud.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-gridfs.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-retryable-reads.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-retryable-writes.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-sessions.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-transactions.json diff --git a/source/unified-test-format/tests/invalid/collectionData-additionalProperties.json b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.json new file mode 100644 index 0000000000..2d85093109 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-additionalProperties.json @@ -0,0 +1,40 @@ +{ + "description": "collectionData-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo", + "documents": [], + "foo": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-required.json b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.json new file mode 100644 index 0000000000..040dd86a1c --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-required.json @@ -0,0 +1,38 @@ +{ + "description": "collectionData-collectionName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "databaseName": "foo", + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-collectionName-type.json b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.json new file mode 100644 index 0000000000..676d822e5e --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-collectionName-type.json @@ -0,0 +1,39 @@ +{ + "description": "collectionData-collectionName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": 0, + "databaseName": "foo", + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-required.json b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.json new file mode 100644 index 0000000000..7548f9d5be --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-required.json @@ -0,0 +1,38 @@ +{ + "description": "collectionData-databaseName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-databaseName-type.json b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.json new file mode 100644 index 0000000000..ef719bbf6a --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-databaseName-type.json @@ -0,0 +1,39 @@ +{ + "description": "collectionData-databaseName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": 0, + "documents": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-items.json b/source/unified-test-format/tests/invalid/collectionData-documents-items.json new file mode 100644 index 0000000000..2916718d50 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-items.json @@ -0,0 +1,41 @@ +{ + "description": "collectionData-documents-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo", + "documents": [ + 0 + ] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-required.json b/source/unified-test-format/tests/invalid/collectionData-documents-required.json new file mode 100644 index 0000000000..7b8a7ead2a --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-required.json @@ -0,0 +1,38 @@ +{ + "description": "collectionData-documents-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionData-documents-type.json b/source/unified-test-format/tests/invalid/collectionData-documents-type.json new file mode 100644 index 0000000000..953cabae6e --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionData-documents-type.json @@ -0,0 +1,39 @@ +{ + "description": "collectionData-documents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "initialData": [ + { + "collectionName": "foo", + "databaseName": "foo", + "documents": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json new file mode 100644 index 0000000000..beef260eed --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-additionalProperties.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "foo": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json new file mode 100644 index 0000000000..1b9f4bcbea --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readConcern-type.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-readConcern-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "readConcern": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json new file mode 100644 index 0000000000..988b594d13 --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-readPreference-type.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-readPreference-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "readPreference": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json new file mode 100644 index 0000000000..bd2157c5cb --- /dev/null +++ b/source/unified-test-format/tests/invalid/collectionOrDatabaseOptions-writeConcern-type.json @@ -0,0 +1,27 @@ +{ + "description": "collectionOrDatabaseOptions-writeConcern-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": { + "writeConcern": 0 + } + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/createEntities-items.json b/source/unified-test-format/tests/invalid/createEntities-items.json new file mode 100644 index 0000000000..8e9d6ff702 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-items.json @@ -0,0 +1,13 @@ +{ + "description": "createEntities-items", + "schemaVersion": "1.0", + "createEntities": [ + 0 + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/createEntities-minItems.json b/source/unified-test-format/tests/invalid/createEntities-minItems.json new file mode 100644 index 0000000000..3654923d28 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "createEntities-minItems", + "schemaVersion": "1.0", + "createEntities": [], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/createEntities-type.json b/source/unified-test-format/tests/invalid/createEntities-type.json new file mode 100644 index 0000000000..ce3c382c93 --- /dev/null +++ b/source/unified-test-format/tests/invalid/createEntities-type.json @@ -0,0 +1,11 @@ +{ + "description": "createEntities-type", + "schemaVersion": "1.0", + "createEntities": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/description-required.json b/source/unified-test-format/tests/invalid/description-required.json new file mode 100644 index 0000000000..e4e0d0efdf --- /dev/null +++ b/source/unified-test-format/tests/invalid/description-required.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-additionalProperties.json new file mode 100644 index 0000000000..38b8898787 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-additionalProperties.json @@ -0,0 +1,15 @@ +{ + "description": "entity-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "foo": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json new file mode 100644 index 0000000000..46f9b4038e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-additionalProperties.json @@ -0,0 +1,31 @@ +{ + "description": "entity-bucket-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json new file mode 100644 index 0000000000..c3d7423e65 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-bucketOptions-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-bucket-bucketOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0", + "bucketOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-required.json b/source/unified-test-format/tests/invalid/entity-bucket-database-required.json new file mode 100644 index 0000000000..1fde5a96c9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-required.json @@ -0,0 +1,29 @@ +{ + "description": "entity-bucket-database-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-database-type.json b/source/unified-test-format/tests/invalid/entity-bucket-database-type.json new file mode 100644 index 0000000000..798d273fb0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-database-type.json @@ -0,0 +1,30 @@ +{ + "description": "entity-bucket-database-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": "bucket0", + "database": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-required.json b/source/unified-test-format/tests/invalid/entity-bucket-id-required.json new file mode 100644 index 0000000000..c547d8ea3c --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-required.json @@ -0,0 +1,29 @@ +{ + "description": "entity-bucket-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "database": "database0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-bucket-id-type.json b/source/unified-test-format/tests/invalid/entity-bucket-id-type.json new file mode 100644 index 0000000000..f4e10ee630 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-bucket-id-type.json @@ -0,0 +1,30 @@ +{ + "description": "entity-bucket-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "bucket": { + "id": 0, + "database": "database0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.json new file mode 100644 index 0000000000..467e1d6ae1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-additionalProperties.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-id-required.json b/source/unified-test-format/tests/invalid/entity-client-id-required.json new file mode 100644 index 0000000000..4be2fbf8e8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-required.json @@ -0,0 +1,15 @@ +{ + "description": "entity-client-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": {} + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-id-type.json b/source/unified-test-format/tests/invalid/entity-client-id-type.json new file mode 100644 index 0000000000..cdc7cbc0e7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-id-type.json @@ -0,0 +1,17 @@ +{ + "description": "entity-client-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json new file mode 100644 index 0000000000..1252ac82d7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-items.json @@ -0,0 +1,20 @@ +{ + "description": "entity-client-ignoreCommandMonitoringEvents-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "ignoreCommandMonitoringEvents": [ + 0 + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json new file mode 100644 index 0000000000..e78068a442 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-minItems.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-ignoreCommandMonitoringEvents-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "ignoreCommandMonitoringEvents": [] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json new file mode 100644 index 0000000000..5ac2b340c5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-ignoreCommandMonitoringEvents-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-ignoreCommandMonitoringEvents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "ignoreCommandMonitoringEvents": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json new file mode 100644 index 0000000000..c39c94eee2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-enum.json @@ -0,0 +1,20 @@ +{ + "description": "entity-client-observeEvents-enum", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "foo" + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json new file mode 100644 index 0000000000..3aee11e3d5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-items.json @@ -0,0 +1,20 @@ +{ + "description": "entity-client-observeEvents-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + 0 + ] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json new file mode 100644 index 0000000000..e70d90c0a7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-minItems.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-observeEvents-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [] + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json new file mode 100644 index 0000000000..c144e32369 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-observeEvents-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-observeEvents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json new file mode 100644 index 0000000000..4252480e98 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-uriOptions-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-uriOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "uriOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json new file mode 100644 index 0000000000..e429cd71f8 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-client-useMultipleMongoses-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-client-useMultipleMongoses-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json new file mode 100644 index 0000000000..90ee2b1ca0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-additionalProperties.json @@ -0,0 +1,32 @@ +{ + "description": "entity-collection-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json new file mode 100644 index 0000000000..2446722e5e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-required.json @@ -0,0 +1,30 @@ +{ + "description": "entity-collection-collectionName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json new file mode 100644 index 0000000000..ccad66aac9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionName-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-collection-collectionName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json new file mode 100644 index 0000000000..52220c1cd1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-collectionOptions-type.json @@ -0,0 +1,32 @@ +{ + "description": "entity-collection-collectionOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "foo", + "collectionOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-required.json b/source/unified-test-format/tests/invalid/entity-collection-database-required.json new file mode 100644 index 0000000000..ba96b43f76 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-required.json @@ -0,0 +1,30 @@ +{ + "description": "entity-collection-database-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-database-type.json b/source/unified-test-format/tests/invalid/entity-collection-database-type.json new file mode 100644 index 0000000000..b87134498d --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-database-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-collection-database-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": "collection0", + "database": 0, + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-required.json b/source/unified-test-format/tests/invalid/entity-collection-id-required.json new file mode 100644 index 0000000000..84e5352ead --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-required.json @@ -0,0 +1,30 @@ +{ + "description": "entity-collection-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "database": "database0", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-collection-id-type.json b/source/unified-test-format/tests/invalid/entity-collection-id-type.json new file mode 100644 index 0000000000..f0821e5250 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-collection-id-type.json @@ -0,0 +1,31 @@ +{ + "description": "entity-collection-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + }, + { + "collection": { + "id": 0, + "database": "database0", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.json new file mode 100644 index 0000000000..964cd27966 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-additionalProperties.json @@ -0,0 +1,25 @@ +{ + "description": "entity-database-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-client-required.json b/source/unified-test-format/tests/invalid/entity-database-client-required.json new file mode 100644 index 0000000000..54f99cf13e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-required.json @@ -0,0 +1,23 @@ +{ + "description": "entity-database-client-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-client-type.json b/source/unified-test-format/tests/invalid/entity-database-client-type.json new file mode 100644 index 0000000000..ff4584c405 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-client-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-database-client-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": 0, + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-required.json b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.json new file mode 100644 index 0000000000..64cca95c49 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-required.json @@ -0,0 +1,23 @@ +{ + "description": "entity-database-databaseName-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseName-type.json b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.json new file mode 100644 index 0000000000..bd01aef781 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseName-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-database-databaseName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json new file mode 100644 index 0000000000..bc22ad3129 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-databaseOptions-type.json @@ -0,0 +1,25 @@ +{ + "description": "entity-database-databaseOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo", + "databaseOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-id-required.json b/source/unified-test-format/tests/invalid/entity-database-id-required.json new file mode 100644 index 0000000000..0b65cf1159 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-required.json @@ -0,0 +1,23 @@ +{ + "description": "entity-database-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "client": "client0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-database-id-type.json b/source/unified-test-format/tests/invalid/entity-database-id-type.json new file mode 100644 index 0000000000..98b5789d04 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-database-id-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-database-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": 0, + "client": "client0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-maxProperties.json b/source/unified-test-format/tests/invalid/entity-maxProperties.json new file mode 100644 index 0000000000..f4a6b7c914 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-maxProperties.json @@ -0,0 +1,22 @@ +{ + "description": "entity-maxProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + }, + "database": { + "id": "database0", + "client": "client0", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-minProperties.json b/source/unified-test-format/tests/invalid/entity-minProperties.json new file mode 100644 index 0000000000..d89949ce30 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-minProperties.json @@ -0,0 +1,13 @@ +{ + "description": "entity-minProperties", + "schemaVersion": "1.0", + "createEntities": [ + {} + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.json new file mode 100644 index 0000000000..ab4cd2014f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-additionalProperties.json @@ -0,0 +1,24 @@ +{ + "description": "entity-session-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0", + "client": "client0", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-client-required.json b/source/unified-test-format/tests/invalid/entity-session-client-required.json new file mode 100644 index 0000000000..8c9ed72e99 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-required.json @@ -0,0 +1,22 @@ +{ + "description": "entity-session-client-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-client-type.json b/source/unified-test-format/tests/invalid/entity-session-client-type.json new file mode 100644 index 0000000000..b5ccc3f60f --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-client-type.json @@ -0,0 +1,23 @@ +{ + "description": "entity-session-client-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0", + "client": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-id-required.json b/source/unified-test-format/tests/invalid/entity-session-id-required.json new file mode 100644 index 0000000000..3e5d5c5439 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-required.json @@ -0,0 +1,22 @@ +{ + "description": "entity-session-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "client": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-id-type.json b/source/unified-test-format/tests/invalid/entity-session-id-type.json new file mode 100644 index 0000000000..dcd46e5be7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-id-type.json @@ -0,0 +1,23 @@ +{ + "description": "entity-session-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": 0, + "client": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json new file mode 100644 index 0000000000..0ee15891eb --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-session-sessionOptions-type.json @@ -0,0 +1,24 @@ +{ + "description": "entity-session-sessionOptions-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "session": { + "id": "session0", + "client": "client0", + "sessionOptions": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json new file mode 100644 index 0000000000..c8e76e9985 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-additionalProperties.json @@ -0,0 +1,19 @@ +{ + "description": "entity-stream-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0", + "hexBytes": "FF", + "foo": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json new file mode 100644 index 0000000000..7381893b55 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-pattern.json @@ -0,0 +1,18 @@ +{ + "description": "entity-stream-hexBytes-pattern", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0", + "hexBytes": "FFF" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json new file mode 100644 index 0000000000..cc3bf09b20 --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-required.json @@ -0,0 +1,17 @@ +{ + "description": "entity-stream-hexBytes-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json new file mode 100644 index 0000000000..e6e2299eac --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-hexBytes-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-stream-hexBytes-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": "stream0", + "hexBytes": 0 + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-required.json b/source/unified-test-format/tests/invalid/entity-stream-id-required.json new file mode 100644 index 0000000000..ff814d4e9c --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-required.json @@ -0,0 +1,17 @@ +{ + "description": "entity-stream-id-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "hexBytes": "FF" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/entity-stream-id-type.json b/source/unified-test-format/tests/invalid/entity-stream-id-type.json new file mode 100644 index 0000000000..5fc654d97e --- /dev/null +++ b/source/unified-test-format/tests/invalid/entity-stream-id-type.json @@ -0,0 +1,18 @@ +{ + "description": "entity-stream-id-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "stream": { + "id": 0, + "hexBytes": "FF" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.json new file mode 100644 index 0000000000..3a79df8e34 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-additionalProperties.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "foo": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCode-type.json b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.json new file mode 100644 index 0000000000..b6b6f5d05a --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCode-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorCode-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorCode": "foo" + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json new file mode 100644 index 0000000000..3ac5e43045 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorCodeName-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorCodeName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorCodeName": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorContains-type.json b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.json new file mode 100644 index 0000000000..847a987dff --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorContains-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorContains-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorContains": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json new file mode 100644 index 0000000000..4eab56ad18 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-items.json @@ -0,0 +1,27 @@ +{ + "description": "expectedError-errorLabelsContain-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsContain": [ + 0 + ] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json new file mode 100644 index 0000000000..48162110aa --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-minItems.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsContain-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsContain": [] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json new file mode 100644 index 0000000000..a0aba918b5 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsContain-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsContain-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsContain": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json new file mode 100644 index 0000000000..6c94d07135 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-items.json @@ -0,0 +1,27 @@ +{ + "description": "expectedError-errorLabelsOmit-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsOmit": [ + 0 + ] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json new file mode 100644 index 0000000000..88c6582028 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-minItems.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsOmit-minItems", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsOmit": [] + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json new file mode 100644 index 0000000000..5f57114fea --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-errorLabelsOmit-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-errorLabelsOmit-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "errorLabelsOmit": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-isClientError-type.json b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.json new file mode 100644 index 0000000000..bfcc06679b --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isClientError-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-isClientError-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isClientError": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-const.json b/source/unified-test-format/tests/invalid/expectedError-isError-const.json new file mode 100644 index 0000000000..6a398bbf22 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-const.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-isError-const", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": false + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-isError-type.json b/source/unified-test-format/tests/invalid/expectedError-isError-type.json new file mode 100644 index 0000000000..354aff31f4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-isError-type.json @@ -0,0 +1,25 @@ +{ + "description": "expectedError-isError-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": 0 + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedError-minProperties.json b/source/unified-test-format/tests/invalid/expectedError-minProperties.json new file mode 100644 index 0000000000..10e0b89ab7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedError-minProperties.json @@ -0,0 +1,23 @@ +{ + "description": "expectedError-minProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": {} + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json new file mode 100644 index 0000000000..2c4f7d27e7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-additionalProperties.json @@ -0,0 +1,32 @@ +{ + "description": "expectedEvent-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "foo": 0 + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json new file mode 100644 index 0000000000..ea6078faae --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandFailedEvent-commandName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandFailedEvent-commandName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandFailedEvent": { + "commandName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json new file mode 100644 index 0000000000..ee6eb50658 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-additionalProperties.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "foo": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json new file mode 100644 index 0000000000..4c9483caf3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-command-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-command-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json new file mode 100644 index 0000000000..a5a66096a0 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-commandName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-commandName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "commandName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json new file mode 100644 index 0000000000..dc040ec108 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandStartedEvent-databaseName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandStartedEvent-databaseName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json new file mode 100644 index 0000000000..4a20e906b9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-commandName-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandSucceededEvent-commandName-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandSucceededEvent": { + "commandName": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json new file mode 100644 index 0000000000..5464542751 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-commandSucceededEvent-reply-type.json @@ -0,0 +1,34 @@ +{ + "description": "expectedEvent-commandSucceededEvent-reply-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandSucceededEvent": { + "reply": 0 + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json new file mode 100644 index 0000000000..f01441946f --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-maxProperties.json @@ -0,0 +1,33 @@ +{ + "description": "expectedEvent-maxProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": {}, + "commandSucceededEvent": {} + } + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEvent-minProperties.json b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.json new file mode 100644 index 0000000000..ebcc494894 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEvent-minProperties.json @@ -0,0 +1,30 @@ +{ + "description": "expectedEvent-minProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + {} + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json new file mode 100644 index 0000000000..6ecf5931fb --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-additionalProperties.json @@ -0,0 +1,29 @@ +{ + "description": "expectedEventsForClient-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [], + "foo": 0 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json new file mode 100644 index 0000000000..b879db8598 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-required.json @@ -0,0 +1,27 @@ +{ + "description": "expectedEventsForClient-client-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "events": [] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json new file mode 100644 index 0000000000..4ee5427df1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-client-type.json @@ -0,0 +1,28 @@ +{ + "description": "expectedEventsForClient-client-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": 0, + "events": [] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json new file mode 100644 index 0000000000..ee8ce4a403 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-items.json @@ -0,0 +1,30 @@ +{ + "description": "expectedEventsForClient-events-items", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": [ + 0 + ] + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json new file mode 100644 index 0000000000..7f1bc6fb53 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-required.json @@ -0,0 +1,27 @@ +{ + "description": "expectedEventsForClient-events-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0" + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json new file mode 100644 index 0000000000..f171fc2b93 --- /dev/null +++ b/source/unified-test-format/tests/invalid/expectedEventsForClient-events-type.json @@ -0,0 +1,28 @@ +{ + "description": "expectedEventsForClient-events-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": [ + { + "client": "client0", + "events": 0 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/initialData-items.json b/source/unified-test-format/tests/invalid/initialData-items.json new file mode 100644 index 0000000000..9c27d554f9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-items.json @@ -0,0 +1,13 @@ +{ + "description": "initialData-items", + "schemaVersion": "1.0", + "initialData": [ + 0 + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/initialData-minItems.json b/source/unified-test-format/tests/invalid/initialData-minItems.json new file mode 100644 index 0000000000..984100a2be --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "initialData-minItems", + "schemaVersion": "1.0", + "initialData": [], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/initialData-type.json b/source/unified-test-format/tests/invalid/initialData-type.json new file mode 100644 index 0000000000..c33585e03a --- /dev/null +++ b/source/unified-test-format/tests/invalid/initialData-type.json @@ -0,0 +1,11 @@ +{ + "description": "initialData-type", + "schemaVersion": "1.0", + "initialData": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-additionalProperties.json b/source/unified-test-format/tests/invalid/operation-additionalProperties.json new file mode 100644 index 0000000000..8f2f1434ec --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-additionalProperties.json @@ -0,0 +1,23 @@ +{ + "description": "operation-additionalProperties", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "foo": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-arguments-type.json b/source/unified-test-format/tests/invalid/operation-arguments-type.json new file mode 100644 index 0000000000..a22f3921c3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-arguments-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-arguments-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "arguments": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json new file mode 100644 index 0000000000..bc15fbac76 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_expectResult.json @@ -0,0 +1,26 @@ +{ + "description": "operation-expectError-conflicts_with_expectResult", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": true + }, + "expectResult": {} + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json new file mode 100644 index 0000000000..dead4a3b9d --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-conflicts_with_saveResultAsEntity.json @@ -0,0 +1,26 @@ +{ + "description": "operation-expectError-conflicts_with_saveResultAsEntity", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": { + "isError": true + }, + "saveResultAsEntity": "foo" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectError-type.json b/source/unified-test-format/tests/invalid/operation-expectError-type.json new file mode 100644 index 0000000000..b224ba3535 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectError-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-expectError-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectError": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-expectEvents-type.json b/source/unified-test-format/tests/invalid/operation-expectEvents-type.json new file mode 100644 index 0000000000..ecd4c011a9 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-expectEvents-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-expectEvents-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "expectEvents": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-name-required.json b/source/unified-test-format/tests/invalid/operation-name-required.json new file mode 100644 index 0000000000..42fcb3a308 --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-required.json @@ -0,0 +1,21 @@ +{ + "description": "operation-name-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "object": "client0" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-name-type.json b/source/unified-test-format/tests/invalid/operation-name-type.json new file mode 100644 index 0000000000..2f91da078a --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-name-type.json @@ -0,0 +1,22 @@ +{ + "description": "operation-name-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": 0, + "object": "client0" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-object-required.json b/source/unified-test-format/tests/invalid/operation-object-required.json new file mode 100644 index 0000000000..c0410ce3fd --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-required.json @@ -0,0 +1,21 @@ +{ + "description": "operation-object-required", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo" + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-object-type.json b/source/unified-test-format/tests/invalid/operation-object-type.json new file mode 100644 index 0000000000..edb0a0b51a --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-object-type.json @@ -0,0 +1,22 @@ +{ + "description": "operation-object-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json new file mode 100644 index 0000000000..65ead94c7a --- /dev/null +++ b/source/unified-test-format/tests/invalid/operation-saveResultAsEntity-type.json @@ -0,0 +1,23 @@ +{ + "description": "operation-saveResultAsEntity-type", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [ + { + "name": "foo", + "object": "client0", + "saveResultAsEntity": 0 + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json new file mode 100644 index 0000000000..79fa687e45 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-additionalProperties.json @@ -0,0 +1,16 @@ +{ + "description": "runOnRequirement-additionalProperties", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "foo": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json new file mode 100644 index 0000000000..78766eb925 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-pattern.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-maxServerVersion-pattern", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "maxServerVersion": "1.2.3.4" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json new file mode 100644 index 0000000000..ffc9118ba2 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-maxServerVersion-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-maxServerVersion-type", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "maxServerVersion": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json new file mode 100644 index 0000000000..c2bfed3be7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minProperties.json @@ -0,0 +1,13 @@ +{ + "description": "runOnRequirement-minProperties", + "schemaVersion": "1.0", + "runOnRequirements": [ + {} + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json new file mode 100644 index 0000000000..19abc1755f --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-pattern.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-minServerVersion-pattern", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "1.2.3.4" + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json new file mode 100644 index 0000000000..688d1c67ee --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-minServerVersion-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-minServerVersion-type", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json new file mode 100644 index 0000000000..f62e5040d4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-enum.json @@ -0,0 +1,17 @@ +{ + "description": "runOnRequirement-topologies-enum", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": [ + "foo" + ] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json new file mode 100644 index 0000000000..a205b3293d --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-items.json @@ -0,0 +1,17 @@ +{ + "description": "runOnRequirement-topologies-items", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": [ + 0 + ] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json new file mode 100644 index 0000000000..16f29b3f4b --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-minItems.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-topologies-minItems", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": [] + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json new file mode 100644 index 0000000000..f6d147cd6f --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirement-topologies-type.json @@ -0,0 +1,15 @@ +{ + "description": "runOnRequirement-topologies-type", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "topologies": 0 + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-items.json b/source/unified-test-format/tests/invalid/runOnRequirements-items.json new file mode 100644 index 0000000000..40ec84a3f3 --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-items.json @@ -0,0 +1,13 @@ +{ + "description": "runOnRequirements-items", + "schemaVersion": "1.0", + "runOnRequirements": [ + 0 + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-minItems.json b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.json new file mode 100644 index 0000000000..4ca9f99b5d --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "runOnRequirements-minItems", + "schemaVersion": "1.0", + "runOnRequirements": [], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/runOnRequirements-type.json b/source/unified-test-format/tests/invalid/runOnRequirements-type.json new file mode 100644 index 0000000000..98b859f3ea --- /dev/null +++ b/source/unified-test-format/tests/invalid/runOnRequirements-type.json @@ -0,0 +1,11 @@ +{ + "description": "runOnRequirements-type", + "schemaVersion": "1.0", + "runOnRequirements": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/schemaVersion-pattern.json b/source/unified-test-format/tests/invalid/schemaVersion-pattern.json new file mode 100644 index 0000000000..bcb8980516 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-pattern.json @@ -0,0 +1,10 @@ +{ + "description": "schemaVersion-pattern", + "schemaVersion": "1.2.3.4", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/schemaVersion-required.json b/source/unified-test-format/tests/invalid/schemaVersion-required.json new file mode 100644 index 0000000000..7388ff0bf1 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-required.json @@ -0,0 +1,9 @@ +{ + "description": "schemaVersion-required", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/schemaVersion-type.json b/source/unified-test-format/tests/invalid/schemaVersion-type.json new file mode 100644 index 0000000000..646473a209 --- /dev/null +++ b/source/unified-test-format/tests/invalid/schemaVersion-type.json @@ -0,0 +1,10 @@ +{ + "description": "schemaVersion-type", + "schemaVersion": 0, + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-additionalProperties.json b/source/unified-test-format/tests/invalid/test-additionalProperties.json new file mode 100644 index 0000000000..a699319c30 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-additionalProperties.json @@ -0,0 +1,11 @@ +{ + "description": "test-additionalProperties", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "foo": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-description-required.json b/source/unified-test-format/tests/invalid/test-description-required.json new file mode 100644 index 0000000000..8bf23014d4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-required.json @@ -0,0 +1,9 @@ +{ + "description": "test-description-required", + "schemaVersion": "1.0", + "tests": [ + { + "operation": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-description-type.json b/source/unified-test-format/tests/invalid/test-description-type.json new file mode 100644 index 0000000000..bba3690449 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-description-type.json @@ -0,0 +1,10 @@ +{ + "description": "test-description-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": 0, + "operation": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-items.json b/source/unified-test-format/tests/invalid/test-expectEvents-items.json new file mode 100644 index 0000000000..394f74746c --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-items.json @@ -0,0 +1,13 @@ +{ + "description": "test-expectEvents-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-expectEvents-type.json b/source/unified-test-format/tests/invalid/test-expectEvents-type.json new file mode 100644 index 0000000000..1569f0a0d7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-expectEvents-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-expectEvents-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "expectEvents": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-operations-items.json b/source/unified-test-format/tests/invalid/test-operations-items.json new file mode 100644 index 0000000000..00af8e7453 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-items.json @@ -0,0 +1,12 @@ +{ + "description": "test-operations-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-operations-required.json b/source/unified-test-format/tests/invalid/test-operations-required.json new file mode 100644 index 0000000000..67c6f83044 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-required.json @@ -0,0 +1,9 @@ +{ + "description": "test-operations-required", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo" + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-operations-type.json b/source/unified-test-format/tests/invalid/test-operations-type.json new file mode 100644 index 0000000000..1e8b5b2496 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-operations-type.json @@ -0,0 +1,10 @@ +{ + "description": "test-operations-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-outcome-items.json b/source/unified-test-format/tests/invalid/test-outcome-items.json new file mode 100644 index 0000000000..cf6bb54f87 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-items.json @@ -0,0 +1,13 @@ +{ + "description": "test-outcome-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "outcome": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-outcome-minItems.json b/source/unified-test-format/tests/invalid/test-outcome-minItems.json new file mode 100644 index 0000000000..aadf8e514a --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "test-outcome-minItems", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "outcome": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-outcome-type.json b/source/unified-test-format/tests/invalid/test-outcome-type.json new file mode 100644 index 0000000000..e60c119d7e --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-outcome-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-outcome-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "outcome": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-items.json b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.json new file mode 100644 index 0000000000..866bebb51f --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-items.json @@ -0,0 +1,13 @@ +{ + "description": "test-runOnRequirements-items", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "runOnRequirements": [ + 0 + ] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json new file mode 100644 index 0000000000..d61f063849 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-minItems.json @@ -0,0 +1,11 @@ +{ + "description": "test-runOnRequirements-minItems", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "runOnRequirements": [] + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-runOnRequirements-type.json b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.json new file mode 100644 index 0000000000..5b25b1005d --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-runOnRequirements-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-runOnRequirements-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "runOnRequirements": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/test-skipReason-type.json b/source/unified-test-format/tests/invalid/test-skipReason-type.json new file mode 100644 index 0000000000..0408e76834 --- /dev/null +++ b/source/unified-test-format/tests/invalid/test-skipReason-type.json @@ -0,0 +1,11 @@ +{ + "description": "test-skipReason-type", + "schemaVersion": "1.0", + "tests": [ + { + "description": "foo", + "operations": [], + "skipReason": 0 + } + ] +} diff --git a/source/unified-test-format/tests/invalid/tests-items.json b/source/unified-test-format/tests/invalid/tests-items.json new file mode 100644 index 0000000000..11f37469e4 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-items.json @@ -0,0 +1,7 @@ +{ + "description": "tests-items", + "schemaVersion": "1.0", + "tests": [ + 0 + ] +} diff --git a/source/unified-test-format/tests/invalid/tests-minItems.json b/source/unified-test-format/tests/invalid/tests-minItems.json new file mode 100644 index 0000000000..3f74f94af7 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-minItems.json @@ -0,0 +1,5 @@ +{ + "description": "tests-minItems", + "schemaVersion": "1.0", + "tests": [] +} diff --git a/source/unified-test-format/tests/invalid/tests-required.json b/source/unified-test-format/tests/invalid/tests-required.json new file mode 100644 index 0000000000..de4b2fd063 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-required.json @@ -0,0 +1,4 @@ +{ + "description": "tests-required", + "schemaVersion": "1.0" +} diff --git a/source/unified-test-format/tests/invalid/tests-type.json b/source/unified-test-format/tests/invalid/tests-type.json new file mode 100644 index 0000000000..62d8194a41 --- /dev/null +++ b/source/unified-test-format/tests/invalid/tests-type.json @@ -0,0 +1,5 @@ +{ + "description": "tests-type", + "schemaVersion": "1.0", + "tests": 0 +} diff --git a/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json new file mode 100644 index 0000000000..7f7f1978c3 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-bucket-database-undefined.json @@ -0,0 +1,18 @@ +{ + "description": "entity-bucket-database-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "bucket": { + "id": "bucket0", + "database": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json new file mode 100644 index 0000000000..20b0733e34 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-collection-database-undefined.json @@ -0,0 +1,19 @@ +{ + "description": "entity-collection-database-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "collection": { + "id": "collection0", + "database": "foo", + "collectionName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json new file mode 100644 index 0000000000..0f8110e6d3 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-database-client-undefined.json @@ -0,0 +1,19 @@ +{ + "description": "entity-database-client-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "database": { + "id": "database0", + "client": "foo", + "databaseName": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json new file mode 100644 index 0000000000..260356436a --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/entity-session-client-undefined.json @@ -0,0 +1,18 @@ +{ + "description": "entity-session-client-undefined", + "schemaVersion": "1.0", + "createEntities": [ + { + "session": { + "id": "session0", + "client": "foo" + } + } + ], + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json new file mode 100644 index 0000000000..ea425fb568 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/returnDocument-enum-invalid.json @@ -0,0 +1,66 @@ +{ + "description": "returnDocument-enum-invalid", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "test" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + } + ], + "tests": [ + { + "description": "FindOneAndReplace returnDocument invalid enum value", + "operations": [ + { + "name": "findOneAndReplace", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "_id": 1, + "x": 111 + }, + "returnDocument": "invalid" + } + } + ] + }, + { + "description": "FindOneAndUpdate returnDocument invalid enum value", + "operations": [ + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "invalid" + } + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json new file mode 100644 index 0000000000..ceb5532917 --- /dev/null +++ b/source/unified-test-format/tests/valid-fail/schemaVersion-unsupported.json @@ -0,0 +1,10 @@ +{ + "description": "schemaVersion-unsupported", + "schemaVersion": "0.1", + "tests": [ + { + "description": "foo", + "operations": [] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-change-streams.json b/source/unified-test-format/tests/valid-pass/poc-change-streams.json new file mode 100644 index 0000000000..dc6e332e3e --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-change-streams.json @@ -0,0 +1,410 @@ +{ + "description": "poc-change-streams", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ], + "ignoreCommandMonitoringEvents": [ + "getMore", + "killCursors" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "change-stream-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "client": { + "id": "client1", + "useMultipleMongoses": false + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "change-stream-tests" + } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "change-stream-tests-2" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "test" + } + }, + { + "collection": { + "id": "collection2", + "database": "database1", + "collectionName": "test2" + } + }, + { + "collection": { + "id": "collection3", + "database": "database2", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "change-stream-tests", + "documents": [] + }, + { + "collectionName": "test2", + "databaseName": "change-stream-tests", + "documents": [] + }, + { + "collectionName": "test", + "databaseName": "change-stream-tests-2", + "documents": [] + } + ], + "tests": [ + { + "description": "Executing a watch helper on a MongoClient results in notifications for changes to all collections in all databases in the cluster.", + "runOnRequirements": [ + { + "minServerVersion": "3.8.0", + "topologies": [ + "replicaset" + ] + } + ], + "operations": [ + { + "name": "createChangeStream", + "object": "client0", + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection2", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "insertOne", + "object": "collection3", + "arguments": { + "document": { + "y": 1 + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "z": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test2" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests-2", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "y": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "z": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": 1, + "cursor": {}, + "pipeline": [ + { + "$changeStream": { + "allChangesForCluster": true, + "fullDocument": { + "$$unsetOrMatches": "default" + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "Test consecutive resume", + "runOnRequirements": [ + { + "minServerVersion": "4.1.7", + "topologies": [ + "replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "getMore" + ], + "closeConnection": true + } + } + } + }, + { + "name": "createChangeStream", + "object": "collection0", + "arguments": { + "batchSize": 1 + }, + "saveResultAsEntity": "changeStream0" + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "x": 1 + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "x": 2 + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "x": 3 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 1 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 2 + } + } + }, + { + "name": "iterateUntilDocumentOrError", + "object": "changeStream0", + "expectResult": { + "operationType": "insert", + "ns": { + "db": "change-stream-tests", + "coll": "test" + }, + "fullDocument": { + "_id": { + "$$type": "objectId" + }, + "x": 3 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "test", + "cursor": { + "batchSize": 1 + }, + "pipeline": [ + { + "$changeStream": { + "fullDocument": { + "$$unsetOrMatches": "default" + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "change-stream-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "aggregate": "test", + "cursor": { + "batchSize": 1 + }, + "pipeline": [ + { + "$changeStream": { + "fullDocument": { + "$$unsetOrMatches": "default" + }, + "resumeAfter": { + "$$exists": true + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "change-stream-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "aggregate": "test", + "cursor": { + "batchSize": 1 + }, + "pipeline": [ + { + "$changeStream": { + "fullDocument": { + "$$unsetOrMatches": "default" + }, + "resumeAfter": { + "$$exists": true + } + } + } + ] + }, + "commandName": "aggregate", + "databaseName": "change-stream-tests" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-command-monitoring.json b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.json new file mode 100644 index 0000000000..499396e0ba --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-command-monitoring.json @@ -0,0 +1,222 @@ +{ + "description": "poc-command-monitoring", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "command-monitoring-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "command-monitoring-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + }, + { + "_id": 5, + "x": 55 + } + ] + } + ], + "tests": [ + { + "description": "A successful find event with a getmore and the server kills the cursor", + "runOnRequirements": [ + { + "minServerVersion": "3.1", + "topologies": [ + "single", + "replicaset" + ] + } + ], + "operations": [ + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3, + "limit": 4 + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": { + "$gte": 1 + } + }, + "sort": { + "_id": 1 + }, + "batchSize": 3, + "limit": 4 + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": { + "$$type": [ + "int", + "long" + ] + }, + "ns": "command-monitoring-tests.test", + "firstBatch": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + }, + "commandName": "find" + } + }, + { + "commandStartedEvent": { + "command": { + "getMore": { + "$$type": [ + "int", + "long" + ] + }, + "collection": "test", + "batchSize": 1 + }, + "commandName": "getMore", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandSucceededEvent": { + "reply": { + "ok": 1, + "cursor": { + "id": 0, + "ns": "command-monitoring-tests.test", + "nextBatch": [ + { + "_id": 4, + "x": 44 + } + ] + } + }, + "commandName": "getMore" + } + } + ] + } + ] + }, + { + "description": "A failed find event", + "operations": [ + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "$or": true + } + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "$or": true + } + }, + "commandName": "find", + "databaseName": "command-monitoring-tests" + } + }, + { + "commandFailedEvent": { + "commandName": "find" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-crud.json b/source/unified-test-format/tests/valid-pass/poc-crud.json new file mode 100644 index 0000000000..2ed86d6150 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-crud.json @@ -0,0 +1,446 @@ +{ + "description": "poc-crud", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "admin" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "coll1" + } + }, + { + "collection": { + "id": "collection2", + "database": "database0", + "collectionName": "coll2", + "collectionOptions": { + "readConcern": { + "level": "majority" + } + } + } + } + ], + "initialData": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + }, + { + "collectionName": "coll1", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + }, + { + "collectionName": "coll2", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + }, + { + "collectionName": "aggregate_out", + "databaseName": "crud-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "BulkWrite with mixed ordered operations", + "operations": [ + { + "name": "bulkWrite", + "object": "collection0", + "arguments": { + "requests": [ + { + "insertOne": { + "document": { + "_id": 3, + "x": 33 + } + } + }, + { + "updateOne": { + "filter": { + "_id": 2 + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "updateMany": { + "filter": { + "_id": { + "$gt": 1 + } + }, + "update": { + "$inc": { + "x": 1 + } + } + } + }, + { + "insertOne": { + "document": { + "_id": 4, + "x": 44 + } + } + }, + { + "deleteMany": { + "filter": { + "x": { + "$nin": [ + 24, + 34 + ] + } + } + } + }, + { + "replaceOne": { + "filter": { + "_id": 4 + }, + "replacement": { + "_id": 4, + "x": 44 + }, + "upsert": true + } + } + ], + "ordered": true + }, + "expectResult": { + "deletedCount": 2, + "insertedCount": 2, + "insertedIds": { + "$$unsetOrMatches": { + "0": 3, + "3": 4 + } + }, + "matchedCount": 3, + "modifiedCount": 3, + "upsertedCount": 1, + "upsertedIds": { + "5": 4 + } + } + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 24 + }, + { + "_id": 3, + "x": 34 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "InsertMany continue-on-error behavior with unordered (duplicate key in requests)", + "operations": [ + { + "name": "insertMany", + "object": "collection1", + "arguments": { + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ], + "ordered": false + }, + "expectError": { + "expectResult": { + "deletedCount": 0, + "insertedCount": 2, + "matchedCount": 0, + "modifiedCount": 0, + "upsertedCount": 0, + "upsertedIds": {} + } + } + } + ], + "outcome": [ + { + "collectionName": "coll1", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "ReplaceOne prohibits atomic modifiers", + "operations": [ + { + "name": "replaceOne", + "object": "collection1", + "arguments": { + "filter": { + "_id": 1 + }, + "replacement": { + "$set": { + "x": 22 + } + } + }, + "expectError": { + "isClientError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [] + } + ], + "outcome": [ + { + "collectionName": "coll1", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "x": 11 + } + ] + } + ] + }, + { + "description": "readConcern majority with out stage", + "runOnRequirements": [ + { + "minServerVersion": "4.1.0", + "topologies": [ + "replicaset", + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "aggregate", + "object": "collection2", + "arguments": { + "pipeline": [ + { + "$sort": { + "x": 1 + } + }, + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$out": "aggregate_out" + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "coll2", + "pipeline": [ + { + "$sort": { + "x": 1 + } + }, + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$out": "aggregate_out" + } + ], + "readConcern": { + "level": "majority" + } + }, + "commandName": "aggregate", + "databaseName": "crud-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "aggregate_out", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + }, + { + "description": "Aggregate with $listLocalSessions", + "runOnRequirements": [ + { + "minServerVersion": "3.6.0" + } + ], + "operations": [ + { + "name": "aggregate", + "object": "database1", + "arguments": { + "pipeline": [ + { + "$listLocalSessions": {} + }, + { + "$limit": 1 + }, + { + "$addFields": { + "dummy": "dummy field" + } + }, + { + "$project": { + "_id": 0, + "dummy": 1 + } + } + ] + }, + "expectResult": [ + { + "dummy": "dummy field" + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-gridfs.json b/source/unified-test-format/tests/valid-pass/poc-gridfs.json new file mode 100644 index 0000000000..c04ed89a7c --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-gridfs.json @@ -0,0 +1,299 @@ +{ + "description": "poc-gridfs", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 10, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "57d83cd477bfb1ccd975ab33d827a92b", + "filename": "length-10", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000006" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000007" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2, + "data": { + "$binary": { + "base64": "mao=", + "subType": "00" + } + } + } + ] + } + ], + "tests": [ + { + "description": "Delete when length is 10", + "operations": [ + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + } + } + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [] + } + ] + }, + { + "description": "Download when there are three chunks", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectResult": { + "$$matchesHexBytes": "112233445566778899aa" + } + } + ] + }, + { + "description": "Download when files entry does not exist", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000000" + } + }, + "expectError": { + "isError": true + } + } + ] + }, + { + "description": "Download when an intermediate chunk is missing", + "operations": [ + { + "name": "deleteOne", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 + } + }, + "expectResult": { + "deletedCount": 1 + } + }, + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectError": { + "isError": true + } + } + ] + }, + { + "description": "Upload when length is 5", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "1122334455" + }, + "chunkSizeBytes": 4 + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "oid0" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {}, + "sort": { + "uploadDate": -1 + }, + "limit": 1 + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "oid0" + }, + "length": 5, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": "283d4fea5dded59cf837d3047328f5af", + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "_id": { + "$gt": { + "$oid": "000000000000000000000007" + } + } + }, + "sort": { + "n": 1 + } + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "oid0" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } + }, + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "oid0" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VQ==", + "subType": "00" + } + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-reads.json b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.json new file mode 100644 index 0000000000..2b65d501a7 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-reads.json @@ -0,0 +1,433 @@ +{ + "description": "poc-retryable-reads", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "single", + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "uriOptions": { + "retryReads": false + }, + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-reads-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "retryable-reads-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-reads-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "tests": [ + { + "description": "Aggregate succeeds after InterruptedAtShutdown", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "aggregate" + ], + "errorCode": 11600 + } + } + } + }, + { + "name": "aggregate", + "object": "collection0", + "arguments": { + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$sort": { + "x": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "aggregate": "coll", + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$sort": { + "x": 1 + } + } + ] + }, + "databaseName": "retryable-reads-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "aggregate": "coll", + "pipeline": [ + { + "$match": { + "_id": { + "$gt": 1 + } + } + }, + { + "$sort": { + "x": 1 + } + } + ] + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "Find succeeds on second attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true + } + } + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": {}, + "sort": { + "_id": 1 + }, + "limit": 2 + }, + "expectResult": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {}, + "sort": { + "_id": 1 + }, + "limit": 2 + }, + "databaseName": "retryable-reads-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {}, + "sort": { + "_id": 1 + }, + "limit": 2 + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "Find fails on first attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true + } + } + } + }, + { + "name": "find", + "object": "collection1", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client1", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {} + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "Find fails on second attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "find" + ], + "closeConnection": true + } + } + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": {} + }, + "expectError": { + "isError": true + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {} + }, + "databaseName": "retryable-reads-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "coll", + "filter": {} + }, + "databaseName": "retryable-reads-tests" + } + } + ] + } + ] + }, + { + "description": "ListDatabases succeeds on second attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "listDatabases" + ], + "closeConnection": true + } + } + } + }, + { + "name": "listDatabases", + "object": "client0" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "listDatabases": 1 + } + } + }, + { + "commandStartedEvent": { + "command": { + "listDatabases": 1 + } + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-retryable-writes.json b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.json new file mode 100644 index 0000000000..e64ce1bcee --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-retryable-writes.json @@ -0,0 +1,481 @@ +{ + "description": "poc-retryable-writes", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "3.6", + "topologies": [ + "replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "uriOptions": { + "retryWrites": false + }, + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "retryable-writes-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "retryable-writes-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ], + "tests": [ + { + "description": "FindOneAndUpdate is committed on first attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 1 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + }, + "expectResult": { + "_id": 1, + "x": 11 + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "FindOneAndUpdate is not committed on first attempt", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 1 + }, + "data": { + "failBeforeCommitExceptionCode": 1 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + }, + "expectResult": { + "_id": 1, + "x": 11 + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 12 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "FindOneAndUpdate is never committed", + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "onPrimaryTransactionalWrite", + "mode": { + "times": 2 + }, + "data": { + "failBeforeCommitExceptionCode": 1 + } + } + } + }, + { + "name": "findOneAndUpdate", + "object": "collection0", + "arguments": { + "filter": { + "_id": 1 + }, + "update": { + "$inc": { + "x": 1 + } + }, + "returnDocument": "Before" + }, + "expectError": { + "isError": true + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "InsertMany succeeds after PrimarySteppedDown", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 189, + "errorLabels": [ + "RetryableWriteError" + ] + } + } + } + }, + { + "name": "insertMany", + "object": "collection0", + "arguments": { + "documents": [ + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ], + "ordered": true + }, + "expectResult": { + "insertedCount": 2, + "insertedIds": { + "$$unsetOrMatches": { + "0": 3, + "1": 4 + } + } + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + }, + { + "_id": 4, + "x": 44 + } + ] + } + ] + }, + { + "description": "InsertOne fails after connection failure when retryWrites option is false", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client1", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + } + } + }, + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + }, + "expectError": { + "errorLabelsOmit": [ + "RetryableWriteError" + ] + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + } + ] + } + ] + }, + { + "description": "InsertOne fails after multiple retryable writeConcernErrors", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.7", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 2 + }, + "data": { + "failCommands": [ + "insert" + ], + "writeConcernError": { + "code": 91, + "errmsg": "Replication is being shut down" + } + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 3, + "x": 33 + } + }, + "expectError": { + "errorLabelsContain": [ + "RetryableWriteError" + ] + } + } + ], + "outcome": [ + { + "collectionName": "coll", + "databaseName": "retryable-writes-tests", + "documents": [ + { + "_id": 1, + "x": 11 + }, + { + "_id": 2, + "x": 22 + }, + { + "_id": 3, + "x": 33 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-sessions.json b/source/unified-test-format/tests/valid-pass/poc-sessions.json new file mode 100644 index 0000000000..75f3489428 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-sessions.json @@ -0,0 +1,466 @@ +{ + "description": "poc-sessions", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "3.6.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": false, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "session-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ], + "tests": [ + { + "description": "Server supports explicit sessions", + "operations": [ + { + "name": "assertSessionNotDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + }, + { + "name": "assertSessionNotDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "endSession", + "object": "session0" + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": -1 + } + }, + "expectResult": [] + }, + { + "name": "assertSameLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client0" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + } + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": -1 + }, + "lsid": { + "$$sessionLsid": "session0" + } + }, + "commandName": "find", + "databaseName": "session-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "Server supports implicit sessions", + "operations": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "document": { + "_id": 2 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": -1 + } + }, + "expectResult": [] + }, + { + "name": "assertSameLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client0" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$type": "object" + } + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": -1 + }, + "lsid": { + "$$type": "object" + } + }, + "commandName": "find", + "databaseName": "session-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + }, + { + "description": "Dirty explicit session is discarded", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "failPoint", + "object": "testRunner", + "arguments": { + "client": "client0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + } + } + }, + { + "name": "assertSessionNotDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 2 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 2 + } + } + } + }, + { + "name": "assertSessionDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "assertSessionDirty", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "endSession", + "object": "session0" + }, + { + "name": "find", + "object": "collection0", + "arguments": { + "filter": { + "_id": -1 + } + }, + "expectResult": [] + }, + { + "name": "assertDifferentLsidOnLastTwoCommands", + "object": "testRunner", + "arguments": { + "client": "client0" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1 + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 2 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1 + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 3 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 2 + }, + "commandName": "insert", + "databaseName": "session-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "find": "test", + "filter": { + "_id": -1 + }, + "lsid": { + "$$type": "object" + } + }, + "commandName": "find", + "databaseName": "session-tests" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "session-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json new file mode 100644 index 0000000000..820ed65927 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-convenient-api.json @@ -0,0 +1,505 @@ +{ + "description": "poc-transactions-convenient-api", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "client": { + "id": "client1", + "uriOptions": { + "readConcernLevel": "local", + "w": 1 + }, + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "database": { + "id": "database1", + "client": "client1", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "collection": { + "id": "collection1", + "database": "database1", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + }, + { + "session": { + "id": "session1", + "client": "client1" + } + }, + { + "session": { + "id": "session2", + "client": "client0", + "sessionOptions": { + "defaultTransactionOptions": { + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "w": 1 + } + } + } + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "withTransaction and no transaction options set", + "operations": [ + { + "name": "withTransaction", + "object": "session0", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "$$exists": false + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "autocommit": false, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "withTransaction inherits transaction options from client", + "operations": [ + { + "name": "withTransaction", + "object": "session1", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection1", + "arguments": { + "session": "session1", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client1", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session1" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "level": "local" + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session1" + }, + "txnNumber": 1, + "autocommit": false, + "writeConcern": { + "w": 1 + }, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "withTransaction inherits transaction options from defaultTransactionOptions", + "operations": [ + { + "name": "withTransaction", + "object": "session2", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session2", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ] + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session2" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session2" + }, + "txnNumber": 1, + "autocommit": false, + "writeConcern": { + "w": 1 + }, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + }, + { + "description": "withTransaction explicit transaction options", + "operations": [ + { + "name": "withTransaction", + "object": "session0", + "arguments": { + "callback": [ + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 1 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 1 + } + } + } + } + ], + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "w": 1 + } + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 1 + } + ], + "ordered": true, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "readConcern": { + "level": "majority" + }, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "autocommit": false, + "writeConcern": { + "w": 1 + }, + "readConcern": { + "$$exists": false + }, + "startTransaction": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json new file mode 100644 index 0000000000..a0b297d59a --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions-mongos-pin-auto.json @@ -0,0 +1,409 @@ +{ + "description": "poc-transactions-mongos-pin-auto", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "useMultipleMongoses": true, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ], + "tests": [ + { + "description": "remain pinned after non-transient Interrupted error on insertOne", + "operations": [ + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "errorCode": 11601 + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 4 + } + }, + "expectError": { + "errorLabelsOmit": [ + "TransientTransactionError", + "UnknownTransactionCommitResult" + ], + "errorCodeName": "Interrupted" + } + }, + { + "name": "assertSessionPinned", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "commitTransaction", + "object": "session0" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 3 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 4 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "recoveryToken": { + "$$type": "object" + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + }, + { + "_id": 3 + } + ] + } + ] + }, + { + "description": "unpin after transient error within a transaction", + "operations": [ + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 3 + } + }, + "expectResult": { + "$$unsetOrMatches": { + "insertedId": { + "$$unsetOrMatches": 3 + } + } + } + }, + { + "name": "targetedFailPoint", + "object": "testRunner", + "arguments": { + "session": "session0", + "failPoint": { + "configureFailPoint": "failCommand", + "mode": { + "times": 1 + }, + "data": { + "failCommands": [ + "insert" + ], + "closeConnection": true + } + } + } + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": 4 + } + }, + "expectError": { + "errorLabelsContain": [ + "TransientTransactionError" + ], + "errorLabelsOmit": [ + "UnknownTransactionCommitResult" + ] + } + }, + { + "name": "assertSessionUnpinned", + "object": "testRunner", + "arguments": { + "session": "session0" + } + }, + { + "name": "abortTransaction", + "object": "session0" + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 3 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "insert": "test", + "documents": [ + { + "_id": 4 + } + ], + "ordered": true, + "readConcern": { + "$$exists": false + }, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "insert", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "abortTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + }, + "recoveryToken": { + "$$type": "object" + } + }, + "commandName": "abortTransaction", + "databaseName": "admin" + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [ + { + "_id": 1 + }, + { + "_id": 2 + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-transactions.json b/source/unified-test-format/tests/valid-pass/poc-transactions.json new file mode 100644 index 0000000000..62528f9ce1 --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-transactions.json @@ -0,0 +1,322 @@ +{ + "description": "poc-transactions", + "schemaVersion": "1.0", + "runOnRequirements": [ + { + "minServerVersion": "4.0", + "topologies": [ + "replicaset" + ] + }, + { + "minServerVersion": "4.1.8", + "topologies": [ + "sharded-replicaset" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "transaction-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "test" + } + }, + { + "session": { + "id": "session0", + "client": "client0" + } + } + ], + "initialData": [ + { + "collectionName": "test", + "databaseName": "transaction-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "Client side error in command starting transaction", + "operations": [ + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "insertOne", + "object": "collection0", + "arguments": { + "session": "session0", + "document": { + "_id": { + ".": "." + } + } + }, + "expectError": { + "isClientError": true + } + }, + { + "name": "assertSessionTransactionState", + "object": "testRunner", + "arguments": { + "session": "session0", + "state": "starting" + } + } + ] + }, + { + "description": "explicitly create collection using create command", + "runOnRequirements": [ + { + "minServerVersion": "4.3.4", + "topologies": [ + "replicaset", + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "dropCollection", + "object": "database0", + "arguments": { + "collection": "test" + } + }, + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "createCollection", + "object": "database0", + "arguments": { + "session": "session0", + "collection": "test" + } + }, + { + "name": "assertCollectionNotExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test" + } + }, + { + "name": "commitTransaction", + "object": "session0" + }, + { + "name": "assertCollectionExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "test", + "writeConcern": { + "$$exists": false + } + }, + "commandName": "drop", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "create": "test", + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "create", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + }, + { + "description": "create index on a non-existing collection", + "runOnRequirements": [ + { + "minServerVersion": "4.3.4", + "topologies": [ + "replicaset", + "sharded-replicaset" + ] + } + ], + "operations": [ + { + "name": "dropCollection", + "object": "database0", + "arguments": { + "collection": "test" + } + }, + { + "name": "startTransaction", + "object": "session0" + }, + { + "name": "createIndex", + "object": "collection0", + "arguments": { + "session": "session0", + "name": "x_1", + "keys": { + "x": 1 + } + } + }, + { + "name": "assertIndexNotExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test", + "indexName": "x_1" + } + }, + { + "name": "commitTransaction", + "object": "session0" + }, + { + "name": "assertIndexExists", + "object": "testRunner", + "arguments": { + "databaseName": "transaction-tests", + "collectionName": "test", + "indexName": "x_1" + } + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "command": { + "drop": "test", + "writeConcern": { + "$$exists": false + } + }, + "commandName": "drop", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "createIndexes": "test", + "indexes": [ + { + "name": "x_1", + "key": { + "x": 1 + } + } + ], + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": true, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "createIndexes", + "databaseName": "transaction-tests" + } + }, + { + "commandStartedEvent": { + "command": { + "commitTransaction": 1, + "lsid": { + "$$sessionLsid": "session0" + }, + "txnNumber": 1, + "startTransaction": { + "$$exists": false + }, + "autocommit": false, + "writeConcern": { + "$$exists": false + } + }, + "commandName": "commitTransaction", + "databaseName": "admin" + } + } + ] + } + ] + } + ] +} From 58de923f9db9aa42e1fa73e3f9077fe23f4557e0 Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Sat, 10 Oct 2020 22:43:22 +0800 Subject: [PATCH 90/90] Add schema checks to Travis CI --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6337236621..bfd530288e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,3 +20,13 @@ matrix: script: - python3 ./source/server-discovery-and-monitoring/tests/errors/generate-error-tests.py - cd source && make && git diff --exit-code + + - name: "Unifed Test Format schema checks" + dist: xenial + language: python + python: + - 3.7 + install: + - npm install -g ajv-cli + script: + - cd source/unified-test-format/tests && make