DRIVERS-709: Unified Test Format

source/unified-test-format/schema-1.0.json

@@ -0,0 +1,286 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+
+  "title": "Unified Test Format",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["description", "schemaVersion", "tests"],
+  "properties": {
+    "description": { "type": "string" },
+    "schemaVersion": { "$ref": "#/definitions/version" },
+    "runOnRequirements": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/runOnRequirement" }
+    },
+    "createEntities": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/entity" }
+    },
+    "initialData": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/collectionData" }
+    },
+    "tests": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/test" }
+    }
+  },
+
+  "definitions": {
+    "version": {
+      "type": "string",
+      "pattern": "^[0-9]+(\\.[0-9]+){1,2}$"
+    },
+
+    "runOnRequirement": {
+      "type": "object",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "maxServerVersion": { "$ref": "#/definitions/version" },
+        "minServerVersion": { "$ref": "#/definitions/version" },
+        "topologies": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "string",
+            "enum": ["single", "replicaset", "sharded", "sharded-replicaset"]
+          }
+        }
+      }
+    },
+
+    "entity": {
+      "type": "object",
+      "additionalProperties": false,
+      "maxProperties": 1,
+      "minProperties": 1,
+      "properties": {
+        "client": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["id"],
+          "properties": {
+            "id": { "type": "string" },
+            "uriOptions": { "type": "object" },
+            "useMultipleMongoses": { "type": "boolean" },
+            "observeEvents": {
+              "type": "array",
+              "minItems": 1,
+              "items": {
+                "type": "string",
+                "enum": ["commandStartedEvent", "commandSucceededEvent", "commandFailedEvent"]
+              }
+            },
+            "ignoreCommandMonitoringEvents": {
+              "type": "array",
+              "minItems": 1,
+              "items": { "type": "string" }
+            }
+          }
+        },
+        "database": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["id", "client", "databaseName"],
+          "properties": {
+            "id": { "type": "string" },
+            "client": { "type": "string" },
+            "databaseName": { "type": "string" },
+            "databaseOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" }
+          }
+        },
+        "collection": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["id", "database", "collectionName"],
+          "properties": {
+            "id": { "type": "string" },
+            "database": { "type": "string" },
+            "collectionName": { "type": "string" },
+            "collectionOptions": { "$ref": "#/definitions/collectionOrDatabaseOptions" }
+          }
+        },
+        "session": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["id", "client"],
+          "properties": {
+            "id": { "type": "string" },
+            "client": { "type": "string" },
+            "sessionOptions": { "type": "object" }
+          }
+        },
+        "bucket": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["id", "database"],
+          "properties": {
+            "id": { "type": "string" },
+            "database": { "type": "string" },
+            "bucketOptions": { "type": "object" }
+          }
+        },
+        "stream": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["id", "hexBytes"],
+          "properties": {
+            "id": { "type": "string" },
+            "hexBytes": {
+              "type": "string",
+              "pattern": "^([0-9A-Fa-f]{2})*$"
+            }
+          }
+        }
+      }
+    },
+
+    "collectionData": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["collectionName", "databaseName", "documents"],
+      "properties": {
+        "collectionName": { "type": "string" },
+        "databaseName": { "type": "string" },
+        "documents": {
+          "type": "array",
+          "items": { "type": "object" }
+        }
+      }
+    },
+
+    "expectedEventsForClient": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["client", "events"],
+      "properties": {
+        "client": { "type": "string" },
+        "events": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/expectedEvent" }
+        }
+      }
+    },
+
+    "expectedEvent": {
+      "type": "object",
+      "additionalProperties": false,
+      "maxProperties": 1,
+      "minProperties": 1,
+      "properties": {
+        "commandStartedEvent": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "command": { "type": "object" },
+            "commandName": { "type": "string" },
+            "databaseName": { "type": "string" }
+          }
+        },
+        "commandSucceededEvent": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "reply": { "type": "object" },
+            "commandName": { "type": "string" }
+          }
+        },
+        "commandFailedEvent": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "commandName": { "type": "string" }
+          }
+        }
+      }
+    },
+
+    "collectionOrDatabaseOptions": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "readConcern": { "type": "object" },
+        "readPreference": { "type": "object" },
+        "writeConcern": { "type": "object" }
+      }
+    },
+
+    "operation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "object"],
+      "properties": {
+        "name": { "type": "string" },
+        "object": { "type": "string" },
+        "arguments": { "type": "object" },
+        "expectError":  { "$ref": "#/definitions/expectedError" },
+        "expectResult": {},
+        "saveResultAsEntity": { "type": "string" }
+      },
+      "allOf": [
+        { "not": { "required": ["expectError", "expectResult"] } },
+        { "not": { "required": ["expectError", "saveResultAsEntity"] } }
+      ]
+    },
+
+    "expectedError": {
+      "type": "object",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "isError": {
+          "type": "boolean",
+          "const": true
+        },
+        "isClientError": { "type": "boolean" },
+        "errorContains": { "type": "string" },
+        "errorCode": { "type": "integer" },
+        "errorCodeName": { "type": "string" },
+        "errorLabelsContain": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string" }
+        },
+        "errorLabelsOmit": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string" }
+        },
+        "expectResult": {}
+      }
+    },
+
+    "test": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["description", "operations"],
+      "properties": {
+        "description": { "type": "string" },
+        "runOnRequirements": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/definitions/runOnRequirement" }
+        },
+        "skipReason": { "type": "string" },
+        "operations": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/operation" }
+        },
+        "expectEvents": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/expectedEventsForClient" }
+        },
+        "outcome": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/definitions/collectionData" }
+        }
+      }
+    }
+  }
+}

source/unified-test-format/schema-testing-example.json

@@ -0,0 +1,63 @@
+{
+  "tests": [
+    {
+      "description": "valid because error:true and result has only expected fields",
+      "operations": [
+        {
+          "object": "collection",
+          "name": "runCommand",
+          "error": true,
+          "result": {
+            "errorContains": "string"
+          }
+        }
+      ]
+    },
+    {
+      "description": "valid because error == false. result can be any type (non-object)",
+      "operations": [
+        {
+          "object": "collection",
+          "name": "operation",
+          "error": false,
+          "result": 1
+        }
+      ]
+    },
+    {
+      "description": "valid because error == false. result can be any type (object)",
+      "operations": [
+        {
+          "object": "collection",
+          "name": "operation",
+          "error": false,
+          "result": {
+            "foo": "bar"
+          }
+        }
+      ]
+    },
+    {
+      "description": "valid because error != true. result can be any type (non-object)",
+      "operations": [
+        {
+          "object": "collection",
+          "name": "operation",
+          "result": 1
+        }
+      ]
+    },
+    {
+      "description": "valid because error != true. result can be any type (object)",
+      "operations": [
+        {
+          "object": "collection",
+          "name": "operation",
+          "result": {
+            "foo": "bar"
+          }
+        }
+      ]
+    }
+  ]
+}

source/unified-test-format/tests/Makefile

@@ -0,0 +1,21 @@
+SCHEMA=../schema-1.0.json
+
+.PHONY: all invalid valid-fail valid-pass HAS_AJV
+
+all: invalid valid-fail valid-pass
+
+invalid: HAS_AJV
+	@# Redirect stdout to hide expected validation errors
+	@ajv test -s $(SCHEMA) -d "invalid/*.yml" --invalid > /dev/null && echo "invalid/*.yml passed test"
+
+valid-fail: HAS_AJV
+	@ajv test -s $(SCHEMA) -d "valid-fail/*.yml" --valid
+
+valid-pass: HAS_AJV
+	@ajv test -s $(SCHEMA) -d "valid-pass/*.yml" --valid
+
+HAS_AJV:
+	@if ! command -v ajv > /dev/null; then                \
+	    echo 'Error: need "npm install -g ajv-cli"' 1>&2; \
+	    exit 1;                                           \
+	fi