Merge pull request #4173 from ralfhandl/main-schema-test-coverage

main: schema test coverage

Author: handrews

package.json

@@ -14,7 +14,7 @@
   "license": "Apache-2.0",
   "scripts": {
     "build": "bash ./scripts/md2html/build.sh",
-    "test": "c8 --100 vitest --watch=false"
+    "test": "c8 --100 vitest --watch=false && bash scripts/schema-test-coverage.sh"
   },
   "readmeFilename": "README.md",
   "files": [

scripts/schema-test-coverage.mjs

@@ -0,0 +1,132 @@
+import { readdir, readFile } from "node:fs/promises";
+import YAML from "yaml";
+import { join } from "node:path";
+import { argv } from "node:process";
+import "@hyperjump/json-schema/draft-2020-12";
+import "@hyperjump/json-schema/draft-04";
+import {
+  compile,
+  getSchema,
+  interpret,
+  Validation,
+  BASIC,
+} from "@hyperjump/json-schema/experimental";
+import * as Instance from "@hyperjump/json-schema/instance/experimental";
+
+/**
+ * @import { AST } from "@hyperjump/json-schema/experimental"
+ * @import { Json } from "@hyperjump/json-schema"
+ */
+
+import contentTypeParser from "content-type";
+import { addMediaTypePlugin } from "@hyperjump/browser";
+import { buildSchemaDocument } from "@hyperjump/json-schema/experimental";
+
+addMediaTypePlugin("application/schema+yaml", {
+  parse: async (response) => {
+    const contentType = contentTypeParser.parse(
+      response.headers.get("content-type") ?? "",
+    );
+    const contextDialectId =
+      contentType.parameters.schema ?? contentType.parameters.profile;
+
+    const foo = YAML.parse(await response.text());
+    return buildSchemaDocument(foo, response.url, contextDialectId);
+  },
+  fileMatcher: (path) => path.endsWith(".yaml"),
+});
+
+/** @type (testDirectory: string) => AsyncGenerator<[string,Json]> */
+const tests = async function* (testDirectory) {
+  for (const file of await readdir(testDirectory, {
+    recursive: true,
+    withFileTypes: true,
+  })) {
+    if (!file.isFile() || !file.name.endsWith(".yaml")) {
+      continue;
+    }
+
+    const testPath = join(file.parentPath, file.name);
+    const testJson = await readFile(testPath, "utf8");
+
+    yield [testPath, YAML.parse(testJson)];
+  }
+};
+
+/** @type (testDirectory: string) => Promise<void> */
+const runTests = async (testDirectory) => {
+  for await (const [name, test] of tests(testDirectory)) {
+    const instance = Instance.fromJs(test);
+
+    const result = interpret(compiled, instance, BASIC);
+
+    if (!result.valid) {
+      console.log("Failed:", name, result.errors);
+    }
+  }
+};
+
+/** @type (ast: AST) => string[] */
+const keywordLocations = (ast) => {
+  /** @type string[] */
+  const locations = [];
+  for (const schemaLocation in ast) {
+    if (schemaLocation === "metaData") {
+      continue;
+    }
+
+    if (Array.isArray(ast[schemaLocation])) {
+      for (const keyword of ast[schemaLocation]) {
+        if (Array.isArray(keyword)) {
+          locations.push(keyword[1]);
+        }
+      }
+    }
+  }
+
+  return locations;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
+const schema = await getSchema(argv[2]);
+const compiled = await compile(schema);
+
+/** @type Set<string> */
+const visitedLocations = new Set();
+const baseInterpret = Validation.interpret;
+Validation.interpret = (url, instance, ast, dynamicAnchors, quiet) => {
+  if (Array.isArray(ast[url])) {
+    for (const keywordNode of ast[url]) {
+      if (Array.isArray(keywordNode)) {
+        visitedLocations.add(keywordNode[1]);
+      }
+    }
+  }
+  return baseInterpret(url, instance, ast, dynamicAnchors, quiet);
+};
+
+await runTests(argv[3]);
+Validation.interpret = baseInterpret;
+
+// console.log("Covered:", visitedLocations);
+
+const allKeywords = keywordLocations(compiled.ast);
+const notCovered = allKeywords.filter(
+  (location) => !visitedLocations.has(location),
+);
+if (notCovered.length > 0) {
+  console.log("NOT Covered:", notCovered.length, "of", allKeywords.length);
+  const maxNotCovered = 20;
+  const firstNotCovered = notCovered.slice(0, maxNotCovered);
+  if (notCovered.length > maxNotCovered) firstNotCovered.push("...");
+  console.log(firstNotCovered);
+}
+
+console.log(
+  "Covered:",
+  visitedLocations.size,
+  "of",
+  allKeywords.length,
+  "(" + Math.floor((visitedLocations.size / allKeywords.length) * 100) + "%)",
+);

scripts/schema-test-coverage.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Author: @ralfhandl
+
+# Run this script from the root of the repo
+
+echo
+echo "Schema Test Coverage"
+echo
+
+for schemaDir in schemas/v3* ; do
+  version=$(basename "$schemaDir")
+  echo $version
+
+  node scripts/schema-test-coverage.mjs $schemaDir/schema.yaml tests/$version/pass
+
+  echo
+done