Merge pull request #19 from arethetypeswrong/snapshot-testing

Add snapshot testing for end-to-end testing of core on real data

Author: andrewbranch

.github/workflows/ci.yml

@@ -11,7 +11,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
-      - run: npm install -g [email protected]
+        with:
+          node-version: 19
       - run: npm install
       - run: npm run tsc
       - run: npm run build

.github/workflows/deploy-web.yml

@@ -9,7 +9,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
-      - run: npm install -g [email protected]
+        with:
+          node-version: 19
       - run: npm install
       - run: npm run tsc
       - run: npm run build

.github/workflows/publish-core.yml

@@ -10,7 +10,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
-      - run: npm install -g [email protected]
+        with:
+          node-version: 19
       - run: npm install
         working-directory: ./packages/core
       - run: npm publish

package.json

@@ -21,5 +21,8 @@
     "@types/node": "^18.11.18",
     "prettier": "^2.8.3",
     "typescript": "^5.0.0-dev.20230207"
+  },
+  "volta": {
+    "node": "19.8.1"
   }
 }

packages/core/package.json

@@ -20,6 +20,7 @@
     "tsc": "tsc",
     "pretest": "tsc -b test",
     "test": "node --test test/dist",
+    "snapshot": "node scripts/createSnapshotFixture.js",
     "patch": "npm version patch && npm run commit-patch",
     "commit-patch": "git commit -am \"Bump core to v$npm_package_version\" && git tag \"v$npm_package_version\"",
     "prepublishOnly": "npm run tsc && npm run test"
@@ -52,5 +53,8 @@
   },
   "devDependencies": {
     "@types/ts-expose-internals": "npm:[email protected]"
+  },
+  "volta": {
+    "node": "19.8.1"
   }
 }

packages/core/scripts/createSnapshotFixture.js

@@ -0,0 +1,34 @@
+import { writeFile } from "fs/promises";
+import createFetch from "fetch-ponyfill";
+const { fetch } = createFetch();
+
+if (import.meta.url === "file://" + process.argv[1]) {
+  const [, , packageSpec] = process.argv;
+  const manifest = await fetchManifest(packageSpec);
+  const { name, version } = manifest;
+  console.log(`Resolved '${name}@${version}'`);
+  const localUrl = await writePackage(manifest);
+  console.log(`Wrote '${localUrl}'`);
+}
+
+/**
+ * @param {string} packageSpec 
+ */
+async function fetchManifest(packageSpec) {
+  const versionIndex = packageSpec.indexOf("@", 1);
+  const packageName = packageSpec.substring(0, versionIndex < 0 ? undefined : versionIndex);
+  const version = versionIndex < 0 ? "latest" : packageSpec.substring(versionIndex + 1);
+  const manifestUrl = `https://registry.npmjs.org/${packageName}/${version}`;
+  return fetch(manifestUrl).then((r) => r.json());
+}
+
+/**
+ * @param {any} manifest
+ */
+async function writePackage(manifest) {
+  const localUrl = new URL(`../test/fixtures/${manifest.name.replace("/", "__")}@${manifest.version}.tgz`, import.meta.url);
+  const tarballUrl = manifest.dist.tarball;
+  const packageBuffer = await fetch(tarballUrl).then((r) => r.arrayBuffer());
+  await writeFile(localUrl, Buffer.from(packageBuffer));
+  return localUrl;
+}

packages/core/scripts/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "checkJs": true,
+    "strict": true,
+    "noEmit": true,
+    "target": "esnext",
+    "module": "nodenext",
+    "types": [
+      "node"
+    ]
+  }
+}

packages/core/test/fixtures/@[email protected]

@@ -0,0 +1,66 @@
+import fs from "fs";
+import { access, readFile, writeFile } from "fs/promises";
+import assert from "node:assert";
+import { after, describe, test } from "node:test";
+import { checkTgz, getProblems, summarizeProblems } from "@arethetypeswrong/core";
+
+const updateSnapshots = process.env.UPDATE_SNAPSHOTS;
+
+describe("snapshots", async () => {
+  const snapshotsWritten: URL[] = [];
+
+  after(() => {
+    if (updateSnapshots && snapshotsWritten.length > 0) {
+      console.log(`Updated ${snapshotsWritten.length} snapshots:`);
+    } else if (snapshotsWritten.length > 0) {
+      console.log(`Wrote ${snapshotsWritten.length} snapshots:`);
+    }
+
+    if (snapshotsWritten.length > 0) {
+      console.log(snapshotsWritten.map((url) => `  ${url.pathname}`).join("\n"));
+    }
+  });
+
+  for (const fixture of fs.readdirSync(new URL("../fixtures", import.meta.url))) {
+    test(fixture, async () => {
+      const tarball = await readFile(new URL(`../fixtures/${fixture}`, import.meta.url));
+      const analysis = await checkTgz(tarball);
+      assert(analysis.containsTypes);
+      const problems = getProblems(analysis);
+      const summary = summarizeProblems(problems, analysis);
+      const snapshotURL = new URL(`../snapshots/${fixture}.md`, import.meta.url);
+      const expectedSnapshot = [
+        `# ${fixture}`,
+        "",
+        "## Summary",
+        "",
+        "```json",
+        JSON.stringify(summary, null, 2),
+        "```",
+        "",
+        "## Problems",
+        "",
+        "```json",
+        JSON.stringify(problems, null, 2),
+        "```",
+      ].join("\n");
+
+      if (
+        await access(snapshotURL)
+          .then(() => true)
+          .catch(() => false)
+      ) {
+        const snapshot = await readFile(snapshotURL, "utf8");
+        if (updateSnapshots) {
+          await writeFile(snapshotURL, expectedSnapshot);
+          snapshotsWritten.push(snapshotURL);
+        } else {
+          assert.strictEqual(snapshot, expectedSnapshot);
+        }
+      } else {
+        await writeFile(snapshotURL, expectedSnapshot);
+        snapshotsWritten.push(snapshotURL);
+      }
+    });
+  }
+});

packages/core/test/fixtures/[email protected]

@@ -0,0 +1,30 @@
+# @[email protected]
+
+## Summary
+
+```json
+[
+  {
+    "kind": "FalseCJS",
+    "title": "Types are CJS, but implementation is ESM",
+    "messages": [
+      {
+        "messageText": "Imports of the package under the `node16` module resolution setting when the importing module is ESM (its extension is `.mts` or `.mjs`, or it has a `.ts` or `.js` extension and is in scope of a `package.json` that contains `\"type\": \"module\"`) resolved to CJS types, but ESM implementations.",
+        "messageHtml": "Imports of the package under the <code>node16</code> module resolution setting when the importing module is ESM (its extension is <code>.mts</code> or <code>.mjs</code>, or it has a <code>.ts</code> or <code>.js</code> extension and is in scope of a <code>package.json</code> that contains <code>\"type\": \"module\"</code>) resolved to CJS types, but ESM implementations."
+      }
+    ]
+  }
+]
+```
+
+## Problems
+
+```json
+[
+  {
+    "kind": "FalseCJS",
+    "entrypoint": ".",
+    "resolutionKind": "node16-esm"
+  }
+]
+```