feat(vm): support wasm module (#5131)

Author: hi-ogawa

eslint.config.js

@@ -11,6 +11,7 @@ export default antfu(
       '**/bench.json',
       '**/fixtures',
       'test/core/src/self',
+      'test/core/src/wasm-bindgen-no-cyclic',
       'test/workspaces/results.json',
       'test/reporters/fixtures/with-syntax-error.test.js',
       'test/network-imports/public/[email protected]',

packages/vitest/src/runtime/external-executor.ts

@@ -27,7 +27,7 @@ export interface ExternalModulesExecutorOptions {
 }
 
 interface ModuleInformation {
-  type: 'data' | 'builtin' | 'vite' | 'module' | 'commonjs'
+  type: 'data' | 'builtin' | 'vite' | 'wasm' | 'module' | 'commonjs'
   url: string
   path: string
 }
@@ -165,7 +165,7 @@ export class ExternalModulesExecutor {
     const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier
     const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString()
 
-    let type: 'module' | 'commonjs' | 'vite'
+    let type: 'module' | 'commonjs' | 'vite' | 'wasm'
     if (this.vite.canResolve(fileUrl)) {
       type = 'vite'
     }
@@ -175,6 +175,11 @@ export class ExternalModulesExecutor {
     else if (extension === '.cjs') {
       type = 'commonjs'
     }
+    else if (extension === '.wasm') {
+      // still experimental on NodeJS --experimental-wasm-modules
+      // cf. ESM_FILE_FORMAT(url) in https://nodejs.org/docs/latest-v20.x/api/esm.html#resolution-algorithm
+      type = 'wasm'
+    }
     else {
       const pkgData = this.findNearestPackageData(normalize(pathUrl))
       type = pkgData.type === 'module' ? 'module' : 'commonjs'
@@ -188,7 +193,7 @@ export class ExternalModulesExecutor {
 
     // create ERR_MODULE_NOT_FOUND on our own since latest NodeJS's import.meta.resolve doesn't throw on non-existing namespace or path
     // https://github.com/nodejs/node/pull/49038
-    if ((type === 'module' || type === 'commonjs') && !existsSync(path)) {
+    if ((type === 'module' || type === 'commonjs' || type === 'wasm') && !existsSync(path)) {
       const error = new Error(`Cannot find module '${path}'`)
       ;(error as any).code = 'ERR_MODULE_NOT_FOUND'
       throw error
@@ -203,6 +208,8 @@ export class ExternalModulesExecutor {
       }
       case 'vite':
         return await this.vite.createViteModule(url)
+      case 'wasm':
+        return await this.esm.createWebAssemblyModule(url, this.fs.readBuffer(path))
       case 'module':
         return await this.esm.createEsModule(url, this.fs.readFile(path))
       case 'commonjs': {

packages/vitest/src/runtime/vm/esm-executor.ts

@@ -77,6 +77,15 @@ export class EsmExecutor {
     return m
   }
 
+  public async createWebAssemblyModule(fileUrl: string, code: Buffer) {
+    const cached = this.moduleCache.get(fileUrl)
+    if (cached)
+      return cached
+    const m = this.loadWebAssemblyModule(code, fileUrl)
+    this.moduleCache.set(fileUrl, m)
+    return m
+  }
+
   public async loadWebAssemblyModule(source: Buffer, identifier: string) {
     const cached = this.moduleCache.get(identifier)
     if (cached)
@@ -90,23 +99,21 @@ export class EsmExecutor {
     const moduleLookup: Record<string, VMModule> = {}
     for (const { module } of imports) {
       if (moduleLookup[module] === undefined) {
-        const resolvedModule = await this.executor.resolveModule(
+        moduleLookup[module] = await this.executor.resolveModule(
           module,
           identifier,
         )
-
-        moduleLookup[module] = await this.evaluateModule(resolvedModule)
       }
     }
 
     const syntheticModule = new SyntheticModule(
       exports.map(({ name }) => name),
-      () => {
+      async () => {
         const importsObject: WebAssembly.Imports = {}
         for (const { module, name } of imports) {
           if (!importsObject[module])
             importsObject[module] = {}
-
+          await this.evaluateModule(moduleLookup[module])
           importsObject[module][name] = (moduleLookup[module].namespace as any)[name]
         }
         const wasmInstance = new WebAssembly.Instance(
@@ -150,7 +157,7 @@ export class EsmExecutor {
       if (encoding !== 'base64')
         throw new Error(`Invalid data URI encoding: ${encoding}`)
 
-      const module = await this.loadWebAssemblyModule(
+      const module = this.loadWebAssemblyModule(
         Buffer.from(match.groups.code, 'base64'),
         identifier,
       )

test/core/src/wasm-bindgen-no-cyclic/README.md

@@ -0,0 +1,15 @@
+The recent version of the wasm-bindgen bundler output does not use cyclic imports between wasm and js.
+
+For this non-cyclic version to work, both `index_bg.js` and `index_bg.wasm` need to be externalized
+since otherwise a dual package hazard on `index_bg.js` would make it non-functional.
+
+The code is copied from https://github.com/rustwasm/wasm-bindgen/tree/8198d2d25920e1f4fc593e9f8eb9d199e004d731/examples/hello_world
+
+```sh
+npm i
+npm run build
+# then
+# 1. copy `examples/hello_world/pkg` to this directory
+# 2. add { "type": "module" } to `package.json`
+#    (this will be automatically included after https://github.com/rustwasm/wasm-pack/pull/1061)
+```

test/core/src/wasm-bindgen-no-cyclic/index.d.ts

@@ -0,0 +1,6 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+* @param {string} name
+*/
+export function greet(name: string): void;

test/core/src/wasm-bindgen-no-cyclic/index.js

@@ -0,0 +1,4 @@
+import * as wasm from "./index_bg.wasm";
+import { __wbg_set_wasm } from "./index_bg.js";
+__wbg_set_wasm(wasm);
+export * from "./index_bg.js";

test/core/src/wasm-bindgen-no-cyclic/index_bg.js

@@ -0,0 +1,117 @@
+let wasm;
+export function __wbg_set_wasm(val) {
+    wasm = val;
+}
+
+
+const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder;
+
+let cachedTextDecoder = new lTextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+
+cachedTextDecoder.decode();
+
+let cachedUint8Memory0 = null;
+
+function getUint8Memory0() {
+    if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
+        cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
+    }
+    return cachedUint8Memory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+}
+
+function logError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        let error = (function () {
+            try {
+                return e instanceof Error ? `${e.message}\n\nStack:\n${e.stack}` : e.toString();
+            } catch(_) {
+                return "<failed to stringify thrown value>";
+            }
+        }());
+        console.error("wasm-bindgen: imported JS function that was not marked as `catch` threw an error:", error);
+        throw e;
+    }
+}
+
+let WASM_VECTOR_LEN = 0;
+
+const lTextEncoder = typeof TextEncoder === 'undefined' ? (0, module.require)('util').TextEncoder : TextEncoder;
+
+let cachedTextEncoder = new lTextEncoder('utf-8');
+
+const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
+    ? function (arg, view) {
+    return cachedTextEncoder.encodeInto(arg, view);
+}
+    : function (arg, view) {
+    const buf = cachedTextEncoder.encode(arg);
+    view.set(buf);
+    return {
+        read: arg.length,
+        written: buf.length
+    };
+});
+
+function passStringToWasm0(arg, malloc, realloc) {
+
+    if (typeof(arg) !== 'string') throw new Error('expected a string argument');
+
+    if (realloc === undefined) {
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = malloc(buf.length, 1) >>> 0;
+        getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
+        WASM_VECTOR_LEN = buf.length;
+        return ptr;
+    }
+
+    let len = arg.length;
+    let ptr = malloc(len, 1) >>> 0;
+
+    const mem = getUint8Memory0();
+
+    let offset = 0;
+
+    for (; offset < len; offset++) {
+        const code = arg.charCodeAt(offset);
+        if (code > 0x7F) break;
+        mem[ptr + offset] = code;
+    }
+
+    if (offset !== len) {
+        if (offset !== 0) {
+            arg = arg.slice(offset);
+        }
+        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+        const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
+        const ret = encodeString(arg, view);
+        if (ret.read !== arg.length) throw new Error('failed to pass whole string');
+        offset += ret.written;
+    }
+
+    WASM_VECTOR_LEN = offset;
+    return ptr;
+}
+/**
+* @param {string} name
+*/
+export function greet(name) {
+    const ptr0 = passStringToWasm0(name, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+    const len0 = WASM_VECTOR_LEN;
+    wasm.greet(ptr0, len0);
+}
+
+export function __wbg_alert_9ea5a791b0d4c7a3() { return logError(function (arg0, arg1) {
+    alert(getStringFromWasm0(arg0, arg1));
+}, arguments) };
+
+export function __wbindgen_throw(arg0, arg1) {
+    throw new Error(getStringFromWasm0(arg0, arg1));
+};
+

test/core/src/wasm-bindgen-no-cyclic/index_bg.wasm

@@ -0,0 +1,6 @@
+/* tslint:disable */
+/* eslint-disable */
+export const memory: WebAssembly.Memory;
+export function greet(a: number, b: number): void;
+export function __wbindgen_malloc(a: number, b: number): number;
+export function __wbindgen_realloc(a: number, b: number, c: number, d: number): number;

test/core/src/wasm-bindgen-no-cyclic/index_bg.wasm.d.ts

@@ -0,0 +1,20 @@
+{
+  "type": "module",
+  "name": "hello_world",
+  "collaborators": [
+    "The wasm-bindgen Developers"
+  ],
+  "version": "0.1.0",
+  "files": [
+    "index_bg.wasm",
+    "index.js",
+    "index_bg.js",
+    "index.d.ts"
+  ],
+  "module": "index.js",
+  "types": "index.d.ts",
+  "sideEffects": [
+    "./index.js",
+    "./snippets/*"
+  ]
+}

test/core/src/wasm-bindgen-no-cyclic/package.json

@@ -7,7 +7,7 @@ import { expect, test, vi } from 'vitest'
 // @ts-expect-error wasm is not typed
 import { add } from '../src/add.wasm'
 
-const wasmFileBuffer = readFileSync(resolve(__dirname, './src/add.wasm'))
+const wasmFileBuffer = readFileSync(resolve(__dirname, '../src/add.wasm'))
 
 test('supports native wasm imports', () => {
   expect(add(1, 2)).toBe(3)
@@ -54,7 +54,7 @@ test('imports from "data:application/wasm" URI with invalid encoding fail', asyn
   ).rejects.toThrow('Invalid data URI encoding: charset=utf-8')
 })
 
-test('supports wasm files that import js resources (wasm-bindgen)', async () => {
+test('supports wasm/js cyclic import (old wasm-bindgen output)', async () => {
   globalThis.alert = vi.fn()
 
   // @ts-expect-error not typed
@@ -63,3 +63,12 @@ test('supports wasm files that import js resources (wasm-bindgen)', async () =>
 
   expect(globalThis.alert).toHaveBeenCalledWith('Hello, World!')
 })
+
+test('supports wasm-bindgen', async () => {
+  globalThis.alert = vi.fn()
+
+  const { greet } = await import('../src/wasm-bindgen-no-cyclic/index.js')
+  greet('No Cyclic')
+
+  expect(globalThis.alert).toHaveBeenCalledWith('Hello, No Cyclic!')
+})

test/core/test/vm-wasm.test.ts

@@ -45,7 +45,7 @@ export default defineConfig({
   },
   test: {
     name: 'core',
-    exclude: ['**/fixtures/**', '**/vm-wasm.test.ts', ...defaultExclude],
+    exclude: ['**/fixtures/**', ...defaultExclude],
     slowTestThreshold: 1000,
     testTimeout: 2000,
     setupFiles: [
@@ -75,7 +75,7 @@ export default defineConfig({
     },
     server: {
       deps: {
-        external: ['tinyspy', /src\/external/, /esm\/esm/, /\.wasm$/],
+        external: ['tinyspy', /src\/external/, /esm\/esm/, /\.wasm$/, /\/wasm-bindgen-no-cyclic\/index_bg/],
         inline: ['inline-lib'],
       },
     },