fix: convert all modules running in loader thread to ESM

This is a second attempt at fixing the double-loader issue in nodejs#205. The
tests from that commit are included.

Author: bengl

hook.js renamed to create-hook.mjs

@@ -2,9 +2,10 @@
 //
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
 
-const { URL, fileURLToPath } = require('url')
-const { inspect } = require('util')
-const { builtinModules } = require('module')
+import { URL, fileURLToPath } from 'url'
+import { inspect } from 'util'
+import { builtinModules } from 'module'
+
 const specifiers = new Map()
 const isWin = process.platform === 'win32'
 let experimentalPatchInternals = false
@@ -21,7 +22,7 @@ let entrypoint
 
 let getExports
 if (NODE_MAJOR >= 20 || (NODE_MAJOR === 18 && NODE_MINOR >= 19)) {
-  getExports = require('./lib/get-exports.js')
+  getExports = (await import('./lib/get-exports.mjs')).default
 } else {
   getExports = (url) => import(url).then(Object.keys)
 }
@@ -35,7 +36,7 @@ function hasIitm (url) {
 }
 
 function isIitm (url, meta) {
-  return url === meta.url || url === meta.url.replace('hook.mjs', 'hook.js')
+  return url === meta.url || url === meta.url.replace('hook.mjs', 'create-hook.mjs')
 }
 
 function deleteIitm (url) {
@@ -284,7 +285,7 @@ function addIitm (url) {
   return needsToAddFileProtocol(urlObj) ? 'file:' + urlObj.href : urlObj.href
 }
 
-function createHook (meta) {
+export function createHook (meta) {
   let cachedResolve
   const iitmURL = new URL('lib/register.js', meta.url).toString()
   let includeModules, excludeModules
@@ -432,6 +433,7 @@ register(${JSON.stringify(realUrl)}, _, set, get, ${JSON.stringify(specifiers.ge
 `
         }
       } catch (cause) {
+        process._rawDebug(cause)
         // If there are other ESM loader hooks registered as well as iitm,
         // depending on the order they are registered, source might not be
         // JavaScript.
@@ -494,5 +496,3 @@ register(${JSON.stringify(realUrl)}, _, set, get, ${JSON.stringify(specifiers.ge
     }
   }
 }
-
-module.exports = { createHook }

hook.mjs

@@ -2,7 +2,7 @@
 //
 // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
 
-import { createHook } from './hook.js'
+import { createHook } from './create-hook.mjs'
 
 const { initialize, load, resolve, getFormat, getSource } = createHook(import.meta)
 

lib/get-esm-exports.js renamed to lib/get-esm-exports.mjs

@@ -1,7 +1,7 @@
 'use strict'
 
-const { Parser } = require('acorn')
-const { importAttributesOrAssertions } = require('acorn-import-attributes')
+import { Parser } from 'acorn'
+import { importAttributesOrAssertions } from 'acorn-import-attributes'
 
 const acornOpts = {
   ecmaVersion: 'latest',
@@ -32,7 +32,7 @@ function warn (txt) {
  * @returns {Set<string>} The identifiers exported by the module along with any
  * custom directives.
  */
-function getEsmExports (moduleSource) {
+export default function getEsmExports (moduleSource) {
   const exportedNames = new Set()
   const tree = parser.parse(moduleSource, acornOpts)
   for (const node of tree.body) {
@@ -114,5 +114,3 @@ function parseSpecifiers (node, exportedNames) {
     }
   }
 }
-
-module.exports = getEsmExports

lib/get-exports.js renamed to lib/get-exports.mjs

@@ -1,11 +1,13 @@
 'use strict'
 
-const getEsmExports = require('./get-esm-exports.js')
-const { parse: parseCjs } = require('cjs-module-lexer')
-const { readFileSync, existsSync } = require('fs')
-const { builtinModules } = require('module')
-const { fileURLToPath, pathToFileURL } = require('url')
-const { dirname, join } = require('path')
+import getEsmExports from './get-esm-exports.mjs'
+import { parse as parseCjs, init as parserInit } from 'cjs-module-lexer'
+import { readFileSync } from 'fs'
+import { builtinModules, createRequire } from 'module'
+import { fileURLToPath, pathToFileURL } from 'url'
+import { dirname, join } from 'path'
+
+await parserInit()
 
 function addDefault (arr) {
   return new Set(['default', ...arr])
@@ -14,9 +16,15 @@ function addDefault (arr) {
 // Cached exports for Node built-in modules
 const BUILT_INS = new Map()
 
+let require
+
 function getExportsForNodeBuiltIn (name) {
   let exports = BUILT_INS.get()
 
+  if (!require) {
+    require = createRequire(import.meta.url)
+  }
+
   if (!exports) {
     exports = new Set(addDefault(Object.keys(require(name))))
     BUILT_INS.set(name, exports)
@@ -110,6 +118,10 @@ async function getCjsExports (url, context, parentLoad, source) {
           [url, re] = resolved
         }
 
+        // Resolve the re-exported module relative to the current module.
+        if (!require) {
+          require = createRequire(import.meta.url)
+        }
         const newUrl = pathToFileURL(require.resolve(re, { paths: [dirname(fileURLToPath(url))] })).href
 
         if (newUrl.endsWith('.node') || newUrl.endsWith('.json')) {
@@ -144,7 +156,7 @@ async function getCjsExports (url, context, parentLoad, source) {
  * Please see {@link getEsmExports} for caveats on special identifiers that may
  * be included in the result set.
  */
-async function getExports (url, context, parentLoad) {
+export default async function getExports (url, context, parentLoad) {
   // `parentLoad` gives us the possibility of getting the source
   // from an upstream loader. This doesn't always work though,
   // so later on we fall back to reading it from disk.
@@ -188,5 +200,3 @@ async function getExports (url, context, parentLoad) {
     throw err
   }
 }
-
-module.exports = getExports

test/fixtures/double-loader.mjs

@@ -0,0 +1,9 @@
+import { readFile } from 'fs/promises'
+
+export async function load (url, context, nextLoad) {
+  const result = await nextLoad(url, context)
+  if (!result.source && url.startsWith('file:')) {
+    result.source = await readFile(new URL(url))
+  }
+  return result
+}

test/get-esm-exports/v18.19-get-esm-exports.js renamed to test/get-esm-exports/v18.19-get-esm-exports.mjs

@@ -1,3 +1,3 @@
 // v18.19.0 backported ESM hook execution to a separate thread,
 // thus being equivalent to >=v20.
-require('./v20-get-esm-exports')
+import './v20-get-esm-exports.mjs'

test/get-esm-exports/v20-get-esm-exports.js renamed to test/get-esm-exports/v20-get-esm-exports.mjs

@@ -1,11 +1,14 @@
 'use strict'
 
-const getEsmExports = require('../../lib/get-esm-exports.js')
-const fs = require('fs')
-const assert = require('assert')
-const path = require('path')
+import getEsmExports from '../../lib/get-esm-exports.mjs'
+import fs from 'fs'
+import assert from 'assert'
+import path from 'path'
+import { fileURLToPath } from 'url'
 
-const fixturePath = path.join(__dirname, '../fixtures/esm-exports.txt')
+const dirname = path.dirname(fileURLToPath(import.meta.url))
+
+const fixturePath = path.join(dirname, '../fixtures/esm-exports.txt')
 const fixture = fs.readFileSync(fixturePath, 'utf8')
 
 fixture.split('\n').forEach(line => {

test/other/double-loading.mjs

@@ -0,0 +1,15 @@
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc.
+
+import { execSync } from 'child_process'
+import { doesNotThrow } from 'assert'
+
+const env = {
+  ...process.env,
+  NODE_OPTIONS: '--no-warnings --experimental-loader ./test/fixtures/double-loader.mjs --experimental-loader ./hook.mjs'
+}
+
+doesNotThrow(() => {
+  execSync('node -p 0', { env })
+})