From 18791064fd775bd06bc6b9aae601c09eac4239a7 Mon Sep 17 00:00:00 2001 From: Gal Schlezinger Date: Mon, 13 Mar 2023 17:58:24 +0200 Subject: [PATCH 1/5] Allow to import node:buffer from Edge --- .../src/build/polyfills/edge/node-buffer.ts | 7 +++++ packages/next/src/build/webpack-config.ts | 5 ++++ .../next/src/server/web/sandbox/context.ts | 14 ++++++++++ .../app/buffer/route.js | 17 +++++++++++ .../app/layout.tsx | 7 +++++ .../edge-runtime-node-compatibility.test.ts | 28 +++++++++++++++++++ .../next.config.js | 8 ++++++ .../tsconfig.json | 24 ++++++++++++++++ 8 files changed, 110 insertions(+) create mode 100644 packages/next/src/build/polyfills/edge/node-buffer.ts create mode 100644 test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js create mode 100644 test/e2e/app-dir/edge-runtime-node-compatibility/app/layout.tsx create mode 100644 test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts create mode 100644 test/e2e/app-dir/edge-runtime-node-compatibility/next.config.js create mode 100644 test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json diff --git a/packages/next/src/build/polyfills/edge/node-buffer.ts b/packages/next/src/build/polyfills/edge/node-buffer.ts new file mode 100644 index 0000000000000..a34539070517d --- /dev/null +++ b/packages/next/src/build/polyfills/edge/node-buffer.ts @@ -0,0 +1,7 @@ +// @ts-ignore will be populated by context.ts +const fromGlobal = global.__nextjs__node_compat__.buffer + +module.exports = { + ...fromGlobal, + default: fromGlobal, +} diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 7c2a6d6797496..01b8fa9fb0faf 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -2076,6 +2076,11 @@ export default async function getBaseWebpackConfig( clientRouterFilters, }) ), + isEdgeServer && + new webpack.NormalModuleReplacementPlugin( + /^node:buffer$/, + require.resolve('./polyfills/edge/node-buffer') + ), isClient && new ReactLoadablePlugin({ filename: REACT_LOADABLE_MANIFEST, diff --git a/packages/next/src/server/web/sandbox/context.ts b/packages/next/src/server/web/sandbox/context.ts index eb0ba9fcae1c3..3311390ebca58 100644 --- a/packages/next/src/server/web/sandbox/context.ts +++ b/packages/next/src/server/web/sandbox/context.ts @@ -16,6 +16,7 @@ import { fetchInlineAsset } from './fetch-inline-assets' import type { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin' import { UnwrapPromise } from '../../../lib/coalesced-function' import { runInContext } from 'vm' +import * as BufferImplementation from 'node:buffer' const WEBPACK_HASH_REGEX = /__webpack_require__\.h = function\(\) \{ return "[0-9a-f]+"; \}/g @@ -155,6 +156,19 @@ async function createModuleContext(options: ModuleContextOptions) { extend: (context) => { context.process = createProcessPolyfill(options) + Object.defineProperty(context, '__nextjs__node_compat__', { + enumerable: false, + value: { + buffer: pick(BufferImplementation, [ + 'constants', + 'kMaxLength', + 'kStringMaxLength', + 'Buffer', + 'SlowBuffer', + ]), + }, + }) + context.__next_eval__ = function __next_eval__(fn: Function) { const key = fn.toString() if (!warnedEvals.has(key)) { diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js b/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js new file mode 100644 index 0000000000000..858c51b88f9b3 --- /dev/null +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js @@ -0,0 +1,17 @@ +import B from 'node:buffer' + +/** + * @param {Request} req + */ +export async function POST(req) { + const text = await req.text() + const buf = B.Buffer.from(text) + return new Response( + JSON.stringify({ + encoded: buf.toString('base64'), + exposedKeys: Object.keys(B), + }) + ) +} + +export const runtime = 'edge' diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/app/layout.tsx b/test/e2e/app-dir/edge-runtime-node-compatibility/app/layout.tsx new file mode 100644 index 0000000000000..e7077399c03ce --- /dev/null +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/app/layout.tsx @@ -0,0 +1,7 @@ +export default function Root({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts b/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts new file mode 100644 index 0000000000000..2a5c45f9d9e4c --- /dev/null +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts @@ -0,0 +1,28 @@ +import { createNextDescribe } from 'e2e-utils' + +createNextDescribe( + 'edge runtime node compatibility', + { + files: __dirname, + }, + ({ next }) => { + it('supports node:buffer', async () => { + const res = await next.fetch('/buffer', { + method: 'POST', + body: 'Hello, world!', + }) + const json = await res.json() + expect(json).toEqual({ + encoded: Buffer.from('Hello, world!').toString('base64'), + exposedKeys: [ + 'constants', + 'kMaxLength', + 'kStringMaxLength', + 'Buffer', + 'SlowBuffer', + 'default', + ], + }) + }) + } +) diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/next.config.js b/test/e2e/app-dir/edge-runtime-node-compatibility/next.config.js new file mode 100644 index 0000000000000..bf49894afd400 --- /dev/null +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/next.config.js @@ -0,0 +1,8 @@ +/** + * @type {import('next').NextConfig} + */ +const nextConfig = { + experimental: { appDir: true }, +} + +module.exports = nextConfig diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json b/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json new file mode 100644 index 0000000000000..d2bc2ac5e3cea --- /dev/null +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": false, + "forceConsistentCasingInFileNames": true, + "noEmit": true, + "incremental": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "plugins": [ + { + "name": "next" + } + ] + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] +} From 87e70ca759f57f881dfd927c80d4bddba9afd258 Mon Sep 17 00:00:00 2001 From: Gal Schlezinger Date: Tue, 14 Mar 2023 11:32:57 +0200 Subject: [PATCH 2/5] Force `buffer` to resolve to `node:buffer` in edge --- packages/next/src/build/webpack-config.ts | 2 +- .../app/buffer/route.js | 12 ++++++------ .../edge-runtime-node-compatibility.test.ts | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 01b8fa9fb0faf..90e1d21b20b5c 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -2078,7 +2078,7 @@ export default async function getBaseWebpackConfig( ), isEdgeServer && new webpack.NormalModuleReplacementPlugin( - /^node:buffer$/, + /^(node:)?buffer$/, require.resolve('./polyfills/edge/node-buffer') ), isClient && diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js b/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js index 858c51b88f9b3..f7c4a783f2da3 100644 --- a/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/app/buffer/route.js @@ -1,4 +1,5 @@ import B from 'node:buffer' +import { NextResponse } from 'next/server' /** * @param {Request} req @@ -6,12 +7,11 @@ import B from 'node:buffer' export async function POST(req) { const text = await req.text() const buf = B.Buffer.from(text) - return new Response( - JSON.stringify({ - encoded: buf.toString('base64'), - exposedKeys: Object.keys(B), - }) - ) + return NextResponse.json({ + 'Buffer === B.Buffer': B.Buffer === Buffer, + encoded: buf.toString('base64'), + exposedKeys: Object.keys(B), + }) } export const runtime = 'edge' diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts b/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts index 2a5c45f9d9e4c..7f36e4d746dae 100644 --- a/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts @@ -13,6 +13,7 @@ createNextDescribe( }) const json = await res.json() expect(json).toEqual({ + 'Buffer === B.Buffer': true, encoded: Buffer.from('Hello, world!').toString('base64'), exposedKeys: [ 'constants', From f7aa6407c1ee9379a25ae0eccd5d56ebb061e3ca Mon Sep 17 00:00:00 2001 From: Gal Schlezinger Date: Tue, 14 Mar 2023 18:05:40 +0200 Subject: [PATCH 3/5] use externals to denote the polyfills --- .../src/build/polyfills/edge/node-buffer.ts | 7 ----- packages/next/src/build/webpack-config.ts | 9 ++---- .../webpack/plugins/middleware-plugin.ts | 17 +++++++++- .../next/src/server/web/sandbox/context.ts | 31 +++++++++++++------ .../edge-runtime-node-compatibility.test.ts | 26 ++++++++++++++-- .../pages/api/buffer.js | 22 +++++++++++++ .../tsconfig.json | 3 +- 7 files changed, 89 insertions(+), 26 deletions(-) delete mode 100644 packages/next/src/build/polyfills/edge/node-buffer.ts create mode 100644 test/e2e/app-dir/edge-runtime-node-compatibility/pages/api/buffer.js diff --git a/packages/next/src/build/polyfills/edge/node-buffer.ts b/packages/next/src/build/polyfills/edge/node-buffer.ts deleted file mode 100644 index a34539070517d..0000000000000 --- a/packages/next/src/build/polyfills/edge/node-buffer.ts +++ /dev/null @@ -1,7 +0,0 @@ -// @ts-ignore will be populated by context.ts -const fromGlobal = global.__nextjs__node_compat__.buffer - -module.exports = { - ...fromGlobal, - default: fromGlobal, -} diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 90e1d21b20b5c..ba681afa43066 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -35,6 +35,7 @@ import { finalizeEntrypoint } from './entries' import * as Log from './output/log' import { buildConfiguration } from './webpack/config' import MiddlewarePlugin, { + getEdgePolyfilledModules, handleWebpackExternalForEdgeRuntime, } from './webpack/plugins/middleware-plugin' import BuildManifestPlugin from './webpack/plugins/build-manifest-plugin' @@ -1450,6 +1451,7 @@ export default async function getBaseWebpackConfig( './cjs/react-dom-server-legacy.browser.development.js': '{}', }, + getEdgePolyfilledModules(), handleWebpackExternalForEdgeRuntime, ] : []), @@ -2061,7 +2063,7 @@ export default async function getBaseWebpackConfig( // Buffer is used by getInlineScriptSource Buffer: [require.resolve('buffer'), 'Buffer'], // Avoid process being overridden when in web run time - ...(isClient && { process: [require.resolve('process')] }), + process: [require.resolve('process')], }), new webpack.DefinePlugin( getDefineEnv({ @@ -2076,11 +2078,6 @@ export default async function getBaseWebpackConfig( clientRouterFilters, }) ), - isEdgeServer && - new webpack.NormalModuleReplacementPlugin( - /^(node:)?buffer$/, - require.resolve('./polyfills/edge/node-buffer') - ), isClient && new ReactLoadablePlugin({ filename: REACT_LOADABLE_MANIFEST, diff --git a/packages/next/src/build/webpack/plugins/middleware-plugin.ts b/packages/next/src/build/webpack/plugins/middleware-plugin.ts index 229c105f7fe55..c9292a3d627c6 100644 --- a/packages/next/src/build/webpack/plugins/middleware-plugin.ts +++ b/packages/next/src/build/webpack/plugins/middleware-plugin.ts @@ -858,6 +858,17 @@ export default class MiddlewarePlugin { } } +const supportedEdgePolyfills = new Set(['buffer']) + +export function getEdgePolyfilledModules() { + const records: Record = {} + for (const mod of supportedEdgePolyfills) { + records[mod] = `commonjs node:${mod}` + records[`node:${mod}`] = `commonjs node:${mod}` + } + return records +} + export async function handleWebpackExternalForEdgeRuntime({ request, context, @@ -869,7 +880,11 @@ export async function handleWebpackExternalForEdgeRuntime({ contextInfo: any getResolve: () => any }) { - if (contextInfo.issuerLayer === 'middleware' && isNodeJsModule(request)) { + if ( + contextInfo.issuerLayer === 'middleware' && + isNodeJsModule(request) && + !supportedEdgePolyfills.has(request) + ) { // allows user to provide and use their polyfills, as we do with buffer. try { await getResolve()(context, request) diff --git a/packages/next/src/server/web/sandbox/context.ts b/packages/next/src/server/web/sandbox/context.ts index 3311390ebca58..fcdb13e9a606a 100644 --- a/packages/next/src/server/web/sandbox/context.ts +++ b/packages/next/src/server/web/sandbox/context.ts @@ -140,6 +140,21 @@ function getDecorateUnhandledRejection(runtime: EdgeRuntime) { } } +const NativeModuleMap = new Map([ + [ + 'node:buffer', + { + ...pick(BufferImplementation, [ + 'constants', + 'kMaxLength', + 'kStringMaxLength', + 'Buffer', + 'SlowBuffer', + ]), + }, + ], +]) + /** * Create a module cache specific for the provided parameters. It includes * a runtime context, require cache and paths cache. @@ -156,16 +171,14 @@ async function createModuleContext(options: ModuleContextOptions) { extend: (context) => { context.process = createProcessPolyfill(options) - Object.defineProperty(context, '__nextjs__node_compat__', { + Object.defineProperty(context, 'require', { enumerable: false, - value: { - buffer: pick(BufferImplementation, [ - 'constants', - 'kMaxLength', - 'kStringMaxLength', - 'Buffer', - 'SlowBuffer', - ]), + value: (id: string) => { + const value = NativeModuleMap.get(id) + if (!value) { + throw TypeError('Native module not found: ' + id) + } + return value }, }) diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts b/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts index 7f36e4d746dae..ccc7f33aec946 100644 --- a/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts @@ -6,7 +6,7 @@ createNextDescribe( files: __dirname, }, ({ next }) => { - it('supports node:buffer', async () => { + it('[app] supports node:buffer', async () => { const res = await next.fetch('/buffer', { method: 'POST', body: 'Hello, world!', @@ -21,7 +21,29 @@ createNextDescribe( 'kStringMaxLength', 'Buffer', 'SlowBuffer', - 'default', + ], + }) + }) + + it('[pages/api] supports node:buffer', async () => { + const res = await next.fetch('/api/buffer', { + method: 'POST', + body: 'Hello, world!', + }) + const json = await res.json() + expect(json).toEqual({ + 'B2.Buffer === B.Buffer': true, + 'Buffer === B.Buffer': true, + 'typeof B.Buffer': 'function', + 'typeof B2.Buffer': 'function', + 'typeof Buffer': 'function', + encoded: 'SGVsbG8sIHdvcmxkIQ==', + exposedKeys: [ + 'constants', + 'kMaxLength', + 'kStringMaxLength', + 'Buffer', + 'SlowBuffer', ], }) }) diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/pages/api/buffer.js b/test/e2e/app-dir/edge-runtime-node-compatibility/pages/api/buffer.js new file mode 100644 index 0000000000000..d847e990af7a9 --- /dev/null +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/pages/api/buffer.js @@ -0,0 +1,22 @@ +import B from 'node:buffer' +import B2 from 'buffer' +import { NextResponse } from 'next/server' + +export const config = { runtime: 'edge' } + +/** + * @param {Request} req + */ +export default async function (req) { + const text = await req.text() + const buf = B.Buffer.from(text) + return NextResponse.json({ + 'Buffer === B.Buffer': B.Buffer === Buffer, + 'B2.Buffer === B.Buffer': B.Buffer === B2.Buffer, + 'typeof Buffer': typeof Buffer, + 'typeof B.Buffer': typeof B.Buffer, + 'typeof B2.Buffer': typeof B2.Buffer, + encoded: buf.toString('base64'), + exposedKeys: Object.keys(B), + }) +} diff --git a/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json b/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json index d2bc2ac5e3cea..9c9b16c24b793 100644 --- a/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json +++ b/test/e2e/app-dir/edge-runtime-node-compatibility/tsconfig.json @@ -17,7 +17,8 @@ { "name": "next" } - ] + ], + "strictNullChecks": true }, "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] From 408e9070a00225c43ddaa71254725a34aace3d40 Mon Sep 17 00:00:00 2001 From: Gal Schlezinger Date: Wed, 15 Mar 2023 12:22:42 +0200 Subject: [PATCH 4/5] remove the change to process polyfill --- packages/next/src/build/webpack-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index ba681afa43066..1fd8b3af185d4 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -2063,7 +2063,7 @@ export default async function getBaseWebpackConfig( // Buffer is used by getInlineScriptSource Buffer: [require.resolve('buffer'), 'Buffer'], // Avoid process being overridden when in web run time - process: [require.resolve('process')], + ...(isClient && { process: [require.resolve('process')] }), }), new webpack.DefinePlugin( getDefineEnv({ From 08e7b38466ef919701a5df10f215d8ca55bc81c9 Mon Sep 17 00:00:00 2001 From: Gal Schlezinger Date: Fri, 17 Mar 2023 11:35:40 +0200 Subject: [PATCH 5/5] add rest of modules, untested. I think we should have it a little more type safe, so the implementation will be a `Record` and we will have to implement it and the compiler will guide us if we decide to add another module --- .../webpack/plugins/middleware-plugin.ts | 8 ++- .../next/src/server/web/sandbox/context.ts | 50 +++++++++++++++---- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/packages/next/src/build/webpack/plugins/middleware-plugin.ts b/packages/next/src/build/webpack/plugins/middleware-plugin.ts index c9292a3d627c6..648f63ebaade8 100644 --- a/packages/next/src/build/webpack/plugins/middleware-plugin.ts +++ b/packages/next/src/build/webpack/plugins/middleware-plugin.ts @@ -858,7 +858,13 @@ export default class MiddlewarePlugin { } } -const supportedEdgePolyfills = new Set(['buffer']) +const supportedEdgePolyfills = new Set([ + 'buffer', + 'events', + 'assert', + 'util', + 'async_hooks', +]) export function getEdgePolyfilledModules() { const records: Record = {} diff --git a/packages/next/src/server/web/sandbox/context.ts b/packages/next/src/server/web/sandbox/context.ts index fcdb13e9a606a..86b13cd98f0f3 100644 --- a/packages/next/src/server/web/sandbox/context.ts +++ b/packages/next/src/server/web/sandbox/context.ts @@ -16,7 +16,11 @@ import { fetchInlineAsset } from './fetch-inline-assets' import type { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin' import { UnwrapPromise } from '../../../lib/coalesced-function' import { runInContext } from 'vm' -import * as BufferImplementation from 'node:buffer' +import BufferImplementation from 'node:buffer' +import EventsImplementation from 'node:events' +import AssertImplementation from 'node:assert' +import UtilImplementation from 'node:util' +import AsyncHooksImplementation from 'node:async_hooks' const WEBPACK_HASH_REGEX = /__webpack_require__\.h = function\(\) \{ return "[0-9a-f]+"; \}/g @@ -140,18 +144,42 @@ function getDecorateUnhandledRejection(runtime: EdgeRuntime) { } } -const NativeModuleMap = new Map([ +const NativeModuleMap = new Map([ [ 'node:buffer', - { - ...pick(BufferImplementation, [ - 'constants', - 'kMaxLength', - 'kStringMaxLength', - 'Buffer', - 'SlowBuffer', - ]), - }, + pick(BufferImplementation, [ + 'constants', + 'kMaxLength', + 'kStringMaxLength', + 'Buffer', + 'SlowBuffer', + ]), + ], + [ + 'node:events', + pick(EventsImplementation, [ + 'EventEmitter', + 'captureRejectionSymbol', + 'defaultMaxListeners', + 'errorMonitor', + 'listenerCount', + 'on', + 'once', + ]), + ], + [ + 'node:async_hooks', + pick(AsyncHooksImplementation, ['AsyncLocalStorage', 'AsyncResource']), + ], + [ + 'node:assert', + // TODO: check if need to pick specific properties + AssertImplementation, + ], + [ + 'node:util', + // TODO: check if need to pick specific properties + UtilImplementation, ], ])