feat!: use fetch (#365)

Co-authored-by: Antoine du Hamel <[email protected]>

Author: merceyz

.eslintrc.js

@@ -2,4 +2,11 @@ module.exports = {
   extends: [
     `@yarnpkg`,
   ],
+  rules: {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    'no-restricted-globals': [`error`, {
+      name: `fetch`,
+      message: `Use fetch from sources/httpUtils.ts`,
+    }],
+  },
 };

.vscode/settings.json

@@ -7,6 +7,6 @@
   "eslint.nodePath": ".yarn/sdks",
   "typescript.enablePromptUseWorkspaceTsdk": true,
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   }
 }

package.json

@@ -25,6 +25,7 @@
     "@types/debug": "^4.1.5",
     "@types/jest": "^29.0.0",
     "@types/node": "^20.4.6",
+    "@types/proxy-from-env": "^1",
     "@types/semver": "^7.1.0",
     "@types/tar": "^6.0.0",
     "@types/which": "^3.0.0",
@@ -40,13 +41,13 @@
     "eslint": "^8.0.0",
     "eslint-plugin-arca": "^0.16.0",
     "jest": "^29.0.0",
-    "nock": "^13.0.4",
-    "proxy-agent": "^6.2.2",
+    "proxy-from-env": "^1.1.0",
     "semver": "^7.5.2",
     "supports-color": "^9.0.0",
     "tar": "^6.0.1",
     "ts-node": "^10.0.0",
     "typescript": "^5.0.4",
+    "undici": "^6.4.0",
     "v8-compile-cache": "^2.3.0",
     "which": "^4.0.0"
   },
@@ -94,8 +95,5 @@
       "./shims/yarnpkg",
       "./shims/yarnpkg.ps1"
     ]
-  },
-  "resolutions": {
-    "vm2": "portal:./vm2"
   }
 }

sources/httpUtils.ts

@@ -1,21 +1,17 @@
-import {UsageError}                          from 'clipanion';
-import {once}                                from 'events';
-import type {RequestOptions}                 from 'https';
-import type {IncomingMessage, ClientRequest} from 'http';
-import {stderr, stdin}                       from 'process';
+import assert          from 'assert';
+import {UsageError}    from 'clipanion';
+import {once}          from 'events';
+import {stderr, stdin} from 'process';
+import {Readable}      from 'stream';
 
-export async function fetchUrlStream(url: string, options: RequestOptions = {}) {
+export async function fetch(input: string | URL, init?: RequestInit) {
   if (process.env.COREPACK_ENABLE_NETWORK === `0`)
-    throw new UsageError(`Network access disabled by the environment; can't reach ${url}`);
+    throw new UsageError(`Network access disabled by the environment; can't reach ${input}`);
 
-  const {default: https} = await import(`https`);
-
-  const {ProxyAgent} = await import(`proxy-agent`);
-
-  const proxyAgent = new ProxyAgent();
+  const agent = await getProxyAgent(input);
 
   if (process.env.COREPACK_ENABLE_DOWNLOAD_PROMPT === `1`) {
-    console.error(`Corepack is about to download ${url}.`);
+    console.error(`Corepack is about to download ${input}.`);
     if (stdin.isTTY && !process.env.CI) {
       stderr.write(`\nDo you want to continue? [Y/n] `);
       stdin.resume();
@@ -30,60 +26,55 @@ export async function fetchUrlStream(url: string, options: RequestOptions = {})
     }
   }
 
-  return new Promise<IncomingMessage>((resolve, reject) => {
-    const createRequest = (url: string) => {
-      const request: ClientRequest = https.get(url, {...options, agent: proxyAgent}, response => {
-        const statusCode = response.statusCode;
-
-        if ([301, 302, 307, 308].includes(statusCode as number) && response.headers.location)
-          return createRequest(response.headers.location as string);
-
-        if (statusCode != null && statusCode >= 200 && statusCode < 300)
-          return resolve(response);
-
-        return reject(new Error(`Server answered with HTTP ${statusCode} when performing the request to ${url}; for troubleshooting help, see https://github.com/nodejs/corepack#troubleshooting`));
-      });
+  let response;
+  try {
+    response = await globalThis.fetch(input, {
+      ...init,
+      dispatcher: agent,
+    });
+  } catch (error) {
+    throw new Error(
+      `Error when performing the request to ${input}; for troubleshooting help, see https://github.com/nodejs/corepack#troubleshooting`,
+      {cause: error},
+    );
+  }
 
-      request.on(`error`, err => {
-        reject(new Error(`Error when performing the request to ${url}; for troubleshooting help, see https://github.com/nodejs/corepack#troubleshooting`));
-      });
-    };
+  if (!response.ok) {
+    await response.arrayBuffer();
+    throw new Error(
+      `Server answered with HTTP ${response.status} when performing the request to ${input}; for troubleshooting help, see https://github.com/nodejs/corepack#troubleshooting`,
+    );
+  }
 
-    createRequest(url);
-  });
+  return response;
 }
 
-export async function fetchAsBuffer(url: string, options?: RequestOptions) {
-  const response = await fetchUrlStream(url, options);
-
-  return new Promise<Buffer>((resolve, reject) => {
-    const chunks: Array<Buffer> = [];
+export async function fetchAsJson(input: string | URL, init?: RequestInit) {
+  const response = await fetch(input, init);
+  return response.json() as Promise<any>;
+}
 
-    response.on(`data`, chunk => {
-      chunks.push(chunk);
-    });
+export async function fetchUrlStream(input: string | URL, init?: RequestInit) {
+  const response = await fetch(input, init);
+  const webStream = response.body;
+  assert(webStream, `Expected stream to be set`);
+  const stream = Readable.fromWeb(webStream);
+  return stream;
+}
 
-    response.on(`error`, error => {
-      reject(error);
-    });
+async function getProxyAgent(input: string | URL) {
+  const {getProxyForUrl} = await import(`proxy-from-env`);
 
-    response.on(`end`, () => {
-      resolve(Buffer.concat(chunks));
-    });
-  });
-}
+  // @ts-expect-error - The internal implementation is compatible with a WHATWG URL instance
+  const proxy = getProxyForUrl(input);
 
-export async function fetchAsJson(url: string, options?: RequestOptions) {
-  const buffer = await fetchAsBuffer(url, options);
-  const asText = buffer.toString();
+  if (!proxy) return undefined;
 
-  try {
-    return JSON.parse(asText);
-  } catch (error) {
-    const truncated = asText.length > 30
-      ? `${asText.slice(0, 30)}...`
-      : asText;
+  // Doing a deep import here since undici isn't tree-shakeable
+  const {default: ProxyAgent} = (await import(
+    // @ts-expect-error No types for this specific file
+    `undici/lib/proxy-agent.js`
+  )) as { default: typeof import('undici').ProxyAgent };
 
-    throw new Error(`Couldn't parse JSON data: ${JSON.stringify(truncated)}`);
-  }
+  return new ProxyAgent(proxy);
 }

sources/npmRegistryUtils.ts

@@ -1,11 +1,10 @@
-import {UsageError}          from 'clipanion';
-import {OutgoingHttpHeaders} from 'http2';
+import {UsageError}   from 'clipanion';
 
-import * as httpUtils        from './httpUtils';
+import * as httpUtils from './httpUtils';
 
 // load abbreviated metadata as that's all we need for these calls
 // see: https://github.com/npm/registry/blob/cfe04736f34db9274a780184d1cdb2fb3e4ead2a/docs/responses/package-metadata.md
-export const DEFAULT_HEADERS: OutgoingHttpHeaders = {
+export const DEFAULT_HEADERS: Record<string, string> = {
   [`Accept`]: `application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8`,
 };
 export const DEFAULT_NPM_REGISTRY_URL = `https://registry.npmjs.org`;