(build) include ESM distributables in CDN build (#3209)

Co-authored-by: Josh Goebel <[email protected]>

Author: aduh95

.github/workflows/node.js.yml

@@ -33,6 +33,11 @@ jobs:
       run: |
         npm run test-browser
         node test/builds/browser_build_as_commonjs.js
+    # CDN build should be easily importable
+    - if: contains(matrix.build-how, 'cdn')
+      name: Test that we can import CDN esm build
+      run: |
+        node test/builds/cdn_build_as_esm.mjs
 
     - if: contains(matrix.build-how, 'node')
       name: Test Node.js build

test/builds/cdn_build_as_esm.mjs

@@ -0,0 +1,24 @@
+import hljs from "../../build/es/highlight.js";
+
+const API = [
+  "getLanguage",
+  "registerLanguage",
+  "highlight",
+  "highlightAuto",
+  "highlightAll",
+  "highlightElement"
+];
+
+const assert = (f,msg) => {
+  if (!f()) {
+    console.error(msg);
+    process.exit(1);
+  }
+};
+const keys = Object.keys(hljs);
+
+API.forEach(n => {
+  assert(_ => keys.includes(n), `API should include ${n}`);
+});
+
+console.log("Pass: browser build works with Node.js just fine.")

tools/build_browser.js

@@ -14,9 +14,14 @@ const log = (...args) => console.log(...args);
 const { rollupCode } = require("./lib/bundling.js");
 const bundling = require('./lib/bundling.js');
 const Table = require('cli-table');
-const { result } = require('lodash');
 
-function buildHeader(args) {
+const getDefaultHeader = () => ({
+  ...require('../package.json'),
+  git_sha : child_process
+    .execSync("git rev-parse --short=10 HEAD")
+    .toString().trim(),
+});
+function buildHeader(args = getDefaultHeader()) {
   return "/*!\n" +
   `  Highlight.js v${args.version} (git: ${args.git_sha})\n` +
   `  (c) ${config.copyrightYears} ${args.author.name} and other contributors\n` +
@@ -68,14 +73,12 @@ async function buildBrowser(options) {
 
   detailedGrammarSizes(languages);
 
-  const size = await buildBrowserHighlightJS(languages, { minify: options.minify });
+  const size = await buildCore("highlight", languages, { minify: options.minify, format: "cjs" });
 
   log("-----");
-  log("Core                :", size.core, "bytes");
-  if (options.minify) { log("Core (min)          :", size.core_min, "bytes"); }
   log("Languages (raw)     :",
     languages.map((el) => el.data.length).reduce((acc, curr) => acc + curr, 0), "bytes");
-  log("highlight.js        :", size.full, "bytes");
+  log("highlight.js        :", size.fullSize, "bytes");
   if (options.minify) {
     log("highlight.min.js    :", size.minified, "bytes");
     log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes");
@@ -175,93 +178,76 @@ function installDemoStyles() {
   });
 }
 
-async function buildBrowserHighlightJS(languages, { minify }) {
-  log("Building highlight.js.");
-
-  const git_sha = child_process
-    .execSync("git rev-parse HEAD")
-    .toString().trim()
-    .slice(0, 10);
-  const versionDetails = { ...require("../package"), git_sha };
-  const header = buildHeader(versionDetails);
-
-  const outFile = `${process.env.BUILD_DIR}/highlight.js`;
-  const minifiedFile = outFile.replace(/js$/, "min.js");
-
-  const built_in_langs = {
-    name: "dynamicLanguages",
-    resolveId: (source) => {
-      if (source == "builtInLanguages") { return "builtInLanguages"}
-      return null;
-    },
-    load: (id) => {
-      if (id == "builtInLanguages") {
-        const escape = (s) => "grmr_" + s.replace("-", "_");
-        let src = "";
-        src += languages.map((x) => `import ${escape(x.name)} from '${x.path}'`).join("\n");
-        src += `\nexport {${languages.map((x) => escape(x.name)).join(",")}}`;
-        return src;
-      }
-      return null;
+const builtInLanguagesPlugin = (languages) => ({
+  name: "hljs-index",
+  resolveId(source) {
+    if (source === "builtInLanguages") {
+      return source; // this signals that rollup should not ask other plugins or check the file system to find this id
     }
+    return null; // other ids should be handled as usually
+  },
+  load(id) {
+    const escape = (s) => "grmr_" + s.replace("-", "_");
+    if (id === "builtInLanguages") {
+      return languages.map((lang) =>
+        `export { default as ${escape(lang.name)} } from ${JSON.stringify(lang.path)};`
+      ).join("\n");
+    }
+    return null;
   }
-
-  const plugins = [...config.rollup.browser_core.input.plugins, built_in_langs];
-
-  const input = { ...config.rollup.browser_core.input, input: `src/stub.js`, plugins };
-  const output = { ...config.rollup.browser_core.output, file: outFile };
-  let librarySrc = await rollupCode(input, output);
-
-
-  // we don't use this, we just use it to get a size approximation for the build stats
-  const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output);
-  const coreSize = coreSrc.length;
-
-  // strip off the original top comment
-  librarySrc = librarySrc.replace(/\/\*.*?\*\//s, "");
-
-  const fullSrc = [
-    header, librarySrc,
-    // ...languages.map((lang) => lang.module)
-  ].join("\n");
-
-  const tasks = [];
-  tasks.push(fs.writeFile(outFile, fullSrc, { encoding: "utf8" }));
-  const shas = {
-    "highlight.js": bundling.sha384(fullSrc)
+});
+
+async function buildCore(name, languages, options) {
+  const header = buildHeader();
+  let relativePath = "";
+  const input = {
+    ...config.rollup.core.input,
+    input: `src/stub.js`
+  };
+  input.plugins = [
+    ...input.plugins,
+    builtInLanguagesPlugin(languages)
+  ];
+  const output = {
+    ...(options.format === "es" ? config.rollup.node.output : config.rollup.browser_iife.output),
+    file: `${process.env.BUILD_DIR}/${name}.js`
   };
 
-  let core_min = [];
-  let minifiedSrc = "";
-
-  if (minify) {
-    const tersed = await Terser.minify(librarySrc, config.terser);
-    const tersedCore = await Terser.minify(coreSrc, config.terser);
+  // optimize for no languages by not including the language loading stub
+  if (languages.length === 0) {
+    input.input = "src/highlight.js";
+  }
 
-    minifiedSrc = [
-      header, tersed.code,
-      // ...languages.map((lang) => lang.minified)
-    ].join("\n");
+  if (options.format === "es") {
+    output.format = "es";
+    output.file = `${process.env.BUILD_DIR}/es/${name}.js`;
+    relativePath = "es/";
+  }
 
-    // get approximate core minified size
-    core_min = [header, tersedCore.code].join().length;
+  log(`Building ${relativePath}${name}.js.`);
 
-    tasks.push(fs.writeFile(minifiedFile, minifiedSrc, { encoding: "utf8" }));
-    shas["highlight.min.js"] = bundling.sha384(minifiedSrc);
+  const index = await rollupCode(input, output);
+  const sizeInfo = { shas: [] };
+  const writePromises = [];
+  if (options.minify) {
+    const { code } = await Terser.minify(index, {...config.terser, module: (options.format === "es") });
+    const src = `${header}\n${code}`;
+    writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), src));
+    sizeInfo.minified = src.length;
+    sizeInfo.minifiedSrc = src;
+    sizeInfo.shas[`${relativePath}${name}.min.js`] = bundling.sha384(src)
   }
-
-  await Promise.all(tasks);
-  return {
-    core: coreSize,
-    core_min: core_min,
-    minified: Buffer.byteLength(minifiedSrc, 'utf8'),
-    minifiedSrc,
-    fullSrc,
-    shas,
-    full: Buffer.byteLength(fullSrc, 'utf8')
-  };
+  {
+    const src = `${header}\n${index}`;
+    writePromises.push(fs.writeFile(output.file, src));
+    sizeInfo.fullSize = src.length;
+    sizeInfo.fullSrc = src;
+    sizeInfo.shas[`${relativePath}${name}.js`] = bundling.sha384(src)
+  }
+  await Promise.all(writePromises);
+  return sizeInfo;
 }
 
 // CDN build uses the exact same highlight.js distributable
-module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS;
+module.exports.buildCore = buildCore;
 module.exports.build = buildBrowser;