Make the staging files, for deployments, as lean as possible (#23049)

* Experiment with making the tarball smaller

Part of #1248

* try this

* stop debugging

* delete translations too

* delete heavy search indexes too

* push and popd

* try this hack

* delete but leave directory

* debug more

* faster delete of translations

* less loud

* async await

* async await

* no tree

* simplify

* experimenting more

* unfinished

* only the large files

* change order

* brotli with level 6

* cope better with decorated rest json files

* tidying

* keep images

* cleaning

* cleaning up

* refactored function

* try this

* better comment

* remove console logging

* more important changes

* improved fs.accessSync check

Author: Peter Bengtsson

.github/actions-scripts/compress-large-files.js

@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+
+import path from 'path'
+import fs from 'fs'
+import zlib from 'zlib'
+import walk from 'walk-sync'
+
+const DRY_RUN = Boolean(JSON.parse(process.env.DRY_RUN || 'false'))
+// Roughly 100KiB means about 25 files at the moment.
+// Set this too low and the overheads will be more than the disk and
+// network I/O that this intends to serve.
+const MIN_GZIP_SIZE = Number(process.env.MIN_GZIP_SIZE || 1024 * 100)
+
+const BROTLI_OPTIONS = {
+  params: {
+    [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT,
+    [zlib.constants.BROTLI_PARAM_QUALITY]: 6,
+  },
+}
+main()
+
+async function main() {
+  compressFromPattern('lib/**/static/**/*.json')
+}
+
+async function compressFromPattern(pattern) {
+  const glob = pattern.includes('*') ? pattern.split(path.sep).slice(1).join(path.sep) : undefined
+  const walkOptions = {
+    globs: glob ? [glob] : undefined,
+    directories: false,
+    includeBasePath: true,
+  }
+  const root = path.resolve(pattern.includes('*') ? pattern.split(path.sep)[0] : pattern)
+  const filePaths = walk(root, walkOptions).filter((filePath) => {
+    return fs.statSync(filePath).size > MIN_GZIP_SIZE
+  })
+
+  if (!DRY_RUN) {
+    console.time(`Compress ${filePaths.length} files`)
+    const compressed = await Promise.all(filePaths.map(compressFile))
+    console.timeEnd(`Compress ${filePaths.length} files`)
+
+    console.time(`Delete ${compressed.length} files`)
+    compressed.forEach((filePath) => fs.unlinkSync(filePath))
+    console.timeEnd(`Delete ${compressed.length} files`)
+  }
+}
+
+function compressFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const contentStream = fs.createReadStream(filePath)
+    const newFilePath = `${filePath}.br`
+    const writeStream = fs.createWriteStream(newFilePath)
+    const compressor = zlib.createBrotliCompress(BROTLI_OPTIONS)
+    contentStream
+      .pipe(compressor)
+      .pipe(writeStream)
+      .on('finish', (err) => {
+        if (err) return reject(err)
+        resolve(filePath)
+      })
+  })
+}

.github/workflows/staging-build-pr.yml

@@ -30,15 +30,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  debug:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Dump full context for debugging
-        run: |
-          cat << EOF
-          ${{ toJSON(github) }}
-          EOF
-
   build-pr:
     if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }}
     runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }}
@@ -117,11 +108,43 @@ jobs:
         run: npm set-script heroku-postbuild "echo 'Application was pre-built!'"
 
       - name: Delete heavy things we won't need deployed
+        if: ${{ github.repository == 'github/docs-internal' }}
         run: |
+
           # The dereferenced file is not used in runtime once the
           # decorated file has been created from it.
           rm -fr lib/rest/static/dereferenced
 
+          # Translations are never tested in Staging builds
+          # but let's keep the empty directory.
+          rm -fr translations
+          mkdir translations
+
+          # Delete all the big search indexes that are NOT English (`*-en-*`)
+          pushd lib/search/indexes
+          ls | grep -v '\-en\-' | xargs rm
+          popd
+
+          # Note! Some day it would be nice to be able to delete
+          # all the heavy assets because they bloat the tarball.
+          # But it's not obvious how to test it then. For now, we'll have
+          # to accept that every staging build has a copy of the images.
+
+          # The assumption here is that a staging build will not
+          # need these legacy redirects. Only the redirects from
+          # front-matter will be at play.
+          # These static redirects json files are notoriously large
+          # and they make the tarball unnecessarily large.
+          echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json
+          echo '{}' > lib/redirects/static/developer.json
+          echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json
+
+          # This will turn every `lib/**/static/*.json` into
+          # an equivalent `lib/**/static/*.json.br` file.
+          # Once the server starts, it'll know to fall back to reading
+          # the `.br` equivalent if the `.json` file isn't present.
+          node .github/actions-scripts/compress-large-files.js
+
       - name: Create an archive
         # Only bother if this is actually a pull request
         if: ${{ github.event.pull_request.number }}

lib/read-json-file.js

@@ -1,6 +1,32 @@
 import fs from 'fs'
-import path from 'path'
+import { brotliDecompressSync } from 'zlib'
 
 export default function readJsonFile(xpath) {
-  return JSON.parse(fs.readFileSync(path.join(process.cwd(), xpath), 'utf8'))
+  return JSON.parse(fs.readFileSync(xpath, 'utf8'))
+}
+
+export function readCompressedJsonFile(xpath) {
+  if (!xpath.endsWith('.br')) {
+    xpath += '.br'
+  }
+  return JSON.parse(brotliDecompressSync(fs.readFileSync(xpath)))
+}
+
+// Ask it to read a `foo.json` file and it will automatically
+// first see if there's a `foo.json.br` and only if it's not,
+// will fallback to reading the `foo.json` file.
+// The reason for this is that staging builds needs to as small as
+// possible (in terms of disk) for them to deploy faster. So the
+// staging deployment process will compress a bunch of large
+// `.json` files before packaging it up.
+export function readCompressedJsonFileFallback(xpath) {
+  try {
+    return readCompressedJsonFile(xpath)
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return readJsonFile(xpath)
+    } else {
+      throw err
+    }
+  }
 }

lib/redirects/precompile.js

@@ -4,7 +4,7 @@ import path from 'path'
 import { isPromise } from 'util/types'
 import { fileURLToPath } from 'url'
 
-import readJsonFile from '../read-json-file.js'
+import { readCompressedJsonFileFallback } from '../read-json-file.js'
 import { latest } from '../../lib/enterprise-server-releases.js'
 import getExceptionRedirects from './exception-redirects.js'
 import { languageKeys } from '../languages.js'
@@ -45,7 +45,7 @@ const DISK_CACHE_FILEPATH = path.join(__dirname, `.redirects-cache_${languageKey
 // This function runs at server warmup and precompiles possible redirect routes.
 // It outputs them in key-value pairs within a neat Javascript object: { oldPath: newPath }
 const precompileRedirects = diskMemoize(DISK_CACHE_FILEPATH, async (pageList) => {
-  const allRedirects = readJsonFile('./lib/redirects/static/developer.json')
+  const allRedirects = readCompressedJsonFileFallback('./lib/redirects/static/developer.json')
 
   // Replace hardcoded 'latest' with real value in the redirected path
   Object.entries(allRedirects).forEach(([oldPath, newPath]) => {

lib/rest/index.js

@@ -3,12 +3,18 @@ import path from 'path'
 import fs from 'fs'
 import { chain, get, groupBy } from 'lodash-es'
 import { allVersions, allVersionKeys } from '../all-versions.js'
+import { readCompressedJsonFileFallback } from '../read-json-file.js'
+
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const schemasPath = path.join(__dirname, 'static/decorated')
+
 export const operations = {}
 fs.readdirSync(schemasPath).forEach((filename) => {
-  const key = filename.replace('.json', '')
-  const value = JSON.parse(fs.readFileSync(path.join(schemasPath, filename)))
+  // In staging deploys, the `.json` files might have been converted to
+  // to `.json.br`. The `readCompressedJsonFileFallback()` function
+  // can handle both but you need to call it with the `.json` filename.
+  const key = path.parse(filename).name
+  const value = readCompressedJsonFileFallback(path.join(schemasPath, filename))
   operations[key] = value
 })
 

middleware/archived-enterprise-versions.js

@@ -9,15 +9,34 @@ import patterns from '../lib/patterns.js'
 import versionSatisfiesRange from '../lib/version-satisfies-range.js'
 import isArchivedVersion from '../lib/is-archived-version.js'
 import got from 'got'
-import readJsonFile from '../lib/read-json-file.js'
+import { readCompressedJsonFileFallback } from '../lib/read-json-file.js'
 import { cacheControlFactory } from './cache-control.js'
 
 function readJsonFileLazily(xpath) {
   const cache = new Map()
   // This will throw if the file isn't accessible at all, e.g. ENOENT
-  fs.accessSync(xpath)
+  // But, the file might have been replaced by one called `SAMENAME.json.br`
+  // because in staging, we ship these files compressed to make the
+  // deployment faster. So, in our file-presence check, we need to
+  // account for that.
+  try {
+    fs.accessSync(xpath)
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      try {
+        fs.accessSync(xpath + '.br')
+      } catch (err) {
+        if (err.code === 'ENOENT') {
+          throw new Error(`Neither ${xpath} nor ${xpath}.br is accessible`)
+        }
+        throw err
+      }
+    } else {
+      throw err
+    }
+  }
   return () => {
-    if (!cache.has(xpath)) cache.set(xpath, readJsonFile(xpath))
+    if (!cache.has(xpath)) cache.set(xpath, readCompressedJsonFileFallback(xpath))
     return cache.get(xpath)
   }
 }

middleware/contextualizers/graphql.js

@@ -1,12 +1,14 @@
-import fs from 'fs'
-import path from 'path'
-import readJsonFile from '../../lib/read-json-file.js'
+import { readCompressedJsonFileFallback } from '../../lib/read-json-file.js'
 import { allVersions } from '../../lib/all-versions.js'
-const previews = readJsonFile('./lib/graphql/static/previews.json')
-const upcomingChanges = readJsonFile('./lib/graphql/static/upcoming-changes.json')
-const changelog = readJsonFile('./lib/graphql/static/changelog.json')
-const prerenderedObjects = readJsonFile('./lib/graphql/static/prerendered-objects.json')
-const prerenderedInputObjects = readJsonFile('./lib/graphql/static/prerendered-input-objects.json')
+const previews = readCompressedJsonFileFallback('./lib/graphql/static/previews.json')
+const upcomingChanges = readCompressedJsonFileFallback('./lib/graphql/static/upcoming-changes.json')
+const changelog = readCompressedJsonFileFallback('./lib/graphql/static/changelog.json')
+const prerenderedObjects = readCompressedJsonFileFallback(
+  './lib/graphql/static/prerendered-objects.json'
+)
+const prerenderedInputObjects = readCompressedJsonFileFallback(
+  './lib/graphql/static/prerendered-input-objects.json'
+)
 
 const explorerUrl =
   process.env.NODE_ENV === 'production'
@@ -27,8 +29,8 @@ export default function graphqlContext(req, res, next) {
   const graphqlVersion = currentVersionObj.miscVersionName
 
   req.context.graphql = {
-    schemaForCurrentVersion: JSON.parse(
-      fs.readFileSync(path.join(process.cwd(), `lib/graphql/static/schema-${graphqlVersion}.json`))
+    schemaForCurrentVersion: readCompressedJsonFileFallback(
+      `lib/graphql/static/schema-${graphqlVersion}.json`
     ),
     previewsForCurrentVersion: previews[graphqlVersion],
     upcomingChangesForCurrentVersion: upcomingChanges[graphqlVersion],