Repo sync

data/reusables/code-scanning/codeql-query-tables/javascript.md

@@ -90,16 +90,16 @@
 | [Failure to abandon session](https://codeql.github.com/codeql-query-help/javascript/js-session-fixation/) | 384 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 | [File data in outbound network request](https://codeql.github.com/codeql-query-help/javascript/js-file-access-to-http/) | 200 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 | [Hard-coded data interpreted as code](https://codeql.github.com/codeql-query-help/javascript/js-hardcoded-data-interpreted-as-code/) | 506 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Indirect uncontrolled command line](https://codeql.github.com/codeql-query-help/javascript/js-indirect-command-line-injection/) | 078, 088 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Insecure temporary file](https://codeql.github.com/codeql-query-help/javascript/js-insecure-temporary-file/) | 377, 378 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Log injection](https://codeql.github.com/codeql-query-help/javascript/js-log-injection/) | 117 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Missing origin verification in `postMessage` handler](https://codeql.github.com/codeql-query-help/javascript/js-missing-origin-check/) | 020, 940 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Missing regular expression anchor](https://codeql.github.com/codeql-query-help/javascript/js-regex-missing-regexp-anchor/) | 020 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
+| [Indirect uncontrolled command line](https://codeql.github.com/codeql-query-help/javascript/js-indirect-command-line-injection/) | 078, 088 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Insecure temporary file](https://codeql.github.com/codeql-query-help/javascript/js-insecure-temporary-file/) | 377, 378 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Log injection](https://codeql.github.com/codeql-query-help/javascript/js-log-injection/) | 117 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Missing origin verification in `postMessage` handler](https://codeql.github.com/codeql-query-help/javascript/js-missing-origin-check/) | 020, 940 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Missing regular expression anchor](https://codeql.github.com/codeql-query-help/javascript/js-regex-missing-regexp-anchor/) | 020 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
 | [Network data written to file](https://codeql.github.com/codeql-query-help/javascript/js-http-to-file-access/) | 912, 434 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 | [Password in configuration file](https://codeql.github.com/codeql-query-help/javascript/js-password-in-configuration-file/) | 256, 260, 313, 522 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 | [Potential file system race condition](https://codeql.github.com/codeql-query-help/javascript/js-file-system-race/) | 367 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 | [Remote property injection](https://codeql.github.com/codeql-query-help/javascript/js-remote-property-injection/) | 250, 400 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Sensitive cookie without SameSite restrictions](https://codeql.github.com/codeql-query-help/javascript/js-samesite-none-cookie/) | 1275 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
+| [Sensitive cookie without SameSite restrictions](https://codeql.github.com/codeql-query-help/javascript/js-samesite-none-cookie/) | 1275 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
 | [Unsafe code constructed from library input](https://codeql.github.com/codeql-query-help/javascript/js-unsafe-code-construction/) | 094, 079, 116 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 | [User-controlled bypass of security check](https://codeql.github.com/codeql-query-help/javascript/js-user-controlled-bypass/) | 807, 290 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
 

data/reusables/code-scanning/codeql-query-tables/python.md

@@ -37,13 +37,13 @@
 | [XML external entity expansion](https://codeql.github.com/codeql-query-help/python/py-xxe/) | 611, 827 | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
 | [XML internal entity expansion](https://codeql.github.com/codeql-query-help/python/py-xml-bomb/) | 776, 400 | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
 | [XPath query built from user-controlled sources](https://codeql.github.com/codeql-query-help/python/py-xpath-injection/) | 643 | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
-| [Arbitrary file write during tarfile extraction](https://codeql.github.com/codeql-query-help/python/py-tarslip/) | 022 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
+| [Arbitrary file write during tarfile extraction](https://codeql.github.com/codeql-query-help/python/py-tarslip/) | 022 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
 | [Hard-coded credentials](https://codeql.github.com/codeql-query-help/python/py-hardcoded-credentials/) | 259, 321, 798 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Jinja2 templating with autoescape=False](https://codeql.github.com/codeql-query-help/python/py-jinja2-autoescape-false/) | 079 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Log Injection](https://codeql.github.com/codeql-query-help/python/py-log-injection/) | 117 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Overly permissive file permissions](https://codeql.github.com/codeql-query-help/python/py-overly-permissive-file/) | 732 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Partial server-side request forgery](https://codeql.github.com/codeql-query-help/python/py-partial-ssrf/) | 918 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Request without certificate validation](https://codeql.github.com/codeql-query-help/python/py-request-without-cert-validation/) | 295 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
-| [Unsafe shell command constructed from library input](https://codeql.github.com/codeql-query-help/python/py-shell-command-constructed-from-input/) | 078, 088, 073 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "x" aria-label="Not included" %} |
+| [Jinja2 templating with autoescape=False](https://codeql.github.com/codeql-query-help/python/py-jinja2-autoescape-false/) | 079 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Log Injection](https://codeql.github.com/codeql-query-help/python/py-log-injection/) | 117 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Overly permissive file permissions](https://codeql.github.com/codeql-query-help/python/py-overly-permissive-file/) | 732 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Partial server-side request forgery](https://codeql.github.com/codeql-query-help/python/py-partial-ssrf/) | 918 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Request without certificate validation](https://codeql.github.com/codeql-query-help/python/py-request-without-cert-validation/) | 295 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
+| [Unsafe shell command constructed from library input](https://codeql.github.com/codeql-query-help/python/py-shell-command-constructed-from-input/) | 078, 088, 073 | {% octicon "x" aria-label="Not included" %} | {% octicon "check" aria-label="Included" %} | {% octicon "check" aria-label="Included" %} |
 
 {% endrowheaders %}

src/content-render/scripts/all-documents/lib.ts

@@ -1,3 +1,4 @@
+import type { Page } from '@/types'
 import contextualize from '@/frame/middleware/context/context.js'
 import features from '@/versions/middleware/features.js'
 import shortVersions from '@/versions/middleware/short-versions.js'
@@ -19,24 +20,6 @@ export type AllDocument = {
   documents: Document[]
 }
 
-type Permalink = {
-  languageCode: string
-  pageVersion: string
-  title: string
-  href: string
-}
-
-type Page = {
-  permalinks: Permalink[]
-  fullPath: string
-  title: string
-  shortTitle?: string
-  intro: string
-  languageCode: string
-  documentType: string
-  renderProp: (prop: string, context: any, opts: any) => Promise<string>
-}
-
 type Options = {
   languages: string[]
   versions: string[]

src/data-directory/scripts/find-orphaned-features/find.ts

@@ -34,6 +34,7 @@ import path from 'path'
 import chalk from 'chalk'
 import { TokenizationError } from 'liquidjs'
 
+import type { Page } from '@/types'
 import warmServer from '@/frame/lib/warm-server.js'
 import { getDeepDataByLanguage } from '@/data-directory/lib/get-data.js'
 import { getLiquidTokens } from '@/content-linter/lib/helpers/liquid-utils.js'
@@ -51,22 +52,6 @@ type Options = {
   verbose?: boolean
 }
 
-type Page = {
-  permalinks: Permalink[]
-  relativePath: string
-  fullPath: string
-  title: string
-  shortTitle?: string
-  intro: string
-  markdown: string
-  languageCode: string
-  versions: Record<string, string>
-}
-type Permalink = {
-  href: string
-  languageCode: string
-}
-
 export async function find(options: Options) {
   const { sourceDirectory } = options
   if (process.env.ENABLED_LANGUAGES && process.env.ENABLED_LANGUAGES === 'en') {

src/frame/lib/warm-server.d.ts

@@ -1,9 +1,3 @@
-type Site = {
-  pages: Record<String, Page>
-  redirects: Record<string, string>
-  unversionedTree: Record<string, string>
-  siteTree: Record<string, string>
-  pageList: Page[]
-}
+import type { Site } from '@/types'
 
 export default function warmServer(languages: string[]): Promise<Site>

src/frame/middleware/index.ts

@@ -18,7 +18,7 @@ import {
 import handleErrors from '@/observability/middleware/handle-errors'
 import handleNextDataPath from './handle-next-data-path'
 import detectLanguage from '@/languages/middleware/detect-language'
-import reloadTree from './reload-tree.js'
+import reloadTree from './reload-tree'
 import context from './context/context.js'
 import shortVersions from '@/versions/middleware/short-versions.js'
 import languageCodeRedirects from '@/redirects/middleware/language-code-redirects.js'

src/frame/middleware/reload-tree.js → src/frame/middleware/reload-tree.ts

@@ -16,43 +16,47 @@
 
 import path from 'path'
 
-import languages, { languageKeys } from '#src/languages/lib/languages.js'
-import createTree from '#src/frame/lib/create-tree.js'
-import warmServer from '#src/frame/lib/warm-server.js'
-import { loadSiteTree, loadPages, loadPageMap } from '#src/frame/lib/page-data.js'
-import loadRedirects from '#src/redirects/lib/precompile.js'
+import type { Response, NextFunction } from 'express'
+
+import type { ExtendedRequest, UnversionedTree, SiteTree } from '@/types'
+import languages, { languageKeys } from '@/languages/lib/languages.js'
+import createTree from '@/frame/lib/create-tree.js'
+import warmServer from '@/frame/lib/warm-server.js'
+import { loadSiteTree, loadPages, loadPageMap } from '@/frame/lib/page-data.js'
+import loadRedirects from '@/redirects/lib/precompile.js'
 
 const languagePrefixRegex = new RegExp(`^/(${languageKeys.join('|')})(/|$)`)
 const englishPrefixRegex = /^\/en(\/|$)/
 
 const isDev = process.env.NODE_ENV === 'development'
 
-export default async function reloadTree(req, res, next) {
+export default async function reloadTree(req: ExtendedRequest, res: Response, next: NextFunction) {
   if (!isDev) return next()
   // Filter out things like `/will/redirect` or `/_next/data/...`
-  if (!languagePrefixRegex.test(req.pagePath)) return next()
+  if (!req.pagePath || !languagePrefixRegex.test(req.pagePath)) return next()
   // We only bother if the loaded URL is something `/en/...`
   if (!englishPrefixRegex.test(req.pagePath)) return next()
 
-  const warmed = await warmServer()
+  const warmed = await warmServer([])
+
   // For all the real English content, this usually takes about 30-60ms on
   // an Intel MacBook Pro.
   const before = getMtimes(warmed.unversionedTree.en)
-  warmed.unversionedTree.en = await createTree(
+  warmed.unversionedTree.en = (await createTree(
     path.join(languages.en.dir, 'content'),
     undefined,
     warmed.unversionedTree.en,
-  )
+  )) as UnversionedTree // Note! Have to use `as` until create-tree.js is JS
   const after = getMtimes(warmed.unversionedTree.en)
   // The next couple of operations are much slower (in total) than
   // refreshing the tree. So we want to know if the tree changed before
   // bothering.
   // If refreshing of the `.en` part of the `unversionedTree` takes 40ms
   // then the following operations takes about 140ms.
   if (before !== after) {
-    warmed.siteTree = await loadSiteTree(warmed.unversionedTree)
+    warmed.siteTree = (await loadSiteTree(warmed.unversionedTree)) as SiteTree
     warmed.pageList = await loadPages(warmed.unversionedTree)
-    warmed.pageMap = await loadPageMap(warmed.pageList)
+    warmed.pages = await loadPageMap(warmed.pageList)
     warmed.redirects = await loadRedirects(warmed.pageList)
   }
 
@@ -63,7 +67,7 @@ export default async function reloadTree(req, res, next) {
 // in the tree.
 // You can use this to compute it before and after the tree is (maybe)
 // mutated and if the numbers *change* you can know the tree changed.
-function getMtimes(tree) {
+function getMtimes(tree: UnversionedTree) {
   let mtimes = tree.page.mtime
   for (const child of tree.childPages || []) {
     mtimes += getMtimes(child)

src/languages/scripts/count-translation-corruptions.ts

@@ -8,7 +8,8 @@ import walk from 'walk-sync'
 
 import { getLiquidTokens } from '@/content-linter/lib/helpers/liquid-utils.js'
 import languages from '@/languages/lib/languages.js'
-import warmServer, { type Site } from '@/frame/lib/warm-server.js'
+import warmServer from '@/frame/lib/warm-server.js'
+import type { Site } from '@/types'
 import { correctTranslatedContentStrings } from '@/languages/lib/correct-translation-content.js'
 
 program
@@ -17,16 +18,6 @@ program
   .action(main)
 program.parse(process.argv)
 
-type Page = {
-  relativePath: string
-  fullPath: string
-  title: string
-  shortTitle?: string
-  intro: string
-  markdown: string
-  languageCode: string
-}
-
 type Reusables = Map<string, string>
 
 async function main(languageCodes: string[]) {
@@ -80,7 +71,7 @@ function run(languageCode: string, site: Site, englishReusables: Reusables) {
 
   console.log(`--- Tallying liquid corruptions in ${languageCode} (${language.name}) ---`)
 
-  const pageList: Page[] = site.pageList
+  const pageList = site.pageList
   const errors = new Map<string, number>()
   const wheres = new Map<string, number>()
   const illegalTags = new Map<string, number>()

src/links/lib/validate-docs-urls.ts

@@ -8,25 +8,16 @@ import features from '@/versions/middleware/features.js'
 import findPage from '@/frame/middleware/find-page.js'
 import { createMinimalProcessor } from '@/content-render/unified/processor.js'
 import getRedirect from '@/redirects/lib/get-redirect.js'
+import type { Page } from '@/types'
 
 export type DocsUrls = {
   [identifier: string]: string
 }
 
-type Page = {
-  permalinks: Permalink[]
-  relativePath: string
-  rawIntro: string
-  rawPermissions?: string
-  markdown: string
-}
 type Permalink = {
   href: string
   languageCode: string
 }
-type PageMap = {
-  [href: string]: Page
-}
 type Redirects = {
   [from: string]: string
 }
@@ -48,7 +39,7 @@ export type Check = {
 
 export async function validateDocsUrl(docsUrls: DocsUrls, { checkFragments = false } = {}) {
   const site = await warmServer(['en'])
-  const pages: PageMap = site.pages
+  const pages = site.pages
   const redirects: Redirects = site.redirects
 
   const checks: Check[] = []

src/types.ts

@@ -26,3 +26,64 @@ type Language = {
 export type Languages = {
   [key: string]: Language
 }
+
+type Permalink = {
+  languageCode: string
+  pageVersion: string
+  title: string
+  href: string
+}
+
+type Versions = {
+  feature?: string
+  fpt?: string
+  ghec?: string
+  ghes?: string
+}
+
+export type Page = {
+  mtime: number
+  permalinks: Permalink[]
+  fullPath: string
+  title: string
+  shortTitle?: string
+  intro: string
+  languageCode: string
+  documentType: string
+  renderProp: (prop: string, context: any, opts: any) => Promise<string>
+  markdown: string
+  versions: Versions
+}
+
+export type Tree = {
+  page: Page
+  children: string[] | undefined
+  href: string
+  childPages?: Tree[]
+}
+export type VersionedTree = {
+  [version: string]: Tree
+}
+
+export type SiteTree = {
+  [languageCode: string]: VersionedTree
+}
+
+export type UnversionedTree = {
+  page: Page
+  children: string[]
+  childPages: UnversionedTree[]
+}
+
+export type UnversionLanguageTree = {
+  [languageCode: string]: UnversionedTree
+}
+
+export type Site = {
+  pages: Record<string, Page>
+  redirects: Record<string, string>
+  unversionedTree: UnversionLanguageTree
+  siteTree: SiteTree
+  pageList: Page[]
+  pageMap: Record<string, Page>
+}