GH-115869: Reference implementation for hosting JIT stencils

.gitattributes

@@ -20,6 +20,9 @@
 # Specific binary files
 PC/classicAppCompat.* binary
 
+# JIT stencils are generated by the build and should not be modified
+Tools/jit/stencils/* text eol=lf
+
 # Text files that should not be subject to eol conversion
 [attr]noeol -text
 

.github/workflows/jit.yml

@@ -58,7 +58,7 @@ jobs:
           - aarch64-pc-windows-msvc/msvc
           - x86_64-apple-darwin/clang
           - aarch64-apple-darwin/clang
-          - x86_64-unknown-linux-gnu/gcc
+          - x86_64-pc-linux-gnu/gcc
           - aarch64-unknown-linux-gnu/gcc
         debug:
           - true
@@ -81,7 +81,7 @@ jobs:
           - target: aarch64-apple-darwin/clang
             architecture: aarch64
             runner: macos-14
-          - target: x86_64-unknown-linux-gnu/gcc
+          - target: x86_64-pc-linux-gnu/gcc
             architecture: x86_64
             runner: ubuntu-24.04
           - target: aarch64-unknown-linux-gnu/gcc
@@ -133,6 +133,71 @@ jobs:
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
+      - name: Check if stencils need regeneration
+        id: check-stencils
+        if: ${{!matrix.debug }}
+        shell: bash
+        run: |
+          git add Tools/jit/stencils
+
+          if ! git diff --staged --exit-code --quiet; then
+            git diff --staged > jit_stencils.patch
+            exit 1
+          fi
+
+      - name: Format target name
+        if: ${{ failure() && steps.check-stencils.conclusion == 'failure' && !matrix.debug }}
+        id: strip-target
+        shell: bash
+        run: |
+          target=${{ matrix.target }}
+          target="${target%%/*}"
+          echo "target=$target" >> $GITHUB_OUTPUT
+
+      - name: Upload stencil patch
+        if: ${{ failure() && steps.check-stencils.conclusion == 'failure' && !matrix.debug }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.strip-target.outputs.target }}-jit-stencils
+          path: jit_stencils.patch
+
+  aggregate-stencil-patches:
+    name: Aggregate stencil patches
+    needs: jit
+    runs-on: ubuntu-24.04
+    if: ${{ failure() }}
+    steps:
+      - name: Download stencil artifacts
+        run: |
+          mkdir -p artifacts
+          gh run download ${{ github.run_id }} --pattern '*jit-stencils*' --dir artifacts --repo ${{ github.repository }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Aggregate stencil patches
+        run: |
+          temp_file="$(mktemp)"
+          : > "$temp_file"
+
+          find artifacts -mindepth 1 -maxdepth 1 -type d | while read -r dir; do
+              find "$dir" -type f -name "*.patch" -exec cat {} + >> "$temp_file"
+          done
+
+          sed_command="s/[[:space:]]*$//"
+          if [[ "$(uname)" == "Darwin" ]]; then
+              sed -i '' "$sed_command" "$temp_file"
+          else
+              sed -i "$sed_command" "$temp_file"
+          fi
+
+          mv "$temp_file" aggregated_jit_stencils.patch
+
+      - name: Upload aggregated stencil patch
+        uses: actions/upload-artifact@v4
+        with:
+          name: aggregated-jit-stencils
+          path: aggregated_jit_stencils.patch
+
   jit-with-disabled-gil:
     name: Free-Threaded (Debug)
     needs: interpreter

.gitignore

@@ -138,6 +138,7 @@ Tools/unicode/data/
 /.ccache
 /cross-build/
 /jit_stencils.h
+/jit_stencils.h.digest
 /platform
 /profile-clean-stamp
 /profile-run-stamp

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
         args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
-        files: ^Tools/jit/
+        files: ^Tools/jit/(?!stencils/).*
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0

Misc/NEWS.d/next/Core_and_Builtins/2025-01-27-02-40-30.gh-issue-115869.QeXSGb.rst

@@ -0,0 +1 @@
+Removes the LLVM build-time requirement for JIT-enabled builds of Python

PCbuild/regen.targets

@@ -30,7 +30,7 @@
     <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" />
     <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: -->
     <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/>
-    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/>
+    <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h;$(PySourcePath)Tools\jit\stencils\**"/>
     <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/>
     <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/>
     <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" />

Tools/jit/README.md

@@ -1,19 +1,33 @@
 The JIT Compiler
 ================
 
-This version of CPython can be built with an experimental just-in-time compiler[^pep-744]. While most everything you already know about building and using CPython is unchanged, you will probably need to install a compatible version of LLVM first.
+This version of CPython can be built with an experimental just-in-time compiler[^pep-744].
+
+## Building CPython with the JIT enabled
+
+For `PCbuild`-based builds, pass the new `--experimental-jit` option to `build.bat`.
+
+For all other builds, pass the new `--enable-experimental-jit` option to `configure`.
+
+Otherwise, just configure and build as you normally would. Cross-compiling "just works", since the JIT is built for the host platform. 
+
+The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/whatsnew/3.13.html#experimental-jit-compiler).
 
 Python 3.11 or newer is required to build the JIT.
 
-## Installing LLVM
+## Contributing to the JIT
+
+While LLVM is not a build-time dependency as stencils are hosted in `Tools/jit/stencils`, you may still want to install LLVM to simplify your local development process (e.g. not have to wait for a CI run to regenerate the stencil for your platform).
+
+###  Installing LLVM
 
 The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon).
 
 LLVM version 19 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
 
 It's easy to install all of the required tools:
 
-### Linux
+#### Linux
 
 Install LLVM 19 on Ubuntu/Debian:
 
@@ -29,7 +43,7 @@ Install LLVM 19 on Fedora Linux 40 or newer:
 sudo dnf install 'clang(major) = 19' 'llvm(major) = 19'
 ```
 
-### macOS
+#### macOS
 
 Install LLVM 19 with [Homebrew](https://brew.sh):
 
@@ -39,16 +53,21 @@ brew install llvm@19
 
 Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them.
 
-### Windows
+#### Windows
 
 Install LLVM 19 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=19), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
 
 Alternatively, you can use [chocolatey](https://chocolatey.org):
 
 ```sh
 choco install llvm --version=19.1.0
+
 ```
 
+### Applying stencil patches from CI
+
+Stencil files are validated and updated when changes are made to JIT-related files in CI as part of the `jit.yml` workflow. The final step in the `jit` job diffs the current
+stencils in the repo against those generated in CI. If there is a diff for a platform’s stencil, a patch file for the updated stencil is generated and the step will fail. After CI is finished running across all platforms, the patches are aggregated into a single patch file for convenience. You can download this aggregated patch, apply it locally (i.e. `git apply`), and commit the updated stencils back to your branch. Then, the subsequent CI run will pass.
 
 ## Building
 

Tools/jit/_targets.py

@@ -7,7 +7,9 @@
 import os
 import pathlib
 import re
+import shutil
 import sys
+import sysconfig
 import tempfile
 import typing
 
@@ -21,10 +23,19 @@
 
 TOOLS_JIT_BUILD = pathlib.Path(__file__).resolve()
 TOOLS_JIT = TOOLS_JIT_BUILD.parent
+TOOLS_JIT_STENCILS = TOOLS_JIT / "stencils"
 TOOLS = TOOLS_JIT.parent
 CPYTHON = TOOLS.parent
 PYTHON_EXECUTOR_CASES_C_H = CPYTHON / "Python" / "executor_cases.c.h"
 TOOLS_JIT_TEMPLATE_C = TOOLS_JIT / "template.c"
+SUPPORTED_TRIPLES = {
+    "aarch64-apple-darwin",
+    "aarch64-unknown-linux-gnu",
+    "i686-pc-windows-msvc",
+    "x86_64-apple-darwin",
+    "x86_64-pc-windows-msvc",
+    "x86_64-pc-linux-gnu",
+}
 
 _S = typing.TypeVar("_S", _schema.COFFSection, _schema.ELFSection, _schema.MachOSection)
 _R = typing.TypeVar(
@@ -43,6 +54,7 @@ class _Target(typing.Generic[_S, _R]):
     debug: bool = False
     verbose: bool = False
     known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
+    stencil_name: str = ""
 
     def _compute_digest(self, out: pathlib.Path) -> str:
         hasher = hashlib.sha256()
@@ -52,6 +64,8 @@ def _compute_digest(self, out: pathlib.Path) -> str:
         hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes())
         hasher.update((out / "pyconfig.h").read_bytes())
         for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)):
+            if pathlib.Path(dirpath) == TOOLS_JIT_STENCILS:
+                continue
             for filename in filenames:
                 hasher.update(pathlib.Path(dirpath, filename).read_bytes())
         return hasher.hexdigest()
@@ -176,41 +190,72 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]:
             )
         return stencil_groups
 
-    def build(
-        self, out: pathlib.Path, *, comment: str = "", force: bool = False
-    ) -> None:
+    def build(self, out: pathlib.Path, *, force: bool = False) -> None:
         """Build jit_stencils.h in the given directory."""
         if not self.stable:
             warning = f"JIT support for {self.triple} is still experimental!"
             request = "Please report any issues you encounter.".center(len(warning))
             outline = "=" * len(warning)
             print("\n".join(["", outline, warning, request, outline, ""]))
-        digest = f"// {self._compute_digest(out)}\n"
+        digest = f"{self._compute_digest(out)}\n"
         jit_stencils = out / "jit_stencils.h"
+        jit_stencils_digest = out / "jit_stencils.h.digest"
+        hosted_stencil = TOOLS_JIT_STENCILS / f"{self.stencil_name}.h"
+
         if (
             not force
+            and jit_stencils_digest.exists()
             and jit_stencils.exists()
-            and jit_stencils.read_text().startswith(digest)
+            and hosted_stencil.exists()
         ):
-            return
+            if jit_stencils_digest.read_text() == digest:
+                print("Skipping JIT stencil generation")
+                return
+
         stencil_groups = asyncio.run(self._build_stencils())
         jit_stencils_new = out / "jit_stencils.h.new"
         try:
-            with jit_stencils_new.open("w") as file:
-                file.write(digest)
-                if comment:
-                    file.write(f"// {comment}\n")
-                file.write("\n")
+            with jit_stencils_new.open("w", newline="\n") as file:
                 for line in _writer.dump(stencil_groups, self.known_symbols):
                     file.write(f"{line}\n")
             try:
                 jit_stencils_new.replace(jit_stencils)
+
+                if "windows" in self.triple:
+                    JIT_ARGS = {
+                        "--experimental-jit"
+                    }  # TODO: Need to figure out the right flags here for Windows
+                    copy_stencils = True
+
+                else:
+                    # TODO: Need to revisit which flags are actually needed here
+                    # JIT_ARGS = {
+                    #     "--enable-experimental-jit",
+                    #     "--with-lto",
+                    #     "--enable-optimizations",
+                    # }
+                    makefile = out / "Makefile"
+                    match = re.search(r"CONFIG_ARGS\s*=\s*'(.*)'", makefile.read_text())
+                    assert match is not None
+                    config_args = match.group(1)
+                    if config_args:
+                        # copy_stencils = all(
+                        #     arg in JIT_ARGS for arg in config_args.split()
+                        # )
+                        copy_stencils = not ("--with-debug" in config_args) and not (
+                            "--disable-gil" in config_args
+                        )
+
+                copy_stencils = copy_stencils and self.stencil_name in SUPPORTED_TRIPLES
+                if copy_stencils:
+                    shutil.copy(jit_stencils, hosted_stencil)
             except FileNotFoundError:
                 # another process probably already moved the file
                 if not jit_stencils.is_file():
                     raise
         finally:
             jit_stencils_new.unlink(missing_ok=True)
+            jit_stencils_digest.write_text(digest)
 
 
 class _COFF(
@@ -497,9 +542,12 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
     """Build a _Target for the given host "triple" and options."""
     target: _COFF | _ELF | _MachO
     if re.fullmatch(r"aarch64-apple-darwin.*", host):
-        target = _MachO(host, alignment=8, prefix="_")
+        target = _MachO(
+            host, alignment=8, prefix="_", stencil_name="aarch64-apple-darwin"
+        )
     elif re.fullmatch(r"aarch64-pc-windows-msvc", host):
         args = ["-fms-runtime-lib=dll"]
+        # stencil_name is omitted since aarch64-pc-windows-msvc is Tier 3
         target = _COFF(host, alignment=8, args=args)
     elif re.fullmatch(r"aarch64-.*-linux-gnu", host):
         args = [
@@ -508,22 +556,24 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
             # was required to disable them.
             "-mno-outline-atomics",
         ]
-        target = _ELF(host, alignment=8, args=args)
+        target = _ELF(
+            host, alignment=8, args=args, stencil_name="aarch64-unknown-linux-gnu"
+        )
     elif re.fullmatch(r"i686-pc-windows-msvc", host):
         args = [
             "-DPy_NO_ENABLE_SHARED",
             # __attribute__((preserve_none)) is not supported
             "-Wno-ignored-attributes",
         ]
-        target = _COFF(host, args=args, prefix="_")
+        target = _COFF(host, args=args, prefix="_", stencil_name="i686-pc-windows-msvc")
     elif re.fullmatch(r"x86_64-apple-darwin.*", host):
-        target = _MachO(host, prefix="_")
+        target = _MachO(host, prefix="_", stencil_name="x86_64-apple-darwin")
     elif re.fullmatch(r"x86_64-pc-windows-msvc", host):
         args = ["-fms-runtime-lib=dll"]
-        target = _COFF(host, args=args)
+        target = _COFF(host, args=args, stencil_name="x86_64-pc-windows-msvc")
     elif re.fullmatch(r"x86_64-.*-linux-gnu", host):
         args = ["-fpic"]
-        target = _ELF(host, args=args)
+        target = _ELF(host, args=args, stencil_name="x86_64-pc-linux-gnu")
     else:
         raise ValueError(host)
     return target

Tools/jit/_writer.py

@@ -48,7 +48,7 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator
     yield "{"
     for part, stencil in [("code", group.code), ("data", group.data)]:
         for line in stencil.disassembly:
-            yield f"    // {line}"
+            yield f"    // {line}".rstrip()
         stripped = stencil.body.rstrip(b"\x00")
         if stripped:
             yield f"    const unsigned char {part}_body[{len(stencil.body)}] = {{"