From 0f07af3df734c990ca3f1ef0b1a29611384c14fa Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 24 Oct 2024 09:53:59 -0700 Subject: [PATCH 1/6] Keep full paths from leaking into the generated file --- Tools/jit/_targets.py | 9 +++++++-- Tools/jit/build.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index d8dce0a905c0f8..0ed8ad30a7d833 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -61,10 +61,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: args = ["--disassemble", "--reloc", f"{path}"] output = await _llvm.maybe_run("llvm-objdump", args, echo=self.verbose) if output is not None: + # Make sure that full paths don't leak out (for reproducibility): + long, short = str(path), str(path.name) group.code.disassembly.extend( - line.expandtabs().strip() + line.expandtabs().strip().replace(long, short) for line in output.splitlines() - if not line.isspace() ) args = [ "--elf-output-style=JSON", @@ -122,6 +123,10 @@ async def _compile( f"-I{CPYTHON / 'Tools' / 'jit'}", "-O3", "-c", + # Shorten full absolute file paths in the generated code (like the + # __FILE__ macro and assert failure messages) for reproducibility: + f"-ffile-prefix-map={CPYTHON}=.", + f"-ffile-prefix-map={tempdir}=.", # This debug info isn't necessary, and bloats out the JIT'ed code. # We *may* be able to re-enable this, process it, and JIT it for a # nicer debugging experience... but that needs a lot more research: diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4a23c6f0afa74a..a8cb0f67c36363 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -8,7 +8,7 @@ import _targets if __name__ == "__main__": - comment = f"$ {shlex.join([sys.executable] + sys.argv)}" + comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "target", type=_targets.get_target, help="a PEP 11 target triple to compile for" From 379c22bda3bcd724fcc18b7359fce7f52bd55c91 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 24 Oct 2024 12:16:32 -0700 Subject: [PATCH 2/6] Fix the order in which we process holes --- Tools/jit/_targets.py | 15 +++++++++++---- Tools/jit/_writer.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 0ed8ad30a7d833..1951a54b2f1d55 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -91,9 +91,6 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) - group.process_relocations( - known_symbols=self.known_symbols, alignment=self.alignment - ) return group def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: @@ -172,7 +169,17 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: c.write_text(template.replace("CASE", case)) coro = self._compile(opname, c, work) tasks.append(group.create_task(coro, name=opname)) - return {task.get_name(): task.result() for task in tasks} + stencil_groups = {task.get_name(): task.result() for task in tasks} + for stencil_group in stencil_groups.values(): + # TODO: Get rid of this once we always have a trampoline (LLVM 19): + if not stencil_group.code.body: + continue + # TODO: Make this (and the other method it calls) a method on _Target: + stencil_group.process_relocations( + known_symbols=self.known_symbols, alignment=self.alignment + ) + return stencil_groups + def build( self, out: pathlib.Path, *, comment: str = "", force: bool = False diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 81a9f08db31703..5588784544ee00 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -77,6 +77,6 @@ def dump( groups: dict[str, _stencils.StencilGroup], symbols: dict[str, int] ) -> typing.Iterator[str]: """Yield a JIT compiler line-by-line as a C header file.""" - for opname, group in sorted(groups.items()): + for opname, group in groups.items(): yield from _dump_stencil(opname, group) yield from _dump_footer(groups, symbols) From 102ef6406e8fabde3a5f97858ac7689892558f4f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 24 Oct 2024 12:16:45 -0700 Subject: [PATCH 3/6] Fix empty padding --- Tools/jit/_stencils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 61be8fd3bbdf55..ee761a73fa808a 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -202,7 +202,8 @@ def pad(self, alignment: int) -> None: """Pad the stencil to the given alignment.""" offset = len(self.body) padding = -offset % alignment - self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") + if padding: + self.disassembly.append(f"{offset:x}: {' '.join(['00'] * padding)}") self.body.extend([0] * padding) def remove_jump(self, *, alignment: int = 1) -> None: From a1b1e0da59dbd6f78ba1aaec24e4fc0d83803203 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 7 Nov 2024 16:04:44 -0800 Subject: [PATCH 4/6] Lint --- Tools/jit/_targets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 1951a54b2f1d55..d82aa7832b3bf1 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -180,7 +180,6 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: ) return stencil_groups - def build( self, out: pathlib.Path, *, comment: str = "", force: bool = False ) -> None: From 64804663eb9bad28e357a7c3356fd6389a8cdb4c Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Sat, 16 Nov 2024 15:36:08 -0800 Subject: [PATCH 5/6] Clean up the diff --- Tools/jit/_targets.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index d82aa7832b3bf1..d23ced19842347 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -171,10 +171,6 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: tasks.append(group.create_task(coro, name=opname)) stencil_groups = {task.get_name(): task.result() for task in tasks} for stencil_group in stencil_groups.values(): - # TODO: Get rid of this once we always have a trampoline (LLVM 19): - if not stencil_group.code.body: - continue - # TODO: Make this (and the other method it calls) a method on _Target: stencil_group.process_relocations( known_symbols=self.known_symbols, alignment=self.alignment ) From 2ee672089f06fb82419228daab934aa7443a97e5 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 22 Nov 2024 08:46:57 -0800 Subject: [PATCH 6/6] blurb add --- .../next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst diff --git a/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst b/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst new file mode 100644 index 00000000000000..9e8a078983f20b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-22-08-46-46.gh-issue-115869.UVLSKd.rst @@ -0,0 +1 @@ +Make ``jit_stencils.h`` (which is produced during JIT builds) reproducible.