Skip to content

Commit a47a043

Browse files
savannahostrowskiSonicField
savannahostrowski
authored and
SonicField
committed
pythonGH-118306: Update JIT to use LLVM 18 (pythonGH-118307)
1 parent aa0d7d2 commit a47a043

File tree

8 files changed

+55
-37
lines changed

8 files changed

+55
-37
lines changed

‎.github/workflows/jit.yml

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
- true
4242
- false
4343
llvm:
44-
- 16
44+
- 18
4545
include:
4646
- target: i686-pc-windows-msvc/msvc
4747
architecture: Win32
@@ -94,6 +94,7 @@ jobs:
9494
- name: Native Windows
9595
if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
9696
run: |
97+
choco upgrade llvm -y
9798
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
9899
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
99100
./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
@@ -102,27 +103,31 @@ jobs:
102103
- name: Emulated Windows
103104
if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
104105
run: |
106+
choco upgrade llvm -y
105107
choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
106108
./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
107109
108110
- name: Native macOS
109111
if: runner.os == 'macOS'
110112
run: |
113+
brew update
111114
brew install llvm@${{ matrix.llvm }}
112115
SDKROOT="$(xcrun --show-sdk-path)" \
113116
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
114117
make all --jobs 4
115118
./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
116119
120+
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
117121
- name: Native Linux
118122
if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
119123
run: |
120124
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
121125
export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
122-
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
126+
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
123127
make all --jobs 4
124128
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
125129
130+
# --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
126131
- name: Emulated Linux
127132
if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
128133
run: |
@@ -139,6 +144,6 @@ jobs:
139144
CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
140145
CPP="$CC --preprocess" \
141146
HOSTRUNNER=qemu-${{ matrix.architecture }} \
142-
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
147+
./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
143148
make all --jobs 4
144149
./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3

‎Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Update JIT compilation to use LLVM 18

‎Python/jit.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
234234
assert((int64_t)value < (1LL << 31));
235235
*loc32 = (uint32_t)value;
236236
continue;
237+
case HoleKind_ARM64_RELOC_BRANCH26:
237238
case HoleKind_IMAGE_REL_ARM64_BRANCH26:
238239
case HoleKind_R_AARCH64_CALL26:
239240
case HoleKind_R_AARCH64_JUMP26:
@@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
278279
case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21:
279280
case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21:
280281
case HoleKind_R_AARCH64_ADR_GOT_PAGE:
282+
case HoleKind_R_AARCH64_ADR_PREL_PG_HI21:
281283
// 21-bit count of pages between this page and an absolute address's
282284
// page... I know, I know, it's weird. Pairs nicely with
283285
// ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below).
@@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[])
341343
case HoleKind_ARM64_RELOC_PAGEOFF12:
342344
case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A:
343345
case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L:
346+
case HoleKind_R_AARCH64_ADD_ABS_LO12_NC:
344347
case HoleKind_R_AARCH64_LD64_GOT_LO12_NC:
345348
// 12-bit low part of an absolute address. Pairs nicely with
346349
// ARM64_RELOC_GOT_LOAD_PAGE21 (above).

‎Tools/jit/README.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,21 @@ sudo ./llvm.sh 16
2323

2424
### macOS
2525

26-
Install LLVM 16 with [Homebrew](https://brew.sh):
26+
Install LLVM 18 with [Homebrew](https://brew.sh):
2727

2828
```sh
29-
brew install llvm@16
29+
brew install llvm@18
3030
```
3131

3232
Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them.
3333

3434
### Windows
3535

36-
Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
36+
Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".**
37+
38+
### Dev Containers
39+
40+
If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box.
3741

3842
## Building
3943

‎Tools/jit/_llvm.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import subprocess
88
import typing
99

10-
_LLVM_VERSION = 16
10+
_LLVM_VERSION = 18
1111
_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+")
1212

1313
_P = typing.ParamSpec("_P")

‎Tools/jit/_schema.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import typing
33

44
HoleKind: typing.TypeAlias = typing.Literal[
5+
"ARM64_RELOC_BRANCH26",
56
"ARM64_RELOC_GOT_LOAD_PAGE21",
67
"ARM64_RELOC_GOT_LOAD_PAGEOFF12",
78
"ARM64_RELOC_PAGE21",
@@ -16,8 +17,10 @@
1617
"IMAGE_REL_I386_REL32",
1718
"R_AARCH64_ABS64",
1819
"R_AARCH64_ADR_GOT_PAGE",
20+
"R_AARCH64_ADR_PREL_PG_HI21",
1921
"R_AARCH64_CALL26",
2022
"R_AARCH64_JUMP26",
23+
"R_AARCH64_ADD_ABS_LO12_NC",
2124
"R_AARCH64_LD64_GOT_LO12_NC",
2225
"R_AARCH64_MOVW_UABS_G0_NC",
2326
"R_AARCH64_MOVW_UABS_G1_NC",
@@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict):
6871

6972

7073
class _ELFSymbol(typing.TypedDict):
71-
Name: dict[typing.Literal["Value"], str]
74+
Name: dict[typing.Literal["Name"], str]
7275
Value: int
7376

7477

7578
class _MachOSymbol(typing.TypedDict):
76-
Name: dict[typing.Literal["Value"], str]
79+
Name: dict[typing.Literal["Name"], str]
7780
Value: int
7881

7982

@@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict):
99102
Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]]
100103
SectionData: dict[typing.Literal["Bytes"], list[int]]
101104
Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]]
102-
Type: dict[typing.Literal["Value"], str]
105+
Type: dict[typing.Literal["Name"], str]
103106

104107

105108
class MachOSection(typing.TypedDict):

‎Tools/jit/_stencils.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,9 @@ def process_relocations(self, *, alignment: int = 1) -> None:
203203
"""Fix up all GOT and internal relocations for this stencil group."""
204204
for hole in self.code.holes.copy():
205205
if (
206-
hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"}
206+
hole.kind in {
207+
"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"
208+
}
207209
and hole.value is HoleValue.ZERO
208210
):
209211
self.code.pad(alignment)

‎Tools/jit/_targets.py

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ def _handle_relocation(
221221
case {
222222
"Offset": offset,
223223
"Symbol": s,
224-
"Type": {"Value": "IMAGE_REL_I386_DIR32" as kind},
224+
"Type": {"Name": "IMAGE_REL_I386_DIR32" as kind},
225225
}:
226226
offset += base
227227
value, symbol = self._unwrap_dllimport(s)
@@ -230,7 +230,7 @@ def _handle_relocation(
230230
"Offset": offset,
231231
"Symbol": s,
232232
"Type": {
233-
"Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
233+
"Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind
234234
},
235235
}:
236236
offset += base
@@ -242,7 +242,7 @@ def _handle_relocation(
242242
"Offset": offset,
243243
"Symbol": s,
244244
"Type": {
245-
"Value": "IMAGE_REL_ARM64_BRANCH26"
245+
"Name": "IMAGE_REL_ARM64_BRANCH26"
246246
| "IMAGE_REL_ARM64_PAGEBASE_REL21"
247247
| "IMAGE_REL_ARM64_PAGEOFFSET_12A"
248248
| "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind
@@ -262,7 +262,7 @@ class _ELF(
262262
def _handle_section(
263263
self, section: _schema.ELFSection, group: _stencils.StencilGroup
264264
) -> None:
265-
section_type = section["Type"]["Value"]
265+
section_type = section["Type"]["Name"]
266266
flags = {flag["Name"] for flag in section["Flags"]["Flags"]}
267267
if section_type == "SHT_RELA":
268268
assert "SHF_INFO_LINK" in flags, flags
@@ -290,7 +290,7 @@ def _handle_section(
290290
for wrapped_symbol in section["Symbols"]:
291291
symbol = wrapped_symbol["Symbol"]
292292
offset = len(stencil.body) + symbol["Value"]
293-
name = symbol["Name"]["Value"]
293+
name = symbol["Name"]["Name"]
294294
name = name.removeprefix(self.prefix)
295295
group.symbols[name] = value, offset
296296
stencil.body.extend(section["SectionData"]["Bytes"])
@@ -312,9 +312,9 @@ def _handle_relocation(
312312
case {
313313
"Addend": addend,
314314
"Offset": offset,
315-
"Symbol": {"Value": s},
315+
"Symbol": {"Name": s},
316316
"Type": {
317-
"Value": "R_AARCH64_ADR_GOT_PAGE"
317+
"Name": "R_AARCH64_ADR_GOT_PAGE"
318318
| "R_AARCH64_LD64_GOT_LO12_NC"
319319
| "R_X86_64_GOTPCREL"
320320
| "R_X86_64_GOTPCRELX"
@@ -327,8 +327,8 @@ def _handle_relocation(
327327
case {
328328
"Addend": addend,
329329
"Offset": offset,
330-
"Symbol": {"Value": s},
331-
"Type": {"Value": kind},
330+
"Symbol": {"Name": s},
331+
"Type": {"Name": kind},
332332
}:
333333
offset += base
334334
s = s.removeprefix(self.prefix)
@@ -371,7 +371,7 @@ def _handle_section(
371371
for wrapped_symbol in section["Symbols"]:
372372
symbol = wrapped_symbol["Symbol"]
373373
offset = symbol["Value"] - start_address
374-
name = symbol["Name"]["Value"]
374+
name = symbol["Name"]["Name"]
375375
name = name.removeprefix(self.prefix)
376376
group.symbols[name] = value, offset
377377
assert "Relocations" in section
@@ -387,9 +387,9 @@ def _handle_relocation(
387387
match relocation:
388388
case {
389389
"Offset": offset,
390-
"Symbol": {"Value": s},
390+
"Symbol": {"Name": s},
391391
"Type": {
392-
"Value": "ARM64_RELOC_GOT_LOAD_PAGE21"
392+
"Name": "ARM64_RELOC_GOT_LOAD_PAGE21"
393393
| "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind
394394
},
395395
}:
@@ -399,8 +399,8 @@ def _handle_relocation(
399399
addend = 0
400400
case {
401401
"Offset": offset,
402-
"Symbol": {"Value": s},
403-
"Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
402+
"Symbol": {"Name": s},
403+
"Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind},
404404
}:
405405
offset += base
406406
s = s.removeprefix(self.prefix)
@@ -410,13 +410,13 @@ def _handle_relocation(
410410
)
411411
case {
412412
"Offset": offset,
413-
"Section": {"Value": s},
414-
"Type": {"Value": "X86_64_RELOC_SIGNED" as kind},
413+
"Section": {"Name": s},
414+
"Type": {"Name": "X86_64_RELOC_SIGNED" as kind},
415415
} | {
416416
"Offset": offset,
417-
"Symbol": {"Value": s},
417+
"Symbol": {"Name": s},
418418
"Type": {
419-
"Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
419+
"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind
420420
},
421421
}:
422422
offset += base
@@ -427,12 +427,12 @@ def _handle_relocation(
427427
)
428428
case {
429429
"Offset": offset,
430-
"Section": {"Value": s},
431-
"Type": {"Value": kind},
430+
"Section": {"Name": s},
431+
"Type": {"Name": kind},
432432
} | {
433433
"Offset": offset,
434-
"Symbol": {"Value": s},
435-
"Type": {"Value": kind},
434+
"Symbol": {"Name": s},
435+
"Type": {"Name": kind},
436436
}:
437437
offset += base
438438
s = s.removeprefix(self.prefix)
@@ -446,13 +446,12 @@ def _handle_relocation(
446446
def get_target(host: str) -> _COFF | _ELF | _MachO:
447447
"""Build a _Target for the given host "triple" and options."""
448448
if re.fullmatch(r"aarch64-apple-darwin.*", host):
449-
args = ["-mcmodel=large"]
450-
return _MachO(host, alignment=8, args=args, prefix="_")
449+
return _MachO(host, alignment=8, prefix="_")
451450
if re.fullmatch(r"aarch64-pc-windows-msvc", host):
452451
args = ["-fms-runtime-lib=dll"]
453452
return _COFF(host, alignment=8, args=args)
454453
if re.fullmatch(r"aarch64-.*-linux-gnu", host):
455-
args = ["-mcmodel=large"]
454+
args = ["-fpic"]
456455
return _ELF(host, alignment=8, args=args)
457456
if re.fullmatch(r"i686-pc-windows-msvc", host):
458457
args = ["-DPy_NO_ENABLE_SHARED"]
@@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO:
463462
args = ["-fms-runtime-lib=dll"]
464463
return _COFF(host, args=args)
465464
if re.fullmatch(r"x86_64-.*-linux-gnu", host):
466-
return _ELF(host)
465+
args = ["-fpic"]
466+
return _ELF(host, args=args)
467467
raise ValueError(host)

0 commit comments

Comments
 (0)