From 5967648b3fa95b81b603883a9ecc7d4edaec9c0f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 19 Nov 2023 02:42:18 +0000 Subject: [PATCH 01/18] Allow perf to work without frame pointers Signed-off-by: Pablo Galindo DWARF Change encoding --- Include/internal/pycore_ceval.h | 1 + Lib/test/test_perf_profiler.py | 137 ++++++-- Lib/test/test_perfmaps.py | 23 +- Modules/_testinternalcapi.c | 4 +- Python/initconfig.c | 14 + Python/lel.c | 8 + Python/perf_trampoline.c | 557 ++++++++++++++++++++++++++++++++ Python/pylifecycle.c | 9 +- Python/sysmodule.c | 10 + 9 files changed, 721 insertions(+), 42 deletions(-) create mode 100644 Python/lel.c diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a357bfa3a26064..d38d809c904835 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -105,6 +105,7 @@ extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; +extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; #endif static inline PyObject* diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 040be63da11447..14ffab3344848b 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -5,6 +5,7 @@ import sysconfig import os import pathlib +import shutil from test import support from test.support.script_helper import ( make_script, @@ -76,14 +77,27 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - perf_lines = perf_file_contents.splitlines(); - expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + perf_lines = perf_file_contents.splitlines() + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] for expected_symbol in expected_symbols: - perf_line = next((line for line in perf_lines if expected_symbol in line), None) - self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_line = next( + (line for line in perf_lines if expected_symbol in line), None + ) + self.assertIsNotNone( + perf_line, f"Could not find {expected_symbol} in perf file" + ) perf_addr = perf_line.split(" ")[0] - self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") - self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") + self.assertFalse( + perf_addr.startswith("0x"), "Address should not be prefixed with 0x" + ) + self.assertTrue( + set(perf_addr).issubset(string.hexdigits), + "Address should contain only hex characters", + ) def test_trampoline_works_with_forks(self): code = """if 1: @@ -212,7 +226,7 @@ def test_sys_api_get_status(self): assert_python_ok("-c", code) -def is_unwinding_reliable(): +def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False @@ -259,14 +273,27 @@ def perf_command_works(): return True -def run_perf(cwd, *args, **env_vars): +def run_perf(cwd, *args, use_jit=False, **env_vars): if env_vars: env = os.environ.copy() env.update(env_vars) else: env = None output_file = cwd + "/perf_output.perf" - base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + if not use_jit: + base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + else: + base_cmd = ( + "perf", + "record", + "-g", + "--call-graph=dwarf,65528", + "-F99", + "-k1", + "-o", + output_file, + "--", + ) proc = subprocess.run( base_cmd + args, stdout=subprocess.PIPE, @@ -274,9 +301,21 @@ def run_perf(cwd, *args, **env_vars): env=env, ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") + if use_jit: + jit_output_file = cwd + "/jit_output.dump" + command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) + proc = subprocess.run( + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + if proc.returncode: + print(proc.stderr) + raise ValueError(f"Perf failed with return code {proc.returncode}") + # Copy the jit_output_file to the output_file + os.rename(jit_output_file, output_file) + base_cmd = ("perf", "script") proc = subprocess.run( ("perf", "script", "-i", output_file), @@ -290,20 +329,9 @@ def run_perf(cwd, *args, **env_vars): ) -@unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(is_unwinding_reliable(), "Unwinding is unreliable") -class TestPerfProfiler(unittest.TestCase): - def setUp(self): - super().setUp() - self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) - - def tearDown(self) -> None: - super().tearDown() - files_to_delete = ( - set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files - ) - for file in files_to_delete: - file.unlink() +class TestPerfProfilerMixin: + def run_perf(self, script_dir, perf_mode, script): + raise NotImplementedError() def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: @@ -322,14 +350,14 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, "-Xperf", script) + stdout, stderr = self.run_perf(script_dir, script) self.assertEqual(stderr, "") self.assertIn(f"py::foo:{script}", stdout) self.assertIn(f"py::bar:{script}", stdout) self.assertIn(f"py::baz:{script}", stdout) - def test_python_calls_do_not_appear_in_the_stack_if_perf_activated(self): + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: def foo(n): @@ -346,13 +374,38 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, script) + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=False + ) self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + is_unwinding_reliable_with_frame_pointers(), + "Unwinding is unreliable with frame pointers", +) +class TestPerfProfiler(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf(script_dir, sys.executable, "-Xperf", script) + return run_perf(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + def test_pre_fork_compile(self): code = """if 1: import sys @@ -370,7 +423,7 @@ def bar_fork(): foo_fork() def foo(): - pass + import time; time.sleep(1) def bar(): foo() @@ -423,12 +476,32 @@ def compile_trampolines_for_all_functions(): # identical in both the parent and child perf-map files. perf_file_lines = perf_file_contents.split("\n") for line in perf_file_lines: - if ( - f"py::foo_fork:{script}" in line - or f"py::bar_fork:{script}" in line - ): + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf( + script_dir, sys.executable, "-Xperfjit", script, use_jit=True + ) + return run_perf(script_dir, sys.executable, script, use_jit=True) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("jit*.dump")) + self.perf_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete = files_to_delete - self.perf_files + for file in files_to_delete: + file.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index a17adb89f55360..5899daaa958172 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -7,13 +7,22 @@ if sys.platform != 'linux': raise unittest.SkipTest('Linux only') +def entry1(): + pass + +def entry2(): + pass class TestPerfMapWriting(unittest.TestCase): def test_write_perf_map_entry(self): - self.assertEqual(write_perf_map_entry(0x1234, 5678, "entry1"), 0) - self.assertEqual(write_perf_map_entry(0x2345, 6789, "entry2"), 0) - with open(f"/tmp/perf-{os.getpid()}.map") as f: - perf_file_contents = f.read() - self.assertIn("1234 162e entry1", perf_file_contents) - self.assertIn("2345 1a85 entry2", perf_file_contents) - perf_map_state_teardown() + sys.activate_stack_trampoline("perf") + try: + self.assertEqual(write_perf_map_entry(0x1234, 5678, entry1.__code__), 0) + self.assertEqual(write_perf_map_entry(0x2345, 6789, entry2.__code__), 0) + with open(f"/tmp/perf-{os.getpid()}.map") as f: + perf_file_contents = f.read() + self.assertIn("1234 162e py::entry1", perf_file_contents) + self.assertIn("2345 1a85 py::entry2", perf_file_contents) + perf_map_state_teardown() + finally: + sys.deactivate_stack_trampoline() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 7d277df164d3ec..645c85df208945 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -902,9 +902,9 @@ write_perf_map_entry(PyObject *self, PyObject *args) PyObject *code_addr_v; const void *code_addr; unsigned int code_size; - const char *entry_name; + PyCodeObject *entry_name; - if (!PyArg_ParseTuple(args, "OIs", &code_addr_v, &code_size, &entry_name)) + if (!PyArg_ParseTuple(args, "OIO", &code_addr_v, &code_size, &entry_name)) return NULL; code_addr = PyLong_AsVoidPtr(code_addr_v); if (code_addr == NULL) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 06e317907b8ec9..1b3c3f2813ff83 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1702,6 +1702,20 @@ config_init_perf_profiling(PyConfig *config) if (xoption) { config->perf_profiling = 1; } + env = config_get_env(config, "PYTHONPERFJITSUPPORT"); + if (env) { + if (_Py_str_to_int(env, &active) != 0) { + active = 0; + } + if (active) { + config->perf_profiling = 2; + } + } + xoption = config_get_xoption(config, L"perfjit"); + if (xoption) { + config->perf_profiling = 2; + } + return _PyStatus_OK(); } diff --git a/Python/lel.c b/Python/lel.c new file mode 100644 index 00000000000000..da52f1f7cd72fc --- /dev/null +++ b/Python/lel.c @@ -0,0 +1,8 @@ +typedef void* (py_evaluator)(void* ts, void* f, int t); + +void* +trampoline(void *ts, void *f, + int throwflag, py_evaluator evaluator) +{ + return evaluator(ts, f, throwflag); +} diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 750ba18d3510ed..c4c6aca99dec70 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -143,6 +143,8 @@ any DWARF information available for them). #include // mmap() #include #include // sysconf() +#include // gettimeofday() + #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__) #define PY_HAVE_INVALIDATE_ICACHE @@ -236,6 +238,561 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { &perf_map_free_state, }; + +// Perf jitdump API + +// Jitdump perf format + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; + void* mapped_buffer; + size_t mapped_size; +} PerfMapJitState; + + +static PerfMapJitState perf_jit_map_state; + +typedef uint64_t uword; +typedef const char* CodeComments; + +#define Pd "d" +#define MB (1024 * 1024) + +#define EM_386 3 +#define EM_X86_64 62 +#define EM_ARM 40 +#define EM_AARCH64 183 +#define EM_RISCV 243 + +#define TARGET_ARCH_IA32 0 +#define TARGET_ARCH_X64 0 +#define TARGET_ARCH_ARM 0 +#define TARGET_ARCH_ARM64 0 +#define TARGET_ARCH_RISCV32 0 +#define TARGET_ARCH_RISCV64 0 + +#define FLAG_generate_perf_jitdump 0 +#define FLAG_write_protect_code 0 +#define FLAG_write_protect_vm_isolate 0 +#define FLAG_code_comments 0 + +#define UNREACHABLE() + +static uword GetElfMachineArchitecture(void) { +#if TARGET_ARCH_IA32 + return EM_386; +#elif TARGET_ARCH_X64 + return EM_X86_64; +#elif TARGET_ARCH_ARM + return EM_ARM; +#elif TARGET_ARCH_ARM64 + return EM_AARCH64; +#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 + return EM_RISCV; +#else + UNREACHABLE(); + return 0; +#endif +} + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} Header; + + enum PerfEvent { + kLoad = 0, + kMove = 1, + kDebugInfo = 2, + kClose = 3, + kUnwindingInfo = 4 +}; + +struct BaseEvent { + uint32_t event; + uint32_t size; + uint64_t time_stamp; + }; + +typedef struct { + struct BaseEvent base; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} CodeLoadEvent; + +typedef struct { + struct BaseEvent base; + uint64_t unwind_data_size; + uint64_t eh_frame_hdr_size; + uint64_t mapped_size; +} CodeUnwindingInfoEvent; + +const intptr_t kMillisecondsPerSecond = 1000; +const intptr_t kMicrosecondsPerMillisecond = 1000; +const intptr_t kMicrosecondsPerSecond = (kMicrosecondsPerMillisecond * kMillisecondsPerSecond); +const intptr_t kNanosecondsPerMicrosecond = 1000; +const intptr_t kNanosecondsPerMillisecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerMillisecond); +const intptr_t kNanosecondsPerSecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerSecond); + +// Dwarf encoding constants + +static int code_id_ = 0; + +uint8_t kUData4 = 0x03; +uint8_t kSData4 = 0x0b; +uint8_t kPcRel = 0x10; +uint8_t kDataRel = 0x30; +uint8_t kOmit = 0xff; +static const int kEhFrameHdrSize = 20; +typedef struct { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + int32_t eh_frame_ptr; + int32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; + +static int64_t get_current_monotonic_ticks(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + UNREACHABLE(); + return 0; + } + // Convert to nanoseconds. + int64_t result = ts.tv_sec; + result *= kNanosecondsPerSecond; + result += ts.tv_nsec; + return result; +} + +int64_t get_current_time_microseconds(void) { + // gettimeofday has microsecond resolution. + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + UNREACHABLE(); + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; +} + + +static int64_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + + +static void perf_map_jit_write_fully(const void* buffer, size_t size) { + FILE* out_file = perf_jit_map_state.perf_map; + const char* ptr = (const char*)(buffer); + while (size > 0) { + const size_t written = fwrite(ptr, 1, size, out_file); + if (written == 0) { + UNREACHABLE(); + break; + } + size -= written; + ptr += written; + } +} + +static void perf_map_jit_write_header(int pid, FILE* out_file) { + Header header; + header.magic = 0x4A695444; + header.version = 1; + header.size = sizeof(Header); + header.elf_mach_target = GetElfMachineArchitecture(); + header.process_id = pid; + header.time_stamp = get_current_time_microseconds(); + header.flags = 0; + perf_map_jit_write_fully(&header, sizeof(header)); +} + +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; + } + + const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) + if (page_size == -1) { + close(fd); + return NULL; + } + + // The perf jit interface forces us to map the first page of the file + // to signal that we are using the interface. + perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; + } + perf_jit_map_state.mapped_size = page_size; + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; + } + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; + } + return &perf_jit_map_state; +} + +/* DWARF definitions. */ +#define DWRF_CIE_VERSION 1 + +enum { + DWRF_CFA_nop = 0x0, + DWRF_CFA_offset_extended = 0x5, + DWRF_CFA_def_cfa = 0xc, + DWRF_CFA_def_cfa_offset = 0xe, + DWRF_CFA_offset_extended_sf = 0x11, + DWRF_CFA_advance_loc = 0x40, + DWRF_CFA_offset = 0x80 +}; + +enum + { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* FDE data encoding. */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* FDE flags. */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + + DWRF_EH_PE_indirect = 0x80 + }; + +enum { DWRF_TAG_compile_unit = 0x11 }; + +enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; + +enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; + +enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; + +enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; + +enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; + +enum { +#ifdef __x86_64__ + /* Yes, the order is strange, but correct. */ + DWRF_REG_AX, + DWRF_REG_DX, + DWRF_REG_CX, + DWRF_REG_BX, + DWRF_REG_SI, + DWRF_REG_DI, + DWRF_REG_BP, + DWRF_REG_SP, + DWRF_REG_8, + DWRF_REG_9, + DWRF_REG_10, + DWRF_REG_11, + DWRF_REG_12, + DWRF_REG_13, + DWRF_REG_14, + DWRF_REG_15, + DWRF_REG_RA, +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_REG_SP = 31, + DWRF_REG_RA = 30, +#else +# error "Unsupported target architecture" +#endif +}; + +typedef struct ELFObjectContext +{ + uint8_t* p; /* Pointer to next address in obj.space. */ + uint8_t* startp; /* Pointer to start address in obj.space. */ + uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ + uint32_t code_size; /* Size of machine code. */ +} ELFObjectContext; + +/* Append a null-terminated string. */ +static uint32_t +elfctx_append_string(ELFObjectContext* ctx, const char* str) +{ + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Append a SLEB128 value. */ +static void +elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) +{ + uint8_t* p = ctx->p; + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); + } + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +/* Append a ULEB128 to buffer. */ +static void +elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) +{ + uint8_t* p = ctx->p; + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); + } + *p++ = (char)v; + ctx->p = p; +} + +/* Shortcuts to generate DWARF structures. */ +#define DWRF_U8(x) (*p++ = (x)) +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +/* Initialize .eh_frame section. */ +static void +elf_init_ehframe(ELFObjectContext* ctx) +{ + uint8_t* p = ctx->p; + uint8_t* framep = p; + + /* Emit DWARF EH CIE. */ + DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); /* Augmentation. */ + DWRF_UV(1); /* Code alignment factor. */ + DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DWRF_U8(DWRF_REG_RA); /* Return address register. */ + DWRF_UV(1); + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ + DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->eh_frame_p = p; + + /* Emit DWARF EH FDE. */ + DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ + DWRF_U32(-0x30); /* Machine code offset relative to .text. */ + DWRF_U32(ctx->code_size); /* Machine code length. */ + DWRF_U8(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#ifdef __x86_64__ + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_advance_loc | 6); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); + /* Extra registers saved for JIT-compiled code. */ +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); + DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); + DWRF_U8(DWRF_CFA_advance_loc | 12); + DWRF_U8(DWRF_CFA_offset | -(64 - 29)); + DWRF_U8(DWRF_CFA_offset | -(64 - 30)); + DWRF_U8(DWRF_CFA_def_cfa_offset); + DWRF_UV(0); +#else +# error "Unsupported target architecture" +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t));) + + ctx->p = p; +} + +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + + if (perf_jit_map_state.perf_map == NULL) { + void* ret = perf_map_jit_init(); + if(ret == NULL){ + return; + } + } + + const char *entry = ""; + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + const char *filename = ""; + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + + + size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); + if (perf_map_entry == NULL) { + return; + } + snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); + + const size_t name_length = strlen(perf_map_entry); + uword base = (uword)code_addr; + uword size = code_size; + + // Write the code unwinding info event. + CodeUnwindingInfoEvent ev2; + EhFrameHeader f; + + char buffer[1024]; + ELFObjectContext ctx; + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + elf_init_ehframe(&ctx); + + int eh_frame_size = ctx.p - ctx.startp; + + + ev2.base.event = kUnwindingInfo; + ev2.base.time_stamp = get_current_monotonic_ticks(); + ev2.unwind_data_size = sizeof(f) + eh_frame_size; + ev2.eh_frame_hdr_size = sizeof(f); + ev2.mapped_size = ev2.unwind_data_size; + int content_size = sizeof(ev2) + sizeof(f) + eh_frame_size; + int padding_size = round_up(content_size, 8) - content_size; + ev2.base.size = content_size + padding_size; + perf_map_jit_write_fully(&ev2, sizeof(ev2)); + + + f.version = 1; + f.eh_frame_ptr_enc = kSData4 | kPcRel; + f.fde_count_enc = kUData4; + f.table_enc = kSData4 | kDataRel; + f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_fde_count = 1; + f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; + f.to = -(eh_frame_size - cie_size); + + perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(&f, sizeof(f)); + + char padding_bytes[] = "\0\0\0\0\0\0\0\0"; + perf_map_jit_write_fully(&padding_bytes, padding_size); + + // Write the code load event. + CodeLoadEvent ev; + ev.base.event = kLoad; + ev.base.size = sizeof(ev) + (name_length+1) + size; + ev.base.time_stamp = get_current_monotonic_ticks(); + ev.process_id = getpid(); + ev.thread_id = gettid(); + ev.vma = base; + ev.code_address = base; + ev.code_size = size; + ev.code_id = code_id_++; + + perf_map_jit_write_fully(&ev, sizeof(ev)); + perf_map_jit_write_fully(perf_map_entry, name_length+1); + perf_map_jit_write_fully((void*)(base), size); + return; +} + +static int perf_map_jit_fini(void* state) { + if (perf_jit_map_state.perf_map != NULL) { + // close the file + PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); + fclose(perf_jit_map_state.perf_map); + PyThread_release_lock(perf_jit_map_state.map_lock); + + // clean up the lock and state + PyThread_free_lock(perf_jit_map_state.map_lock); + perf_jit_map_state.perf_map = NULL; + } + if (perf_jit_map_state.mapped_buffer != NULL) { + munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + } + trampoline_api.state = NULL; + return 0; +} + + +_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { + &perf_map_jit_init, + &perf_map_jit_write_entry, + &perf_map_jit_fini, +}; + + +// TRAMPOLINE MANAGEMENT API + static int new_code_arena(void) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 1d8af26e4a1cb7..9c5163abb9a136 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1168,7 +1168,14 @@ init_interp_main(PyThreadState *tstate) #ifdef PY_HAVE_PERF_TRAMPOLINE if (config->perf_profiling) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_callbacks) < 0 || + _PyPerf_Callbacks *cur_cb; + if (config->perf_profiling == 1) { + cur_cb = &_Py_perfmap_callbacks; + } + else { + cur_cb = &_Py_perfmap_jit_callbacks; + } + if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || _PyPerfTrampoline_Init(config->perf_profiling) < 0) { return _PyStatus_ERR("can't initialize the perf trampoline"); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index c2de4ecdc8ce0f..b2263ed79dc060 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2236,6 +2236,16 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } + else if (strcmp(backend, "perfjit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; + } + } + } } else { PyErr_Format(PyExc_ValueError, "invalid backend: %s", backend); From 3adad265f3026a8eea5e51da98edf3bec656a593 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 31 Dec 2023 15:44:27 +0000 Subject: [PATCH 02/18] Fixes for new perf --- Python/perf_trampoline.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index c4c6aca99dec70..c743e8f470588f 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -724,7 +724,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev2.base.time_stamp = get_current_monotonic_ticks(); ev2.unwind_data_size = sizeof(f) + eh_frame_size; ev2.eh_frame_hdr_size = sizeof(f); - ev2.mapped_size = ev2.unwind_data_size; + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); int content_size = sizeof(ev2) + sizeof(f) + eh_frame_size; int padding_size = round_up(content_size, 8) - content_size; ev2.base.size = content_size + padding_size; @@ -813,6 +813,7 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; + size_t chunk_size = round_up(code_size + 0x50, 16); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -821,9 +822,9 @@ new_code_arena(void) // measurable performance improvement by rounding trampolines up to 32-bit // or 64-bit alignment. - size_t n_copies = mem_size / code_size; + size_t n_copies = mem_size / chunk_size; for (size_t i = 0; i < n_copies; i++) { - memcpy(memory + i * code_size, start, code_size * sizeof(char)); + memcpy(memory + i * chunk_size, start, code_size * sizeof(char)); } // Some systems may prevent us from creating executable code on the fly. int res = mprotect(memory, mem_size, PROT_READ | PROT_EXEC); @@ -877,8 +878,8 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - code_arena->size_left -= code_arena->code_size; - code_arena->current_addr += code_arena->code_size; + code_arena->size_left -= round_up(code_arena->code_size + 0x50, 16); + code_arena->current_addr += round_up(code_arena->code_size + 0x50, 16); return trampoline; } @@ -886,7 +887,7 @@ static inline py_trampoline compile_trampoline(void) { if ((perf_code_arena == NULL) || - (perf_code_arena->size_left <= perf_code_arena->code_size)) { + (perf_code_arena->size_left <= round_up(perf_code_arena->code_size+0x50, 16))) { if (new_code_arena() < 0) { return NULL; } From 0c43cfcaf4c32ca445fdb9e017199d3471a3833d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 31 Dec 2023 16:07:51 +0000 Subject: [PATCH 03/18] Fixes --- Include/internal/pycore_ceval_state.h | 1 + Lib/test/test_perfmaps.py | 23 +++++++---------------- Modules/_testinternalcapi.c | 4 ++-- Python/perf_trampoline.c | 12 ++++++++++++ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 28738980eb49be..5d4da92b0f4a87 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -52,6 +52,7 @@ struct _ceval_runtime_state { struct { #ifdef PY_HAVE_PERF_TRAMPOLINE perf_status_t status; + int perf_trampoline_type; Py_ssize_t extra_code_index; struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index 5899daaa958172..a17adb89f55360 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -7,22 +7,13 @@ if sys.platform != 'linux': raise unittest.SkipTest('Linux only') -def entry1(): - pass - -def entry2(): - pass class TestPerfMapWriting(unittest.TestCase): def test_write_perf_map_entry(self): - sys.activate_stack_trampoline("perf") - try: - self.assertEqual(write_perf_map_entry(0x1234, 5678, entry1.__code__), 0) - self.assertEqual(write_perf_map_entry(0x2345, 6789, entry2.__code__), 0) - with open(f"/tmp/perf-{os.getpid()}.map") as f: - perf_file_contents = f.read() - self.assertIn("1234 162e py::entry1", perf_file_contents) - self.assertIn("2345 1a85 py::entry2", perf_file_contents) - perf_map_state_teardown() - finally: - sys.deactivate_stack_trampoline() + self.assertEqual(write_perf_map_entry(0x1234, 5678, "entry1"), 0) + self.assertEqual(write_perf_map_entry(0x2345, 6789, "entry2"), 0) + with open(f"/tmp/perf-{os.getpid()}.map") as f: + perf_file_contents = f.read() + self.assertIn("1234 162e entry1", perf_file_contents) + self.assertIn("2345 1a85 entry2", perf_file_contents) + perf_map_state_teardown() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 645c85df208945..7d277df164d3ec 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -902,9 +902,9 @@ write_perf_map_entry(PyObject *self, PyObject *args) PyObject *code_addr_v; const void *code_addr; unsigned int code_size; - PyCodeObject *entry_name; + const char *entry_name; - if (!PyArg_ParseTuple(args, "OIO", &code_addr_v, &code_size, &entry_name)) + if (!PyArg_ParseTuple(args, "OIs", &code_addr_v, &code_size, &entry_name)) return NULL; code_addr = PyLong_AsVoidPtr(code_addr_v); if (code_addr == NULL) { diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index c743e8f470588f..dad6203e013a75 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -189,12 +189,19 @@ struct code_arena_st { typedef struct code_arena_st code_arena_t; typedef struct trampoline_api_st trampoline_api_t; +enum perf_trampoline_type { + PERF_TRAMPOLINE_UNSET = 0, + PERF_TRAMPOLINE_TYPE_MAP = 1, + PERF_TRAMPOLINE_TYPE_JITDUMP = 2, +}; + #define perf_status _PyRuntime.ceval.perf.status #define extra_code_index _PyRuntime.ceval.perf.extra_code_index #define perf_code_arena _PyRuntime.ceval.perf.code_arena #define trampoline_api _PyRuntime.ceval.perf.trampoline_api #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork +#define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type static void perf_map_write_entry(void *state, const void *code_addr, @@ -222,6 +229,7 @@ static void* perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); + perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -1038,6 +1046,7 @@ _PyPerfTrampoline_Fini(void) } if (perf_status == PERF_STATUS_OK) { trampoline_api.free_state(trampoline_api.state); + perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } extra_code_index = -1; perf_status = PERF_STATUS_NO_INIT; @@ -1066,6 +1075,9 @@ _PyPerfTrampoline_AfterFork_Child(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE if (persist_after_fork) { + if (perf_trampoline_type != PERF_TRAMPOLINE_TYPE_MAP) { + return PyStatus_Error("Failed to copy perf map file as perf trampoline type is not type map."); + } _PyPerfTrampoline_Fini(); char filename[256]; pid_t parent_pid = getppid(); From eb9c3bac50a2d9d4a52a9b4acdcbbd8348f0c6e0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 31 Dec 2023 16:33:00 +0000 Subject: [PATCH 04/18] Fix smelly --- Python/perf_trampoline.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index dad6203e013a75..a720f562f005e7 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -346,12 +346,12 @@ typedef struct { uint64_t mapped_size; } CodeUnwindingInfoEvent; -const intptr_t kMillisecondsPerSecond = 1000; -const intptr_t kMicrosecondsPerMillisecond = 1000; -const intptr_t kMicrosecondsPerSecond = (kMicrosecondsPerMillisecond * kMillisecondsPerSecond); -const intptr_t kNanosecondsPerMicrosecond = 1000; -const intptr_t kNanosecondsPerMillisecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerMillisecond); -const intptr_t kNanosecondsPerSecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerSecond); +static const intptr_t kMillisecondsPerSecond = 1000; +static const intptr_t kMicrosecondsPerMillisecond = 1000; +static const intptr_t kMicrosecondsPerSecond = (kMicrosecondsPerMillisecond * kMillisecondsPerSecond); +static const intptr_t kNanosecondsPerMicrosecond = 1000; +static const intptr_t kNanosecondsPerMillisecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerMillisecond); +static const intptr_t kNanosecondsPerSecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerSecond); // Dwarf encoding constants @@ -387,7 +387,7 @@ static int64_t get_current_monotonic_ticks(void) { return result; } -int64_t get_current_time_microseconds(void) { +static int64_t get_current_time_microseconds(void) { // gettimeofday has microsecond resolution. struct timeval tv; if (gettimeofday(&tv, NULL) < 0) { From 4e010e3f44f4f49da30beb7aead0aca284ff9913 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 31 Dec 2023 16:37:28 +0000 Subject: [PATCH 05/18] Fix types --- Python/perf_trampoline.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index a720f562f005e7..4ca51fbafeb46c 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -249,8 +249,6 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { // Perf jitdump API -// Jitdump perf format - typedef struct { FILE* perf_map; PyThread_type_lock map_lock; @@ -258,7 +256,6 @@ typedef struct { size_t mapped_size; } PerfMapJitState; - static PerfMapJitState perf_jit_map_state; typedef uint64_t uword; @@ -346,12 +343,7 @@ typedef struct { uint64_t mapped_size; } CodeUnwindingInfoEvent; -static const intptr_t kMillisecondsPerSecond = 1000; -static const intptr_t kMicrosecondsPerMillisecond = 1000; -static const intptr_t kMicrosecondsPerSecond = (kMicrosecondsPerMillisecond * kMillisecondsPerSecond); -static const intptr_t kNanosecondsPerMicrosecond = 1000; -static const intptr_t kNanosecondsPerMillisecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerMillisecond); -static const intptr_t kNanosecondsPerSecond = (kNanosecondsPerMicrosecond * kMicrosecondsPerSecond); +static const intptr_t nanoseconds_per_second = 1000000000; // Dwarf encoding constants @@ -362,7 +354,6 @@ uint8_t kSData4 = 0x0b; uint8_t kPcRel = 0x10; uint8_t kDataRel = 0x30; uint8_t kOmit = 0xff; -static const int kEhFrameHdrSize = 20; typedef struct { unsigned char version; unsigned char eh_frame_ptr_enc; @@ -382,7 +373,7 @@ static int64_t get_current_monotonic_ticks(void) { } // Convert to nanoseconds. int64_t result = ts.tv_sec; - result *= kNanosecondsPerSecond; + result *= nanoseconds_per_second; result += ts.tv_nsec; return result; } @@ -398,7 +389,7 @@ static int64_t get_current_time_microseconds(void) { } -static int64_t round_up(int64_t value, int64_t multiple) { +static size_t round_up(int64_t value, int64_t multiple) { if (multiple == 0) { // Avoid division by zero return value; From 449db3a00664f84991301a57c3fc5d7266f10834 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 16:37:17 +0200 Subject: [PATCH 06/18] Refactor --- Include/internal/pycore_ceval_state.h | 1 + Makefile.pre.in | 1 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/perf_jit_trampoline.c | 614 +++++++++++++++++++++++++ Python/perf_trampoline.c | 554 +--------------------- 8 files changed, 631 insertions(+), 547 deletions(-) create mode 100644 Python/perf_jit_trampoline.c diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 5d4da92b0f4a87..6c28549f7c1cec 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -45,6 +45,7 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; + Py_ssize_t code_padding; }; #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 6a64547e97d266..0f700e969a3744 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -467,6 +467,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/perf_jit_trampoline.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index f8c5fafa561efa..49774b7142d176 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -232,6 +232,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 1c5a6d623f4dad..3ae5f19fef6397 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -91,6 +91,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index c90ad1a3592f67..c6fe7cdeea4503 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -589,6 +589,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index a96ca24cf08b66..4a964de2987a62 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1352,6 +1352,9 @@ Python + + Python + Python diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c new file mode 100644 index 00000000000000..f60faa2acfda09 --- /dev/null +++ b/Python/perf_jit_trampoline.c @@ -0,0 +1,614 @@ +#include "Python.h" +#include "pycore_ceval.h" // _PyPerf_Callbacks +#include "pycore_frame.h" +#include "pycore_interp.h" + + +#ifdef PY_HAVE_PERF_TRAMPOLINE + +#include +#include +#include +#include // mmap() +#include +#include // sysconf() +#include // gettimeofday() + +// ---------------------------------- +// Perf jitdump API +// ---------------------------------- + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; + void* mapped_buffer; + size_t mapped_size; +} PerfMapJitState; + +static PerfMapJitState perf_jit_map_state; + +/* +Usually the binary and libraries are mapped in separate region like below: + + address -> + --+---------------------+--//--+---------------------+-- + | .text | .data | ... | | .text | .data | ... | + --+---------------------+--//--+---------------------+-- + myprog libc.so + +So it'd be easy and straight-forward to find a mapped binary or library from an +address. + +But for JIT code, the code arena only cares about the code section. But the +resulting DSOs (which is generated by perf inject -j) contain ELF headers and +unwind info too. Then it'd generate following address space with synthesized +MMAP events. Let's say it has a sample between address B and C. + + sample + | + address -> A B v C + --------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + --------------------------------------------------------------------------------------------------- + +If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see +the unwind info. If it maps both .text section and unwind sections, the sample +could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing +which one is right. So to make perf happy we have non-overlapping ranges for each +DSO: + + address -> + ------------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + ------------------------------------------------------------------------------------------------------- + +As the trampolines are constant, we add a constant padding but in general the padding needs to have the +size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + +#define PERF_JIT_CODE_PADDING = 0x50; + +typedef uint64_t uword; +typedef const char* CodeComments; + +#define Pd "d" +#define MB (1024 * 1024) + +#define EM_386 3 +#define EM_X86_64 62 +#define EM_ARM 40 +#define EM_AARCH64 183 +#define EM_RISCV 243 + +#define TARGET_ARCH_IA32 0 +#define TARGET_ARCH_X64 0 +#define TARGET_ARCH_ARM 0 +#define TARGET_ARCH_ARM64 0 +#define TARGET_ARCH_RISCV32 0 +#define TARGET_ARCH_RISCV64 0 + +#define FLAG_generate_perf_jitdump 0 +#define FLAG_write_protect_code 0 +#define FLAG_write_protect_vm_isolate 0 +#define FLAG_code_comments 0 + +#define UNREACHABLE() + +static uword GetElfMachineArchitecture(void) { +#if TARGET_ARCH_IA32 + return EM_386; +#elif TARGET_ARCH_X64 + return EM_X86_64; +#elif TARGET_ARCH_ARM + return EM_ARM; +#elif TARGET_ARCH_ARM64 + return EM_AARCH64; +#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 + return EM_RISCV; +#else + UNREACHABLE(); + return 0; +#endif +} + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} Header; + + enum PerfEvent { + PerfLoad = 0, + PerfMove = 1, + PerfDebugInfo = 2, + PerfClose = 3, + PerfUnwindingInfo = 4 +}; + +struct BaseEvent { + uint32_t event; + uint32_t size; + uint64_t time_stamp; + }; + +typedef struct { + struct BaseEvent base; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} CodeLoadEvent; + +typedef struct { + struct BaseEvent base; + uint64_t unwind_data_size; + uint64_t eh_frame_hdr_size; + uint64_t mapped_size; +} CodeUnwindingInfoEvent; + +static const intptr_t nanoseconds_per_second = 1000000000; + +// Dwarf encoding constants + +static int code_id_ = 0; + +uint8_t DwarfUData4 = 0x03; +uint8_t DwarfSData4 = 0x0b; +uint8_t DwarfPcRel = 0x10; +uint8_t DwarfDataRel = 0x30; +uint8_t DwarfOmit = 0xff; +typedef struct { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + int32_t eh_frame_ptr; + int32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; + +static int64_t get_current_monotonic_ticks(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + UNREACHABLE(); + return 0; + } + // Convert to nanoseconds. + int64_t result = ts.tv_sec; + result *= nanoseconds_per_second; + result += ts.tv_nsec; + return result; +} + +static int64_t get_current_time_microseconds(void) { + // gettimeofday has microsecond resolution. + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + UNREACHABLE(); + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; +} + + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + + +static void perf_map_jit_write_fully(const void* buffer, size_t size) { + FILE* out_file = perf_jit_map_state.perf_map; + const char* ptr = (const char*)(buffer); + while (size > 0) { + const size_t written = fwrite(ptr, 1, size, out_file); + if (written == 0) { + UNREACHABLE(); + break; + } + size -= written; + ptr += written; + } +} + +static void perf_map_jit_write_header(int pid, FILE* out_file) { + Header header; + header.magic = 0x4A695444; + header.version = 1; + header.size = sizeof(Header); + header.elf_mach_target = GetElfMachineArchitecture(); + header.process_id = pid; + header.time_stamp = get_current_time_microseconds(); + header.flags = 0; + perf_map_jit_write_fully(&header, sizeof(header)); +} + +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; + } + + const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) + if (page_size == -1) { + close(fd); + return NULL; + } + + // The perf jit interface forces us to map the first page of the file + // to signal that we are using the interface. + perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; + } + perf_jit_map_state.mapped_size = page_size; + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; + } + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; + } + + + trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + return &perf_jit_map_state; +} + +/* DWARF definitions. */ + +#define DWRF_CIE_VERSION 1 + +enum { + DWRF_CFA_nop = 0x0, + DWRF_CFA_offset_extended = 0x5, + DWRF_CFA_def_cfa = 0xc, + DWRF_CFA_def_cfa_offset = 0xe, + DWRF_CFA_offset_extended_sf = 0x11, + DWRF_CFA_advance_loc = 0x40, + DWRF_CFA_offset = 0x80 +}; + +enum + { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* FDE data encoding. */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* FDE flags. */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + + DWRF_EH_PE_indirect = 0x80 + }; + +enum { DWRF_TAG_compile_unit = 0x11 }; + +enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; + +enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; + +enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; + +enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; + +enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; + +enum { +#ifdef __x86_64__ + /* Yes, the order is strange, but correct. */ + DWRF_REG_AX, + DWRF_REG_DX, + DWRF_REG_CX, + DWRF_REG_BX, + DWRF_REG_SI, + DWRF_REG_DI, + DWRF_REG_BP, + DWRF_REG_SP, + DWRF_REG_8, + DWRF_REG_9, + DWRF_REG_10, + DWRF_REG_11, + DWRF_REG_12, + DWRF_REG_13, + DWRF_REG_14, + DWRF_REG_15, + DWRF_REG_RA, +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_REG_SP = 31, + DWRF_REG_RA = 30, +#else +# error "Unsupported target architecture" +#endif +}; + +typedef struct ELFObjectContext +{ + uint8_t* p; /* Pointer to next address in obj.space. */ + uint8_t* startp; /* Pointer to start address in obj.space. */ + uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ + uint32_t code_size; /* Size of machine code. */ +} ELFObjectContext; + +/* Append a null-terminated string. */ +static uint32_t +elfctx_append_string(ELFObjectContext* ctx, const char* str) +{ + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Append a SLEB128 value. */ +static void +elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) +{ + uint8_t* p = ctx->p; + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); + } + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +/* Append a ULEB128 to buffer. */ +static void +elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) +{ + uint8_t* p = ctx->p; + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); + } + *p++ = (char)v; + ctx->p = p; +} + +/* Shortcuts to generate DWARF structures. */ +#define DWRF_U8(x) (*p++ = (x)) +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +/* Initialize .eh_frame section. */ +static void +elf_init_ehframe(ELFObjectContext* ctx) +{ + uint8_t* p = ctx->p; + uint8_t* framep = p; + + /* Emit DWARF EH CIE. */ + DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); /* Augmentation. */ + DWRF_UV(1); /* Code alignment factor. */ + DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DWRF_U8(DWRF_REG_RA); /* Return address register. */ + DWRF_UV(1); + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ + DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->eh_frame_p = p; + + /* Emit DWARF EH FDE. */ + DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ + DWRF_U32(-0x30); /* Machine code offset relative to .text. */ + DWRF_U32(ctx->code_size); /* Machine code length. */ + DWRF_U8(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#ifdef __x86_64__ + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_advance_loc | 6); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); + /* Extra registers saved for JIT-compiled code. */ +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); + DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); + DWRF_U8(DWRF_CFA_advance_loc | 12); + DWRF_U8(DWRF_CFA_offset | -(64 - 29)); + DWRF_U8(DWRF_CFA_offset | -(64 - 30)); + DWRF_U8(DWRF_CFA_def_cfa_offset); + DWRF_UV(0); +#else +# error "Unsupported target architecture" +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t));) + + ctx->p = p; +} + +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + + if (perf_jit_map_state.perf_map == NULL) { + void* ret = perf_map_jit_init(); + if(ret == NULL){ + return; + } + } + + const char *entry = ""; + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + const char *filename = ""; + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + + + size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); + if (perf_map_entry == NULL) { + return; + } + snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); + + const size_t name_length = strlen(perf_map_entry); + uword base = (uword)code_addr; + uword size = code_size; + + // Write the code unwinding info event. + + // Create unwinding information (eh frame) + ELFObjectContext ctx; + char buffer[1024]; + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + elf_init_ehframe(&ctx); + int eh_frame_size = ctx.p - ctx.startp; + + // Populate the unwind info event for perf + CodeUnwindingInfoEvent ev2; + ev2.base.event = PerfUnwindingInfo; + ev2.base.time_stamp = get_current_monotonic_ticks(); + ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + // Ensure we have enough space between DSOs when perf maps them + assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; + int padding_size = round_up(content_size, 8) - content_size; + ev2.base.size = content_size + padding_size; + perf_map_jit_write_fully(&ev2, sizeof(ev2)); + + + // Populate the eh Frame header + EhFrameHeader f; + f.version = 1; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; + f.fde_count_enc = DwarfUData4; + f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_fde_count = 1; + f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; + f.to = -(eh_frame_size - cie_size); + + perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(&f, sizeof(f)); + + char padding_bytes[] = "\0\0\0\0\0\0\0\0"; + perf_map_jit_write_fully(&padding_bytes, padding_size); + + // Write the code load event. + CodeLoadEvent ev; + ev.base.event = PerfLoad; + ev.base.size = sizeof(ev) + (name_length+1) + size; + ev.base.time_stamp = get_current_monotonic_ticks(); + ev.process_id = getpid(); + ev.thread_id = gettid(); + ev.vma = base; + ev.code_address = base; + ev.code_size = size; + ev.code_id = code_id_++; + + perf_map_jit_write_fully(&ev, sizeof(ev)); + perf_map_jit_write_fully(perf_map_entry, name_length+1); + perf_map_jit_write_fully((void*)(base), size); + return; +} + +static int perf_map_jit_fini(void* state) { + if (perf_jit_map_state.perf_map != NULL) { + // close the file + PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); + fclose(perf_jit_map_state.perf_map); + PyThread_release_lock(perf_jit_map_state.map_lock); + + // clean up the lock and state + PyThread_free_lock(perf_jit_map_state.map_lock); + perf_jit_map_state.perf_map = NULL; + } + if (perf_jit_map_state.mapped_buffer != NULL) { + munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + } + trampoline_api.state = NULL; + return 0; +} + +_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { + &perf_map_jit_init, + &perf_map_jit_write_entry, + &perf_map_jit_fini, +}; + +#endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 4ca51fbafeb46c..58abb2424cf2d9 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -229,6 +229,7 @@ static void* perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); + trampoline_api.code_padding = 0; perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -247,549 +248,6 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { }; -// Perf jitdump API - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; - void* mapped_buffer; - size_t mapped_size; -} PerfMapJitState; - -static PerfMapJitState perf_jit_map_state; - -typedef uint64_t uword; -typedef const char* CodeComments; - -#define Pd "d" -#define MB (1024 * 1024) - -#define EM_386 3 -#define EM_X86_64 62 -#define EM_ARM 40 -#define EM_AARCH64 183 -#define EM_RISCV 243 - -#define TARGET_ARCH_IA32 0 -#define TARGET_ARCH_X64 0 -#define TARGET_ARCH_ARM 0 -#define TARGET_ARCH_ARM64 0 -#define TARGET_ARCH_RISCV32 0 -#define TARGET_ARCH_RISCV64 0 - -#define FLAG_generate_perf_jitdump 0 -#define FLAG_write_protect_code 0 -#define FLAG_write_protect_vm_isolate 0 -#define FLAG_code_comments 0 - -#define UNREACHABLE() - -static uword GetElfMachineArchitecture(void) { -#if TARGET_ARCH_IA32 - return EM_386; -#elif TARGET_ARCH_X64 - return EM_X86_64; -#elif TARGET_ARCH_ARM - return EM_ARM; -#elif TARGET_ARCH_ARM64 - return EM_AARCH64; -#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 - return EM_RISCV; -#else - UNREACHABLE(); - return 0; -#endif -} - -typedef struct { - uint32_t magic; - uint32_t version; - uint32_t size; - uint32_t elf_mach_target; - uint32_t reserved; - uint32_t process_id; - uint64_t time_stamp; - uint64_t flags; -} Header; - - enum PerfEvent { - kLoad = 0, - kMove = 1, - kDebugInfo = 2, - kClose = 3, - kUnwindingInfo = 4 -}; - -struct BaseEvent { - uint32_t event; - uint32_t size; - uint64_t time_stamp; - }; - -typedef struct { - struct BaseEvent base; - uint32_t process_id; - uint32_t thread_id; - uint64_t vma; - uint64_t code_address; - uint64_t code_size; - uint64_t code_id; -} CodeLoadEvent; - -typedef struct { - struct BaseEvent base; - uint64_t unwind_data_size; - uint64_t eh_frame_hdr_size; - uint64_t mapped_size; -} CodeUnwindingInfoEvent; - -static const intptr_t nanoseconds_per_second = 1000000000; - -// Dwarf encoding constants - -static int code_id_ = 0; - -uint8_t kUData4 = 0x03; -uint8_t kSData4 = 0x0b; -uint8_t kPcRel = 0x10; -uint8_t kDataRel = 0x30; -uint8_t kOmit = 0xff; -typedef struct { - unsigned char version; - unsigned char eh_frame_ptr_enc; - unsigned char fde_count_enc; - unsigned char table_enc; - int32_t eh_frame_ptr; - int32_t eh_fde_count; - int32_t from; - int32_t to; -} EhFrameHeader; - -static int64_t get_current_monotonic_ticks(void) { - struct timespec ts; - if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { - UNREACHABLE(); - return 0; - } - // Convert to nanoseconds. - int64_t result = ts.tv_sec; - result *= nanoseconds_per_second; - result += ts.tv_nsec; - return result; -} - -static int64_t get_current_time_microseconds(void) { - // gettimeofday has microsecond resolution. - struct timeval tv; - if (gettimeofday(&tv, NULL) < 0) { - UNREACHABLE(); - return 0; - } - return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; -} - - -static size_t round_up(int64_t value, int64_t multiple) { - if (multiple == 0) { - // Avoid division by zero - return value; - } - - int64_t remainder = value % multiple; - if (remainder == 0) { - // Value is already a multiple of 'multiple' - return value; - } - - // Calculate the difference to the next multiple - int64_t difference = multiple - remainder; - - // Add the difference to the value - int64_t rounded_up_value = value + difference; - - return rounded_up_value; -} - - -static void perf_map_jit_write_fully(const void* buffer, size_t size) { - FILE* out_file = perf_jit_map_state.perf_map; - const char* ptr = (const char*)(buffer); - while (size > 0) { - const size_t written = fwrite(ptr, 1, size, out_file); - if (written == 0) { - UNREACHABLE(); - break; - } - size -= written; - ptr += written; - } -} - -static void perf_map_jit_write_header(int pid, FILE* out_file) { - Header header; - header.magic = 0x4A695444; - header.version = 1; - header.size = sizeof(Header); - header.elf_mach_target = GetElfMachineArchitecture(); - header.process_id = pid; - header.time_stamp = get_current_time_microseconds(); - header.flags = 0; - perf_map_jit_write_fully(&header, sizeof(header)); -} - -static void* perf_map_jit_init(void) { - char filename[100]; - int pid = getpid(); - snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); - const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); - if (fd == -1) { - return NULL; - } - - const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) - if (page_size == -1) { - close(fd); - return NULL; - } - - // The perf jit interface forces us to map the first page of the file - // to signal that we are using the interface. - perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); - if (perf_jit_map_state.mapped_buffer == NULL) { - close(fd); - return NULL; - } - perf_jit_map_state.mapped_size = page_size; - perf_jit_map_state.perf_map = fdopen(fd, "w+"); - if (perf_jit_map_state.perf_map == NULL) { - close(fd); - return NULL; - } - setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); - perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); - - perf_jit_map_state.map_lock = PyThread_allocate_lock(); - if (perf_jit_map_state.map_lock == NULL) { - fclose(perf_jit_map_state.perf_map); - return NULL; - } - return &perf_jit_map_state; -} - -/* DWARF definitions. */ -#define DWRF_CIE_VERSION 1 - -enum { - DWRF_CFA_nop = 0x0, - DWRF_CFA_offset_extended = 0x5, - DWRF_CFA_def_cfa = 0xc, - DWRF_CFA_def_cfa_offset = 0xe, - DWRF_CFA_offset_extended_sf = 0x11, - DWRF_CFA_advance_loc = 0x40, - DWRF_CFA_offset = 0x80 -}; - -enum - { - DWRF_EH_PE_absptr = 0x00, - DWRF_EH_PE_omit = 0xff, - - /* FDE data encoding. */ - DWRF_EH_PE_uleb128 = 0x01, - DWRF_EH_PE_udata2 = 0x02, - DWRF_EH_PE_udata4 = 0x03, - DWRF_EH_PE_udata8 = 0x04, - DWRF_EH_PE_sleb128 = 0x09, - DWRF_EH_PE_sdata2 = 0x0a, - DWRF_EH_PE_sdata4 = 0x0b, - DWRF_EH_PE_sdata8 = 0x0c, - DWRF_EH_PE_signed = 0x08, - - /* FDE flags. */ - DWRF_EH_PE_pcrel = 0x10, - DWRF_EH_PE_textrel = 0x20, - DWRF_EH_PE_datarel = 0x30, - DWRF_EH_PE_funcrel = 0x40, - DWRF_EH_PE_aligned = 0x50, - - DWRF_EH_PE_indirect = 0x80 - }; - -enum { DWRF_TAG_compile_unit = 0x11 }; - -enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; - -enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; - -enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; - -enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; - -enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; - -enum { -#ifdef __x86_64__ - /* Yes, the order is strange, but correct. */ - DWRF_REG_AX, - DWRF_REG_DX, - DWRF_REG_CX, - DWRF_REG_BX, - DWRF_REG_SI, - DWRF_REG_DI, - DWRF_REG_BP, - DWRF_REG_SP, - DWRF_REG_8, - DWRF_REG_9, - DWRF_REG_10, - DWRF_REG_11, - DWRF_REG_12, - DWRF_REG_13, - DWRF_REG_14, - DWRF_REG_15, - DWRF_REG_RA, -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_REG_SP = 31, - DWRF_REG_RA = 30, -#else -# error "Unsupported target architecture" -#endif -}; - -typedef struct ELFObjectContext -{ - uint8_t* p; /* Pointer to next address in obj.space. */ - uint8_t* startp; /* Pointer to start address in obj.space. */ - uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ - uint32_t code_size; /* Size of machine code. */ -} ELFObjectContext; - -/* Append a null-terminated string. */ -static uint32_t -elfctx_append_string(ELFObjectContext* ctx, const char* str) -{ - uint8_t* p = ctx->p; - uint32_t ofs = (uint32_t)(p - ctx->startp); - do { - *p++ = (uint8_t)*str; - } while (*str++); - ctx->p = p; - return ofs; -} - -/* Append a SLEB128 value. */ -static void -elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) -{ - uint8_t* p = ctx->p; - for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { - *p++ = (uint8_t)((v & 0x7f) | 0x80); - } - *p++ = (uint8_t)(v & 0x7f); - ctx->p = p; -} - -/* Append a ULEB128 to buffer. */ -static void -elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) -{ - uint8_t* p = ctx->p; - for (; v >= 0x80; v >>= 7) { - *p++ = (char)((v & 0x7f) | 0x80); - } - *p++ = (char)v; - ctx->p = p; -} - -/* Shortcuts to generate DWARF structures. */ -#define DWRF_U8(x) (*p++ = (x)) -#define DWRF_I8(x) (*(int8_t*)p = (x), p++) -#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) -#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) -#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) -#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) -#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) -#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) -#define DWRF_ALIGNNOP(s) \ - while ((uintptr_t)p & ((s)-1)) { \ - *p++ = DWRF_CFA_nop; \ - } -#define DWRF_SECTION(name, stmt) \ - { \ - uint32_t* szp_##name = (uint32_t*)p; \ - p += 4; \ - stmt; \ - *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ - } - -/* Initialize .eh_frame section. */ -static void -elf_init_ehframe(ELFObjectContext* ctx) -{ - uint8_t* p = ctx->p; - uint8_t* framep = p; - - /* Emit DWARF EH CIE. */ - DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ - DWRF_U8(DWRF_CIE_VERSION); - DWRF_STR("zR"); /* Augmentation. */ - DWRF_UV(1); /* Code alignment factor. */ - DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ - DWRF_U8(DWRF_REG_RA); /* Return address register. */ - DWRF_UV(1); - DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ - DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); - DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); - DWRF_ALIGNNOP(sizeof(uintptr_t)); - ) - - ctx->eh_frame_p = p; - - /* Emit DWARF EH FDE. */ - DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ - DWRF_U32(-0x30); /* Machine code offset relative to .text. */ - DWRF_U32(ctx->code_size); /* Machine code length. */ - DWRF_U8(0); /* Augmentation data. */ - /* Registers saved in CFRAME. */ -#ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 4); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); - DWRF_U8(DWRF_CFA_advance_loc | 6); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); - /* Extra registers saved for JIT-compiled code. */ -#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_U8(DWRF_CFA_advance_loc | 4); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); - DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); - DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); - DWRF_U8(DWRF_CFA_advance_loc | 12); - DWRF_U8(DWRF_CFA_offset | -(64 - 29)); - DWRF_U8(DWRF_CFA_offset | -(64 - 30)); - DWRF_U8(DWRF_CFA_def_cfa_offset); - DWRF_UV(0); -#else -# error "Unsupported target architecture" -#endif - DWRF_ALIGNNOP(sizeof(uintptr_t));) - - ctx->p = p; -} - -static void perf_map_jit_write_entry(void *state, const void *code_addr, - unsigned int code_size, PyCodeObject *co) -{ - - if (perf_jit_map_state.perf_map == NULL) { - void* ret = perf_map_jit_init(); - if(ret == NULL){ - return; - } - } - - const char *entry = ""; - if (co->co_qualname != NULL) { - entry = PyUnicode_AsUTF8(co->co_qualname); - } - const char *filename = ""; - if (co->co_filename != NULL) { - filename = PyUnicode_AsUTF8(co->co_filename); - } - - - size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; - char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); - if (perf_map_entry == NULL) { - return; - } - snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); - - const size_t name_length = strlen(perf_map_entry); - uword base = (uword)code_addr; - uword size = code_size; - - // Write the code unwinding info event. - CodeUnwindingInfoEvent ev2; - EhFrameHeader f; - - char buffer[1024]; - ELFObjectContext ctx; - ctx.code_size = code_size; - ctx.startp = ctx.p = (uint8_t*)buffer; - elf_init_ehframe(&ctx); - - int eh_frame_size = ctx.p - ctx.startp; - - - ev2.base.event = kUnwindingInfo; - ev2.base.time_stamp = get_current_monotonic_ticks(); - ev2.unwind_data_size = sizeof(f) + eh_frame_size; - ev2.eh_frame_hdr_size = sizeof(f); - ev2.mapped_size = round_up(ev2.unwind_data_size, 16); - int content_size = sizeof(ev2) + sizeof(f) + eh_frame_size; - int padding_size = round_up(content_size, 8) - content_size; - ev2.base.size = content_size + padding_size; - perf_map_jit_write_fully(&ev2, sizeof(ev2)); - - - f.version = 1; - f.eh_frame_ptr_enc = kSData4 | kPcRel; - f.fde_count_enc = kUData4; - f.table_enc = kSData4 | kDataRel; - f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); - f.eh_fde_count = 1; - f.from = -(round_up(code_size, 8) + eh_frame_size); - int cie_size = ctx.eh_frame_p - ctx.startp; - f.to = -(eh_frame_size - cie_size); - - perf_map_jit_write_fully(ctx.startp, eh_frame_size); - perf_map_jit_write_fully(&f, sizeof(f)); - - char padding_bytes[] = "\0\0\0\0\0\0\0\0"; - perf_map_jit_write_fully(&padding_bytes, padding_size); - - // Write the code load event. - CodeLoadEvent ev; - ev.base.event = kLoad; - ev.base.size = sizeof(ev) + (name_length+1) + size; - ev.base.time_stamp = get_current_monotonic_ticks(); - ev.process_id = getpid(); - ev.thread_id = gettid(); - ev.vma = base; - ev.code_address = base; - ev.code_size = size; - ev.code_id = code_id_++; - - perf_map_jit_write_fully(&ev, sizeof(ev)); - perf_map_jit_write_fully(perf_map_entry, name_length+1); - perf_map_jit_write_fully((void*)(base), size); - return; -} - -static int perf_map_jit_fini(void* state) { - if (perf_jit_map_state.perf_map != NULL) { - // close the file - PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); - fclose(perf_jit_map_state.perf_map); - PyThread_release_lock(perf_jit_map_state.map_lock); - - // clean up the lock and state - PyThread_free_lock(perf_jit_map_state.map_lock); - perf_jit_map_state.perf_map = NULL; - } - if (perf_jit_map_state.mapped_buffer != NULL) { - munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); - } - trampoline_api.state = NULL; - return 0; -} - - -_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { - &perf_map_jit_init, - &perf_map_jit_write_entry, - &perf_map_jit_fini, -}; - - // TRAMPOLINE MANAGEMENT API static int @@ -812,7 +270,7 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; - size_t chunk_size = round_up(code_size + 0x50, 16); + size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -877,16 +335,18 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - code_arena->size_left -= round_up(code_arena->code_size + 0x50, 16); - code_arena->current_addr += round_up(code_arena->code_size + 0x50, 16); + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + code_arena->size_left -= total_code_size; + code_arena->current_addr += total_code_size; return trampoline; } static inline py_trampoline compile_trampoline(void) { + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); if ((perf_code_arena == NULL) || - (perf_code_arena->size_left <= round_up(perf_code_arena->code_size+0x50, 16))) { + (perf_code_arena->size_left <= total_code_size)) { if (new_code_arena() < 0) { return NULL; } From 4762581cea7e0e8fed99ff5e5d4f48449cca0073 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 18:59:30 +0200 Subject: [PATCH 07/18] Fixes --- Python/perf_jit_trampoline.c | 6 +++--- Python/perf_trampoline.c | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index f60faa2acfda09..0bbebb38eca8dd 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -72,7 +72,8 @@ As the trampolines are constant, we add a constant padding but in general the pa size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 */ -#define PERF_JIT_CODE_PADDING = 0x50; +#define PERF_JIT_CODE_PADDING 0x50 +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api typedef uint64_t uword; typedef const char* CodeComments; @@ -290,8 +291,7 @@ static void* perf_map_jit_init(void) { return NULL; } - - trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + // trampoline_api.code_padding = PERF_JIT_CODE_PADDING; return &perf_jit_map_state; } diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 58abb2424cf2d9..f144f7d436fe68 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -248,6 +248,27 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { }; +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + // TRAMPOLINE MANAGEMENT API static int @@ -344,7 +365,7 @@ code_arena_new_code(code_arena_t *code_arena) static inline py_trampoline compile_trampoline(void) { - size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + size_t total_code_size = round_up(perf_code_arena->code_size + trampoline_api.code_padding, 16); if ((perf_code_arena == NULL) || (perf_code_arena->size_left <= total_code_size)) { if (new_code_arena() < 0) { From 91a4cdee236617f2f850fc3cd91688fb234ae075 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 19:00:48 +0200 Subject: [PATCH 08/18] Fixes --- Include/internal/pycore_ceval_state.h | 2 +- Python/perf_jit_trampoline.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 6c28549f7c1cec..22ca148c508ff2 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -45,7 +45,7 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; - Py_ssize_t code_padding; + Py_ssize_t code_padding; }; #endif diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 0bbebb38eca8dd..515bcfe9a31440 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -60,7 +60,7 @@ could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing which one is right. So to make perf happy we have non-overlapping ranges for each DSO: - address -> + address -> ------------------------------------------------------------------------------------------------------- /tmp/jitted-PID-0.so | (headers) | .text | unwind info | /tmp/jitted-PID-1.so | (headers) | .text | unwind info | From 236abf84b039ae63d2ec3af853400b9f3641731e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 19:36:44 +0200 Subject: [PATCH 09/18] Fixes --- Python/perf_jit_trampoline.c | 16 ++++++++-------- Tools/c-analyzer/cpython/ignored.tsv | 2 ++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 515bcfe9a31440..e367aed84353ac 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -23,6 +23,7 @@ typedef struct { PyThread_type_lock map_lock; void* mapped_buffer; size_t mapped_size; + int code_id_ = 0; } PerfMapJitState; static PerfMapJitState perf_jit_map_state; @@ -164,13 +165,11 @@ static const intptr_t nanoseconds_per_second = 1000000000; // Dwarf encoding constants -static int code_id_ = 0; - -uint8_t DwarfUData4 = 0x03; -uint8_t DwarfSData4 = 0x0b; -uint8_t DwarfPcRel = 0x10; -uint8_t DwarfDataRel = 0x30; -uint8_t DwarfOmit = 0xff; +static uint8_t DwarfUData4 = 0x03; +static uint8_t DwarfSData4 = 0x0b; +static uint8_t DwarfPcRel = 0x10; +static uint8_t DwarfDataRel = 0x30; +static uint8_t DwarfOmit = 0xff; typedef struct { unsigned char version; unsigned char eh_frame_ptr_enc; @@ -290,6 +289,7 @@ static void* perf_map_jit_init(void) { fclose(perf_jit_map_state.perf_map); return NULL; } + perf_jit_map_state.code_id_ = 0; // trampoline_api.code_padding = PERF_JIT_CODE_PADDING; return &perf_jit_map_state; @@ -579,7 +579,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev.vma = base; ev.code_address = base; ev.code_size = size; - ev.code_id = code_id_++; + ev.code_id = perf_jit_map_state->code_id_++; perf_map_jit_write_fully(&ev, sizeof(ev)); perf_map_jit_write_fully(perf_map_entry, name_length+1); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 2f9e80d6ab6737..90359791be9f8b 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -362,6 +362,8 @@ Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - From 8082102f13ac2e4591e27eeb6ffbd75a8ee9c45d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 19:44:39 +0200 Subject: [PATCH 10/18] Moar Fixes --- Python/perf_jit_trampoline.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index e367aed84353ac..b73b740e130713 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -23,7 +23,7 @@ typedef struct { PyThread_type_lock map_lock; void* mapped_buffer; size_t mapped_size; - int code_id_ = 0; + int code_id; } PerfMapJitState; static PerfMapJitState perf_jit_map_state; @@ -169,7 +169,7 @@ static uint8_t DwarfUData4 = 0x03; static uint8_t DwarfSData4 = 0x0b; static uint8_t DwarfPcRel = 0x10; static uint8_t DwarfDataRel = 0x30; -static uint8_t DwarfOmit = 0xff; +// static uint8_t DwarfOmit = 0xff; typedef struct { unsigned char version; unsigned char eh_frame_ptr_enc; @@ -289,7 +289,7 @@ static void* perf_map_jit_init(void) { fclose(perf_jit_map_state.perf_map); return NULL; } - perf_jit_map_state.code_id_ = 0; + perf_jit_map_state.code_id = 0; // trampoline_api.code_padding = PERF_JIT_CODE_PADDING; return &perf_jit_map_state; @@ -579,7 +579,8 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev.vma = base; ev.code_address = base; ev.code_size = size; - ev.code_id = perf_jit_map_state->code_id_++; + perf_jit_map_state.code_id += 1; + ev.code_id = perf_jit_map_state.code_id; perf_map_jit_write_fully(&ev, sizeof(ev)); perf_map_jit_write_fully(perf_map_entry, name_length+1); From d269c1a3d023988aac0f61210c42f5e741b007aa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 20:13:31 +0200 Subject: [PATCH 11/18] Moar Fixes --- Tools/c-analyzer/cpython/ignored.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 90359791be9f8b..8f976b6f9820fc 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -362,8 +362,8 @@ Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - -Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - -Python/perf_jit_trampoline.c - perf_jit_map_state - +Python/perf_jit_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - From 4635b9d873e9eb129c7f467f36b144f932e74997 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 20:15:33 +0200 Subject: [PATCH 12/18] Moar Fixes --- Python/lel.c | 8 -------- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 Python/lel.c diff --git a/Python/lel.c b/Python/lel.c deleted file mode 100644 index da52f1f7cd72fc..00000000000000 --- a/Python/lel.c +++ /dev/null @@ -1,8 +0,0 @@ -typedef void* (py_evaluator)(void* ts, void* f, int t); - -void* -trampoline(void *ts, void *f, - int throwflag, py_evaluator evaluator) -{ - return evaluator(ts, f, throwflag); -} diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8f976b6f9820fc..342ce1ba9a6f2f 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -363,7 +363,7 @@ Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/perf_jit_trampoline.c - _Py_perfmap_callbacks - -Python/perf_jit_trampoline.c - perf_jit_map_state - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - From 5db0f557f03a5fae0737cdfa6c62161f4dbfcb18 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 2 May 2024 20:32:53 +0200 Subject: [PATCH 13/18] Add news entry --- .../2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst | 4 ++++ Python/perf_jit_trampoline.c | 8 ++++---- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst new file mode 100644 index 00000000000000..7d4c003019bdef --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst @@ -0,0 +1,4 @@ +Allow the Linux perf support to work without frame pointers using perf's +advanced JIT support. The feature is activated when using the +``PYTHONPERFJITSUPPORT`` environment variable or when running Python with +``-Xperfjit``. Patch by Pablo Galindo diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index b73b740e130713..05c7efc98a5dfc 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -165,10 +165,10 @@ static const intptr_t nanoseconds_per_second = 1000000000; // Dwarf encoding constants -static uint8_t DwarfUData4 = 0x03; -static uint8_t DwarfSData4 = 0x0b; -static uint8_t DwarfPcRel = 0x10; -static uint8_t DwarfDataRel = 0x30; +static const uint8_t DwarfUData4 = 0x03; +static const uint8_t DwarfSData4 = 0x0b; +static const uint8_t DwarfPcRel = 0x10; +static const uint8_t DwarfDataRel = 0x30; // static uint8_t DwarfOmit = 0xff; typedef struct { unsigned char version; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 342ce1ba9a6f2f..c2c3bf498339c8 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -362,7 +362,7 @@ Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - -Python/perf_jit_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - From 4842167901b17265ab35cde365e4089c1cd31c8b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 3 May 2024 10:35:58 +0200 Subject: [PATCH 14/18] Fix dwarf --- Python/perf_jit_trampoline.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 05c7efc98a5dfc..bd06f93814be72 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -73,7 +73,7 @@ As the trampolines are constant, we add a constant padding but in general the pa size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 */ -#define PERF_JIT_CODE_PADDING 0x50 +#define PERF_JIT_CODE_PADDING 0x100 #define trampoline_api _PyRuntime.ceval.perf.trampoline_api typedef uint64_t uword; @@ -471,17 +471,17 @@ elf_init_ehframe(ELFObjectContext* ctx) DWRF_U8(0); /* Augmentation data. */ /* Registers saved in CFRAME. */ #ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_advance_loc | 8); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); DWRF_U8(DWRF_CFA_advance_loc | 6); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); /* Extra registers saved for JIT-compiled code. */ #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_advance_loc | 1); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); - DWRF_U8(DWRF_CFA_advance_loc | 12); + DWRF_U8(DWRF_CFA_advance_loc | 3); DWRF_U8(DWRF_CFA_offset | -(64 - 29)); DWRF_U8(DWRF_CFA_offset | -(64 - 30)); DWRF_U8(DWRF_CFA_def_cfa_offset); From 3beb8a52f85802451129c47e400ab945d7587438 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 3 May 2024 10:47:15 +0200 Subject: [PATCH 15/18] Fix dwarf --- Python/perf_jit_trampoline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index bd06f93814be72..fdce0da3eded72 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -471,7 +471,7 @@ elf_init_ehframe(ELFObjectContext* ctx) DWRF_U8(0); /* Augmentation data. */ /* Registers saved in CFRAME. */ #ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 8); + DWRF_U8(DWRF_CFA_advance_loc | 4); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); DWRF_U8(DWRF_CFA_advance_loc | 6); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); From 653ea34319d4daf699418c89c616a10c069be346 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 3 May 2024 11:06:56 +0200 Subject: [PATCH 16/18] debugging --- Python/perf_jit_trampoline.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index fdce0da3eded72..faa6b4647fae97 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -447,6 +447,7 @@ elf_init_ehframe(ELFObjectContext* ctx) { uint8_t* p = ctx->p; uint8_t* framep = p; + printf("Code size is: %d\n", ctx->code_size); /* Emit DWARF EH CIE. */ DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ @@ -471,7 +472,7 @@ elf_init_ehframe(ELFObjectContext* ctx) DWRF_U8(0); /* Augmentation data. */ /* Registers saved in CFRAME. */ #ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_advance_loc | 8); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); DWRF_U8(DWRF_CFA_advance_loc | 6); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); From 8f7b329fd8393a4cf42d4965f56bd91bc044749d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 3 May 2024 16:05:27 +0200 Subject: [PATCH 17/18] debugging --- Python/perf_jit_trampoline.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index faa6b4647fae97..fdce0da3eded72 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -447,7 +447,6 @@ elf_init_ehframe(ELFObjectContext* ctx) { uint8_t* p = ctx->p; uint8_t* framep = p; - printf("Code size is: %d\n", ctx->code_size); /* Emit DWARF EH CIE. */ DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ @@ -472,7 +471,7 @@ elf_init_ehframe(ELFObjectContext* ctx) DWRF_U8(0); /* Augmentation data. */ /* Registers saved in CFRAME. */ #ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 8); + DWRF_U8(DWRF_CFA_advance_loc | 4); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); DWRF_U8(DWRF_CFA_advance_loc | 6); DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); From dec8a9b6d567d309222e0aa4abe741713d9418fb Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 4 May 2024 19:35:24 +0200 Subject: [PATCH 18/18] Add docs --- Doc/c-api/init_config.rst | 5 ++++- Doc/howto/perf_profiling.rst | 33 +++++++++++++++++++++++++++++++++ Doc/using/cmdline.rst | 21 +++++++++++++++++++++ Doc/whatsnew/3.13.rst | 6 ++++++ Lib/test/test_perf_profiler.py | 9 +++++++++ 5 files changed, 73 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 47a8fbb2cd9c97..63ec25b8e60bad 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1251,7 +1251,10 @@ PyConfig for more information. Set by :option:`-X perf <-X>` command line option and by the - :envvar:`PYTHONPERFSUPPORT` environment variable. + :envvar:`PYTHONPERFSUPPORT` environment variable for perf support + with stack pointers and :option:`-X perfjit <-X>` command line option + and by the :envvar:`PYTHONPERFJITSUPPORT` environment variable for perf + support with DWARF JIT information. Default: ``-1``. diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index bb1c00e0aa51d5..0ce66f31274a5f 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -205,3 +205,36 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. + + +How to work without frame pointers +---------------------------------- + +If you are working with a Python interpreter that has been compiled without frame pointers +you can still use the ``perf`` profiler but the overhead will be a bit higher because Python +needs to generate unwinding information for every Python function call on the fly. Additionally, +``perf`` will take more time to process the data because it will need to use the DWARF debugging +information to unwind the stack and this is a slow process. + +To enable this mode, you can use the environment variable :envvar:`PYTHONPERFJITSUPPORT` or the +:option:`-X perfjit <-X>` option, which will enable the JIT mode for the ``perf`` profiler. + +When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to +call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file. + + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperfjit my_script.py + $ perf inject -i perf.data --jit + $ perf report -g -i perf.data + +or using the environment variable:: + + $ PYTHONPERFJITSUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ perf inject -i perf.data --jit + $ perf report -g -i perf.data + +Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take snapshots of the stack of +the process being profiled and save the information in the ``perf.data`` file. By default the size of +the stack dump is 8192 bytes but the user can change the size by passing the size after comma like +``--call-graph dwarf,4096``. The size of the stack dump is important because if the size is too small +``perf`` will not be able to unwind the stack and the output will be incomplete. + diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index e032a1971bc6d6..17f9e3b1b5c24a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -547,6 +547,12 @@ Miscellaneous options report Python calls. This option is only available on some platforms and will do nothing if is not supported on the current system. The default value is "off". See also :envvar:`PYTHONPERFSUPPORT` and :ref:`perf_profiling`. + * ``-X perfjit`` enables support for the Linux ``perf`` profiler with DWARF + support. When this option is provided, the ``perf`` profiler will be able + to report Python calls using DWARF ifnormation. This option is only available on + some platforms and will do nothing if is not supported on the current + system. The default value is "off". See also :envvar:`PYTHONPERFJITSUPPORT` + and :ref:`perf_profiling`. * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. @@ -1108,6 +1114,21 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHONPERFJITSUPPORT + + If this variable is set to a nonzero value, it enables support for + the Linux ``perf`` profiler so Python calls can be detected by it + using DWARF information. + + If set to ``0``, disable Linux ``perf`` profiler support. + + See also the :option:`-X perfjit <-X>` command-line option + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + + + .. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides the return diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 888ebd0402d0e7..4fc2a66d19588f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -132,6 +132,12 @@ Other Language Changes equivalent of the :option:`-X frozen_modules <-X>` command-line option. (Contributed by Yilei Yang in :gh:`111374`.) +* Add :ref:`support for the perf profiler ` working without + frame pointers through the new environment variable + :envvar:`PYTHONPERFJITSUPPORT` and command-line option :option:`-X perfjit + <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) + + New Modules =========== diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 14ffab3344848b..888ecbd74450a3 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -479,8 +479,17 @@ def compile_trampolines_for_all_functions(): if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) +def _is_kernel_version_at_least(major, minor): + try: + with open("/proc/version") as f: + version = f.readline().split()[2] + except FileNotFoundError: + return False + version = version.split(".") + return int(version[0]) > major or (int(version[0]) == major and int(version[1]) >= minor) @unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless(_is_kernel_version_at_least(6, 6), "perf command may not work due to a perf bug") class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): def run_perf(self, script_dir, script, activate_trampoline=True): if activate_trampoline: