Feature/update python support

.github/workflows/build-on-windows.yml

@@ -10,7 +10,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: true
-
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
       - name: run the build script
         run: .\scripts\windows\build.bat
 

.github/workflows/build-test-deploy.yml

@@ -66,10 +66,10 @@ jobs:
           - debian:10
           - debian:11
     steps:
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: "3.8"
+          python-version: "3.10"
 
       - name: Install Java
         uses: actions/setup-java@v1
@@ -213,15 +213,16 @@ jobs:
     if: ${{ !startsWith(github.ref, 'refs/tags/') }}
     needs: build-container-x64
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       fail-fast: false  # helps detecting flakiness / errors specific to one Python version
       matrix:
         python-version:
-          - "3.8"
-          - "3.9"
-          - "3.10"  # see https://github.com/Granulate/gprofiler/issues/502, we need 3.10.0, others fail PyPerf tests.
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
 
     steps:
     - name: Set up Python ${{ matrix.python-version }}

.github/workflows/linters.yml

@@ -22,9 +22,10 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.8"
-          - "3.9"
           - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13"
 
     runs-on: ubuntu-latest
 

README.md

@@ -424,7 +424,7 @@ The playbook defines 2 more variables:
 * `gprofiler_args` - additional arguments to pass to gProfiler, empty by default. You can use it to pass, for example, `'--profiling-frequency 15'` to change the frequency.
 
 ## Running from source
-gProfiler requires Python 3.8+ to run.
+gProfiler requires Python 3.10+ to run.
 
 ```bash
 pip3 install -r requirements.txt

deploy/docker-compose/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.9"
+version: "3.10"
 
 services:
   gprofiler:

dev-requirements.txt

@@ -1,16 +1,14 @@
-pytest==7.0.1
-flake8==6.0.0
-black==23.3.0
-mypy==0.971
-isort==5.12.0
-PyYAML==6.0
-docker==7.0.0
-types-requests==0.1.9
-types-dataclasses==0.6.6
-types-psutil==5.8.19
-types-PyYAML==6.0.3
-types-pkg-resources==0.1.3
-types-protobuf==3.19.22
-types-toml==0.10.8
-types-retry==0.9.9
-types-beautifulsoup4==4.11.1
+pytest==8.3.5
+flake8==7.2.0
+black==25.1.0
+mypy==1.15.0
+isort==6.0.1
+PyYAML==6.0.2
+docker==7.1.0
+types-requests~=2.32.0
+types-psutil~=7.0.0
+types-PyYAML~=6.0.12
+types-protobuf~=5.29.1
+types-toml~=0.10.8
+types-retry~=0.9.9
+types-beautifulsoup4~=4.12.0

exe-requirements.txt

@@ -1,13 +1,2 @@
-# requirements for the standalone executable
-pyinstaller==4.6; platform.machine == "x86_64"
-# aarch64 requires a later version due to the use of a newer centos version.
-# see https://github.com/pyinstaller/pyinstaller/issues/5540
-pyinstaller==4.10; platform.machine == "aarch64" or sys.platform == "win32"
-# for aarch64 we build a slightly patched version
-# I tried upgrading to 0.13.6 but it crashes when the botoloader is run as non-root :/
-# I got this error in gdb:
-# (gdb) run
-# Starting program: /path/to/build/x86_64/gprofiler
-# During startup program terminated with signal SIGSEGV, Segmentation fault.
-# staying with 0.12.1 for the mean time...
-staticx==0.12.1; platform.machine == "x86_64"
+pyinstaller==6.12.0
+staticx @ git+https://github.com/Granulate/staticx.git@33eefdadc72832d5aa67c0792768c9e76afb746d; platform.machine == "x86_64"

executable.Dockerfile

@@ -46,7 +46,8 @@ RUN mv "/tmp/rbspy/target/$(uname -m)-unknown-linux-musl/release/rbspy" /tmp/rbs
 FROM mcr.microsoft.com/dotnet/sdk${DOTNET_BUILDER} as dotnet-builder
 WORKDIR /tmp
 RUN apt-get update && \
-  dotnet tool install --global dotnet-trace --version 6.0.351802
+  dotnet tool install --global dotnet-trace --version 6.0.351802 && \
+  apt-get install -y --no-install-recommends patchelf
 
 RUN cp -r "$HOME/.dotnet" "/tmp/dotnet"
 COPY scripts/dotnet_prepare_dependencies.sh .
@@ -77,11 +78,12 @@ RUN ./phpspy_build.sh
 # async-profiler glibc
 FROM centos${AP_BUILDER_CENTOS} AS async-profiler-builder-glibc
 WORKDIR /tmp
-COPY scripts/async_profiler_env_glibc.sh scripts/fix_centos7.sh ./
+COPY scripts/async_profiler_env_glibc.sh scripts/fix_centos7.sh scripts/pdeathsigger.c ./
 RUN if grep -q "CentOS Linux 7" /etc/os-release ; then \
       ./fix_centos7.sh; \
     fi
-RUN ./async_profiler_env_glibc.sh
+RUN ./async_profiler_env_glibc.sh && \
+    gcc -static -o pdeathsigger pdeathsigger.c
 
 COPY scripts/async_profiler_build_shared.sh .
 RUN ./async_profiler_build_shared.sh
@@ -124,7 +126,6 @@ COPY --from=perf-builder /bpftool /bpftool
 
 WORKDIR /bcc
 COPY scripts/staticx_for_pyperf_patch.diff .
-COPY scripts/staticx_patch.diff .
 COPY scripts/bcc_helpers_build.sh .
 COPY scripts/pyperf_env.sh .
 RUN ./pyperf_env.sh --with-staticx
@@ -177,6 +178,7 @@ WORKDIR /app
 RUN yum --setopt=skip_missing_names_on_install=False install -y \
         gcc \
         curl \
+        glibc-static \
         libicu && \
     yum clean all
 
@@ -191,10 +193,6 @@ RUN set -e; \
     if [ "$(uname -m)" = "aarch64" ]; then \
          ln -s /usr/lib64/python3.10/lib-dynload /usr/lib/python3.10/lib-dynload; \
     fi
-RUN set -e; \
-    if [ "$(uname -m)" = "aarch64" ]; then \
-        python3 -m pip install --no-cache-dir 'wheel==0.37.0' 'scons==4.2.0'; \
-    fi
 
 # we want the latest pip
 # hadolint ignore=DL3013
@@ -255,6 +253,7 @@ COPY --from=async-profiler-builder-glibc /usr/bin/xargs gprofiler/resources/php/
 
 COPY --from=async-profiler-builder-glibc /tmp/async-profiler/build/bin/asprof gprofiler/resources/java/asprof
 COPY --from=async-profiler-builder-glibc /tmp/async-profiler/build/async-profiler-version gprofiler/resources/java/async-profiler-version
+COPY --from=async-profiler-builder-glibc /tmp/pdeathsigger gprofiler/resources/pdeathsigger
 COPY --from=async-profiler-centos-min-test-glibc /libasyncProfiler.so gprofiler/resources/java/glibc/libasyncProfiler.so
 COPY --from=async-profiler-builder-musl /tmp/async-profiler/build/lib/libasyncProfiler.so gprofiler/resources/java/musl/libasyncProfiler.so
 COPY --from=node-package-builder-musl /tmp/module_build gprofiler/resources/node/module/musl
@@ -275,15 +274,13 @@ RUN pyinstaller pyinstaller.spec \
     && test -f build/pyinstaller/warn-pyinstaller.txt \
     && ./check_pyinstaller.sh
 
-# for aarch64 - build a patched version of staticx 0.13.6. we remove calls to getpwnam and getgrnam, for these end up doing dlopen()s which
-# crash the staticx bootloader. we don't need them anyway (all files in our staticx tar are uid 0 and we don't need the names translation)
-COPY scripts/staticx_patch.diff staticx_patch.diff
+# We need staticx main as version wasn't released yet for PyInstaller hooks throwing on non elfs.
+# Removed build isolation as from some reason there's an issue with scons on the isolated env.
 # hadolint ignore=DL3003
 RUN if [ "$(uname -m)" = "aarch64" ]; then \
-        git clone -b v0.13.6 https://github.com/JonathonReinhart/staticx.git && \
+        git clone  https://github.com/Granulate/staticx.git && \
         cd staticx && \
-        git reset --hard 819d8eafecbaab3646f70dfb1e3e19f6bbc017f8 && \
-        git apply ../staticx_patch.diff && \
+        git checkout 33eefdadc72832d5aa67c0792768c9e76afb746d && \
         ln -s libnss_files.so.2 /lib64/libnss_files.so && \
         ln -s libnss_dns.so.2 /lib64/libnss_dns.so && \
         python3 -m pip install --no-cache-dir . ; \

gprofiler/main.py

@@ -20,13 +20,12 @@
 import logging.handlers
 import os
 import shutil
-import signal
 import sys
 import time
 import traceback
 from pathlib import Path
 from threading import Event
-from types import FrameType, TracebackType
+from types import TracebackType
 from typing import Iterable, List, Optional, Type, cast
 
 import configargparse
@@ -73,6 +72,7 @@
     reset_umask,
     resource_path,
     run_process,
+    setup_signals,
 )
 from gprofiler.utils.fs import escape_filename, mkdir_owned_root_wrapper
 from gprofiler.utils.proxy import get_https_proxy
@@ -97,21 +97,6 @@
 
 UPLOAD_FILE_SUBCOMMAND = "upload-file"
 
-# 1 KeyboardInterrupt raised per this many seconds, no matter how many SIGINTs we get.
-SIGINT_RATELIMIT = 0.5
-
-last_signal_ts: Optional[float] = None
-
-
-def sigint_handler(sig: int, frame: Optional[FrameType]) -> None:
-    global last_signal_ts
-    ts = time.monotonic()
-    # no need for atomicity here: we can't get another SIGINT before this one returns.
-    # https://www.gnu.org/software/libc/manual/html_node/Signals-in-Handler.html#Signals-in-Handler
-    if last_signal_ts is None or ts > last_signal_ts + SIGINT_RATELIMIT:
-        last_signal_ts = ts
-        raise KeyboardInterrupt
-
 
 class GProfiler:
     def __init__(
@@ -212,11 +197,7 @@ def _update_last_output(self, last_output_name: str, output_path: str) -> None:
         atomically_symlink(os.path.basename(output_path), last_output)
         # delete if rotating & there was a link target before.
         if self._rotating_output and os.path.basename(prev_output) != last_output_name:
-            # can't use missing_ok=True, available only from 3.8 :/
-            try:
-                prev_output.unlink()
-            except FileNotFoundError:
-                pass
+            prev_output.unlink(missing_ok=True)
 
     def _generate_output_files(
         self,
@@ -445,7 +426,7 @@ def _submit_profile_logged(
         logger.exception("Error occurred sending profile to server")
     else:
         logger.info("Successfully uploaded profiling data to the server")
-        return response_dict.get("gpid", "")
+        return cast(str, response_dict.get("gpid", ""))
     return ""
 
 
@@ -955,17 +936,6 @@ def verify_preconditions(args: configargparse.Namespace, processes_to_profile: O
             sys.exit(1)
 
 
-def setup_signals() -> None:
-    # When we run under staticx & PyInstaller, both of them forward (some of the) signals to gProfiler.
-    # We catch SIGINTs and ratelimit them, to avoid being interrupted again during the handling of the
-    # first INT.
-    # See my commit message for more information.
-    signal.signal(signal.SIGINT, sigint_handler)
-    # handle SIGTERM in the same manner - gracefully stop gProfiler.
-    # SIGTERM is also forwarded by staticx & PyInstaller, so we need to ratelimit it.
-    signal.signal(signal.SIGTERM, sigint_handler)
-
-
 def log_system_info() -> None:
     system_info = get_static_system_info()
     logger.info(f"gProfiler Python version: {system_info.python_version}")

gprofiler/metadata/py_module_version.py

@@ -57,7 +57,7 @@ def _get_packages_dir(file_path: str) -> Optional[str]:
 def _get_metadata(dist: pkg_resources.Distribution) -> Dict[str, str]:
     """Based on pip._internal.utils.get_metadata"""
     metadata_name = "METADATA"
-    if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(metadata_name):  # type: ignore
+    if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata(metadata_name):
         metadata = dist.get_metadata(metadata_name)
     elif dist.has_metadata("PKG-INFO"):
         metadata_name = "PKG-INFO"
@@ -120,7 +120,7 @@ def _files_from_legacy(dist: pkg_resources.Distribution) -> Optional[Iterator[st
         return None
     paths = (p for p in text.splitlines(keepends=False) if p)
     root = dist.location
-    info = dist.egg_info  # type: ignore
+    info = dist.egg_info
     if root is None or info is None:
         return paths
     try:
@@ -158,8 +158,8 @@ def _get_libpython_path(process: Process) -> Optional[str]:
 
 
 # Matches PY_VERSION in Python's binary
-# Notice that we'll match only 2.7 and 3.5-3.12
-_PY_VERSION_STRING_PATTERN = re.compile(rb"(?<=\D)(?:2\.7|3\.(?:[5-9]|1[0-2]))\.\d\d?(?=\x00)")
+# Notice that we'll match only 2.7 and 3.5-3.13
+_PY_VERSION_STRING_PATTERN = re.compile(rb"(?<=\D)(?:2\.7|3\.(?:[5-9]|1[0-3]))\.\d\d?(?=\x00)")
 
 
 @functools.lru_cache(maxsize=128)
@@ -230,8 +230,8 @@ def _populate_packages_versions(packages_versions: Dict[str, Optional[Tuple[str,
     # This function resolves symlinks and makes paths absolute for comparison purposes which isn't required
     # for our usage.
     if hasattr(pkg_resources, "_normalize_cached"):
-        original__normalize_cache = pkg_resources._normalize_cached  # type: ignore
-        pkg_resources._normalize_cached = lambda path: path  # type: ignore
+        original__normalize_cache = pkg_resources._normalize_cached
+        pkg_resources._normalize_cached = lambda path: path
     else:
         global _warned_no__normalized_cached
         if not _warned_no__normalized_cached:
@@ -261,7 +261,7 @@ def _populate_packages_versions(packages_versions: Dict[str, Optional[Tuple[str,
                 packages_versions[module_path] = package_info
     finally:
         # Don't forget to restore the original implementation in case someone else uses this function
-        pkg_resources._normalize_cached = original__normalize_cache  # type: ignore
+        pkg_resources._normalize_cached = original__normalize_cache
 
 
 _exceptions_logged = 0

gprofiler/metadata/system_metadata.py

@@ -48,7 +48,12 @@ def decode_libc_version(version: bytes) -> str:
 
     try:
         ldd_version = run_process(
-            ["ldd", "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, suppress_log=True, check=False
+            ["ldd", "--version"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            suppress_log=True,
+            check=False,
+            pdeathsigger=False,
         ).stdout
     except FileNotFoundError:
         ldd_version = b"ldd not found"
@@ -259,8 +264,7 @@ def get_static_system_info() -> SystemInfo:
         processors=cpu_count,
         cpu_model_name=cpu_model_name,
         cpu_flags=cpu_flags,
-        memory_capacity_mb=round(psutil.virtual_memory().total / 1024 / 1024),  # type: ignore # virtual_memory doesn't
-        # have a return type is types-psutil
+        memory_capacity_mb=round(psutil.virtual_memory().total / 1024 / 1024),
         hostname=hostname,
         system=platform.system(),
         os_name=os_name,

gprofiler/metadata/versions.py

@@ -35,7 +35,9 @@ def get_exe_version(
     exe_path = f"/proc/{get_process_nspid(process.pid)}/exe"
 
     def _run_get_version() -> "CompletedProcess[bytes]":
-        return run_process([exe_path, version_arg], stop_event=stop_event, timeout=get_version_timeout)
+        return run_process(
+            [exe_path, version_arg], stop_event=stop_event, timeout=get_version_timeout, pdeathsigger=False
+        )
 
     try:
         cp = run_in_ns_wrapper(["pid", "mnt"], _run_get_version, process.pid)

gprofiler/profilers/java.py

@@ -25,7 +25,7 @@
 from subprocess import CompletedProcess
 from threading import Event, Lock
 from types import TracebackType
-from typing import Any, Dict, Iterable, List, Optional, Set, Type, TypeVar, Union, cast
+from typing import Any, Dict, Iterable, List, Optional, Set, Type, TypeVar, Union
 
 import psutil
 from granulate_utils.java import (
@@ -350,11 +350,12 @@ def get_java_version(process: Process, stop_event: Event) -> Optional[str]:
     def _run_java_version() -> "CompletedProcess[bytes]":
         return run_process(
             [
-                cast(str, process_java_path),
+                process_java_path,
                 "-version",
             ],
             stop_event=stop_event,
             timeout=_JAVA_VERSION_TIMEOUT,
+            pdeathsigger=False,
         )
 
     # doesn't work without changing PID NS as well (I'm getting ENOENT for libjli.so)