Merge branch 'main' into add-pycode-verify-stateless

Author: ericsnowcurrently

.github/CODEOWNERS

@@ -188,7 +188,7 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 
 # AST
 Python/ast.c                  @isidentical @JelleZijlstra @eclips4
-Python/ast_opt.c              @isidentical @eclips4
+Python/ast_preprocess.c       @isidentical @eclips4
 Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
 Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
 Lib/ast.py                    @isidentical @JelleZijlstra @eclips4

.github/workflows/jit.yml

@@ -95,10 +95,10 @@ jobs:
         with:
           python-version: '3.11'
 
+      # PCbuild downloads LLVM automatically:
       - name: Windows
         if: runner.os == 'Windows'
         run: |
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
@@ -126,29 +126,30 @@ jobs:
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-  jit-with-disabled-gil:
-    name: Free-Threaded (Debug)
-    needs: interpreter
-    runs-on: ubuntu-24.04
-    timeout-minutes: 90
-    strategy:
-      fail-fast: false
-      matrix:
-        llvm:
-          - 19
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-      - name: Build with JIT enabled and GIL disabled
-        run: |
-          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
-          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit --with-pydebug --disable-gil
-          make all --jobs 4
-      - name: Run tests
-        run: |
-          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+  # XXX: GH-133171
+  # jit-with-disabled-gil:
+  #   name: Free-Threaded (Debug)
+  #   needs: interpreter
+  #   runs-on: ubuntu-24.04
+  #   timeout-minutes: 90
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       llvm:
+  #         - 19
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         persist-credentials: false
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: '3.11'
+  #     - name: Build with JIT enabled and GIL disabled
+  #       run: |
+  #         sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+  #         export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+  #         ./configure --enable-experimental-jit --with-pydebug --disable-gil
+  #         make all --jobs 4
+  #     - name: Run tests
+  #       run: |
+  #         ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3

.github/workflows/reusable-context.yml

@@ -97,6 +97,8 @@ jobs:
       run: python Tools/build/compute-changes.py
       env:
         GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }}
+        CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }}
 
     - name: Compute hash for config cache key
       id: config-hash

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.6
+    rev: v0.11.8
     hooks:
       - id: ruff
         name: Run Ruff (lint) on Doc/
@@ -22,14 +22,14 @@ repos:
         name: Run Ruff (format) on Doc/
         args: [--check]
         files: ^Doc/
+      - id: ruff-format
+        name: Run Ruff (format) on Tools/build/check_warnings.py
+        args: [--check, --config=Tools/build/.ruff.toml]
+        files: ^Tools/build/check_warnings.py
 
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 25.1.0
     hooks:
-      - id: black
-        name: Run Black on Tools/build/check_warnings.py
-        files: ^Tools/build/check_warnings.py
-        args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/

.ruff.toml

@@ -0,0 +1,12 @@
+# Default settings for Ruff in CPython
+
+# PYTHON_FOR_REGEN
+target-version = "py310"
+
+# PEP 8
+line-length = 79
+
+# Enable automatic fixes by default.
+# To override this, use ``fix = false`` in a subdirectory's config file
+# or ``--no-fix`` on the command line.
+fix = true

Android/README.md

@@ -25,11 +25,13 @@ it:
   `android-sdk/cmdline-tools/latest`.
 * `export ANDROID_HOME=/path/to/android-sdk`
 
-The `android.py` script also requires the following commands to be on the `PATH`:
+The `android.py` script will automatically use the SDK's `sdkmanager` to install
+any packages it needs.
+
+The script also requires the following commands to be on the `PATH`:
 
 * `curl`
 * `java` (or set the `JAVA_HOME` environment variable)
-* `tar`
 
 
 ## Building
@@ -97,7 +99,7 @@ similar to the `Android` directory of the CPython source tree.
 The Python test suite can be run on Linux, macOS, or Windows:
 
 * On Linux, the emulator needs access to the KVM virtualization interface, and
-  a DISPLAY environment variable pointing at an X server.
+  a DISPLAY environment variable pointing at an X server. Xvfb is acceptable.
 
 The test suite can usually be run on a device with 2 GB of RAM, but this is
 borderline, so you may need to increase it to 4 GB. As of Android

Android/android.py

@@ -138,19 +138,19 @@ def make_build_python(context):
     run(["make", "-j", str(os.cpu_count())])
 
 
-def unpack_deps(host):
+def unpack_deps(host, prefix_dir):
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
     for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4",
                      "sqlite-3.49.1-0", "xz-5.4.6-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
-        run(["tar", "-xf", filename])
+        shutil.unpack_archive(filename, prefix_dir)
         os.remove(filename)
 
 
 def download(url, target_dir="."):
     out_path = f"{target_dir}/{basename(url)}"
-    run(["curl", "-Lf", "-o", out_path, url])
+    run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url])
     return out_path
 
 
@@ -162,8 +162,7 @@ def configure_host_python(context):
     prefix_dir = host_dir / "prefix"
     if not prefix_dir.exists():
         prefix_dir.mkdir()
-        os.chdir(prefix_dir)
-        unpack_deps(context.host)
+        unpack_deps(context.host, prefix_dir)
 
     os.chdir(host_dir)
     command = [
@@ -241,16 +240,15 @@ def setup_sdk():
 # the Gradle wrapper is not included in the CPython repository. Instead, we
 # extract it from the Gradle GitHub repository.
 def setup_testbed():
-    # The Gradle version used for the build is specified in
-    # testbed/gradle/wrapper/gradle-wrapper.properties. This wrapper version
-    # doesn't need to match, as any version of the wrapper can download any
-    # version of Gradle.
-    version = "8.9.0"
     paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"]
-
     if all((TESTBED_DIR / path).exists() for path in paths):
         return
 
+    # The wrapper version isn't important, as any version of the wrapper can
+    # download any version of Gradle. The Gradle version actually used for the
+    # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties.
+    version = "8.9.0"
+
     for path in paths:
         out_path = TESTBED_DIR / path
         out_path.parent.mkdir(exist_ok=True)

Doc/.ruff.toml

@@ -1,7 +1,6 @@
+extend = "../.ruff.toml"  # Inherit the project-wide settings
+
 target-version = "py312"  # Align with the version in oldest_supported_sphinx
-fix = true
-output-format = "full"
-line-length = 79
 extend-exclude = [
     "includes/*",
     # Temporary exclusions:

Doc/c-api/float.rst

@@ -96,6 +96,9 @@ NaNs (if such things exist on the platform) isn't handled correctly, and
 attempting to unpack a bytes string containing an IEEE INF or NaN will raise an
 exception.
 
+Note that NaNs type may not be preserved on IEEE platforms (silent NaN become
+quiet), for example on x86 systems in 32-bit mode.
+
 On non-IEEE platforms with more precision, or larger dynamic range, than IEEE
 754 supports, not all values can be packed; on non-IEEE platforms with less
 precision, or smaller dynamic range, not all values can be unpacked. What

Doc/c-api/object.rst

@@ -613,6 +613,38 @@ Object Protocol
 
    .. versionadded:: 3.14
 
+.. c:function:: int PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *obj)
+
+   Check if *obj* is a unique temporary object.
+   Returns ``1`` if *obj* is known to be a unique temporary object,
+   and ``0`` otherwise.  This function cannot fail, but the check is
+   conservative, and may return ``0`` in some cases even if *obj* is a unique
+   temporary object.
+
+   If an object is a unique temporary, it is guaranteed that the current code
+   has the only reference to the object. For arguments to C functions, this
+   should be used instead of checking if the reference count is ``1``. Starting
+   with Python 3.14, the interpreter internally avoids some reference count
+   modifications when loading objects onto the operands stack by
+   :term:`borrowing <borrowed reference>` references when possible, which means
+   that a reference count of ``1`` by itself does not guarantee that a function
+   argument uniquely referenced.
+
+   In the example below, ``my_func`` is called with a unique temporary object
+   as its argument::
+
+      my_func([1, 2, 3])
+
+   In the example below, ``my_func`` is **not** called with a unique temporary
+   object as its argument, even if its refcount is ``1``::
+
+      my_list = [1, 2, 3]
+      my_func(my_list)
+
+   See also the function :c:func:`Py_REFCNT`.
+
+   .. versionadded:: 3.14
+
 .. c:function:: int PyUnstable_IsImmortal(PyObject *obj)
 
    This function returns non-zero if *obj* is :term:`immortal`, and zero
@@ -705,3 +737,21 @@ Object Protocol
    caller must hold a :term:`strong reference` to *obj* when calling this.
 
    .. versionadded:: 3.14
+
+.. c:function:: int PyUnstable_Object_IsUniquelyReferenced(PyObject *op)
+
+   Determine if *op* only has one reference.
+
+   On GIL-enabled builds, this function is equivalent to
+   :c:expr:`Py_REFCNT(op) == 1`.
+
+   On a :term:`free threaded <free threading>` build, this checks if *op*'s
+   :term:`reference count` is equal to one and additionally checks if *op*
+   is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not**
+   thread-safe on free threaded builds; prefer this function.
+
+   The caller must hold an :term:`attached thread state`, despite the fact
+   that this function doesn't call into the Python interpreter. This function
+   cannot fail.
+
+   .. versionadded:: 3.14

Doc/c-api/refcounting.rst

@@ -23,6 +23,15 @@ of Python objects.
 
    Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count.
 
+   .. note::
+
+      On :term:`free threaded <free threading>` builds of Python, returning 1
+      isn't sufficient to determine if it's safe to treat *o* as having no
+      access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced`
+      for that instead.
+
+      See also the function :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary()`.
+
    .. versionchanged:: 3.10
       :c:func:`Py_REFCNT()` is changed to the inline static function.
 

Doc/deprecations/index.rst

@@ -5,11 +5,17 @@ Deprecations
 
 .. include:: pending-removal-in-3.16.rst
 
+.. include:: pending-removal-in-3.17.rst
+
+.. include:: pending-removal-in-3.19.rst
+
 .. include:: pending-removal-in-future.rst
 
 C API deprecations
 ------------------
 
 .. include:: c-api-pending-removal-in-3.15.rst
 
+.. include:: c-api-pending-removal-in-3.18.rst
+
 .. include:: c-api-pending-removal-in-future.rst

Doc/deprecations/pending-removal-in-3.16.rst

@@ -61,6 +61,12 @@ Pending removal in Python 3.16
   * Calling the Python implementation of :func:`functools.reduce` with *function*
     or *sequence* as keyword arguments has been deprecated since Python 3.14.
 
+* :mod:`logging`:
+
+  Support for custom logging handlers with the *strm* argument is deprecated
+  and scheduled for removal in Python 3.16. Define handlers with the *stream*
+  argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.)
+
 * :mod:`mimetypes`:
 
   * Valid extensions start with a '.' or are empty for

Doc/deprecations/pending-removal-in-3.17.rst

@@ -0,0 +1,10 @@
+Pending removal in Python 3.17
+------------------------------
+
+* :mod:`typing`:
+
+  - Before Python 3.14, old-style unions were implemented using the private class
+    ``typing._UnionGenericAlias``. This class is no longer needed for the implementation,
+    but it has been retained for backward compatibility, with removal scheduled for Python
+    3.17. Users should use documented introspection helpers like :func:`typing.get_origin`
+    and :func:`typing.get_args` instead of relying on private implementation details.

Doc/deprecations/pending-removal-in-3.19.rst

@@ -0,0 +1,8 @@
+Pending removal in Python 3.19
+------------------------------
+
+* :mod:`ctypes`:
+
+  * Implicitly switching to the MSVC-compatible struct layout by setting
+    :attr:`~ctypes.Structure._pack_` but not :attr:`~ctypes.Structure._layout_`
+    on non-Windows platforms.

Doc/howto/regex.rst

@@ -738,9 +738,12 @@ given location, they can obviously be matched an infinite number of times.
    different: ``\A`` still matches only at the beginning of the string, but ``^``
    may match at any location inside the string that follows a newline character.
 
-``\Z``
+``\z``
    Matches only at the end of the string.
 
+``\Z``
+   The same as ``\z``.  For compatibility with old Python versions.
+
 ``\b``
    Word boundary.  This is a zero-width assertion that matches only at the
    beginning or end of a word.  A word is defined as a sequence of alphanumeric