Merge branch 'main' into incremental-gc-3

Author: markshannon

.devcontainer/Dockerfile

@@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37
 
 ENV CC=clang
 
-ENV WASI_SDK_VERSION=20
+ENV WASI_SDK_VERSION=21
 ENV WASI_SDK_PATH=/opt/wasi-sdk
 
 ENV WASMTIME_HOME=/opt/wasmtime
-ENV WASMTIME_VERSION=18.0.2
+ENV WASMTIME_VERSION=18.0.3
 ENV WASMTIME_CPU_ARCH=x86_64
 
 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \

.github/workflows/build.yml

@@ -97,7 +97,7 @@ jobs:
       - name: Get a list of the changed documentation-related files
         if: github.event_name == 'pull_request'
         id: changed-docs-files
-        uses: Ana06/get-changed-files@v2.2.0
+        uses: Ana06/get-changed-files@v2.3.0
         with:
           filter: |
             Doc/**
@@ -206,6 +206,8 @@ jobs:
     uses: ./.github/workflows/reusable-macos.yml
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
+      # macos-14 is M1, macos-13 is Intel
+      os-matrix: '["macos-14", "macos-13"]'
 
   build_macos_free_threading:
     name: 'macOS (free-threading)'
@@ -215,6 +217,8 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       free-threading: true
+      # macos-14 is M1
+      os-matrix: '["macos-14"]'
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -480,6 +484,24 @@ jobs:
     - name: Tests
       run: xvfb-run make test
 
+  build_tsan:
+    name: 'Thread sanitizer'
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    uses: ./.github/workflows/reusable-tsan.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      options: ./configure --config-cache --with-thread-sanitizer --with-pydebug
+
+  build_tsan_free_threading:
+    name: 'Thread sanitizer (free-threading)'
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    uses: ./.github/workflows/reusable-tsan.yml
+    with:
+      config_hash: ${{ needs.check_source.outputs.config_hash }}
+      options: ./configure --config-cache --disable-gil --with-thread-sanitizer --with-pydebug
+
   # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
   cifuzz:
     name: CIFuzz
@@ -538,6 +560,8 @@ jobs:
     - build_windows_free_threading
     - test_hypothesis
     - build_asan
+    - build_tsan
+    - build_tsan_free_threading
     - cifuzz
 
     runs-on: ubuntu-latest
@@ -571,6 +595,8 @@ jobs:
             build_windows,
             build_windows_free_threading,
             build_asan,
+            build_tsan,
+            build_tsan_free_threading,
             '
             || ''
           }}

.github/workflows/jit.yml

@@ -108,8 +108,8 @@ jobs:
         if: runner.os == 'macOS'
         run: |
           brew install llvm@${{ matrix.llvm }}
-          export SDKROOT="$(xcrun --show-sdk-path)"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          SDKROOT="$(xcrun --show-sdk-path)" \
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
           ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
 
@@ -134,10 +134,10 @@ jobs:
           sudo apt install --yes "gcc-$HOST" qemu-user
           ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
           ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
-          export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
-          export CPP="$CC --preprocess"
-          export HOSTRUNNER=qemu-${{ matrix.architecture }}
           export QEMU_LD_PREFIX="/usr/$HOST"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+          CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
+            CPP="$CC --preprocess" \
+            HOSTRUNNER=qemu-${{ matrix.architecture }} \
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3

.github/workflows/lint.yml

@@ -23,4 +23,4 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
-      - uses: pre-commit/[email protected].0
+      - uses: pre-commit/[email protected].1

.github/workflows/project-updater.yml

@@ -23,7 +23,7 @@ jobs:
           - { project: 32, label: sprint }
 
     steps:
-      - uses: actions/add-to-project@v0.1.0
+      - uses: actions/add-to-project@v0.6.0
         with:
           project-url: https://github.com/orgs/python/projects/${{ matrix.project }}
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}

.github/workflows/reusable-macos.yml

@@ -8,6 +8,9 @@ on:
         required: false
         type: boolean
         default: false
+      os-matrix:
+        required: false
+        type: string
 
 jobs:
   build_macos:
@@ -22,10 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [
-          "macos-14",  # M1
-          "macos-13",  # Intel
-        ]
+        os: ${{fromJson(inputs.os-matrix)}}
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4

.github/workflows/reusable-tsan.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_call:
+    inputs:
+      config_hash:
+        required: true
+        type: string
+      options:
+        required: true
+        type: string
+
+jobs:
+  build_tsan_reusable:
+    name: 'Thread sanitizer'
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v4
+    - name: Runner image version
+      run: echo "IMAGE_VERSION=${ImageVersion}" >> $GITHUB_ENV
+    - name: Restore config.cache
+      uses: actions/cache@v4
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
+    - name: Install Dependencies
+      run: |
+        sudo ./.github/workflows/posix-deps-apt.sh
+        sudo apt install -y clang
+        # Reduce ASLR to avoid TSAN crashing
+        sudo sysctl -w vm.mmap_rnd_bits=28
+    - name: TSAN Option Setup
+      run: |
+        echo "TSAN_OPTIONS=suppressions=${GITHUB_WORKSPACE}/Tools/tsan/supressions.txt" >> $GITHUB_ENV
+        echo "CC=clang" >> $GITHUB_ENV
+        echo "CXX=clang++" >> $GITHUB_ENV
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+      with:
+        save: ${{ github.event_name == 'push' }}
+        max-size: "200M"
+    - name: Configure CPython
+      run: ${{ inputs.options }}
+    - name: Build CPython
+      run: make -j4
+    - name: Display build info
+      run: make pythoninfo
+    - name: Tests
+      run: ./python -m test --tsan -j4

.github/workflows/reusable-wasi.yml

@@ -11,8 +11,8 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     env:
-      WASMTIME_VERSION: 18.0.2
-      WASI_SDK_VERSION: 20
+      WASMTIME_VERSION: 18.0.3
+      WASI_SDK_VERSION: 21
       WASI_SDK_PATH: /opt/wasi-sdk
       CROSS_BUILD_PYTHON: cross-build/build
       CROSS_BUILD_WASI: cross-build/wasm32-wasi

.github/workflows/stale.yml

@@ -2,7 +2,7 @@ name: Mark stale pull requests
 
 on:
   schedule:
-  - cron: "0 0 * * *"
+  - cron: "0 */6 * * *"
 
 permissions:
   pull-requests: write

.gitignore

@@ -142,7 +142,6 @@ Tools/unicode/data/
 /profile-clean-stamp
 /profile-run-stamp
 /profile-bolt-stamp
-/Python/deepfreeze/*.c
 /pybuilddir.txt
 /pyconfig.h
 /python-config

Doc/c-api/code.rst

@@ -34,10 +34,16 @@ bound into a function.
 
    Return the number of free variables in a code object.
 
-.. c:function:: int PyCode_GetFirstFree(PyCodeObject *co)
+.. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co)
 
    Return the position of the first free variable in a code object.
 
+   .. versionchanged:: 3.13
+
+      Renamed from ``PyCode_GetFirstFree`` as part of :ref:`unstable-c-api`.
+      The old name is deprecated, but will remain available until the
+      signature changes again.
+
 .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable)
 
    Return a new code object.  If you need a dummy code object to create a frame,

Doc/c-api/long.rst

@@ -450,7 +450,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    a “fast path” for small integers. For compact values use
    :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a
    :c:func:`PyLong_As* <PyLong_AsSize_t>` function or
-   :c:func:`calling <PyObject_CallMethod>` :meth:`int.to_bytes`.
+   :c:func:`PyLong_AsNativeBytes`.
 
    The speedup is expected to be negligible for most users.
 

Doc/c-api/type.rst

@@ -185,6 +185,21 @@ Type Objects
 
    .. versionadded:: 3.11
 
+.. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type)
+
+   Return the type's fully qualified name. Equivalent to
+   ``f"{type.__module__}.{type.__qualname__}"``, or ``type.__qualname__`` if
+   ``type.__module__`` is not a string or is equal to ``"builtins"``.
+
+   .. versionadded:: 3.13
+
+.. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type)
+
+   Return the type's module name. Equivalent to getting the ``type.__module__``
+   attribute.
+
+   .. versionadded:: 3.13
+
 .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot)
 
    Return the function pointer stored in the given slot. If the

Doc/c-api/unicode.rst

@@ -518,6 +518,26 @@ APIs:
         - :c:expr:`PyObject*`
         - The result of calling :c:func:`PyObject_Repr`.
 
+      * - ``T``
+        - :c:expr:`PyObject*`
+        - Get the fully qualified name of an object type;
+          call :c:func:`PyType_GetFullyQualifiedName`.
+
+      * - ``T#``
+        - :c:expr:`PyObject*`
+        - Similar to ``T`` format, but use a colon (``:``) as separator between
+          the module name and the qualified name.
+
+      * - ``N``
+        - :c:expr:`PyTypeObject*`
+        - Get the fully qualified name of a type;
+          call :c:func:`PyType_GetFullyQualifiedName`.
+
+      * - ``N#``
+        - :c:expr:`PyTypeObject*`
+        - Similar to ``N`` format, but use a colon (``:``) as separator between
+          the module name and the qualified name.
+
    .. note::
       The width formatter unit is number of characters rather than bytes.
       The precision formatter unit is number of bytes or :c:type:`wchar_t`
@@ -553,6 +573,9 @@ APIs:
       In previous versions it caused all the rest of the format string to be
       copied as-is to the result string, and any extra arguments discarded.
 
+   .. versionchanged:: 3.13
+      Support for ``%T``, ``%T#``, ``%N`` and ``%N#`` formats added.
+
 
 .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs)
 

Doc/data/stable_abi.dat

@@ -259,9 +259,11 @@ is evaluated in all cases.
 Why isn't there a switch or case statement in Python?
 -----------------------------------------------------
 
-You can do this easily enough with a sequence of ``if... elif... elif... else``.
-For literal values, or constants within a namespace, you can also use a
-``match ... case`` statement.
+In general, structured switch statements execute one block of code
+when an expression has a particular value or set of values.
+Since Python 3.10 one can easily match literal values, or constants
+within a namespace, with a ``match ... case`` statement.
+An older alternative is a sequence of ``if... elif... elif... else``.
 
 For cases where you need to choose from a very large number of possibilities,
 you can create a dictionary mapping case values to functions to call.  For
@@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in
 this example.  Without such a prefix, if values are coming from an untrusted
 source, an attacker would be able to call any method on your object.
 
+Imitating switch with fallthrough, as with C's switch-case-default,
+is possible, much harder, and less needed.
+
 
 Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation?
 --------------------------------------------------------------------------------------------------------

Doc/faq/design.rst

@@ -50,7 +50,7 @@ to learn Python's C API.
 If you need to interface to some C or C++ library for which no Python extension
 currently exists, you can try wrapping the library's data types and functions
 with a tool such as `SWIG <https://www.swig.org>`_.  `SIP
-<https://riverbankcomputing.com/software/sip/intro>`__, `CXX
+<https://github.com/Python-SIP/sip>`__, `CXX
 <https://cxx.sourceforge.net/>`_ `Boost
 <https://www.boost.org/libs/python/doc/index.html>`_, or `Weave
 <https://github.com/scipy/weave>`_ are also

Doc/faq/extending.rst

@@ -841,10 +841,11 @@ Glossary
       Some named tuples are built-in types (such as the above examples).
       Alternatively, a named tuple can be created from a regular class
       definition that inherits from :class:`tuple` and that defines named
-      fields.  Such a class can be written by hand or it can be created with
-      the factory function :func:`collections.namedtuple`.  The latter
-      technique also adds some extra methods that may not be found in
-      hand-written or built-in named tuples.
+      fields.  Such a class can be written by hand, or it can be created by
+      inheriting :class:`typing.NamedTuple`, or with the factory function
+      :func:`collections.namedtuple`.  The latter techniques also add some
+      extra methods that may not be found in hand-written or built-in named
+      tuples.
 
    namespace
       The place where a variable is stored.  Namespaces are implemented as