Merge branch 'main' into agen

Author: kumaraditya303

.devcontainer/Dockerfile

@@ -2,11 +2,11 @@ FROM docker.io/library/fedora:37
 
 ENV CC=clang
 
-ENV WASI_SDK_VERSION=19
+ENV WASI_SDK_VERSION=20
 ENV WASI_SDK_PATH=/opt/wasi-sdk
 
 ENV WASMTIME_HOME=/opt/wasmtime
-ENV WASMTIME_VERSION=7.0.0
+ENV WASMTIME_VERSION=9.0.1
 ENV WASMTIME_CPU_ARCH=x86_64
 
 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \

.devcontainer/devcontainer.json

@@ -9,7 +9,9 @@
         "-y",
         "which",
         "zsh",
-        "fish"
+        "fish",
+        // For umask fix below.
+        "/usr/bin/setfacl"
     ],
     "updateContentCommand": {
         // Using the shell for `nproc` usage.
@@ -22,6 +24,11 @@
             "html"
         ]
     },
+    "postCreateCommand": {
+        // https://github.com/orgs/community/discussions/26026
+        "umask fix: workspace": ["sudo", "setfacl", "-bnR", "."],
+        "umask fix: /tmp": ["sudo", "setfacl", "-bnR", "/tmp"]
+    },
     "customizations": {
         "vscode": {
             "extensions": [
@@ -31,10 +38,10 @@
                 "maelvalais.autoconf",
                 // C auto-complete.
                 "ms-vscode.cpptools",
-                // To view built docs.
-                "ms-vscode.live-server"
-                // https://github.com/microsoft/vscode-python/issues/18073
-                // "ms-python.python"
+                // To view HTML build of docs.
+                "ms-vscode.live-server",
+                // Python auto-complete.
+                "ms-python.python"
             ],
             "settings": {
                 "C_Cpp.default.compilerPath": "/usr/bin/clang",

.gitattributes

@@ -18,7 +18,6 @@
 *.zip binary
 
 # Specific binary files
-Lib/test/sndhdrdata/sndhdr.* binary
 PC/classicAppCompat.* binary
 
 # Text files that should not be subject to eol conversion

.github/CODEOWNERS

@@ -97,6 +97,8 @@ Include/pytime.h              @pganssle @abalkin
 /Tools/peg_generator/         @pablogsal @lysnikolaou
 /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 /Grammar/python.gram          @pablogsal @lysnikolaou
+/Lib/tokenize.py              @pablogsal @lysnikolaou
+/Lib/test/test_tokenize.py    @pablogsal @lysnikolaou
 
 # AST
 Python/ast.c                  @isidentical

.github/workflows/build.yml

@@ -8,6 +8,7 @@ on:
   push:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'
@@ -16,6 +17,7 @@ on:
   pull_request:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'
@@ -81,6 +83,11 @@ jobs:
     if: needs.check_source.outputs.run_tests == 'true'
     steps:
       - uses: actions/checkout@v3
+      - name: Restore config.cache
+        uses: actions/cache@v3
+        with:
+          path: config.cache
+          key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}
       - uses: actions/setup-python@v3
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
@@ -97,7 +104,7 @@ jobs:
       - name: Configure CPython
         run: |
           # Build Python with the libpython dynamic library
-          ./configure --with-pydebug --enable-shared
+          ./configure --config-cache --with-pydebug --enable-shared
       - name: Regenerate autoconf files with container image
         run: make regen-configure
       - name: Build CPython
@@ -178,6 +185,11 @@ jobs:
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v3
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}
     - name: Install Homebrew dependencies
       run: brew install pkg-config [email protected] xz gdbm tcl-tk
     - name: Configure CPython
@@ -186,6 +198,7 @@ jobs:
         LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \
         PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \
         ./configure \
+          --config-cache \
           --with-pydebug \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix [email protected])"
@@ -238,9 +251,18 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+      run: |
+        ../cpython-ro-srcdir/configure \
+          --config-cache \
+          --with-pydebug \
+          --with-openssl=$OPENSSL_DIR
     - name: Build CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j4
@@ -271,6 +293,11 @@ jobs:
       LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
     steps:
     - uses: actions/checkout@v3
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -295,7 +322,7 @@ jobs:
     - name: Configure ccache action
       uses: hendrikmuhs/[email protected]
     - name: Configure CPython
-      run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
+      run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR
     - name: Build CPython
       run: make -j4
     - name: Display build info
@@ -304,7 +331,7 @@ jobs:
       run: ./python Lib/test/ssltests.py
 
   test_hypothesis:
-    name: "Hypothesis Tests on Ubuntu"
+    name: "Hypothesis tests on Ubuntu"
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     needs: check_source
@@ -345,9 +372,18 @@ jobs:
       run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
     - name: Bind mount sources read-only
       run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}
     - name: Configure CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+      run: |
+        ../cpython-ro-srcdir/configure \
+          --config-cache \
+          --with-pydebug \
+          --with-openssl=$OPENSSL_DIR
     - name: Build CPython out-of-tree
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j4
@@ -368,6 +404,14 @@ jobs:
         echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
         echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
         ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: 'Restore Hypothesis database'
+      id: cache-hypothesis-database
+      uses: actions/cache@v3
+      with:
+        path: ./hypothesis
+        key: hypothesis-database-${{ github.head_ref || github.run_id }}
+        restore-keys: |
+          - hypothesis-database-
     - name: "Run tests"
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -388,6 +432,11 @@ jobs:
           -x test_subprocess \
           -x test_signal \
           -x test_sysconfig
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: hypothesis-example-db
+        path: .hypothesis/examples/
 
 
   build_asan:
@@ -402,6 +451,11 @@ jobs:
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
     - uses: actions/checkout@v3
+    - name: Restore config.cache
+      uses: actions/cache@v3
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}
     - name: Register gcc problem matcher
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
@@ -430,7 +484,7 @@ jobs:
     - name: Configure ccache action
       uses: hendrikmuhs/[email protected]
     - name: Configure CPython
-      run: ./configure --with-address-sanitizer --without-pymalloc
+      run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
     - name: Build CPython
       run: make -j4
     - name: Display build info

.github/workflows/build_msi.yml

@@ -8,12 +8,14 @@ on:
     - '3.*'
     paths:
     - 'Tools/msi/**'
+    - '.github/workflows/build_msi.yml'
   pull_request:
     branches:
     - 'main'
     - '3.*'
     paths:
     - 'Tools/msi/**'
+    - '.github/workflows/build_msi.yml'
 
 permissions:
   contents: read
@@ -33,4 +35,4 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Build CPython installer
-      run: .\Tools\msi\build.bat -${{ matrix.type }}
+      run: .\Tools\msi\build.bat --doc -${{ matrix.type }}

.github/workflows/doc.yml

@@ -5,6 +5,7 @@ on:
   #push:
   #  branches:
   #  - 'main'
+  #  - '3.12'
   #  - '3.11'
   #  - '3.10'
   #  - '3.9'
@@ -15,6 +16,7 @@ on:
   pull_request:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'

.gitignore

@@ -23,6 +23,10 @@
 *.gc??
 *.profclang?
 *.profraw
+# Copies of binaries before BOLT optimizations.
+*.prebolt
+# BOLT profile data.
+*.fdata
 *.dyn
 .gdb_history
 .purify
@@ -57,7 +61,6 @@ Doc/.venv/
 Doc/env/
 Doc/.env/
 Include/pydtrace_probes.h
-Lib/lib2to3/*.pickle
 Lib/site-packages/*
 !Lib/site-packages/README.txt
 Lib/test/data/*
@@ -124,6 +127,7 @@ Tools/unicode/data/
 /platform
 /profile-clean-stamp
 /profile-run-stamp
+/profile-bolt-stamp
 /Python/deepfreeze/*.c
 /pybuilddir.txt
 /pyconfig.h

.readthedocs.yml

@@ -13,6 +13,20 @@ build:
     python: "3"
 
   commands:
+    # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition
+    #
+    # Cancel building pull requests when there aren't changes in the Doc directory.
+    #
+    # If there are no changes (git diff exits with 0) we force the command to return with 183.
+    # This is a special exit code on Read the Docs that will cancel the build immediately.
+    - |
+      if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ];
+      then
+        echo "No changes to Doc/ - exiting the build.";
+        exit 183;
+      fi
+
     - make -C Doc venv html
     - mkdir _readthedocs
     - mv Doc/build/html _readthedocs/html
+

Doc/c-api/long.rst

@@ -322,3 +322,27 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    with :c:func:`PyLong_FromVoidPtr`.
 
    Returns ``NULL`` on error.  Use :c:func:`PyErr_Occurred` to disambiguate.
+
+
+.. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op)
+
+   Return 1 if *op* is compact, 0 otherwise.
+
+   This function makes it possible for performance-critical code to implement
+   a “fast path” for small integers. For compact values use
+   :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a
+   :c:func:`PyLong_As* <PyLong_AsSize_t>` function or
+   :c:func:`calling <PyObject_CallMethod>` :meth:`int.to_bytes`.
+
+   The speedup is expected to be negligible for most users.
+
+   Exactly what values are considered compact is an implementation detail
+   and is subject to change.
+
+.. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op)
+
+   If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`,
+   return its value.
+
+   Otherwise, the return value is undefined.
+