feat: add support for python 3.12 (#1816)

Co-authored-by: omair <[email protected]>
Co-authored-by: Anthonios Partheniou <[email protected]>

Author: ohmayr

packages/gapic-generator/.github/sync-repo-settings.yaml

@@ -7,9 +7,15 @@ branchProtectionRules:
   requiredStatusCheckContexts:
     - 'cla/google'
     - 'docs'
-    - 'mypy'
-    - 'showcase (showcase)'
-    - 'showcase (showcase_alternative_templates)'
+    - 'mypy (3.8)'
+    - 'mypy (3.9)'
+    - 'mypy (3.10)'
+    - 'mypy (3.11)'
+    - 'mypy (3.12)'
+    - 'showcase (3.7, showcase)'
+    - 'showcase (3.12, showcase)'
+    - 'showcase (3.7, showcase_alternative_templates)'
+    - 'showcase (3.12, showcase_alternative_templates)'
     # TODO(dovs): reenable these when the mtls tests have been debugged and fixed
     # See #1218 for details
     # - 'showcase-mtls (showcase_mtls)'
@@ -21,21 +27,25 @@ branchProtectionRules:
     - 'showcase-unit (3.9)'
     - 'showcase-unit (3.10)'
     - 'showcase-unit (3.11)'
+    - 'showcase-unit (3.12)'
     - 'showcase-unit (3.7, _alternative_templates)'
     - 'showcase-unit (3.8, _alternative_templates)'
     - 'showcase-unit (3.9, _alternative_templates)'
     - 'showcase-unit (3.10, _alternative_templates)'
     - 'showcase-unit (3.11, _alternative_templates)'
+    - 'showcase-unit (3.12, _alternative_templates)'
     - 'showcase-unit (3.7, _alternative_templates_mixins)'
     - 'showcase-unit (3.8, _alternative_templates_mixins)'
     - 'showcase-unit (3.9, _alternative_templates_mixins)'
     - 'showcase-unit (3.10, _alternative_templates_mixins)'
     - 'showcase-unit (3.11, _alternative_templates_mixins)'
+    - 'showcase-unit (3.12, _alternative_templates_mixins)'
     - 'showcase-unit (3.7, _mixins)'
     - 'showcase-unit (3.8, _mixins)'
     - 'showcase-unit (3.9, _mixins)'
     - 'showcase-unit (3.10, _mixins)'
     - 'showcase-unit (3.11, _mixins)'
+    - 'showcase-unit (3.12, _mixins)'
     - 'showcase-unit-add-iam-methods'
     - 'integration'
     - 'goldens-lint'
@@ -46,16 +56,19 @@ branchProtectionRules:
     - 'unit (3.9)'
     - 'unit (3.10)'
     - 'unit (3.11)'
+    - 'unit (3.12)'
     - 'fragment (3.7)'
     - 'fragment (3.8)'
     - 'fragment (3.9)'
     - 'fragment (3.10)'
     - 'fragment (3.11)'
+    - 'fragment (3.12)'
     - 'fragment (3.7, _alternative_templates)'
     - 'fragment (3.8, _alternative_templates)'
     - 'fragment (3.9, _alternative_templates)'
     - 'fragment (3.10, _alternative_templates)'
     - 'fragment (3.11, _alternative_templates)'
+    - 'fragment (3.12, _alternative_templates)'
     - 'OwlBot Post Processor'
   requiredApprovingReviewCount: 1
   requiresCodeOwnerReviews: true

packages/gapic-generator/.github/workflows/tests.yaml

@@ -19,43 +19,51 @@ env:
 
 jobs:
   docs:
-    # Don't upgrade python version; there's a bug in 3.10 sphinx
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
+        # Use python 3.10 for docs to match the version for the sphinx plugin
+        # https://github.com/googleapis/synthtool/pull/1891
         with:
-          python-version: "3.9"
+          python-version: "3.10"
           cache: 'pip'
       - name: Install nox.
         run: python -m pip install nox
       - name: Build the documentation.
         run: nox -s docs
   mypy:
+    strategy:
+      matrix:
+        # Run mypy on all of the supported python versions listed in setup.py
+        # https://github.com/python/mypy/blob/master/setup.py
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python "3.11"
+      - name: Set up Python "${{ matrix.python }}"
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "${{ matrix.python }}"
           cache: 'pip'
       - name: Install nox.
         run: python -m pip install nox
       - name: Check type annotations.
-        run: nox -s mypy
+        run: nox -s mypy-${{ matrix.python }}
   showcase:
     strategy:
+      # Run showcase tests on the lowest and highest supported runtimes
       matrix:
+        python: ["3.7", "3.12"]
         target: [showcase, showcase_alternative_templates]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python "3.11"
+      - name: Set up Python "${{ matrix.python }}"
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "${{ matrix.python }}"
           cache: 'pip'
       - name: Install system dependencies.
         run: |
@@ -81,7 +89,7 @@ jobs:
             unzip protoc-${PROTOC_VERSION}.zip
             sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
       - name: Run showcase tests.
-        run: nox -s ${{ matrix.target }}
+        run: nox -s ${{ matrix.target }}-${{ matrix.python }}
   showcase-mtls:
     if: ${{ false }}            # TODO(dovs): reenable when #1218 is fixed
     strategy:
@@ -95,10 +103,10 @@ jobs:
         run: |
           sudo mkdir -p /tmp/workspace/tests/cert/
           sudo chown -R ${USER} /tmp/workspace/
-      - name: Set up Python "3.11"
+      - name: Set up Python "3.12"
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: 'pip'
       - name: Copy mtls files
         run: cp tests/cert/mtls.* /tmp/workspace/tests/cert/
@@ -131,7 +139,7 @@ jobs:
   showcase-unit:
     strategy:
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
         variant: ['', _alternative_templates, _mixins, _alternative_templates_mixins]
     runs-on: ubuntu-latest
     steps:
@@ -161,10 +169,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python "3.11"
+      - name: Set up Python "3.12"
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: 'pip'
       - name: Install system dependencies.
         run: |
@@ -189,10 +197,10 @@ jobs:
         variant: ['', _alternative_templates]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python "3.11"
+      - name: Set up Python "3.12"
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: 'pip'
       - name: Install system dependencies.
         run: |
@@ -214,10 +222,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python "3.11"
+      - name: Set up Python "3.12"
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: 'pip'
       - name: Install system dependencies.
         run: |
@@ -230,7 +238,7 @@ jobs:
   unit:
     strategy:
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -251,7 +259,7 @@ jobs:
   fragment:
     strategy:
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
         variant: ['', _alternative_templates]
     runs-on: ubuntu-latest
     steps:
@@ -303,28 +311,30 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: 'pip'
       - name: Install nox.
         run: |
             python -m pip install nox
       - name: Run blacken and lint on the generated output.
         run: |
-          nox -f tests/integration/goldens/asset/noxfile.py -s mypy-3.11 blacken lint_setup_py lint
-          nox -f tests/integration/goldens/credentials/noxfile.py -s mypy-3.11 blacken lint_setup_py lint
-          nox -f tests/integration/goldens/eventarc/noxfile.py -s mypy-3.11 blacken lint_setup_py lint
-          nox -f tests/integration/goldens/logging/noxfile.py -s mypy-3.11 blacken lint_setup_py lint
-          nox -f tests/integration/goldens/redis/noxfile.py -s mypy-3.11 blacken lint_setup_py lint
+          nox -f tests/integration/goldens/asset/noxfile.py -s mypy-3.12 blacken lint
+          nox -f tests/integration/goldens/credentials/noxfile.py -s mypy-3.12 blacken lint
+          nox -f tests/integration/goldens/eventarc/noxfile.py -s mypy-3.12 blacken lint
+          nox -f tests/integration/goldens/logging/noxfile.py -s mypy-3.12 blacken lint
+          nox -f tests/integration/goldens/redis/noxfile.py -s mypy-3.12 blacken lint
   style-check:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python "3.11"
         uses: actions/setup-python@v4
         with:
+          # Do not upgrade this check to python 3.12 until
+          # https://github.com/hhatto/autopep8/issues/712 is fixed
           python-version: "3.11"
           cache: 'pip'
       - name: Install autopep8

packages/gapic-generator/gapic/ads-templates/noxfile.py.j2

@@ -13,6 +13,7 @@ ALL_PYTHON = [
     "3.9",
     "3.10",
     "3.11",
+    "3.12",
 ]
 
 @nox.session(python=ALL_PYTHON)

packages/gapic-generator/gapic/ads-templates/setup.py.j2

@@ -47,6 +47,7 @@ setuptools.setup(
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
         'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
         'Topic :: Internet',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],

packages/gapic-generator/gapic/schema/metadata.py

@@ -178,7 +178,7 @@ def convert_to_versioned_package(self) -> Tuple[str, ...]:
         # underscore between the module and the version. For example,
         # change google.cloud.documentai.v1 to google.cloud.documentai_v1.
         # Check if the package name contains a version.
-        version_regex = "^v\d[^/]*$"
+        version_regex = r"^v\d[^/]*$"
         regex_match = re.match(version_regex, self.package[-1])
         if regex_match and len(self.package) > 1:
             versioned_module = f"{self.package[-2]}_{regex_match[0]}"

packages/gapic-generator/gapic/templates/noxfile.py.j2

@@ -17,16 +17,17 @@ ALL_PYTHON = [
     "3.9",
     "3.10",
     "3.11",
+    "3.12"
 ]
 
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
 LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt"
-PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8")
+PACKAGE_NAME = '{{ api.naming.warehouse_package_name }}'
 
 BLACK_VERSION = "black==22.3.0"
 BLACK_PATHS = ["docs", "google", "tests", "samples", "noxfile.py", "setup.py"]
-DEFAULT_PYTHON_VERSION = "3.11"
+DEFAULT_PYTHON_VERSION = "3.12"
 
 nox.sessions = [
     "unit",
@@ -37,7 +38,6 @@ nox.sessions = [
     "docs",
     "blacken",
     "lint",
-    "lint_setup_py",
 ]
 
 @nox.session(python=ALL_PYTHON)
@@ -82,7 +82,7 @@ def mypy(session):
     session.install('.')
     session.run(
         'mypy',
-        '--explicit-package-bases',
+        '-p',
         {% if api.naming.module_namespace %}
         '{{ api.naming.module_namespace[0] }}',
         {% else %}
@@ -169,11 +169,4 @@ def blacken(session):
         *BLACK_PATHS,
     )
 
-
-@nox.session(python=DEFAULT_PYTHON_VERSION)
-def lint_setup_py(session):
-    """Verify that setup.py is valid (including RST check)."""
-    session.install("docutils", "pygments")
-    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
-
 {% endblock %}

packages/gapic-generator/gapic/templates/setup.py.j2

@@ -55,8 +55,6 @@ packages = [
     if package.startswith("{{ api.naming.namespace_packages|first }}")
 ]
 
-namespaces = [{% for namespace_packages in api.naming.namespace_packages %}"{{ namespace_packages }}"{% if not loop.last %}, {% endif %}{% endfor %}]
-
 setuptools.setup(
     name=name,
     version=version,
@@ -77,6 +75,7 @@ setuptools.setup(
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Operating System :: OS Independent",
         "Topic :: Internet",
     ],