Switch project setup from setup.py to pyproject.toml (#92)

fnesveda · web-flow · commit 8357c33abbf6 · 2023-05-23T16:55:19.000+02:00
Basically the same as apify/apify-client-python#123. Since we're hiring a new Python engineer hopefully soon, I thought it's time to get rid of some tech debt. This rewrites the project setup to use pyproject.toml instead of setup.py, as pyproject.toml is the modern, recommended way to specify project metadata, and setup.py is being deprecated. I've also updated the publication to PyPI to use the official GitHub action, and to use OIDC for authentication instead of an API token. And I've switched the secrets for building Python Actor base Docker images to use an org-wide GitHub token.
diff --git a/.flake8 b/.flake8
@@ -3,8 +3,7 @@ filename =
     ./docs/*.py,
     ./scripts/*.py,
     ./src/*.py,
-    ./tests/*.py,
-    ./setup.py
+    ./tests/*.py
 per-file-ignores =
     docs/*: D
     scripts/*: D
diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml
@@ -3,7 +3,7 @@ name: Integration tests
 on:
   workflow_call:
     secrets:
-      APIFY_TEST_USER_API_TOKEN:
+      APIFY_TEST_USER_PYTHON_SDK_API_TOKEN:
         description: API token of the Python SDK testing user on Apify
         required: true
 
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -41,10 +41,16 @@ jobs:
     name: Check whether the documentation is up to date
     uses: ./.github/workflows/check_docs.yaml
 
-  deploy:
+  publish_to_pypi:
     name: Publish to PyPI
     needs: [lint_and_type_checks, unit_tests, integration_tests, check_docs]
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/apify
 
     steps:
     - name: Checkout repository
@@ -56,9 +62,7 @@ jobs:
         python-version: 3.8
 
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install --upgrade setuptools twine wheel
+      run: make install-dev
 
     - # Determine if this is a prerelease or latest release
       name: Determine release type
@@ -86,7 +90,7 @@ jobs:
     - # Check whether the released version is listed in CHANGELOG.md
       name: Check whether the released version is listed in the changelog
       if: steps.get-release-type.outputs.release_type != 'alpha'
-      run: python ./scripts/check_version_in_changelog.py
+      run: make check-changelog-entry
 
     - # Check version consistency and increment pre-release version number for prereleases (must be the last step before build)
       name: Bump pre-release version
@@ -95,18 +99,15 @@ jobs:
 
     - # Build a source distribution and a python3-only wheel
       name: Build distribution files
-      run: python setup.py sdist bdist_wheel
+      run: make build
 
     - # Check whether the package description will render correctly on PyPI
       name: Check package rendering on PyPI
-      run: python -m twine check dist/*
+      run: make twine-check
 
     - # Publish package to PyPI using their official GitHub action
       name: Publish package to PyPI
-      run: python -m twine upload --non-interactive --disable-progress-bar dist/*
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+      uses: pypa/gh-action-pypi-publish@release/v1
 
     - # Tag the current commit with the version tag if this is not made from the release event (releases are tagged with the release process)
       name: Tag Version
@@ -132,4 +133,4 @@ jobs:
           -F client_payload[release_tag]=${{ steps.get-release-type.outputs.docker_image_tag }} \
           -F client_payload[apify_version]=$PACKAGE_VERSION
       env:
-        GH_TOKEN: ${{ secrets.BUILD_DOCKER_IMAGES_REPOSITORY_DISPATCH_GH_TOKEN }}
+        GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,7 @@
 Changelog
 =========
 
-[1.1.0](../../releases/tag/v1.1.0) - Unreleased
+[1.1.0](../../releases/tag/v1.1.0) - 2023-05-23
 -----------------------------------------------
 
 ### Added
@@ -15,7 +15,11 @@ Changelog
 - started enforcing local storage to always use the UTF-8 encoding
 - fixed saving key-value store values to local storage with the right extension for a given content type
 
-[1.0.0](../../releases/tag/v1.0.0) - 2022-03-13
+### Internal changes
+
+- switched from `setup.py` to `pyproject.toml` for specifying project setup
+
+[1.0.0](../../releases/tag/v1.0.0) - 2023-03-13
 -----------------------------------------------
 
 ### Internal changes
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -51,13 +51,13 @@ and from Markdown documents in the `docs` folder in the `docs` branch, and then
 
 Publishing new versions to [PyPI](https://pypi.org/project/apify) happens automatically through GitHub Actions.
 
-On each commit to the `master` branch, a new beta release is published, taking the version number from `src/apify/_version.py`
+On each commit to the `master` branch, a new beta release is published, taking the version number from `pyproject.toml`
 and automatically incrementing the beta version suffix by 1 from the last beta release published to PyPI.
 
-A stable version is published when a new release is created using GitHub Releases, again taking the version number from `src/apify/_version.py`.
+A stable version is published when a new release is created using GitHub Releases, again taking the version number from `pyproject.toml`.
 The built package assets are automatically uploaded to the GitHub release.
 
-If there is already a stable version with the same version number as in `src/apify/_version.py` published to PyPI, the publish process fails,
+If there is already a stable version with the same version number as in `pyproject.toml` published to PyPI, the publish process fails,
 so don't forget to update the version number before releasing a new version.
 The release process also fails when the released version is not described in `CHANGELOG.md`,
 so don't forget to describe the changes in the new version there.
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: clean install-dev lint unit-tests integration-tests type-check check-code format docs check-docs check-changelog-entry
+.PHONY: clean install-dev build publish twine-check lint unit-tests integration-tests type-check check-code format docs check-docs check-changelog-entry
 
 # This is default for local testing, but GitHub workflows override it to a higher value in CI
 INTEGRATION_TESTS_CONCURRENCY = 1
@@ -8,10 +8,18 @@ clean:
 
 install-dev:
 	python -m pip install --upgrade pip
-	pip install --upgrade setuptools wheel
 	pip install --no-cache-dir -e ".[dev]"
 	pre-commit install
 
+build:
+	python -m build
+
+publish:
+	python -m twine upload dist/*
+
+twine-check:
+	python -m twine check dist/*
+
 lint:
 	python3 -m flake8
 
diff --git a/mypy.ini b/mypy.ini
@@ -4,8 +4,7 @@ files =
     docs,
     scripts,
     src,
-    tests,
-    setup.py
+    tests
 check_untyped_defs = True
 disallow_incomplete_defs = True
 disallow_untyped_calls = True
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,93 @@
+[project]
+name = "apify"
+version = "1.1.0"
+description = "Apify SDK for Python"
+readme = "README.md"
+license = {text = "Apache Software License"}
+authors = [
+    {name = "Apify Technologies s.r.o.", email = "support@apify.com" }
+]
+keywords = ["apify", "sdk", "actor", "scraping", "automation"]
+
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Software Development :: Libraries",
+]
+
+requires-python = ">=3.8"
+dependencies = [
+    "aiofiles ~= 23.1.0",
+    "aioshutil ~= 1.3",
+    "apify-client ~= 1.2.0",
+    "colorama ~= 0.4.6",
+    "cryptography ~= 40.0.2",
+    "httpx ~= 0.24.1",
+    "psutil ~= 5.9.5",
+    "pyee ~= 9.1.0",
+    "sortedcollections ~= 2.1.0",
+    "typing-extensions ~= 4.6.0",
+    "websockets ~= 11.0.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "autopep8 ~= 2.0.2",
+    "build ~= 0.10.0",
+    "filelock ~= 3.12.0",
+    "flake8 ~= 6.0.0",
+    "flake8-bugbear ~= 23.5.9",
+    "flake8-commas ~= 2.1.0",
+    "flake8-comprehensions ~= 3.12.0",
+    "flake8-datetimez ~= 20.10.0",
+    "flake8-docstrings ~= 1.7.0",
+    "flake8-encodings ~= 0.5.0",
+    "flake8-isort ~= 6.0.0",
+    "flake8-noqa ~= 1.3.1",
+    "flake8-pytest-style ~= 1.7.2",
+    "flake8-quotes ~= 3.3.2",
+    "flake8-unused-arguments ~= 0.0.13",
+    "isort ~= 5.12.0",
+    "mypy ~= 1.3.0",
+    "pep8-naming ~= 0.13.3",
+    "pre-commit ~= 3.3.2",
+    "pytest ~= 7.3.1",
+    "pytest-asyncio ~= 0.21.0",
+    "pytest-only ~= 2.0.0",
+    "pytest-randomly ~= 3.12.0",
+    "pytest-timeout ~= 2.1.0",
+    "pytest-xdist ~= 3.3.1",
+    "respx ~= 0.20.1",
+    "sphinx ~= 6.1.3",
+    "sphinx-autodoc-typehints ~= 1.22",
+    "sphinx-markdown-builder == 0.5.4",  # pinned to 0.5.4, because 0.5.5 has a formatting bug
+    "twine ~= 4.0.2",
+    "types-aiofiles ~= 23.1.0.3",
+    "types-colorama ~= 0.4.15.11",
+    "types-psutil ~= 5.9.5.12",
+]
+
+[project.urls]
+"Homepage" = "https://docs.apify.com/sdk/python/"
+"Documentation" = "https://docs.apify.com/sdk/python/"
+"Source" = "https://github.com/apify/apify-sdk-python"
+"Issue tracker" = "https://github.com/apify/apify-sdk-python/issues"
+"Changelog" = "https://github.com/apify/apify-sdk-python/blob/master/CHANGELOG.md"
+"Apify Homepage" = "https://apify.com"
+
+[build-system]
+requires = ["setuptools>=64.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["apify*"]
+
+[tool.setuptools.package-data]
+apify = ["py.typed"]
diff --git a/scripts/print_current_package_version.py b/scripts/print_current_package_version.py
@@ -2,6 +2,6 @@
 
 from utils import get_current_package_version
 
-# Print the current package version from the src/package_name/_version.py file to stdout
+# Print the current package version from the pyproject.toml file to stdout
 if __name__ == '__main__':
     print(get_current_package_version(), end='')
diff --git a/scripts/update_version_for_prerelease.py b/scripts/update_version_for_prerelease.py
@@ -8,7 +8,7 @@
 from utils import PACKAGE_NAME, get_current_package_version, set_current_package_version
 
 # Checks whether the current package version number was not already used in a published release,
-# and if not, modifies the package version number in src/package_name/_version.py
+# and if not, modifies the package version number in pyproject.toml
 # from a stable release version (X.Y.Z) to a prerelease version (X.Y.ZbN or X.Y.Z.aN or X.Y.Z.rcN)
 if __name__ == '__main__':
     if len(sys.argv) != 2:
@@ -55,6 +55,6 @@
             if prerelease_version > latest_prerelease:
                 latest_prerelease = prerelease_version
 
-    # Write the latest prerelease version number to src/package_name/_version.py
+    # Write the latest prerelease version number to pyproject.toml
     new_prerelease_version_number = f'{current_version}{prerelease_prefix}{latest_prerelease + 1}'
     set_current_package_version(new_prerelease_version_number)
diff --git a/scripts/utils.py b/scripts/utils.py
@@ -2,37 +2,37 @@
 
 PACKAGE_NAME = 'apify'
 REPO_ROOT = pathlib.Path(__file__).parent.resolve() / '..'
-VERSION_FILE_PATH = REPO_ROOT / f'src/{PACKAGE_NAME}/_version.py'
+PYPROJECT_TOML_FILE_PATH = REPO_ROOT / 'pyproject.toml'
 
 
-# Load the current version number from src/package_name/_version.py
-# It is on a line in the format __version__ = 1.2.3
+# Load the current version number from pyproject.toml
+# It is on a line in the format `version = "1.2.3"`
 def get_current_package_version() -> str:
-    with open(VERSION_FILE_PATH, 'r', encoding='utf-8') as version_file:
-        for line in version_file:
-            if line.startswith('__version__'):
+    with open(PYPROJECT_TOML_FILE_PATH, 'r', encoding='utf-8') as pyproject_toml_file:
+        for line in pyproject_toml_file:
+            if line.startswith('version = '):
                 delim = '"' if '"' in line else "'"
                 version = line.split(delim)[1]
                 return version
         else:
             raise RuntimeError('Unable to find version string.')
 
 
-# Write the given version number from src/package_name/_version.py
-# It replaces the version number on the line with the format __version__ = 1.2.3
+# Write the given version number from pyproject.toml
+# It replaces the version number on the line with the format `version = "1.2.3"`
 def set_current_package_version(version: str) -> None:
-    with open(VERSION_FILE_PATH, 'r+', encoding='utf-8') as version_file:
-        updated_version_file_lines = []
+    with open(PYPROJECT_TOML_FILE_PATH, 'r+', encoding='utf-8') as pyproject_toml_file:
+        updated_pyproject_toml_file_lines = []
         version_string_found = False
-        for line in version_file:
-            if line.startswith('__version__'):
+        for line in pyproject_toml_file:
+            if line.startswith('version = '):
                 version_string_found = True
-                line = f"__version__ = '{version}'"
-            updated_version_file_lines.append(line)
+                line = f'version = "{version}"'
+            updated_pyproject_toml_file_lines.append(line)
 
         if not version_string_found:
             raise RuntimeError('Unable to find version string.')
 
-        version_file.seek(0)
-        version_file.write('\n'.join(updated_version_file_lines))
-        version_file.truncate()
+        pyproject_toml_file.seek(0)
+        pyproject_toml_file.write('\n'.join(updated_pyproject_toml_file_lines))
+        pyproject_toml_file.truncate()
diff --git a/setup.py b/setup.py
diff --git a/src/apify/__init__.py b/src/apify/__init__.py
diff --git a/src/apify/_utils.py b/src/apify/_utils.py
diff --git a/src/apify/_version.py b/src/apify/_version.py
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
