From 16e52738edbd7a268d7a1857e7d8deabd3dc3481 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Wed, 4 Oct 2023 13:58:41 +0200 Subject: [PATCH 1/5] Add support for Python 3.12 --- docs/source/index.rst | 1 + pyproject.toml | 3 ++- requirements-dev.txt | 8 +------- src/neo4j/_async_compat/concurrency.py | 2 ++ src/neo4j/_async_compat/shims/__init__.py | 2 +- testkit/Dockerfile | 16 +++++++++++++--- testkit/_common.py | 5 +++++ tests/unit/async_/test_driver.py | 1 + tests/unit/async_/work/test_result.py | 2 +- .../unit/mixed/async_compat/test_concurrency.py | 1 - tests/unit/mixed/async_compat/test_shims.py | 8 ++++---- tests/unit/sync/test_driver.py | 1 + tests/unit/sync/work/test_result.py | 2 +- tox.ini | 13 +++++++++---- 14 files changed, 42 insertions(+), 23 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 084cade61..c7ab462a0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,6 +12,7 @@ Neo4j versions supported: Python versions supported: +* Python 3.12 (added in driver version 5.14.0) * Python 3.11 (added in driver version 5.3.0) * Python 3.10 * Python 3.9 diff --git a/pyproject.toml b/pyproject.toml index b3ce62b0c..2350f7a6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,10 +50,11 @@ pandas = [ "pandas >= 1.1.0, < 3.0.0", "numpy >= 1.7.0, < 2.0.0", ] +pyarrow = ["pyarrow >= 1.0.0"] [build-system] requires = [ - "setuptools~=65.6", + "setuptools>=66.1.0", "tomlkit~=0.11.6", ] build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt index 1845d8e35..5ab2cf72b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # the driver itself --e .[pandas,numpy] +-e .[pandas,numpy,pyarrow] # auto-generate sync driver from async code unasync>=0.5.0 @@ -10,16 +10,10 @@ mypy>=0.971 typing-extensions>=4.3.0 types-pytz>=2022.1.2 -# for packaging -setuptools~=65.6 -# TODO: 6.0 - can be removed once `setup.py` is simplified -tomlkit~=0.11.6 - # needed for running tests coverage[toml]>=5.5 freezegun >= 1.2.2 mock>=4.0.3 -pyarrow>=1.0.0 pytest>=6.2.5 pytest-asyncio>=0.16.0 pytest-benchmark>=3.4.1 diff --git a/src/neo4j/_async_compat/concurrency.py b/src/neo4j/_async_compat/concurrency.py index 3c4a5eae7..f3d23fd6c 100644 --- a/src/neo4j/_async_compat/concurrency.py +++ b/src/neo4j/_async_compat/concurrency.py @@ -136,6 +136,8 @@ async def acquire(self, blocking=True, timeout=-1): try: await wait_for(fut, timeout) except asyncio.CancelledError: + if fut.cancelled(): + raise already_finished = not fut.cancel() if already_finished: # Too late to cancel the acquisition. diff --git a/src/neo4j/_async_compat/shims/__init__.py b/src/neo4j/_async_compat/shims/__init__.py index a1056646f..aac9a0ebe 100644 --- a/src/neo4j/_async_compat/shims/__init__.py +++ b/src/neo4j/_async_compat/shims/__init__.py @@ -30,7 +30,7 @@ # we remove support for Python 3.9. -if sys.version_info >= (3, 8): +if (3, 12) > sys.version_info >= (3, 8): # copied from Python 3.10's asyncio package with applied patch # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, diff --git a/testkit/Dockerfile b/testkit/Dockerfile index 0ae80272c..e407b4533 100644 --- a/testkit/Dockerfile +++ b/testkit/Dockerfile @@ -42,19 +42,29 @@ ENV PYENV_ROOT /.pyenv ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH # Setup python version -ENV PYTHON_VERSIONS 3.7 3.8 3.9 3.10 3.11 +ENV PYTHON_VERSIONS 3.12 3.11 3.10 3.9 3.8 3.7 RUN for version in $PYTHON_VERSIONS; do \ pyenv install $version; \ done RUN pyenv rehash -RUN pyenv global $(pyenv versions --bare --skip-aliases) +RUN pyenv global $(pyenv versions --bare --skip-aliases | sort --version-sort --reverse) # Install Latest pip and setuptools for each environment # + tox and tools for starting the tests # https://pip.pypa.io/en/stable/news/ RUN for version in $PYTHON_VERSIONS; do \ python$version -m pip install -U pip && \ - python$version -m pip install -U setuptools && \ python$version -m pip install -U coverage tox; \ done + +# Installing pyarrow lib until pre-built wheel for Python 3.12 exists +# https://github.com/apache/arrow/issues/37880 +RUN apt update && \ + apt install -y -V lsb-release cmake gcc && \ + distro_name=$(lsb_release --id --short | tr 'A-Z' 'a-z') && \ + code_name=$(lsb_release --codename --short) && \ + wget https://apache.jfrog.io/artifactory/arrow/${distro_name}/apache-arrow-apt-source-latest-${code_name}.deb && \ + apt install -y -V ./apache-arrow-apt-source-latest-${code_name}.deb && \ + apt update && \ + apt install -y -V libarrow-dev # For C++ \ diff --git a/testkit/_common.py b/testkit/_common.py index 1ab0254ed..e1391f204 100644 --- a/testkit/_common.py +++ b/testkit/_common.py @@ -15,6 +15,11 @@ def run(args, env=None): def run_python(args, env=None, warning_as_error=True): cmd = [TEST_BACKEND_VERSION, "-u"] + if sys.version_info >= (3, 12): + # Ignore warnings for Python 3.12 for now + # https://github.com/dateutil/dateutil/issues/1284 needs to be released + # and propagate through our dependency graph + warning_as_error = False if warning_as_error: cmd += ["-W", "error"] cmd += list(args) diff --git a/tests/unit/async_/test_driver.py b/tests/unit/async_/test_driver.py index bee79f798..432aa68f6 100644 --- a/tests/unit/async_/test_driver.py +++ b/tests/unit/async_/test_driver.py @@ -716,6 +716,7 @@ async def test_execute_query_keyword_parameters( @pytest.mark.parametrize("parameters", ( {"_": "a"}, {"foo_": None}, {"foo_": 1, "bar_": 2} )) +@mark_async_test async def test_reserved_query_keyword_parameters( mocker, parameters: t.Dict[str, t.Any], ) -> None: diff --git a/tests/unit/async_/work/test_result.py b/tests/unit/async_/work/test_result.py index 47a4a0808..185eb0331 100644 --- a/tests/unit/async_/work/test_result.py +++ b/tests/unit/async_/work/test_result.py @@ -607,7 +607,7 @@ async def test_data(num_records): record.data.return_value = expected_data[-1] assert await result.data("hello", "world") == expected_data for record in records: - assert record.data.called_once_with("hello", "world") + record.data.assert_called_once_with("hello", "world") @pytest.mark.parametrize("records", ( diff --git a/tests/unit/mixed/async_compat/test_concurrency.py b/tests/unit/mixed/async_compat/test_concurrency.py index ffe08652e..7171ee809 100644 --- a/tests/unit/mixed/async_compat/test_concurrency.py +++ b/tests/unit/mixed/async_compat/test_concurrency.py @@ -225,7 +225,6 @@ async def waiter_non_blocking(): assert fut.exception() is exc awaits += 1 - assert not lock.locked() await asyncio.gather(blocker(), waiter_non_blocking()) assert not lock.locked() diff --git a/tests/unit/mixed/async_compat/test_shims.py b/tests/unit/mixed/async_compat/test_shims.py index f4c66f914..ef6fcef4a 100644 --- a/tests/unit/mixed/async_compat/test_shims.py +++ b/tests/unit/mixed/async_compat/test_shims.py @@ -42,8 +42,8 @@ async def _check_wait_for(wait_for_, should_propagate_cancellation): @pytest.mark.skipif( - sys.version_info < (3, 8), - reason="wait_for is only broken in Python 3.8+" + not (3, 12) > sys.version_info >= (3, 8), + reason="wait_for is only broken in Python 3.8-3.11 (inclusive)" ) @mark_async_test async def test_wait_for_shim_is_necessary_starting_from_3x8(): @@ -56,8 +56,8 @@ async def test_wait_for_shim_is_necessary_starting_from_3x8(): @pytest.mark.skipif( - sys.version_info >= (3, 8), - reason="wait_for is only broken in Python 3.8+" + (3, 12) > sys.version_info >= (3, 8), + reason="wait_for is only broken in Python 3.8-3.11 (inclusive)" ) @mark_async_test async def test_wait_for_shim_is_not_necessary_prior_to_3x8(): diff --git a/tests/unit/sync/test_driver.py b/tests/unit/sync/test_driver.py index 74faa8b64..11a11e131 100644 --- a/tests/unit/sync/test_driver.py +++ b/tests/unit/sync/test_driver.py @@ -715,6 +715,7 @@ def test_execute_query_keyword_parameters( @pytest.mark.parametrize("parameters", ( {"_": "a"}, {"foo_": None}, {"foo_": 1, "bar_": 2} )) +@mark_sync_test def test_reserved_query_keyword_parameters( mocker, parameters: t.Dict[str, t.Any], ) -> None: diff --git a/tests/unit/sync/work/test_result.py b/tests/unit/sync/work/test_result.py index 9899bd875..45904bbe5 100644 --- a/tests/unit/sync/work/test_result.py +++ b/tests/unit/sync/work/test_result.py @@ -607,7 +607,7 @@ def test_data(num_records): record.data.return_value = expected_data[-1] assert result.data("hello", "world") == expected_data for record in records: - assert record.data.called_once_with("hello", "world") + record.data.assert_called_once_with("hello", "world") @pytest.mark.parametrize("records", ( diff --git a/tox.ini b/tox.ini index 82fb5767d..73e9db620 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,20 @@ [tox] -envlist = py{37,38,39,310,311}-{unit,integration,performance} +envlist = py{37,38,39,310,311,312}-{unit,integration,performance} [testenv] passenv = TEST_NEO4J_* deps = -r requirements-dev.txt setenv = COVERAGE_FILE={envdir}/.coverage usedevelop = true +# Ignore warnings for Python 3.12 for now +# https://github.com/dateutil/dateutil/issues/1284 needs to be released +# and propagate through our dependency graph +warnargs = + py{37,38,39,310,311}: -W error + py312: commands = coverage erase - unit: coverage run -m pytest -W error -v {posargs} tests/unit - unit: coverage run -m pytest -v --doctest-modules {posargs} src - integration: coverage run -m pytest -W error -v {posargs} tests/integration + unit: coverage run -m pytest {[testenv]warnargs} -v {posargs} tests/unit + integration: coverage run -m pytest {[testenv]warnargs} -v {posargs} tests/integration performance: python -m pytest --benchmark-autosave -v {posargs} tests/performance unit,integration: coverage report From 307a13c7e0ab9b890dd897c5da4225b32d815be4 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Thu, 5 Oct 2023 12:22:31 +0200 Subject: [PATCH 2/5] Use pyproject.toml to make setuptools pick up the driver version --- pyproject.toml | 11 ++++++----- setup.py | 7 +------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2350f7a6e..ba038a626 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,16 +52,17 @@ pandas = [ ] pyarrow = ["pyarrow >= 1.0.0"] + [build-system] requires = [ - "setuptools>=66.1.0", - "tomlkit~=0.11.6", + "setuptools >= 66.1.0", + # TODO: 6.0 - can be removed once `setup.py` is simplified + "tomlkit ~= 0.11.6", ] build-backend = "setuptools.build_meta" -# still in beta -#[tool.setuptools.dynamic] -#version = {attr = "neo4j._meta.version"} +[tool.setuptools.dynamic] +version = {attr = "neo4j._meta.version"} [tool.coverage] diff --git a/setup.py b/setup.py index 2dcf14864..f6e72313a 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,6 @@ from neo4j._meta import ( deprecated_package as deprecated, package, - version, ) @@ -84,8 +83,4 @@ def changed_package_name(new_name): with changed_package_name(package): - setup( - # until `[tool.setuptools.dynamic]` in pyproject.toml is out of beta - version=version, - long_description=readme, - ) + setup(long_description=readme) From 6d498bc8f3e8a48323fd552c33bf00b805cb4d59 Mon Sep 17 00:00:00 2001 From: Grant Lodge <6323995+thelonelyvulpes@users.noreply.github.com> Date: Fri, 6 Oct 2023 14:08:05 +0200 Subject: [PATCH 3/5] Add python 3.12 tag to package metadata Signed-off-by: Rouven Bauer --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ba038a626..646c7aab4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dynamic = ["version", "readme"] From d501ed65b1027e2c18e9109f3e3caaabac6049ea Mon Sep 17 00:00:00 2001 From: Rouven Bauer Date: Fri, 6 Oct 2023 14:08:29 +0200 Subject: [PATCH 4/5] Comment why wait_for shim not needed in Python 3.12 Signed-off-by: Grant Lodge <6323995+thelonelyvulpes@users.noreply.github.com> --- src/neo4j/_async_compat/shims/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/neo4j/_async_compat/shims/__init__.py b/src/neo4j/_async_compat/shims/__init__.py index aac9a0ebe..79941c38b 100644 --- a/src/neo4j/_async_compat/shims/__init__.py +++ b/src/neo4j/_async_compat/shims/__init__.py @@ -25,9 +25,8 @@ # The shipped wait_for can swallow cancellation errors (starting with 3.8). # See: https://github.com/python/cpython/pull/26097 # and https://github.com/python/cpython/pull/28149 -# Since 3.8 and 3.9 already received their final maintenance release, there -# will be now fix for this. So this patch needs to stick around at least until -# we remove support for Python 3.9. +# Ultimately, this got fixed in https://github.com/python/cpython/pull/98518 +# (released with Python 3.12) by re-doing how wait_for works. if (3, 12) > sys.version_info >= (3, 8): From 03d52efc7d11730866008e79c4c828519d85188f Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Mon, 9 Oct 2023 11:43:28 +0200 Subject: [PATCH 5/5] Silence more deprecation warnings for Python 3.12 --- testkit/_common.py | 12 +++++++++++- testkitbackend/__main__.py | 6 +++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/testkit/_common.py b/testkit/_common.py index e1391f204..76ad479ff 100644 --- a/testkit/_common.py +++ b/testkit/_common.py @@ -1,4 +1,5 @@ import os +import re import subprocess import sys @@ -7,15 +8,24 @@ def run(args, env=None): + print(args) return subprocess.run( args, universal_newlines=True, stdout=sys.stdout, stderr=sys.stderr, check=True, env=env ) +def get_python_version(): + cmd = [TEST_BACKEND_VERSION, "-V"] + res = subprocess.check_output(cmd, universal_newlines=True, + stderr=sys.stderr) + raw_version = re.match(r"(?:.*?)((?:\d+\.)+(?:\d+))", res).group(1) + return tuple(int(e) for e in raw_version.split(".")) + + def run_python(args, env=None, warning_as_error=True): cmd = [TEST_BACKEND_VERSION, "-u"] - if sys.version_info >= (3, 12): + if get_python_version() >= (3, 12): # Ignore warnings for Python 3.12 for now # https://github.com/dateutil/dateutil/issues/1284 needs to be released # and propagate through our dependency graph diff --git a/testkitbackend/__main__.py b/testkitbackend/__main__.py index 94739b13d..4afa32724 100644 --- a/testkitbackend/__main__.py +++ b/testkitbackend/__main__.py @@ -49,7 +49,11 @@ async def main(): if __name__ == "__main__": - warnings.simplefilter("error") + if sys.version_info < (3, 12): + # Ignore warnings for Python 3.12 for now + # https://github.com/dateutil/dateutil/issues/1284 needs to be released + # and propagate through our dependency graph + warnings.simplefilter("error") if len(sys.argv) == 2 and sys.argv[1].lower().strip() == "async": async_main() else: