diff --git a/neo4j/_async/driver.py b/neo4j/_async/driver.py index 36d56732d..a29b17980 100644 --- a/neo4j/_async/driver.py +++ b/neo4j/_async/driver.py @@ -644,7 +644,6 @@ def session(self, **config) -> AsyncSession: """ session_config = SessionConfig(self._default_workspace_config, config) - SessionConfig.consume(config) # Consume the config return AsyncSession(self._pool, session_config) @@ -676,5 +675,4 @@ def __init__(self, pool, default_workspace_config): def session(self, **config) -> AsyncSession: session_config = SessionConfig(self._default_workspace_config, config) - SessionConfig.consume(config) # Consume the config return AsyncSession(self._pool, session_config) diff --git a/neo4j/_async/io/_bolt.py b/neo4j/_async/io/_bolt.py index 8344b2696..fb67d9131 100644 --- a/neo4j/_async/io/_bolt.py +++ b/neo4j/_async/io/_bolt.py @@ -259,19 +259,20 @@ def get_handshake(cls): return b"".join(version.to_bytes() for version in offered_versions).ljust(16, b"\x00") @classmethod - async def ping(cls, address, *, timeout=None, **config): + async def ping(cls, address, *, timeout=None, pool_config=None): """ Attempt to establish a Bolt connection, returning the agreed Bolt protocol version if successful. """ - config = PoolConfig.consume(config) + if pool_config is None: + pool_config = PoolConfig() try: s, protocol_version, handshake, data = \ await AsyncBoltSocket.connect( address, timeout=timeout, - custom_resolver=config.resolver, - ssl_context=config.get_ssl_context(), - keep_alive=config.keep_alive, + custom_resolver=pool_config.resolver, + ssl_context=pool_config.get_ssl_context(), + keep_alive=pool_config.keep_alive, ) except (ServiceUnavailable, SessionExpired, BoltHandshakeError): return None @@ -282,7 +283,7 @@ async def ping(cls, address, *, timeout=None, **config): @classmethod async def open( cls, address, *, auth=None, timeout=None, routing_context=None, - **pool_config + pool_config=None ): """Open a new Bolt connection to a given server address. @@ -305,7 +306,8 @@ def time_remaining(): return t if t > 0 else 0 t0 = perf_counter() - pool_config = PoolConfig.consume(pool_config) + if pool_config is None: + pool_config = PoolConfig() socket_connection_timeout = pool_config.connection_timeout if socket_connection_timeout is None: diff --git a/neo4j/_async/io/_pool.py b/neo4j/_async/io/_pool.py index b530b3a2a..d0a9acd4f 100644 --- a/neo4j/_async/io/_pool.py +++ b/neo4j/_async/io/_pool.py @@ -379,7 +379,7 @@ def open(cls, address, *, auth, pool_config, workspace_config): async def opener(addr, timeout): return await AsyncBolt.open( addr, auth=auth, timeout=timeout, routing_context=None, - **pool_config + pool_config=pool_config ) pool = cls(opener, pool_config, workspace_config, address) @@ -431,7 +431,7 @@ def open(cls, *addresses, auth, pool_config, workspace_config, async def opener(addr, timeout): return await AsyncBolt.open( addr, auth=auth, timeout=timeout, - routing_context=routing_context, **pool_config + routing_context=routing_context, pool_config=pool_config ) pool = cls(opener, pool_config, workspace_config, address) diff --git a/neo4j/_conf.py b/neo4j/_conf.py index 434670853..1fa4ec4ff 100644 --- a/neo4j/_conf.py +++ b/neo4j/_conf.py @@ -300,9 +300,19 @@ def set_attr(k, v): else: raise AttributeError(k) + rejected_keys = [] for key, value in data_dict.items(): if value is not None: - set_attr(key, value) + try: + set_attr(key, value) + except AttributeError as exc: + if not exc.args == (key,): + raise + rejected_keys.append(key) + + if rejected_keys: + raise ConfigurationError("Unexpected config keys: " + + ", ".join(rejected_keys)) def __init__(self, *args, **kwargs): for arg in args: diff --git a/neo4j/_meta.py b/neo4j/_meta.py index 3e41e3467..970ead8a3 100644 --- a/neo4j/_meta.py +++ b/neo4j/_meta.py @@ -17,6 +17,7 @@ import asyncio +import tracemalloc import typing as t from functools import wraps from warnings import warn @@ -130,8 +131,6 @@ def inner(*args, **kwargs): def unclosed_resource_warn(obj): - import tracemalloc - from warnings import warn msg = f"Unclosed {obj!r}." trace = tracemalloc.get_object_traceback(obj) if trace: diff --git a/neo4j/_sync/driver.py b/neo4j/_sync/driver.py index 058928634..604900ef3 100644 --- a/neo4j/_sync/driver.py +++ b/neo4j/_sync/driver.py @@ -643,7 +643,6 @@ def session(self, **config) -> Session: """ session_config = SessionConfig(self._default_workspace_config, config) - SessionConfig.consume(config) # Consume the config return Session(self._pool, session_config) @@ -675,5 +674,4 @@ def __init__(self, pool, default_workspace_config): def session(self, **config) -> Session: session_config = SessionConfig(self._default_workspace_config, config) - SessionConfig.consume(config) # Consume the config return Session(self._pool, session_config) diff --git a/neo4j/_sync/io/_bolt.py b/neo4j/_sync/io/_bolt.py index 7d5ceeb98..eaa30fd1b 100644 --- a/neo4j/_sync/io/_bolt.py +++ b/neo4j/_sync/io/_bolt.py @@ -259,19 +259,20 @@ def get_handshake(cls): return b"".join(version.to_bytes() for version in offered_versions).ljust(16, b"\x00") @classmethod - def ping(cls, address, *, timeout=None, **config): + def ping(cls, address, *, timeout=None, pool_config=None): """ Attempt to establish a Bolt connection, returning the agreed Bolt protocol version if successful. """ - config = PoolConfig.consume(config) + if pool_config is None: + pool_config = PoolConfig() try: s, protocol_version, handshake, data = \ BoltSocket.connect( address, timeout=timeout, - custom_resolver=config.resolver, - ssl_context=config.get_ssl_context(), - keep_alive=config.keep_alive, + custom_resolver=pool_config.resolver, + ssl_context=pool_config.get_ssl_context(), + keep_alive=pool_config.keep_alive, ) except (ServiceUnavailable, SessionExpired, BoltHandshakeError): return None @@ -282,7 +283,7 @@ def ping(cls, address, *, timeout=None, **config): @classmethod def open( cls, address, *, auth=None, timeout=None, routing_context=None, - **pool_config + pool_config=None ): """Open a new Bolt connection to a given server address. @@ -305,7 +306,8 @@ def time_remaining(): return t if t > 0 else 0 t0 = perf_counter() - pool_config = PoolConfig.consume(pool_config) + if pool_config is None: + pool_config = PoolConfig() socket_connection_timeout = pool_config.connection_timeout if socket_connection_timeout is None: diff --git a/neo4j/_sync/io/_pool.py b/neo4j/_sync/io/_pool.py index 947555a01..d031ad8c8 100644 --- a/neo4j/_sync/io/_pool.py +++ b/neo4j/_sync/io/_pool.py @@ -379,7 +379,7 @@ def open(cls, address, *, auth, pool_config, workspace_config): def opener(addr, timeout): return Bolt.open( addr, auth=auth, timeout=timeout, routing_context=None, - **pool_config + pool_config=pool_config ) pool = cls(opener, pool_config, workspace_config, address) @@ -431,7 +431,7 @@ def open(cls, *addresses, auth, pool_config, workspace_config, def opener(addr, timeout): return Bolt.open( addr, auth=auth, timeout=timeout, - routing_context=routing_context, **pool_config + routing_context=routing_context, pool_config=pool_config ) pool = cls(opener, pool_config, workspace_config, address) diff --git a/testkit/Dockerfile b/testkit/Dockerfile index 74068ce46..e3322d281 100644 --- a/testkit/Dockerfile +++ b/testkit/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev \ - libxml2-dev libxmlsec1-dev libffi-dev \ + libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ ca-certificates && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/testkit/backend.py b/testkit/backend.py index 745ca8e02..b0c3c09f8 100644 --- a/testkit/backend.py +++ b/testkit/backend.py @@ -24,7 +24,7 @@ if __name__ == "__main__": - cmd = ["python", "-m", "testkitbackend"] + cmd = ["python", "-W", "error", "-m", "testkitbackend"] if "TEST_BACKEND_SERVER" in os.environ: cmd.append(os.environ["TEST_BACKEND_SERVER"]) subprocess.check_call(cmd, stdout=sys.stdout, stderr=sys.stderr) diff --git a/testkit/integration.py b/testkit/integration.py index 1c8f435d7..81b864f0c 100644 --- a/testkit/integration.py +++ b/testkit/integration.py @@ -27,4 +27,4 @@ def run(args): if __name__ == "__main__": - run(["python", "-m", "tox", "-f", "integration"]) + run(["python", "-W", "error", "-m", "tox", "-f", "integration"]) diff --git a/testkit/unittests.py b/testkit/unittests.py index 262f1bbc0..3cea1355c 100644 --- a/testkit/unittests.py +++ b/testkit/unittests.py @@ -27,4 +27,4 @@ def run(args): if __name__ == "__main__": - run(["python", "-m", "tox", "-f", "unit"]) + run(["python", "-W", "error", "-m", "tox", "-f", "unit"]) diff --git a/testkitbackend/__main__.py b/testkitbackend/__main__.py index a6c7a8ecd..91a4a6e1c 100644 --- a/testkitbackend/__main__.py +++ b/testkitbackend/__main__.py @@ -20,6 +20,7 @@ import asyncio import sys +import warnings from .server import ( AsyncServer, @@ -46,6 +47,7 @@ async def main(): if __name__ == "__main__": + warnings.simplefilter("error") if len(sys.argv) == 2 and sys.argv[1].lower().strip() == "async": async_main() else: diff --git a/testkitbackend/_async/backend.py b/testkitbackend/_async/backend.py index 851865c6b..7f74e024c 100644 --- a/testkitbackend/_async/backend.py +++ b/testkitbackend/_async/backend.py @@ -68,6 +68,23 @@ def __init__(self, rd, wr): self._requestHandlers = dict( [m for m in getmembers(requests, isfunction)]) + async def close(self): + for dict_of_closables in ( + self.transactions, + {key: tracker.session for key, tracker in self.sessions.items()}, + self.drivers, + ): + for key, closable in dict_of_closables.items(): + try: + await closable.close() + except (Neo4jError, DriverError, OSError): + log.error( + "Error during TestKit backend garbage collection. " + "While collecting: (key: %s) %s\n%s", + key, closable, traceback.format_exc() + ) + dict_of_closables.clear() + def next_key(self): self.key = self.key + 1 return self.key diff --git a/testkitbackend/_async/requests.py b/testkitbackend/_async/requests.py index 6f939e9cb..1a8daeb80 100644 --- a/testkitbackend/_async/requests.py +++ b/testkitbackend/_async/requests.py @@ -30,6 +30,7 @@ test_subtest_skips, totestkit, ) +from .._warning_check import warning_check from ..exceptions import MarkdAsDriverException @@ -176,8 +177,14 @@ async def GetServerInfo(backend, data): async def CheckMultiDBSupport(backend, data): driver_id = data["driverId"] driver = backend.drivers[driver_id] + with warning_check( + neo4j.ExperimentalWarning, + "Feature support query, based on Bolt protocol version and Neo4j " + "server version will change in the future." + ): + available = await driver.supports_multi_db() await backend.send_response("MultiDBSupport", { - "id": backend.next_key(), "available": await driver.supports_multi_db() + "id": backend.next_key(), "available": available }) @@ -258,7 +265,14 @@ async def NewBookmarkManager(backend, data): backend, bookmark_manager_id ) - bookmark_manager = neo4j.AsyncGraphDatabase.bookmark_manager(**bmm_kwargs) + with warning_check( + neo4j.ExperimentalWarning, + "The bookmark manager feature is experimental. It might be changed or " + "removed any time even without prior notice." + ): + bookmark_manager = neo4j.AsyncGraphDatabase.bookmark_manager( + **bmm_kwargs + ) backend.bookmark_managers[bookmark_manager_id] = bookmark_manager await backend.send_response("BookmarkManager", {"id": bookmark_manager_id}) @@ -374,7 +388,15 @@ async def NewSession(backend, data): ): if data_name in data: config[conf_name] = data[data_name] - session = driver.session(**config) + if "bookmark_manager" in config: + with warning_check( + neo4j.ExperimentalWarning, + "The 'bookmark_manager' config key is experimental. It might be " + "changed or removed any time even without prior notice." + ): + session = driver.session(**config) + else: + session = driver.session(**config) key = backend.next_key() backend.sessions[key] = SessionTracker(session) await backend.send_response("Session", {"id": key}) @@ -534,6 +556,7 @@ async def ResultSingle(backend, data): async def ResultSingleOptional(backend, data): result = backend.results[data["resultId"]] with warnings.catch_warnings(record=True) as warning_list: + warnings.simplefilter("always") record = await result.single(strict=False) if record: record = totestkit.record(record) diff --git a/testkitbackend/_sync/backend.py b/testkitbackend/_sync/backend.py index 5dae1753d..346c3dd0e 100644 --- a/testkitbackend/_sync/backend.py +++ b/testkitbackend/_sync/backend.py @@ -68,6 +68,23 @@ def __init__(self, rd, wr): self._requestHandlers = dict( [m for m in getmembers(requests, isfunction)]) + def close(self): + for dict_of_closables in ( + self.transactions, + {key: tracker.session for key, tracker in self.sessions.items()}, + self.drivers, + ): + for key, closable in dict_of_closables.items(): + try: + closable.close() + except (Neo4jError, DriverError, OSError): + log.error( + "Error during TestKit backend garbage collection. " + "While collecting: (key: %s) %s\n%s", + key, closable, traceback.format_exc() + ) + dict_of_closables.clear() + def next_key(self): self.key = self.key + 1 return self.key diff --git a/testkitbackend/_sync/requests.py b/testkitbackend/_sync/requests.py index c46b65db9..47be9aff8 100644 --- a/testkitbackend/_sync/requests.py +++ b/testkitbackend/_sync/requests.py @@ -30,6 +30,7 @@ test_subtest_skips, totestkit, ) +from .._warning_check import warning_check from ..exceptions import MarkdAsDriverException @@ -176,8 +177,14 @@ def GetServerInfo(backend, data): def CheckMultiDBSupport(backend, data): driver_id = data["driverId"] driver = backend.drivers[driver_id] + with warning_check( + neo4j.ExperimentalWarning, + "Feature support query, based on Bolt protocol version and Neo4j " + "server version will change in the future." + ): + available = driver.supports_multi_db() backend.send_response("MultiDBSupport", { - "id": backend.next_key(), "available": driver.supports_multi_db() + "id": backend.next_key(), "available": available }) @@ -258,7 +265,14 @@ def NewBookmarkManager(backend, data): backend, bookmark_manager_id ) - bookmark_manager = neo4j.GraphDatabase.bookmark_manager(**bmm_kwargs) + with warning_check( + neo4j.ExperimentalWarning, + "The bookmark manager feature is experimental. It might be changed or " + "removed any time even without prior notice." + ): + bookmark_manager = neo4j.GraphDatabase.bookmark_manager( + **bmm_kwargs + ) backend.bookmark_managers[bookmark_manager_id] = bookmark_manager backend.send_response("BookmarkManager", {"id": bookmark_manager_id}) @@ -374,7 +388,15 @@ def NewSession(backend, data): ): if data_name in data: config[conf_name] = data[data_name] - session = driver.session(**config) + if "bookmark_manager" in config: + with warning_check( + neo4j.ExperimentalWarning, + "The 'bookmark_manager' config key is experimental. It might be " + "changed or removed any time even without prior notice." + ): + session = driver.session(**config) + else: + session = driver.session(**config) key = backend.next_key() backend.sessions[key] = SessionTracker(session) backend.send_response("Session", {"id": key}) @@ -534,6 +556,7 @@ def ResultSingle(backend, data): def ResultSingleOptional(backend, data): result = backend.results[data["resultId"]] with warnings.catch_warnings(record=True) as warning_list: + warnings.simplefilter("always") record = result.single(strict=False) if record: record = totestkit.record(record) diff --git a/testkitbackend/_warning_check.py b/testkitbackend/_warning_check.py new file mode 100644 index 000000000..7fb25aa23 --- /dev/null +++ b/testkitbackend/_warning_check.py @@ -0,0 +1,64 @@ +# Copyright (c) "Neo4j" +# Neo4j Sweden AB [https://neo4j.com] +# +# This file is part of Neo4j. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import re +import warnings +from contextlib import contextmanager + + +@contextmanager +def warning_check(category, message): + with warnings.catch_warnings(record=True) as warn_log: + warnings.filterwarnings("always", category=category, message=message) + yield + if len(warn_log) != 1: + raise AssertionError("Expected 1 warning, found %d: %s" + % (len(warn_log), warn_log)) + + +@contextmanager +def warnings_check(category_message_pairs): + with warnings.catch_warnings(record=True) as warn_log: + for category, message in category_message_pairs: + warnings.filterwarnings("always", category=category, + message=message) + yield + if len(warn_log) != len(category_message_pairs): + raise AssertionError( + "Expected %d warnings, found %d: %s" + % (len(category_message_pairs), len(warn_log), warn_log) + ) + category_message_pairs = [ + (category, re.compile(message, re.I)) + for category, message in category_message_pairs + ] + for category, matcher in category_message_pairs: + match = None + for i, warning in enumerate(warn_log): + if ( + warning.category == category + and matcher.match(warning.message.args[0]) + ): + match = i + break + if match is None: + raise AssertionError( + "Expected warning not found: %r %r" + % (category, matcher.pattern) + ) + warn_log.pop(match) diff --git a/testkitbackend/server.py b/testkitbackend/server.py index f7ec85b9d..bae92cb54 100644 --- a/testkitbackend/server.py +++ b/testkitbackend/server.py @@ -33,8 +33,11 @@ def __init__(self, address): class Handler(StreamRequestHandler): def handle(self): backend = Backend(self.rfile, self.wfile) - while backend.process_request(): - pass + try: + while backend.process_request(): + pass + finally: + backend.close() print("Disconnected") super(Server, self).__init__(address, Handler) @@ -47,8 +50,12 @@ def __init__(self, address): @staticmethod async def _handler(reader, writer): backend = AsyncBackend(reader, writer) - while await backend.process_request(): - pass + try: + while await backend.process_request(): + pass + finally: + writer.close() + await backend.close() print("Disconnected") async def start(self): diff --git a/testkitbackend/totestkit.py b/testkitbackend/totestkit.py index e81ff8fb9..180f5b4e8 100644 --- a/testkitbackend/totestkit.py +++ b/testkitbackend/totestkit.py @@ -34,6 +34,8 @@ Time, ) +from ._warning_check import warning_check + def record(rec): fields = [] @@ -73,18 +75,34 @@ def to(name, val): if isinstance(v, (bytes, bytearray)): return to("CypherBytes", " ".join("{:02x}".format(byte) for byte in v)) if isinstance(v, Node): + with warning_check( + DeprecationWarning, "`id` is deprecated, use `element_id` instead" + ): + id_ = v.id node = { - "id": field(v.id), + "id": field(id_), "labels": field(v.labels), "props": field(v._properties), "elementId": field(v.element_id), } return {"name": "Node", "data": node} if isinstance(v, Relationship): + with warning_check( + DeprecationWarning, "`id` is deprecated, use `element_id` instead" + ): + id_ = v.id + with warning_check( + DeprecationWarning, "`id` is deprecated, use `element_id` instead" + ): + start_id = v.start_node.id + with warning_check( + DeprecationWarning, "`id` is deprecated, use `element_id` instead" + ): + end_id = v.end_node.id rel = { - "id": field(v.id), - "startNodeId": field(v.start_node.id), - "endNodeId": field(v.end_node.id), + "id": field(id_), + "startNodeId": field(start_id), + "endNodeId": field(end_id), "type": field(v.type), "props": field(v._properties), "elementId": field(v.element_id), diff --git a/tests/integration/test_readme.py b/tests/integration/test_readme.py index 872461307..15fa60cc4 100644 --- a/tests/integration/test_readme.py +++ b/tests/integration/test_readme.py @@ -32,6 +32,7 @@ def test_should_run_readme(uri, auth): driver = GraphDatabase.driver("neo4j://localhost:7687", auth=("neo4j", "password")) # === END: README === + driver.close() driver = GraphDatabase.driver(uri, auth=auth) # === START: README === diff --git a/tox.ini b/tox.ini index 203bfb01b..b414ff11f 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ deps = -r tests/requirements.txt commands = coverage erase - unit: coverage run -m pytest -v {posargs} tests/unit - integration: coverage run -m pytest -v {posargs} tests/integration + unit: coverage run -m pytest -W error -v {posargs} tests/unit + integration: coverage run -m pytest -W error -v {posargs} tests/integration performance: python -m pytest --benchmark-autosave -v {posargs} tests/performance unit,integration: coverage report