Skip to content

Commit 3d3035a

Browse files
authored
[4.4] Warnings overhaul (#789)
* Warnings overhaul * Fix falsely emitted `DeprecationWarning` regarding `update_routing_table_timeout`. * Treat warnings as errors during testing * Changed tests to expect and catch warning or to avoid them. * TestKit backend: fix missing Python build dependency * Improve TestKit glue output * Close stub server pipes * ITs: fix 5.0 support: use `WAIT` Cypher keyword when supported * TestKit backend: close resources when done
1 parent 7f6506d commit 3d3035a

29 files changed

+368
-79
lines changed

neo4j/__init__.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -383,13 +383,11 @@ def session(self, **config):
383383
"""
384384
from neo4j.work.simple import Session
385385
session_config = SessionConfig(self._default_workspace_config, config)
386-
SessionConfig.consume(config) # Consume the config
387386
return Session(self._pool, session_config)
388387

389388
def pipeline(self, **config):
390389
from neo4j.work.pipelining import Pipeline, PipelineConfig
391390
pipeline_config = PipelineConfig(self._default_workspace_config, config)
392-
PipelineConfig.consume(config) # Consume the config
393391
return Pipeline(self._pool, pipeline_config)
394392

395393
@experimental("The configuration may change in the future.")
@@ -427,13 +425,11 @@ def __init__(self, pool, default_workspace_config):
427425

428426
def session(self, **config):
429427
session_config = SessionConfig(self._default_workspace_config, config)
430-
SessionConfig.consume(config) # Consume the config
431428
return Session(self._pool, session_config)
432429

433430
def pipeline(self, **config):
434431
from neo4j.work.pipelining import Pipeline, PipelineConfig
435432
pipeline_config = PipelineConfig(self._default_workspace_config, config)
436-
PipelineConfig.consume(config) # Consume the config
437433
return Pipeline(self._pool, pipeline_config)
438434

439435
@experimental("The configuration may change in the future.")

neo4j/conf.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,9 +170,19 @@ def set_attr(k, v):
170170
else:
171171
raise AttributeError(k)
172172

173+
rejected_keys = []
173174
for key, value in data_dict.items():
174175
if value is not None:
175-
set_attr(key, value)
176+
try:
177+
set_attr(key, value)
178+
except AttributeError as exc:
179+
if not exc.args == (key,):
180+
raise
181+
rejected_keys.append(key)
182+
183+
if rejected_keys:
184+
raise ConfigurationError("Unexpected config keys: "
185+
+ ", ".join(rejected_keys))
176186

177187
def __init__(self, *args, **kwargs):
178188
for arg in args:

neo4j/io/__init__.py

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -277,18 +277,19 @@ def get_handshake(cls):
277277
return b"".join(version.to_bytes() for version in offered_versions).ljust(16, b"\x00")
278278

279279
@classmethod
280-
def ping(cls, address, *, timeout=None, **config):
280+
def ping(cls, address, *, timeout=None, pool_config=None):
281281
""" Attempt to establish a Bolt connection, returning the
282282
agreed Bolt protocol version if successful.
283283
"""
284-
config = PoolConfig.consume(config)
284+
if pool_config is None:
285+
pool_config = PoolConfig()
285286
try:
286287
s, protocol_version, handshake, data = BoltSocket.connect(
287288
address,
288289
timeout=timeout,
289-
custom_resolver=config.resolver,
290-
ssl_context=config.get_ssl_context(),
291-
keep_alive=config.keep_alive,
290+
custom_resolver=pool_config.resolver,
291+
ssl_context=pool_config.get_ssl_context(),
292+
keep_alive=pool_config.keep_alive,
292293
)
293294
except (ServiceUnavailable, SessionExpired, BoltHandshakeError):
294295
return None
@@ -297,7 +298,8 @@ def ping(cls, address, *, timeout=None, **config):
297298
return protocol_version
298299

299300
@classmethod
300-
def open(cls, address, *, auth=None, timeout=None, routing_context=None, **pool_config):
301+
def open(cls, address, *, auth=None, timeout=None, routing_context=None,
302+
pool_config=None):
301303
""" Open a new Bolt connection to a given server address.
302304
303305
:param address:
@@ -316,7 +318,8 @@ def time_remaining():
316318
return t if t > 0 else 0
317319

318320
t0 = perf_counter()
319-
pool_config = PoolConfig.consume(pool_config)
321+
if pool_config is None:
322+
pool_config = PoolConfig()
320323

321324
socket_connection_timeout = pool_config.connection_timeout
322325
if socket_connection_timeout is None:
@@ -906,7 +909,7 @@ def open(cls, address, *, auth, pool_config, workspace_config):
906909
def opener(addr, timeout):
907910
return Bolt.open(
908911
addr, auth=auth, timeout=timeout, routing_context=None,
909-
**pool_config
912+
pool_config=pool_config
910913
)
911914

912915
pool = cls(opener, pool_config, workspace_config, address)
@@ -951,7 +954,8 @@ def open(cls, *addresses, auth, pool_config, workspace_config, routing_context=N
951954

952955
def opener(addr, timeout):
953956
return Bolt.open(addr, auth=auth, timeout=timeout,
954-
routing_context=routing_context, **pool_config)
957+
routing_context=routing_context,
958+
pool_config=pool_config)
955959

956960
pool = cls(opener, pool_config, workspace_config, address)
957961
return pool

neo4j/io/_socket.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ def _handshake(cls, s, resolved_address):
259259
def close_socket(cls, socket_):
260260
try:
261261
if isinstance(socket_, BoltSocket):
262-
socket.close()
262+
socket_.close()
263263
else:
264264
socket_.shutdown(SHUT_RDWR)
265265
socket_.close()

neo4j/time/__init__.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ def __mul__(self, other):
482482
:rtype: Duration
483483
"""
484484
if isinstance(other, float):
485-
deprecation_warn("Multiplication with float will be deprecated in "
485+
deprecation_warn("Multiplication with float will be removed in "
486486
"5.0.")
487487
if isinstance(other, (int, float)):
488488
return Duration(
@@ -1627,7 +1627,7 @@ def from_clock_time(cls, clock_time, epoch):
16271627
ts = clock_time.seconds % 86400
16281628
nanoseconds = int(NANO_SECONDS * ts + clock_time.nanoseconds)
16291629
ticks = (epoch.time().ticks_ns + nanoseconds) % (86400 * NANO_SECONDS)
1630-
return Time.from_ticks_ns(ticks)
1630+
return cls.from_ticks_ns(ticks)
16311631

16321632
@classmethod
16331633
def __normalize_hour(cls, hour):
@@ -1657,8 +1657,8 @@ def __normalize_nanosecond(cls, hour, minute, second, nanosecond):
16571657
# TODO 5.0: remove -----------------------------------------------------
16581658
seconds, extra_ns = divmod(second, 1)
16591659
if extra_ns:
1660-
deprecation_warn("Float support second will be removed in 5.0. "
1661-
"Use `nanosecond` instead.")
1660+
deprecation_warn("Float support for `second` will be removed in "
1661+
"5.0. Use `nanosecond` instead.")
16621662
# ----------------------------------------------------------------------
16631663
hour, minute, second = cls.__normalize_second(hour, minute, second)
16641664
nanosecond = int(nanosecond
@@ -1753,7 +1753,7 @@ def nanosecond(self):
17531753
return self.__nanosecond
17541754

17551755
@property
1756-
@deprecated("hour_minute_second will be removed in 5.0. "
1756+
@deprecated("`hour_minute_second` will be removed in 5.0. "
17571757
"Use `hour_minute_second_nanosecond` instead.")
17581758
def hour_minute_second(self):
17591759
"""The time as a tuple of (hour, minute, second).

testkit/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ RUN apt-get update && \
2525
apt-get install -y --no-install-recommends \
2626
make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
2727
libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev \
28-
libxml2-dev libxmlsec1-dev libffi-dev \
28+
libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
2929
ca-certificates && \
3030
apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
3131

testkit/backend.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@
33

44
if __name__ == "__main__":
55
subprocess.check_call(
6-
["python", "-m", "testkitbackend"],
6+
["python", "-W", "error", "-m", "testkitbackend"],
77
stdout=sys.stdout, stderr=sys.stderr
88
)

testkit/stress.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
import subprocess
2-
import os
32
import sys
43

5-
64
if __name__ == "__main__":
75
# Until below works
86
sys.exit(0)
@@ -16,4 +14,4 @@
1614
"NEO4J_PASSWORD": os.environ["TEST_NEO4J_PASS"],
1715
"NEO4J_URI": uri}
1816
subprocess.check_call(cmd, universal_newlines=True,
19-
stderr=subprocess.STDOUT, env=env)
17+
stdout=sys.stdout, stderr=sys.stderr, env=env)

testkit/unittests.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
import subprocess
2+
import sys
23

34

45
def run(args):
56
subprocess.run(
6-
args, universal_newlines=True, stderr=subprocess.STDOUT, check=True)
7+
args, universal_newlines=True, stdout=sys.stdout, stderr=sys.stderr,
8+
check=True
9+
)
710

811

912
if __name__ == "__main__":

testkitbackend/__main__.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1+
import warnings
2+
13
from .server import Server
24

5+
36
if __name__ == "__main__":
7+
warnings.simplefilter("error")
48
server = Server(("0.0.0.0", 9876))
59
while True:
610
server.handle_request()

testkitbackend/_warning_check.py

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright (c) "Neo4j"
2+
# Neo4j Sweden AB [https://neo4j.com]
3+
#
4+
# This file is part of Neo4j.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# https://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
19+
import re
20+
import warnings
21+
from contextlib import contextmanager
22+
23+
24+
@contextmanager
25+
def warning_check(category, message):
26+
with warnings.catch_warnings(record=True) as warn_log:
27+
warnings.filterwarnings("always", category=category, message=message)
28+
yield
29+
if len(warn_log) != 1:
30+
raise AssertionError("Expected 1 warning, found %d: %s"
31+
% (len(warn_log), warn_log))
32+
33+
34+
@contextmanager
35+
def warnings_check(category_message_pairs):
36+
with warnings.catch_warnings(record=True) as warn_log:
37+
for category, message in category_message_pairs:
38+
warnings.filterwarnings("always", category=category,
39+
message=message)
40+
yield
41+
if len(warn_log) != len(category_message_pairs):
42+
raise AssertionError(
43+
"Expected %d warnings, found %d: %s"
44+
% (len(category_message_pairs), len(warn_log), warn_log)
45+
)
46+
category_message_pairs = [
47+
(category, re.compile(message, re.I))
48+
for category, message in category_message_pairs
49+
]
50+
for category, matcher in category_message_pairs:
51+
match = None
52+
for i, warning in enumerate(warn_log):
53+
if (
54+
warning.category == category
55+
and matcher.match(warning.message.args[0])
56+
):
57+
match = i
58+
break
59+
if match is None:
60+
raise AssertionError(
61+
"Expected warning not found: %r %r"
62+
% (category, matcher.pattern)
63+
)
64+
warn_log.pop(match)

testkitbackend/backend.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,23 @@ def __init__(self, rd, wr):
110110
self._requestHandlers = dict(
111111
[m for m in getmembers(requests, isfunction)])
112112

113+
def close(self):
114+
for dict_of_closables in (
115+
self.transactions,
116+
{key: tracker.session for key, tracker in self.sessions.items()},
117+
self.drivers,
118+
):
119+
for key, closable in dict_of_closables.items():
120+
try:
121+
closable.close()
122+
except (Neo4jError, DriverError, OSError):
123+
log.error(
124+
"Error during TestKit backend garbage collection. "
125+
"While collecting: (key: %s) %s\n%s",
126+
key, closable, traceback.format_exc()
127+
)
128+
dict_of_closables.clear()
129+
113130
def next_key(self):
114131
self.key = self.key + 1
115132
return self.key

testkitbackend/requests.py

Lines changed: 36 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1515
# See the License for the specific language governing permissions and
1616
# limitations under the License.
17+
18+
1719
import json
1820
from os import path
1921

@@ -22,6 +24,11 @@
2224
import testkitbackend.totestkit as totestkit
2325
from testkitbackend.fromtestkit import to_meta_and_timeout
2426

27+
from ._warning_check import (
28+
warning_check,
29+
warnings_check,
30+
)
31+
2532

2633
class FrontendError(Exception):
2734
pass
@@ -97,9 +104,21 @@ def NewDriver(backend, data):
97104
data.mark_item_as_read_if_equals("livenessCheckTimeoutMs", None)
98105

99106
data.mark_item_as_read("domainNameResolverRegistered")
100-
driver = neo4j.GraphDatabase.driver(
101-
data["uri"], auth=auth, user_agent=data["userAgent"], **kwargs
102-
)
107+
expected_warnings = []
108+
if "update_routing_table_timeout" in kwargs:
109+
expected_warnings.append((
110+
DeprecationWarning,
111+
"The 'update_routing_table_timeout' config key is deprecated"
112+
))
113+
if "session_connection_timeout" in kwargs:
114+
expected_warnings.append((
115+
DeprecationWarning,
116+
"The 'session_connection_timeout' config key is deprecated"
117+
))
118+
with warnings_check(expected_warnings):
119+
driver = neo4j.GraphDatabase.driver(
120+
data["uri"], auth=auth, user_agent=data["userAgent"], **kwargs
121+
)
103122
key = backend.next_key()
104123
backend.drivers[key] = driver
105124
backend.send_response("Driver", {"id": key})
@@ -108,17 +127,26 @@ def NewDriver(backend, data):
108127
def VerifyConnectivity(backend, data):
109128
driver_id = data["driverId"]
110129
driver = backend.drivers[driver_id]
111-
driver.verify_connectivity()
130+
with warning_check(
131+
neo4j.ExperimentalWarning,
132+
"The configuration may change in the future."
133+
):
134+
driver.verify_connectivity()
112135
backend.send_response("Driver", {"id": driver_id})
113136

114137

115138
def CheckMultiDBSupport(backend, data):
116139
driver_id = data["driverId"]
117140
driver = backend.drivers[driver_id]
118-
backend.send_response(
119-
"MultiDBSupport",
120-
{"id": backend.next_key(), "available": driver.supports_multi_db()}
121-
)
141+
with warning_check(
142+
neo4j.ExperimentalWarning,
143+
"Feature support query, based on Bolt protocol version and Neo4j "
144+
"server version will change in the future."
145+
):
146+
available = driver.supports_multi_db()
147+
backend.send_response("MultiDBSupport", {
148+
"id": backend.next_key(), "available": available
149+
})
122150

123151

124152
def resolution_func(backend, custom_resolver=False, custom_dns_resolver=False):

testkitbackend/server.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@ def __init__(self, address):
2525
class Handler(StreamRequestHandler):
2626
def handle(self):
2727
backend = Backend(self.rfile, self.wfile)
28-
while backend.process_request():
29-
pass
28+
try:
29+
while backend.process_request():
30+
pass
31+
finally:
32+
backend.close()
3033
print("Disconnected")
3134
super(Server, self).__init__(address, Handler)

0 commit comments

Comments
 (0)