Skip to content

Commit fe1f59c

Browse files
committed
Merge remote-tracking branch 'origin' into alter-pexpire
2 parents ced8968 + 6df0019 commit fe1f59c

26 files changed

+618
-176
lines changed

CHANGES

+14-9
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
1-
* Create codeql-analysis.yml (#1988). Thanks @chayim
1+
2+
* Add `items` parameter to `hset` signature
3+
* Create codeql-analysis.yml (#1988). Thanks @chayim
4+
* Add limited support for Lua scripting with RedisCluster
5+
* Implement `.lock()` method on RedisCluster
6+
27
* 4.1.3 (Feb 8, 2022)
3-
* Fix flushdb and flushall (#1926)
4-
* Add redis5 and redis4 dockers (#1871)
5-
* Change json.clear test multi to be up to date with redisjson (#1922)
6-
* Fixing volume for unstable_cluster docker (#1914)
7-
* Update changes file with changes since 4.0.0-beta2 (#1915)
8+
* Fix flushdb and flushall (#1926)
9+
* Add redis5 and redis4 dockers (#1871)
10+
* Change json.clear test multi to be up to date with redisjson (#1922)
11+
* Fixing volume for unstable_cluster docker (#1914)
12+
* Update changes file with changes since 4.0.0-beta2 (#1915)
813
* 4.1.2 (Jan 27, 2022)
914
* Invalid OCSP certificates should raise ConnectionError on failed validation (#1907)
1015
* Added retry mechanism on socket timeouts when connecting to the server (#1895)
@@ -94,10 +99,10 @@
9499
* Removing command on initial connections (#1722)
95100
* Removing hiredis warning when not installed (#1721)
96101
* 4.0.0 (Nov 15, 2021)
97-
* FT.EXPLAINCLI intentionally raising NotImplementedError
102+
* FT.EXPLAINCLI intentionally raising NotImplementedError
98103
* Restoring ZRANGE desc for Redis < 6.2.0 (#1697)
99104
* Response parsing occasionally fails to parse floats (#1692)
100-
* Re-enabling read-the-docs (#1707)
105+
* Re-enabling read-the-docs (#1707)
101106
* Call HSET after FT.CREATE to avoid keyspace scan (#1706)
102107
* Unit tests fixes for compatibility (#1703)
103108
* Improve documentation about Locks (#1701)
@@ -117,7 +122,7 @@
117122
* Sleep for flaky search test (#1680)
118123
* Test function renames, to match standards (#1679)
119124
* Docstring improvements for Redis class (#1675)
120-
* Fix georadius tests (#1672)
125+
* Fix georadius tests (#1672)
121126
* Improvements to JSON coverage (#1666)
122127
* Add python_requires setuptools check for python > 3.6 (#1656)
123128
* SMISMEMBER support (#1667)
+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# produces redisfab/redis-py-modcluster:6.2.6
2+
FROM redislabs/redismod:edge
3+
4+
COPY create_redismod_cluster.sh /create_redismod_cluster.sh
5+
RUN chmod +x /create_redismod_cluster.sh
6+
7+
EXPOSE 46379 46380 46381 46382 46383 46384
8+
9+
ENV START_PORT=46379
10+
ENV END_PORT=46384
11+
ENTRYPOINT []
12+
CMD /create_redismod_cluster.sh
+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#! /bin/bash
2+
3+
mkdir -p /nodes
4+
touch /nodes/nodemap
5+
if [ -z ${START_PORT} ]; then
6+
START_PORT=46379
7+
fi
8+
if [ -z ${END_PORT} ]; then
9+
END_PORT=46384
10+
fi
11+
if [ ! -z "$3" ]; then
12+
START_PORT=$2
13+
START_PORT=$3
14+
fi
15+
echo "STARTING: ${START_PORT}"
16+
echo "ENDING: ${END_PORT}"
17+
18+
for PORT in `seq ${START_PORT} ${END_PORT}`; do
19+
mkdir -p /nodes/$PORT
20+
if [[ -e /redis.conf ]]; then
21+
cp /redis.conf /nodes/$PORT/redis.conf
22+
else
23+
touch /nodes/$PORT/redis.conf
24+
fi
25+
cat << EOF >> /nodes/$PORT/redis.conf
26+
port ${PORT}
27+
cluster-enabled yes
28+
daemonize yes
29+
logfile /redis.log
30+
dir /nodes/$PORT
31+
EOF
32+
33+
set -x
34+
redis-server /nodes/$PORT/redis.conf
35+
if [ $? -ne 0 ]; then
36+
echo "Redis failed to start, exiting."
37+
continue
38+
fi
39+
echo 127.0.0.1:$PORT >> /nodes/nodemap
40+
done
41+
if [ -z "${REDIS_PASSWORD}" ]; then
42+
echo yes | redis-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
43+
else
44+
echo yes | redis-cli -a ${REDIS_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
45+
fi
46+
tail -f /redis.log

docker/redismod_cluster/redis.conf

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
loadmodule /usr/lib/redis/modules/redisai.so
2+
loadmodule /usr/lib/redis/modules/redisearch.so
3+
loadmodule /usr/lib/redis/modules/redisgraph.so
4+
loadmodule /usr/lib/redis/modules/redistimeseries.so
5+
loadmodule /usr/lib/redis/modules/rejson.so
6+
loadmodule /usr/lib/redis/modules/redisbloom.so
7+
loadmodule /var/opt/redislabs/lib/modules/redisgears.so Plugin /var/opt/redislabs/modules/rg/plugin/gears_python.so Plugin /var/opt/redislabs/modules/rg/plugin/gears_jvm.so JvmOptions -Djava.class.path=/var/opt/redislabs/modules/rg/gear_runtime-jar-with-dependencies.jar JvmPath /var/opt/redislabs/modules/rg/OpenJDK/jdk-11.0.9.1+1/
8+

redis/asyncio/client.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,10 @@ def lock(
348348
continue trying forever. ``blocking_timeout`` can be specified as a
349349
float or integer, both representing the number of seconds to wait.
350350
351-
``lock_class`` forces the specified lock implementation.
351+
``lock_class`` forces the specified lock implementation. Note that as
352+
of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
353+
a Lua-based lock). So, it's unlikely you'll need this parameter, unless
354+
you have created your own custom lock class.
352355
353356
``thread_local`` indicates whether the lock token is placed in
354357
thread-local storage. By default, the token is placed in thread local

redis/asyncio/retry.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from asyncio import sleep
2-
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Tuple, TypeVar
2+
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Tuple, Type, TypeVar
33

44
from redis.exceptions import ConnectionError, RedisError, TimeoutError
55

@@ -19,7 +19,7 @@ def __init__(
1919
self,
2020
backoff: "AbstractBackoff",
2121
retries: int,
22-
supported_errors: Tuple[RedisError, ...] = (
22+
supported_errors: Tuple[Type[RedisError], ...] = (
2323
ConnectionError,
2424
TimeoutError,
2525
),

redis/client.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -1082,7 +1082,10 @@ def lock(
10821082
continue trying forever. ``blocking_timeout`` can be specified as a
10831083
float or integer, both representing the number of seconds to wait.
10841084
1085-
``lock_class`` forces the specified lock implementation.
1085+
``lock_class`` forces the specified lock implementation. Note that as
1086+
of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
1087+
a Lua-based lock). So, it's unlikely you'll need this parameter, unless
1088+
you have created your own custom lock class.
10861089
10871090
``thread_local`` indicates whether the lock token is placed in
10881091
thread-local storage. By default, the token is placed in thread local

redis/cluster.py

+110-1
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
TimeoutError,
2929
TryAgainError,
3030
)
31+
from redis.lock import Lock
3132
from redis.utils import (
3233
dict_merge,
3334
list_keys_to_dict,
@@ -227,6 +228,7 @@ class RedisCluster(RedisClusterCommands):
227228
"ACL SETUSER",
228229
"ACL USERS",
229230
"ACL WHOAMI",
231+
"AUTH",
230232
"CLIENT LIST",
231233
"CLIENT SETNAME",
232234
"CLIENT GETNAME",
@@ -282,19 +284,29 @@ class RedisCluster(RedisClusterCommands):
282284
"READONLY",
283285
"READWRITE",
284286
"TIME",
287+
"GRAPH.CONFIG",
285288
],
286289
DEFAULT_NODE,
287290
),
288291
list_keys_to_dict(
289292
[
290293
"FLUSHALL",
291294
"FLUSHDB",
295+
"FUNCTION DELETE",
296+
"FUNCTION FLUSH",
297+
"FUNCTION LIST",
298+
"FUNCTION LOAD",
299+
"FUNCTION RESTORE",
292300
"SCRIPT EXISTS",
293301
"SCRIPT FLUSH",
294302
"SCRIPT LOAD",
295303
],
296304
PRIMARIES,
297305
),
306+
list_keys_to_dict(
307+
["FUNCTION DUMP"],
308+
RANDOM,
309+
),
298310
list_keys_to_dict(
299311
[
300312
"CLUSTER COUNTKEYSINSLOT",
@@ -742,6 +754,76 @@ def pipeline(self, transaction=None, shard_hint=None):
742754
reinitialize_steps=self.reinitialize_steps,
743755
)
744756

757+
def lock(
758+
self,
759+
name,
760+
timeout=None,
761+
sleep=0.1,
762+
blocking_timeout=None,
763+
lock_class=None,
764+
thread_local=True,
765+
):
766+
"""
767+
Return a new Lock object using key ``name`` that mimics
768+
the behavior of threading.Lock.
769+
770+
If specified, ``timeout`` indicates a maximum life for the lock.
771+
By default, it will remain locked until release() is called.
772+
773+
``sleep`` indicates the amount of time to sleep per loop iteration
774+
when the lock is in blocking mode and another client is currently
775+
holding the lock.
776+
777+
``blocking_timeout`` indicates the maximum amount of time in seconds to
778+
spend trying to acquire the lock. A value of ``None`` indicates
779+
continue trying forever. ``blocking_timeout`` can be specified as a
780+
float or integer, both representing the number of seconds to wait.
781+
782+
``lock_class`` forces the specified lock implementation. Note that as
783+
of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
784+
a Lua-based lock). So, it's unlikely you'll need this parameter, unless
785+
you have created your own custom lock class.
786+
787+
``thread_local`` indicates whether the lock token is placed in
788+
thread-local storage. By default, the token is placed in thread local
789+
storage so that a thread only sees its token, not a token set by
790+
another thread. Consider the following timeline:
791+
792+
time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
793+
thread-1 sets the token to "abc"
794+
time: 1, thread-2 blocks trying to acquire `my-lock` using the
795+
Lock instance.
796+
time: 5, thread-1 has not yet completed. redis expires the lock
797+
key.
798+
time: 5, thread-2 acquired `my-lock` now that it's available.
799+
thread-2 sets the token to "xyz"
800+
time: 6, thread-1 finishes its work and calls release(). if the
801+
token is *not* stored in thread local storage, then
802+
thread-1 would see the token value as "xyz" and would be
803+
able to successfully release the thread-2's lock.
804+
805+
In some use cases it's necessary to disable thread local storage. For
806+
example, if you have code where one thread acquires a lock and passes
807+
that lock instance to a worker thread to release later. If thread
808+
local storage isn't disabled in this case, the worker thread won't see
809+
the token set by the thread that acquired the lock. Our assumption
810+
is that these cases aren't common and as such default to using
811+
thread local storage."""
812+
if lock_class is None:
813+
lock_class = Lock
814+
return lock_class(
815+
self,
816+
name,
817+
timeout=timeout,
818+
sleep=sleep,
819+
blocking_timeout=blocking_timeout,
820+
thread_local=thread_local,
821+
)
822+
823+
def set_response_callback(self, command, callback):
824+
"""Set a custom Response Callback"""
825+
self.cluster_response_callbacks[command] = callback
826+
745827
def _determine_nodes(self, *args, **kwargs):
746828
command = args[0]
747829
nodes_flag = kwargs.pop("nodes_flag", None)
@@ -843,6 +925,10 @@ def determine_slot(self, *args):
843925
else:
844926
keys = self._get_command_keys(*args)
845927
if keys is None or len(keys) == 0:
928+
# FCALL can call a function with 0 keys, that means the function
929+
# can be run on any node so we can just return a random slot
930+
if command in ("FCALL", "FCALL_RO"):
931+
return random.randrange(0, REDIS_CLUSTER_HASH_SLOTS)
846932
raise RedisClusterException(
847933
"No way to dispatch this command to Redis Cluster. "
848934
"Missing key.\nYou can execute the command by specifying "
@@ -1113,6 +1199,20 @@ def _process_result(self, command, res, **kwargs):
11131199
else:
11141200
return res
11151201

1202+
def load_external_module(
1203+
self,
1204+
funcname,
1205+
func,
1206+
):
1207+
"""
1208+
This function can be used to add externally defined redis modules,
1209+
and their namespaces to the redis client.
1210+
1211+
``funcname`` - A string containing the name of the function to create
1212+
``func`` - The function, being added to this class.
1213+
"""
1214+
setattr(self, funcname, func)
1215+
11161216

11171217
class ClusterNode:
11181218
def __init__(self, host, port, server_type=None, redis_connection=None):
@@ -1958,7 +2058,13 @@ def _send_cluster_commands(
19582058

19592059
# turn the response back into a simple flat array that corresponds
19602060
# to the sequence of commands issued in the stack in pipeline.execute()
1961-
response = [c.result for c in sorted(stack, key=lambda x: x.position)]
2061+
response = []
2062+
for c in sorted(stack, key=lambda x: x.position):
2063+
if c.args[0] in self.cluster_response_callbacks:
2064+
c.result = self.cluster_response_callbacks[c.args[0]](
2065+
c.result, **c.options
2066+
)
2067+
response.append(c.result)
19622068

19632069
if raise_on_error:
19642070
self.raise_first_error(stack)
@@ -1972,6 +2078,9 @@ def _fail_on_redirect(self, allow_redirections):
19722078
"ASK & MOVED redirection not allowed in this pipeline"
19732079
)
19742080

2081+
def exists(self, *keys):
2082+
return self.execute_command("EXISTS", *keys)
2083+
19752084
def eval(self):
19762085
""" """
19772086
raise RedisClusterException("method eval() is not implemented")

redis/commands/cluster.py

+4
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@
44
from .core import (
55
ACLCommands,
66
DataAccessCommands,
7+
FunctionCommands,
78
ManagementCommands,
89
PubSubCommands,
910
ScriptCommands,
1011
)
1112
from .helpers import list_or_args
13+
from .redismodules import RedisModuleCommands
1214

1315

1416
class ClusterMultiKeyCommands:
@@ -212,6 +214,8 @@ class RedisClusterCommands(
212214
PubSubCommands,
213215
ClusterDataAccessCommands,
214216
ScriptCommands,
217+
FunctionCommands,
218+
RedisModuleCommands,
215219
):
216220
"""
217221
A class for all Redis Cluster commands

0 commit comments

Comments
 (0)