Merge remote-tracking branch 'origin' into alter-pexpire

Author: dogukanteber

CHANGES

@@ -1,10 +1,15 @@
-	* Create codeql-analysis.yml (#1988). Thanks @chayim
+
+    * Add `items` parameter to `hset` signature
+  	* Create codeql-analysis.yml (#1988). Thanks @chayim
+    * Add limited support for Lua scripting with RedisCluster
+    * Implement `.lock()` method on RedisCluster
+
 * 4.1.3 (Feb 8, 2022)
-  * Fix flushdb and flushall (#1926)
-  * Add redis5 and redis4 dockers (#1871)
-  * Change json.clear test multi to be up to date with redisjson (#1922)
-  * Fixing volume for unstable_cluster docker (#1914)
-  * Update changes file with changes since 4.0.0-beta2 (#1915)
+    * Fix flushdb and flushall (#1926)
+    * Add redis5 and redis4 dockers (#1871)
+    * Change json.clear test multi to be up to date with redisjson (#1922)
+    * Fixing volume for unstable_cluster docker (#1914)
+    * Update changes file with changes since 4.0.0-beta2 (#1915)
 * 4.1.2 (Jan 27, 2022)
     * Invalid OCSP certificates should raise ConnectionError on failed validation (#1907)
     * Added retry mechanism on socket timeouts when connecting to the server (#1895)
@@ -94,10 +99,10 @@
     * Removing command on initial connections (#1722)
     * Removing hiredis warning when not installed (#1721)
 * 4.0.0 (Nov 15, 2021)
-    * FT.EXPLAINCLI intentionally raising NotImplementedError 
+    * FT.EXPLAINCLI intentionally raising NotImplementedError
     * Restoring ZRANGE desc for Redis < 6.2.0 (#1697)
     * Response parsing occasionally fails to parse floats (#1692)
-    * Re-enabling read-the-docs (#1707) 
+    * Re-enabling read-the-docs (#1707)
     * Call HSET after FT.CREATE to avoid keyspace scan (#1706)
     * Unit tests fixes for compatibility (#1703)
     * Improve documentation about Locks (#1701)
@@ -117,7 +122,7 @@
     * Sleep for flaky search test (#1680)
     * Test function renames, to match standards (#1679)
     * Docstring improvements for Redis class (#1675)
-    * Fix georadius tests (#1672) 
+    * Fix georadius tests (#1672)
     * Improvements to JSON coverage (#1666)
     * Add python_requires setuptools check for python > 3.6 (#1656)
     * SMISMEMBER support (#1667)

docker/base/Dockerfile.redismod_cluster

@@ -0,0 +1,12 @@
+# produces redisfab/redis-py-modcluster:6.2.6
+FROM redislabs/redismod:edge
+
+COPY create_redismod_cluster.sh /create_redismod_cluster.sh
+RUN chmod +x /create_redismod_cluster.sh
+
+EXPOSE 46379 46380 46381 46382 46383 46384
+
+ENV START_PORT=46379
+ENV END_PORT=46384
+ENTRYPOINT []
+CMD /create_redismod_cluster.sh

docker/base/create_redismod_cluster.sh

@@ -0,0 +1,46 @@
+#! /bin/bash
+
+mkdir -p /nodes
+touch /nodes/nodemap
+if [ -z ${START_PORT} ]; then
+    START_PORT=46379
+fi
+if [ -z ${END_PORT} ]; then
+    END_PORT=46384
+fi
+if [ ! -z "$3" ]; then
+    START_PORT=$2
+    START_PORT=$3
+fi
+echo "STARTING: ${START_PORT}"
+echo "ENDING: ${END_PORT}"
+
+for PORT in `seq ${START_PORT} ${END_PORT}`; do
+  mkdir -p /nodes/$PORT
+  if [[ -e /redis.conf ]]; then
+    cp /redis.conf /nodes/$PORT/redis.conf
+  else
+    touch /nodes/$PORT/redis.conf
+  fi
+  cat << EOF >> /nodes/$PORT/redis.conf
+port ${PORT}
+cluster-enabled yes
+daemonize yes
+logfile /redis.log
+dir /nodes/$PORT
+EOF
+
+  set -x
+  redis-server /nodes/$PORT/redis.conf
+  if [ $? -ne 0 ]; then
+    echo "Redis failed to start, exiting."
+    continue
+  fi
+  echo 127.0.0.1:$PORT >> /nodes/nodemap
+done
+if [ -z "${REDIS_PASSWORD}" ]; then
+    echo yes | redis-cli --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
+else
+    echo yes | redis-cli -a ${REDIS_PASSWORD} --cluster create `seq -f 127.0.0.1:%g ${START_PORT} ${END_PORT}` --cluster-replicas 1
+fi
+tail -f /redis.log

docker/redismod_cluster/redis.conf

@@ -0,0 +1,8 @@
+loadmodule /usr/lib/redis/modules/redisai.so
+loadmodule /usr/lib/redis/modules/redisearch.so
+loadmodule /usr/lib/redis/modules/redisgraph.so
+loadmodule /usr/lib/redis/modules/redistimeseries.so
+loadmodule /usr/lib/redis/modules/rejson.so
+loadmodule /usr/lib/redis/modules/redisbloom.so
+loadmodule /var/opt/redislabs/lib/modules/redisgears.so Plugin /var/opt/redislabs/modules/rg/plugin/gears_python.so Plugin /var/opt/redislabs/modules/rg/plugin/gears_jvm.so JvmOptions -Djava.class.path=/var/opt/redislabs/modules/rg/gear_runtime-jar-with-dependencies.jar JvmPath /var/opt/redislabs/modules/rg/OpenJDK/jdk-11.0.9.1+1/
+

redis/asyncio/client.py

@@ -348,7 +348,10 @@ def lock(
         continue trying forever. ``blocking_timeout`` can be specified as a
         float or integer, both representing the number of seconds to wait.
 
-        ``lock_class`` forces the specified lock implementation.
+        ``lock_class`` forces the specified lock implementation. Note that as
+        of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
+        a Lua-based lock). So, it's unlikely you'll need this parameter, unless
+        you have created your own custom lock class.
 
         ``thread_local`` indicates whether the lock token is placed in
         thread-local storage. By default, the token is placed in thread local

redis/asyncio/retry.py

@@ -1,5 +1,5 @@
 from asyncio import sleep
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Tuple, TypeVar
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Tuple, Type, TypeVar
 
 from redis.exceptions import ConnectionError, RedisError, TimeoutError
 
@@ -19,7 +19,7 @@ def __init__(
         self,
         backoff: "AbstractBackoff",
         retries: int,
-        supported_errors: Tuple[RedisError, ...] = (
+        supported_errors: Tuple[Type[RedisError], ...] = (
             ConnectionError,
             TimeoutError,
         ),

redis/client.py

@@ -1082,7 +1082,10 @@ def lock(
         continue trying forever. ``blocking_timeout`` can be specified as a
         float or integer, both representing the number of seconds to wait.
 
-        ``lock_class`` forces the specified lock implementation.
+        ``lock_class`` forces the specified lock implementation. Note that as
+        of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
+        a Lua-based lock). So, it's unlikely you'll need this parameter, unless
+        you have created your own custom lock class.
 
         ``thread_local`` indicates whether the lock token is placed in
         thread-local storage. By default, the token is placed in thread local

redis/cluster.py

@@ -28,6 +28,7 @@
     TimeoutError,
     TryAgainError,
 )
+from redis.lock import Lock
 from redis.utils import (
     dict_merge,
     list_keys_to_dict,
@@ -227,6 +228,7 @@ class RedisCluster(RedisClusterCommands):
                 "ACL SETUSER",
                 "ACL USERS",
                 "ACL WHOAMI",
+                "AUTH",
                 "CLIENT LIST",
                 "CLIENT SETNAME",
                 "CLIENT GETNAME",
@@ -282,19 +284,29 @@ class RedisCluster(RedisClusterCommands):
                 "READONLY",
                 "READWRITE",
                 "TIME",
+                "GRAPH.CONFIG",
             ],
             DEFAULT_NODE,
         ),
         list_keys_to_dict(
             [
                 "FLUSHALL",
                 "FLUSHDB",
+                "FUNCTION DELETE",
+                "FUNCTION FLUSH",
+                "FUNCTION LIST",
+                "FUNCTION LOAD",
+                "FUNCTION RESTORE",
                 "SCRIPT EXISTS",
                 "SCRIPT FLUSH",
                 "SCRIPT LOAD",
             ],
             PRIMARIES,
         ),
+        list_keys_to_dict(
+            ["FUNCTION DUMP"],
+            RANDOM,
+        ),
         list_keys_to_dict(
             [
                 "CLUSTER COUNTKEYSINSLOT",
@@ -742,6 +754,76 @@ def pipeline(self, transaction=None, shard_hint=None):
             reinitialize_steps=self.reinitialize_steps,
         )
 
+    def lock(
+        self,
+        name,
+        timeout=None,
+        sleep=0.1,
+        blocking_timeout=None,
+        lock_class=None,
+        thread_local=True,
+    ):
+        """
+        Return a new Lock object using key ``name`` that mimics
+        the behavior of threading.Lock.
+
+        If specified, ``timeout`` indicates a maximum life for the lock.
+        By default, it will remain locked until release() is called.
+
+        ``sleep`` indicates the amount of time to sleep per loop iteration
+        when the lock is in blocking mode and another client is currently
+        holding the lock.
+
+        ``blocking_timeout`` indicates the maximum amount of time in seconds to
+        spend trying to acquire the lock. A value of ``None`` indicates
+        continue trying forever. ``blocking_timeout`` can be specified as a
+        float or integer, both representing the number of seconds to wait.
+
+        ``lock_class`` forces the specified lock implementation. Note that as
+        of redis-py 3.0, the only lock class we implement is ``Lock`` (which is
+        a Lua-based lock). So, it's unlikely you'll need this parameter, unless
+        you have created your own custom lock class.
+
+        ``thread_local`` indicates whether the lock token is placed in
+        thread-local storage. By default, the token is placed in thread local
+        storage so that a thread only sees its token, not a token set by
+        another thread. Consider the following timeline:
+
+            time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
+                     thread-1 sets the token to "abc"
+            time: 1, thread-2 blocks trying to acquire `my-lock` using the
+                     Lock instance.
+            time: 5, thread-1 has not yet completed. redis expires the lock
+                     key.
+            time: 5, thread-2 acquired `my-lock` now that it's available.
+                     thread-2 sets the token to "xyz"
+            time: 6, thread-1 finishes its work and calls release(). if the
+                     token is *not* stored in thread local storage, then
+                     thread-1 would see the token value as "xyz" and would be
+                     able to successfully release the thread-2's lock.
+
+        In some use cases it's necessary to disable thread local storage. For
+        example, if you have code where one thread acquires a lock and passes
+        that lock instance to a worker thread to release later. If thread
+        local storage isn't disabled in this case, the worker thread won't see
+        the token set by the thread that acquired the lock. Our assumption
+        is that these cases aren't common and as such default to using
+        thread local storage."""
+        if lock_class is None:
+            lock_class = Lock
+        return lock_class(
+            self,
+            name,
+            timeout=timeout,
+            sleep=sleep,
+            blocking_timeout=blocking_timeout,
+            thread_local=thread_local,
+        )
+
+    def set_response_callback(self, command, callback):
+        """Set a custom Response Callback"""
+        self.cluster_response_callbacks[command] = callback
+
     def _determine_nodes(self, *args, **kwargs):
         command = args[0]
         nodes_flag = kwargs.pop("nodes_flag", None)
@@ -843,6 +925,10 @@ def determine_slot(self, *args):
         else:
             keys = self._get_command_keys(*args)
             if keys is None or len(keys) == 0:
+                # FCALL can call a function with 0 keys, that means the function
+                #  can be run on any node so we can just return a random slot
+                if command in ("FCALL", "FCALL_RO"):
+                    return random.randrange(0, REDIS_CLUSTER_HASH_SLOTS)
                 raise RedisClusterException(
                     "No way to dispatch this command to Redis Cluster. "
                     "Missing key.\nYou can execute the command by specifying "
@@ -1113,6 +1199,20 @@ def _process_result(self, command, res, **kwargs):
         else:
             return res
 
+    def load_external_module(
+        self,
+        funcname,
+        func,
+    ):
+        """
+        This function can be used to add externally defined redis modules,
+        and their namespaces to the redis client.
+
+        ``funcname`` - A string containing the name of the function to create
+        ``func`` - The function, being added to this class.
+        """
+        setattr(self, funcname, func)
+
 
 class ClusterNode:
     def __init__(self, host, port, server_type=None, redis_connection=None):
@@ -1958,7 +2058,13 @@ def _send_cluster_commands(
 
         # turn the response back into a simple flat array that corresponds
         # to the sequence of commands issued in the stack in pipeline.execute()
-        response = [c.result for c in sorted(stack, key=lambda x: x.position)]
+        response = []
+        for c in sorted(stack, key=lambda x: x.position):
+            if c.args[0] in self.cluster_response_callbacks:
+                c.result = self.cluster_response_callbacks[c.args[0]](
+                    c.result, **c.options
+                )
+            response.append(c.result)
 
         if raise_on_error:
             self.raise_first_error(stack)
@@ -1972,6 +2078,9 @@ def _fail_on_redirect(self, allow_redirections):
                 "ASK & MOVED redirection not allowed in this pipeline"
             )
 
+    def exists(self, *keys):
+        return self.execute_command("EXISTS", *keys)
+
     def eval(self):
         """ """
         raise RedisClusterException("method eval() is not implemented")

redis/commands/cluster.py

@@ -4,11 +4,13 @@
 from .core import (
     ACLCommands,
     DataAccessCommands,
+    FunctionCommands,
     ManagementCommands,
     PubSubCommands,
     ScriptCommands,
 )
 from .helpers import list_or_args
+from .redismodules import RedisModuleCommands
 
 
 class ClusterMultiKeyCommands:
@@ -212,6 +214,8 @@ class RedisClusterCommands(
     PubSubCommands,
     ClusterDataAccessCommands,
     ScriptCommands,
+    FunctionCommands,
+    RedisModuleCommands,
 ):
     """
     A class for all Redis Cluster commands