From e06d0a546eee8dd6b511b13822f78195a0d2cbad Mon Sep 17 00:00:00 2001 From: Jake Barnwell Date: Tue, 22 Feb 2022 13:41:55 -0500 Subject: [PATCH 1/7] Add support for .lock() for RedisCluster --- redis/client.py | 5 +++- redis/cluster.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ redis/lock.py | 4 +-- tests/test_lock.py | 2 -- 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/redis/client.py b/redis/client.py index 0eade79e90..b12ad578d6 100755 --- a/redis/client.py +++ b/redis/client.py @@ -1082,7 +1082,10 @@ def lock( continue trying forever. ``blocking_timeout`` can be specified as a float or integer, both representing the number of seconds to wait. - ``lock_class`` forces the specified lock implementation. + ``lock_class`` forces the specified lock implementation. Note that as + of redis-py 3.0, the only lock class we implement is ``Lock`` (which is + a Lua-based lock). So, it's unlikely you'll need this parameter, unless + you have created your own custom lock class. ``thread_local`` indicates whether the lock token is placed in thread-local storage. By default, the token is placed in thread local diff --git a/redis/cluster.py b/redis/cluster.py index b8d6b1997f..3b30a6e0a8 100644 --- a/redis/cluster.py +++ b/redis/cluster.py @@ -28,6 +28,7 @@ TimeoutError, TryAgainError, ) +from redis.lock import Lock from redis.utils import ( dict_merge, list_keys_to_dict, @@ -742,6 +743,72 @@ def pipeline(self, transaction=None, shard_hint=None): reinitialize_steps=self.reinitialize_steps, ) + def lock( + self, + name, + timeout=None, + sleep=0.1, + blocking_timeout=None, + lock_class=None, + thread_local=True, + ): + """ + Return a new Lock object using key ``name`` that mimics + the behavior of threading.Lock. + + If specified, ``timeout`` indicates a maximum life for the lock. + By default, it will remain locked until release() is called. + + ``sleep`` indicates the amount of time to sleep per loop iteration + when the lock is in blocking mode and another client is currently + holding the lock. + + ``blocking_timeout`` indicates the maximum amount of time in seconds to + spend trying to acquire the lock. A value of ``None`` indicates + continue trying forever. ``blocking_timeout`` can be specified as a + float or integer, both representing the number of seconds to wait. + + ``lock_class`` forces the specified lock implementation. Note that as + of redis-py 3.0, the only lock class we implement is ``Lock`` (which is + a Lua-based lock). So, it's unlikely you'll need this parameter, unless + you have created your own custom lock class. + + ``thread_local`` indicates whether the lock token is placed in + thread-local storage. By default, the token is placed in thread local + storage so that a thread only sees its token, not a token set by + another thread. Consider the following timeline: + + time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds. + thread-1 sets the token to "abc" + time: 1, thread-2 blocks trying to acquire `my-lock` using the + Lock instance. + time: 5, thread-1 has not yet completed. redis expires the lock + key. + time: 5, thread-2 acquired `my-lock` now that it's available. + thread-2 sets the token to "xyz" + time: 6, thread-1 finishes its work and calls release(). if the + token is *not* stored in thread local storage, then + thread-1 would see the token value as "xyz" and would be + able to successfully release the thread-2's lock. + + In some use cases it's necessary to disable thread local storage. For + example, if you have code where one thread acquires a lock and passes + that lock instance to a worker thread to release later. If thread + local storage isn't disabled in this case, the worker thread won't see + the token set by the thread that acquired the lock. Our assumption + is that these cases aren't common and as such default to using + thread local storage.""" + if lock_class is None: + lock_class = Lock + return lock_class( + self, + name, + timeout=timeout, + sleep=sleep, + blocking_timeout=blocking_timeout, + thread_local=thread_local, + ) + def _determine_nodes(self, *args, **kwargs): command = args[0] nodes_flag = kwargs.pop("nodes_flag", None) diff --git a/redis/lock.py b/redis/lock.py index 95bb413d7e..74e769bfea 100644 --- a/redis/lock.py +++ b/redis/lock.py @@ -180,7 +180,7 @@ def acquire(self, blocking=None, blocking_timeout=None, token=None): if token is None: token = uuid.uuid1().hex.encode() else: - encoder = self.redis.connection_pool.get_encoder() + encoder = self.redis.get_encoder() token = encoder.encode(token) if blocking is None: blocking = self.blocking @@ -224,7 +224,7 @@ def owned(self): # need to always compare bytes to bytes # TODO: this can be simplified when the context manager is finished if stored_token and not isinstance(stored_token, bytes): - encoder = self.redis.connection_pool.get_encoder() + encoder = self.redis.get_encoder() stored_token = encoder.encode(stored_token) return self.local.token is not None and stored_token == self.local.token diff --git a/tests/test_lock.py b/tests/test_lock.py index 02cca1b522..01ecb880b8 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -9,7 +9,6 @@ from .conftest import _get_client -@pytest.mark.onlynoncluster class TestLock: @pytest.fixture() def r_decoded(self, request): @@ -223,7 +222,6 @@ def test_reacquiring_lock_no_longer_owned_raises_error(self, r): lock.reacquire() -@pytest.mark.onlynoncluster class TestLockClassSelection: def test_lock_class_argument(self, r): class MyLock: From 3cad16fb22cc2756fdf1926ca41c8b2101b5316a Mon Sep 17 00:00:00 2001 From: Jake Barnwell Date: Tue, 22 Feb 2022 15:33:57 -0500 Subject: [PATCH 2/7] Update changelog with lua scripting and lock() changes --- CHANGES | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/CHANGES b/CHANGES index 8da3a8b675..ddf7cfd4e3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,10 +1,12 @@ - * Create codeql-analysis.yml (#1988). Thanks @chayim + * Create codeql-analysis.yml (#1988). Thanks @chayim + * Add limited support for Lua scripting with RedisCluster + * Implement `.lock()` method on RedisCluster * 4.1.3 (Feb 8, 2022) - * Fix flushdb and flushall (#1926) - * Add redis5 and redis4 dockers (#1871) - * Change json.clear test multi to be up to date with redisjson (#1922) - * Fixing volume for unstable_cluster docker (#1914) - * Update changes file with changes since 4.0.0-beta2 (#1915) + * Fix flushdb and flushall (#1926) + * Add redis5 and redis4 dockers (#1871) + * Change json.clear test multi to be up to date with redisjson (#1922) + * Fixing volume for unstable_cluster docker (#1914) + * Update changes file with changes since 4.0.0-beta2 (#1915) * 4.1.2 (Jan 27, 2022) * Invalid OCSP certificates should raise ConnectionError on failed validation (#1907) * Added retry mechanism on socket timeouts when connecting to the server (#1895) @@ -94,10 +96,10 @@ * Removing command on initial connections (#1722) * Removing hiredis warning when not installed (#1721) * 4.0.0 (Nov 15, 2021) - * FT.EXPLAINCLI intentionally raising NotImplementedError + * FT.EXPLAINCLI intentionally raising NotImplementedError * Restoring ZRANGE desc for Redis < 6.2.0 (#1697) * Response parsing occasionally fails to parse floats (#1692) - * Re-enabling read-the-docs (#1707) + * Re-enabling read-the-docs (#1707) * Call HSET after FT.CREATE to avoid keyspace scan (#1706) * Unit tests fixes for compatibility (#1703) * Improve documentation about Locks (#1701) @@ -117,7 +119,7 @@ * Sleep for flaky search test (#1680) * Test function renames, to match standards (#1679) * Docstring improvements for Redis class (#1675) - * Fix georadius tests (#1672) + * Fix georadius tests (#1672) * Improvements to JSON coverage (#1666) * Add python_requires setuptools check for python > 3.6 (#1656) * SMISMEMBER support (#1667) From bb6b9e6994004cab83daeb03d8f590fdc2e1f7b8 Mon Sep 17 00:00:00 2001 From: Jake Barnwell Date: Mon, 28 Feb 2022 12:02:31 -0500 Subject: [PATCH 3/7] Also update asyncio client .lock() doc --- redis/asyncio/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/redis/asyncio/client.py b/redis/asyncio/client.py index 619592ef76..2afad0f4d1 100644 --- a/redis/asyncio/client.py +++ b/redis/asyncio/client.py @@ -348,7 +348,10 @@ def lock( continue trying forever. ``blocking_timeout`` can be specified as a float or integer, both representing the number of seconds to wait. - ``lock_class`` forces the specified lock implementation. + ``lock_class`` forces the specified lock implementation. Note that as + of redis-py 3.0, the only lock class we implement is ``Lock`` (which is + a Lua-based lock). So, it's unlikely you'll need this parameter, unless + you have created your own custom lock class. ``thread_local`` indicates whether the lock token is placed in thread-local storage. By default, the token is placed in thread local From 6979ea3dd21d4f006d6620d94adae3ac6543f1a8 Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Wed, 23 Feb 2022 02:25:54 -0500 Subject: [PATCH 4/7] Add Python 3.6 back to hash verify CI (#2008) --- .github/workflows/integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index bfd1841669..74b25d6b27 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -36,7 +36,7 @@ jobs: strategy: max-parallel: 15 matrix: - python-version: ['3.6','3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] test-type: ['standalone', 'cluster'] connection-type: ['hiredis', 'plain'] env: @@ -80,7 +80,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] steps: - uses: actions/checkout@v2 - name: install python ${{ matrix.python-version }} From f2341bc49794671a2677633ec9ceeef2bca94944 Mon Sep 17 00:00:00 2001 From: Chayim Date: Wed, 23 Feb 2022 09:31:59 +0200 Subject: [PATCH 5/7] Renaming chore as maintenance (#2015) --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/release-drafter-config.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a46d8c9bbc..a3b0b0e4e7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,7 +7,7 @@ _Please make sure to review and check all of these items:_ - [ ] Is the new or changed code fully tested? - [ ] Is a documentation update included (if this change modifies existing APIs, or introduces new ones)? - [ ] Is there an example added to the examples folder (if applicable)? -- [ ] Is the changes added to CHANGES file? +- [ ] Was the change added to CHANGES file? _NOTE: these things are not required to open a PR and can be done afterwards / while the PR is open._ diff --git a/.github/release-drafter-config.yml b/.github/release-drafter-config.yml index a3a5d8392a..aab645f9e4 100644 --- a/.github/release-drafter-config.yml +++ b/.github/release-drafter-config.yml @@ -1,16 +1,16 @@ name-template: 'Version $NEXT_PATCH_VERSION' tag-template: 'v$NEXT_PATCH_VERSION' autolabeler: - - label: 'chore' + - label: 'maintenance' files: - '*.md' - '.github/*' - label: 'bug' branch: - '/bug-.+' - - label: 'chore' + - label: 'maintenance' branch: - - '/chore-.+' + - '/maintenance-.+' - label: 'feature' branch: - '/feature-.+' @@ -28,7 +28,7 @@ categories: - 'bugfix' - 'bug' - title: '🧰 Maintenance' - label: 'chore' + label: 'maintenance' change-template: '- $TITLE (#$NUMBER)' exclude-labels: - 'skip-changelog' From d9fa1b2d1176b9deab0cd93165922f6d42dce7a6 Mon Sep 17 00:00:00 2001 From: Andrew Chen Wang <60190294+Andrew-Chen-Wang@users.noreply.github.com> Date: Thu, 24 Feb 2022 09:12:27 -0500 Subject: [PATCH 6/7] Add AsyncFunctionCommands (#2009) --- redis/commands/core.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/redis/commands/core.py b/redis/commands/core.py index 7fd668e6ae..968b7cc412 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -5485,6 +5485,9 @@ def readonly(self, **kwargs) -> ResponseT: return self.execute_command("READONLY", **kwargs) +AsyncClusterCommands = ClusterCommands + + class FunctionCommands: """ Redis Function commands @@ -5497,7 +5500,7 @@ def function_load( code: str, replace: Optional[bool] = False, description: Optional[str] = None, - ) -> str: + ) -> Union[Awaitable[str], str]: """ Load a library to Redis. :param engine: the name of the execution engine for the library @@ -5517,7 +5520,7 @@ def function_load( pieces.append(code) return self.execute_command("FUNCTION LOAD", *pieces) - def function_delete(self, library: str) -> str: + def function_delete(self, library: str) -> Union[Awaitable[str], str]: """ Delete the library called ``library`` and all its functions. @@ -5525,7 +5528,7 @@ def function_delete(self, library: str) -> str: """ return self.execute_command("FUNCTION DELETE", library) - def function_flush(self, mode: str = "SYNC") -> str: + def function_flush(self, mode: str = "SYNC") -> Union[Awaitable[str], str]: """ Deletes all the libraries. @@ -5535,7 +5538,7 @@ def function_flush(self, mode: str = "SYNC") -> str: def function_list( self, library: Optional[str] = "*", withcode: Optional[bool] = False - ) -> List: + ) -> Union[Awaitable[list], list]: """ Return information about the functions and libraries. :param library: pecify a pattern for matching library names @@ -5549,10 +5552,12 @@ def function_list( def _fcall( self, command: str, function, numkeys: int, *keys_and_args: Optional[List] - ) -> str: + ) -> Union[Awaitable[str], str]: return self.execute_command(command, function, numkeys, *keys_and_args) - def fcall(self, function, numkeys: int, *keys_and_args: Optional[List]) -> str: + def fcall( + self, function, numkeys: int, *keys_and_args: Optional[List] + ) -> Union[Awaitable[str], str]: """ Invoke a function. @@ -5560,7 +5565,9 @@ def fcall(self, function, numkeys: int, *keys_and_args: Optional[List]) -> str: """ return self._fcall("FCALL", function, numkeys, *keys_and_args) - def fcall_ro(self, function, numkeys: int, *keys_and_args: Optional[List]) -> str: + def fcall_ro( + self, function, numkeys: int, *keys_and_args: Optional[List] + ) -> Union[Awaitable[str], str]: """ This is a read-only variant of the FCALL command that cannot execute commands that modify data. @@ -5569,7 +5576,7 @@ def fcall_ro(self, function, numkeys: int, *keys_and_args: Optional[List]) -> st """ return self._fcall("FCALL_RO", function, numkeys, *keys_and_args) - def function_dump(self) -> str: + def function_dump(self) -> Union[Awaitable[str], str]: """ Return the serialized payload of loaded libraries. @@ -5582,7 +5589,9 @@ def function_dump(self) -> str: return self.execute_command("FUNCTION DUMP", **options) - def function_restore(self, payload: str, policy: Optional[str] = "APPEND") -> str: + def function_restore( + self, payload: str, policy: Optional[str] = "APPEND" + ) -> Union[Awaitable[str], str]: """ Restore libraries from the serialized ``payload``. You can use the optional policy argument to provide a policy @@ -5592,7 +5601,7 @@ def function_restore(self, payload: str, policy: Optional[str] = "APPEND") -> st """ return self.execute_command("FUNCTION RESTORE", payload, policy) - def function_kill(self) -> str: + def function_kill(self) -> Union[Awaitable[str], str]: """ Kill a function that is currently executing. @@ -5600,7 +5609,7 @@ def function_kill(self) -> str: """ return self.execute_command("FUNCTION KILL") - def function_stats(self) -> list: + def function_stats(self) -> Union[Awaitable[list], list]: """ Return information about the function that's currently running and information about the available execution engines. @@ -5610,7 +5619,7 @@ def function_stats(self) -> list: return self.execute_command("FUNCTION STATS") -AsyncClusterCommands = ClusterCommands +AsyncFunctionCommands = FunctionCommands class DataAccessCommands( @@ -5671,6 +5680,7 @@ class AsyncCoreCommands( AsyncModuleCommands, AsyncPubSubCommands, AsyncScriptCommands, + AsyncFunctionCommands, ): """ A class containing all of the implemented redis commands. This class is From 53ae580a52e373cdb1ea8883feb491ce937f62d8 Mon Sep 17 00:00:00 2001 From: Jake Barnwell Date: Mon, 28 Feb 2022 12:02:31 -0500 Subject: [PATCH 7/7] Also update asyncio client .lock() doc --- redis/asyncio/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/redis/asyncio/client.py b/redis/asyncio/client.py index 619592ef76..2afad0f4d1 100644 --- a/redis/asyncio/client.py +++ b/redis/asyncio/client.py @@ -348,7 +348,10 @@ def lock( continue trying forever. ``blocking_timeout`` can be specified as a float or integer, both representing the number of seconds to wait. - ``lock_class`` forces the specified lock implementation. + ``lock_class`` forces the specified lock implementation. Note that as + of redis-py 3.0, the only lock class we implement is ``Lock`` (which is + a Lua-based lock). So, it's unlikely you'll need this parameter, unless + you have created your own custom lock class. ``thread_local`` indicates whether the lock token is placed in thread-local storage. By default, the token is placed in thread local