From 114fc0c6b772dce4dbf09bdeda9c1a330e7434ef Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Tue, 4 Jun 2024 09:57:05 -0400 Subject: [PATCH 01/10] basic test suite --- tests/integration/test_connection.py | 33 +++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_connection.py b/tests/integration/test_connection.py index 13347bfe..40ad9aab 100644 --- a/tests/integration/test_connection.py +++ b/tests/integration/test_connection.py @@ -6,13 +6,16 @@ from redis.exceptions import ConnectionError from redisvl.redis.connection import RedisConnectionFactory, get_address_from_env +from redisvl.version import __version__ + +EXPECTED_LIB_NAME = f"redis-py(redisvl_v{__version__})" def test_get_address_from_env(redis_url): assert get_address_from_env() == redis_url -def test_sync_redis_connection(redis_url): +def test_sync_redis_connect(redis_url): client = RedisConnectionFactory.connect(redis_url) assert client is not None assert isinstance(client, Redis) @@ -21,7 +24,7 @@ def test_sync_redis_connection(redis_url): @pytest.mark.asyncio -async def test_async_redis_connection(redis_url): +async def test_async_redis_connect(redis_url): client = RedisConnectionFactory.connect(redis_url, use_async=True) assert client is not None assert isinstance(client, AsyncRedis) @@ -49,11 +52,29 @@ def test_unknown_redis(): bad_client.ping() -def test_required_modules(client): - RedisConnectionFactory.validate_redis_modules(client) +def test_validate_redis(client): + RedisConnectionFactory.validate_redis(client) + lib_name = client.client_info() + assert lib_name["lib-name"] == EXPECTED_LIB_NAME + + +@pytest.mark.asyncio +async def test_validate_async_redis(async_client): + client = await async_client + RedisConnectionFactory.validate_async_redis(client) + lib_name = await client.client_info() + assert lib_name["lib-name"] == EXPECTED_LIB_NAME + + +def test_custom_lib_name(client): + RedisConnectionFactory.validate_redis(client, "langchain_v0.1.0") + lib_name = client.client_info() + assert lib_name["lib-name"] == f"redis-py(redisvl_v{__version__};langchain_v0.1.0)" @pytest.mark.asyncio -async def test_async_required_modules(async_client): +async def test_async_custom_lib_name(async_client): client = await async_client - RedisConnectionFactory.validate_async_redis_modules(client) + RedisConnectionFactory.validate_async_redis(client, "langchain_v0.1.0") + lib_name = await client.client_info() + assert lib_name["lib-name"] == f"redis-py(redisvl_v{__version__};langchain_v0.1.0)" From 284174a75ec5c92a786ca4f593b82d18c72cf8c1 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Tue, 4 Jun 2024 09:57:30 -0400 Subject: [PATCH 02/10] support client pass through lib_name on connection --- redisvl/index/index.py | 27 ++++++++++++++++++--------- redisvl/redis/connection.py | 36 +++++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 2162f4de..11972608 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -84,12 +84,12 @@ def _process(doc: "Document") -> Dict[str, Any]: return [_process(doc) for doc in results.docs] -def check_modules_present(): +def setup_redis(): def decorator(func): @wraps(func) def wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) - RedisConnectionFactory.validate_redis_modules(self._redis_client) + RedisConnectionFactory.validate_redis(self._redis_client, self._lib_name) return result return wrapper @@ -97,12 +97,14 @@ def wrapper(self, *args, **kwargs): return decorator -def check_async_modules_present(): +def setup_async_redis(): def decorator(func): @wraps(func) def wrapper(self, *args, **kwargs): result = func(self, *args, **kwargs) - RedisConnectionFactory.validate_async_redis_modules(self._redis_client) + RedisConnectionFactory.validate_async_redis( + self._redis_client, self._lib_name + ) return result return wrapper @@ -175,6 +177,9 @@ def __init__( self.schema = schema + # set custom lib name + self._lib_name: Optional[str] = kwargs.pop("lib_name", None) + # set up redis connection self._redis_client: Optional[Union[redis.Redis, aredis.Redis]] = None if redis_client is not None: @@ -350,11 +355,13 @@ def connect(self, redis_url: Optional[str] = None, **kwargs): index.connect(redis_url="redis://localhost:6379") """ - client = RedisConnectionFactory.connect(redis_url, use_async=False, **kwargs) + client = RedisConnectionFactory.connect( + redis_url=redis_url, use_async=False, **kwargs + ) return self.set_client(client) - @check_modules_present() - def set_client(self, client: redis.Redis): + @setup_redis() + def set_client(self, client: redis.Redis, **kwargs): """Manually set the Redis client to use with the search index. This method configures the search index to use a specific Redis or @@ -729,10 +736,12 @@ def connect(self, redis_url: Optional[str] = None, **kwargs): index.connect(redis_url="redis://localhost:6379") """ - client = RedisConnectionFactory.connect(redis_url, use_async=True, **kwargs) + client = RedisConnectionFactory.connect( + redis_url=redis_url, use_async=True, **kwargs + ) return self.set_client(client) - @check_async_modules_present() + @setup_async_redis() def set_client(self, client: aredis.Redis): """Manually set the Redis client to use with the search index. diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 79eb3060..2ddc1e5c 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -13,6 +13,7 @@ from redisvl.redis.constants import REDIS_REQUIRED_MODULES from redisvl.redis.utils import convert_bytes +from redisvl.version import __version__ def get_address_from_env() -> str: @@ -26,6 +27,20 @@ def get_address_from_env() -> str: return os.environ["REDIS_URL"] +def make_lib_name(*args) -> str: + """Build the lib name to be reported through the Redis client setinfo + command. + + Returns: + str: Redis client library name + """ + custom_libs = f"redisvl_v{__version__}" + for arg in args: + if arg: + custom_libs += f";{arg}" + return f"redis-py({custom_libs})" + + class RedisConnectionFactory: """Builds connections to a Redis database, supporting both synchronous and asynchronous clients. @@ -108,8 +123,10 @@ def get_async_redis_connection(url: Optional[str] = None, **kwargs) -> AsyncRedi return AsyncRedis.from_url(get_address_from_env(), **kwargs) @staticmethod - def validate_redis_modules( - client: Redis, redis_required_modules: Optional[List[Dict[str, Any]]] = None + def validate_redis( + client: Redis, + lib_name: Optional[str] = None, + redis_required_modules: Optional[List[Dict[str, Any]]] = None, ) -> None: """Validates if the required Redis modules are installed. @@ -119,13 +136,18 @@ def validate_redis_modules( Raises: ValueError: If required Redis modules are not installed. """ - RedisConnectionFactory._validate_redis_modules( + # set client library name + client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) + + # validate available modules + RedisConnectionFactory._validate_modules( convert_bytes(client.module_list()), redis_required_modules ) @staticmethod - def validate_async_redis_modules( + def validate_async_redis( client: AsyncRedis, + lib_name: Optional[str] = None, redis_required_modules: Optional[List[Dict[str, Any]]] = None, ) -> None: """ @@ -150,12 +172,12 @@ def validate_async_redis_modules( **client.connection_pool.connection_kwargs, ) ) - RedisConnectionFactory.validate_redis_modules( - temp_client, redis_required_modules + RedisConnectionFactory.validate_redis( + temp_client, lib_name, redis_required_modules ) @staticmethod - def _validate_redis_modules( + def _validate_modules( installed_modules, redis_required_modules: Optional[List[Dict[str, Any]]] = None ) -> None: """ From 1784e5a4ccff9a13b5ae2bd5e9b15cec139e712d Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Tue, 4 Jun 2024 09:57:45 -0400 Subject: [PATCH 03/10] cleanup --- .gitignore | 1 + conftest.py | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d4204a44..a238288e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,6 @@ scratch wiki_schema.yaml docs/_build/ .venv +.env coverage.xml dist/ \ No newline at end of file diff --git a/conftest.py b/conftest.py index e7975057..204ac177 100644 --- a/conftest.py +++ b/conftest.py @@ -6,12 +6,6 @@ from testcontainers.compose import DockerCompose -# @pytest.fixture(scope="session") -# def event_loop(): -# loop = asyncio.get_event_loop_policy().new_event_loop() -# yield loop -# loop.close() - @pytest.fixture(scope="session", autouse=True) def redis_container(): From 6cdb7e8db66c9e45e5d099fdf3023eca20e17377 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 10:13:38 -0400 Subject: [PATCH 04/10] handle the connections with proper event loop parsing --- redisvl/index/index.py | 24 +++--- redisvl/redis/connection.py | 113 +++++++++++++++++++-------- tests/integration/test_connection.py | 18 +---- 3 files changed, 93 insertions(+), 62 deletions(-) diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 11972608..d25b10f4 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -97,19 +97,19 @@ def wrapper(self, *args, **kwargs): return decorator -def setup_async_redis(): - def decorator(func): - @wraps(func) - def wrapper(self, *args, **kwargs): - result = func(self, *args, **kwargs) - RedisConnectionFactory.validate_async_redis( - self._redis_client, self._lib_name - ) - return result +# def setup_async_redis(): +# def decorator(func): +# @wraps(func) +# def wrapper(self, *args, **kwargs): +# result = func(self, *args, **kwargs) +# RedisConnectionFactory.validate_async_redis( +# self._redis_client, self._lib_name +# ) +# return result - return wrapper +# return wrapper - return decorator +# return decorator def check_index_exists(): @@ -741,7 +741,7 @@ def connect(self, redis_url: Optional[str] = None, **kwargs): ) return self.set_client(client) - @setup_async_redis() + @setup_redis() def set_client(self, client: aredis.Redis): """Manually set the Redis client to use with the search index. diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 2ddc1e5c..0d91559b 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -1,5 +1,6 @@ +import asyncio import os -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type, Union from redis import Redis from redis.asyncio import Redis as AsyncRedis @@ -122,59 +123,105 @@ def get_async_redis_connection(url: Optional[str] = None, **kwargs) -> AsyncRedi # fallback to env var REDIS_URL return AsyncRedis.from_url(get_address_from_env(), **kwargs) - @staticmethod def validate_redis( - client: Redis, + client: Union[Redis, AsyncRedis], lib_name: Optional[str] = None, redis_required_modules: Optional[List[Dict[str, Any]]] = None, ) -> None: - """Validates if the required Redis modules are installed. + """Validates the Redis connection. Args: - client (Redis): Synchronous Redis client. + client (Redis or AsyncRedis): Redis client. + lib_name (str): Library name to set on the Redis client. + redis_required_modules (List[Dict[str, Any]]): List of required modules and their versions. Raises: ValueError: If required Redis modules are not installed. """ - # set client library name + if isinstance(client, AsyncRedis): + print("VALIDATING ASYNC CLIENT", flush=True) + RedisConnectionFactory._run_async( + RedisConnectionFactory._validate_async_redis, + client, + lib_name, + redis_required_modules, + ) + else: + RedisConnectionFactory._validate_sync_redis( + client, lib_name, redis_required_modules + ) + + @staticmethod + def _validate_sync_redis( + client: Redis, + lib_name: Optional[str], + redis_required_modules: Optional[List[Dict[str, Any]]], + ) -> None: + """Validates the sync client.""" + # Set client library name client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) - # validate available modules - RedisConnectionFactory._validate_modules( - convert_bytes(client.module_list()), redis_required_modules - ) + # Get list of modules + modules_list = convert_bytes(client.module_list()) + + # Validate available modules + RedisConnectionFactory._validate_modules(modules_list, redis_required_modules) @staticmethod - def validate_async_redis( + async def _validate_async_redis( client: AsyncRedis, - lib_name: Optional[str] = None, - redis_required_modules: Optional[List[Dict[str, Any]]] = None, + lib_name: Optional[str], + redis_required_modules: Optional[List[Dict[str, Any]]], ) -> None: + """Validates the async client.""" + # Set client library name + res = await client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) + print("SET ASYNC CLIENT NAME", res, flush=True) + + # Get list of modules + modules_list = convert_bytes(await client.module_list()) + + # Validate available modules + RedisConnectionFactory._validate_modules(modules_list, redis_required_modules) + + @staticmethod + def _run_async(coro, *args, **kwargs): """ - Validates if the required Redis modules are installed. + Runs an asynchronous function in the appropriate event loop context. + + This method checks if there is an existing event loop running. If there is, + it schedules the coroutine to be run within the current loop using `asyncio.ensure_future`. + If no event loop is running, it creates a new event loop, runs the coroutine, + and then closes the loop to avoid resource leaks. Args: - client (AsyncRedis): Asynchronous Redis client. + coro (coroutine): The coroutine function to be run. + *args: Positional arguments to pass to the coroutine function. + **kwargs: Keyword arguments to pass to the coroutine function. - Raises: - ValueError: If required Redis modules are not installed. + Returns: + The result of the coroutine if a new event loop is created, + otherwise a task object representing the coroutine execution. """ - # pick the right connection class - connection_class: Type[AbstractConnection] = ( - SSLConnection - if client.connection_pool.connection_class == ASSLConnection - else Connection - ) - # set up a temp sync client - temp_client = Redis( - connection_pool=ConnectionPool( - connection_class=connection_class, - **client.connection_pool.connection_kwargs, - ) - ) - RedisConnectionFactory.validate_redis( - temp_client, lib_name, redis_required_modules - ) + try: + # Try to get the current running event loop + loop = asyncio.get_running_loop() + except RuntimeError: # No running event loop + loop = None + + if loop and loop.is_running(): + # If an event loop is running, schedule the coroutine to run in the existing loop + return asyncio.ensure_future(coro(*args, **kwargs)) + else: + # No event loop is running, create a new event loop + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + try: + # Run the coroutine in the new event loop and wait for it to complete + return loop.run_until_complete(coro(*args, **kwargs)) + finally: + # Close the event loop to release resources + loop.close() @staticmethod def _validate_modules( diff --git a/tests/integration/test_connection.py b/tests/integration/test_connection.py index 40ad9aab..dbc796ad 100644 --- a/tests/integration/test_connection.py +++ b/tests/integration/test_connection.py @@ -58,23 +58,7 @@ def test_validate_redis(client): assert lib_name["lib-name"] == EXPECTED_LIB_NAME -@pytest.mark.asyncio -async def test_validate_async_redis(async_client): - client = await async_client - RedisConnectionFactory.validate_async_redis(client) - lib_name = await client.client_info() - assert lib_name["lib-name"] == EXPECTED_LIB_NAME - - -def test_custom_lib_name(client): +def test_validate_redis_custom_lib_name(client): RedisConnectionFactory.validate_redis(client, "langchain_v0.1.0") lib_name = client.client_info() assert lib_name["lib-name"] == f"redis-py(redisvl_v{__version__};langchain_v0.1.0)" - - -@pytest.mark.asyncio -async def test_async_custom_lib_name(async_client): - client = await async_client - RedisConnectionFactory.validate_async_redis(client, "langchain_v0.1.0") - lib_name = await client.client_info() - assert lib_name["lib-name"] == f"redis-py(redisvl_v{__version__};langchain_v0.1.0)" From 9ba565e3624b435c9557e52602c76c3d2e1ee9a1 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 10:19:29 -0400 Subject: [PATCH 05/10] fix formatting and linting --- redisvl/redis/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 0d91559b..2bdccbad 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -123,6 +123,7 @@ def get_async_redis_connection(url: Optional[str] = None, **kwargs) -> AsyncRedi # fallback to env var REDIS_URL return AsyncRedis.from_url(get_address_from_env(), **kwargs) + @staticmethod def validate_redis( client: Union[Redis, AsyncRedis], lib_name: Optional[str] = None, @@ -159,7 +160,7 @@ def _validate_sync_redis( ) -> None: """Validates the sync client.""" # Set client library name - client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) + client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore # Get list of modules modules_list = convert_bytes(client.module_list()) @@ -175,7 +176,7 @@ async def _validate_async_redis( ) -> None: """Validates the async client.""" # Set client library name - res = await client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) + res = await client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore print("SET ASYNC CLIENT NAME", res, flush=True) # Get list of modules From 48d362c5fbe48249205101e9cff49f72d3014b1c Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 10:27:33 -0400 Subject: [PATCH 06/10] fix formatting again --- .github/workflows/release.yml | 112 +++++++++++++++++----------------- redisvl/redis/connection.py | 4 +- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ca60babc..c26fe8a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,62 +38,62 @@ jobs: name: dist path: dist/ - # test-pypi-publish: - # needs: build - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v4 - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - - # - name: Install Poetry - # uses: snok/install-poetry@v1 - - # - uses: actions/download-artifact@v4 - # with: - # name: dist - # path: dist/ - - # - name: Publish to TestPyPI - # env: - # POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TESTPYPI }} - # run: poetry config repositories.test-pypi https://test.pypi.org/legacy/; poetry config pypi-token.test-pypi $POETRY_PYPI_TOKEN_TESTPYPI; poetry publish --repository test-pypi - - # pre-release-checks: - # needs: test-pypi-publish - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v4 - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - - # - name: Install Poetry - # uses: snok/install-poetry@v1 - - # - name: Install dependencies - # run: | - # poetry install --all-extras - - # - name: Install published package from TestPyPI - # env: - # OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }} - # GCP_LOCATION: ${{ secrets.GCP_LOCATION }} - # GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - # COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - # AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}} - # AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}} - # AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}} - # OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}} - # run: - # poetry run pip install --index-url https://test.pypi.org/simple/ --no-deps redisvl-test; poetry run test-cov + test-pypi-publish: + needs: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish to TestPyPI + env: + POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TESTPYPI }} + run: poetry config repositories.test-pypi https://test.pypi.org/legacy/; poetry config pypi-token.test-pypi $POETRY_PYPI_TOKEN_TESTPYPI; poetry publish --repository test-pypi + + pre-release-checks: + needs: test-pypi-publish + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Poetry + uses: snok/install-poetry@v1 + + - name: Install dependencies + run: | + poetry install --all-extras + + - name: Install published package from TestPyPI + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }} + GCP_LOCATION: ${{ secrets.GCP_LOCATION }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}} + AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}} + AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}} + OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}} + run: + poetry run pip install --index-url https://test.pypi.org/simple/ --no-deps redisvl-test; poetry run test-cov publish: needs: build #pre-release-checks diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 2bdccbad..fac40d97 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -160,7 +160,7 @@ def _validate_sync_redis( ) -> None: """Validates the sync client.""" # Set client library name - client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore + client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore # Get list of modules modules_list = convert_bytes(client.module_list()) @@ -176,7 +176,7 @@ async def _validate_async_redis( ) -> None: """Validates the async client.""" # Set client library name - res = await client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore + res = await client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore print("SET ASYNC CLIENT NAME", res, flush=True) # Get list of modules From 31662406bf6ad9731d73bd70f7f746ad082ab125 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 13:27:58 -0400 Subject: [PATCH 07/10] undo test pypi piece --- .github/workflows/release.yml | 112 +++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c26fe8a6..ca60babc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,62 +38,62 @@ jobs: name: dist path: dist/ - test-pypi-publish: - needs: build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Poetry - uses: snok/install-poetry@v1 - - - uses: actions/download-artifact@v4 - with: - name: dist - path: dist/ - - - name: Publish to TestPyPI - env: - POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TESTPYPI }} - run: poetry config repositories.test-pypi https://test.pypi.org/legacy/; poetry config pypi-token.test-pypi $POETRY_PYPI_TOKEN_TESTPYPI; poetry publish --repository test-pypi - - pre-release-checks: - needs: test-pypi-publish - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Install Poetry - uses: snok/install-poetry@v1 - - - name: Install dependencies - run: | - poetry install --all-extras - - - name: Install published package from TestPyPI - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }} - GCP_LOCATION: ${{ secrets.GCP_LOCATION }} - GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}} - AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}} - AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}} - OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}} - run: - poetry run pip install --index-url https://test.pypi.org/simple/ --no-deps redisvl-test; poetry run test-cov + # test-pypi-publish: + # needs: build + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v4 + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install Poetry + # uses: snok/install-poetry@v1 + + # - uses: actions/download-artifact@v4 + # with: + # name: dist + # path: dist/ + + # - name: Publish to TestPyPI + # env: + # POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.TESTPYPI }} + # run: poetry config repositories.test-pypi https://test.pypi.org/legacy/; poetry config pypi-token.test-pypi $POETRY_PYPI_TOKEN_TESTPYPI; poetry publish --repository test-pypi + + # pre-release-checks: + # needs: test-pypi-publish + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v4 + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + + # - name: Install Poetry + # uses: snok/install-poetry@v1 + + # - name: Install dependencies + # run: | + # poetry install --all-extras + + # - name: Install published package from TestPyPI + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_KEY }} + # GCP_LOCATION: ${{ secrets.GCP_LOCATION }} + # GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + # COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + # AZURE_OPENAI_API_KEY: ${{secrets.AZURE_OPENAI_API_KEY}} + # AZURE_OPENAI_ENDPOINT: ${{secrets.AZURE_OPENAI_ENDPOINT}} + # AZURE_OPENAI_DEPLOYMENT_NAME: ${{secrets.AZURE_OPENAI_DEPLOYMENT_NAME}} + # OPENAI_API_VERSION: ${{secrets.OPENAI_API_VERSION}} + # run: + # poetry run pip install --index-url https://test.pypi.org/simple/ --no-deps redisvl-test; poetry run test-cov publish: needs: build #pre-release-checks From 4be4cad4d8c2cb7cb265553b677bf301d14bd4b0 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 13:28:28 -0400 Subject: [PATCH 08/10] remove helper --- redisvl/index/index.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/redisvl/index/index.py b/redisvl/index/index.py index d25b10f4..3fe79631 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -97,21 +97,6 @@ def wrapper(self, *args, **kwargs): return decorator -# def setup_async_redis(): -# def decorator(func): -# @wraps(func) -# def wrapper(self, *args, **kwargs): -# result = func(self, *args, **kwargs) -# RedisConnectionFactory.validate_async_redis( -# self._redis_client, self._lib_name -# ) -# return result - -# return wrapper - -# return decorator - - def check_index_exists(): def decorator(func): @wraps(func) From a9ce22843f2298820746ddd1c7c1e72f04a1d9e4 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 13:48:17 -0400 Subject: [PATCH 09/10] use fallback method for older redis versions --- redisvl/redis/connection.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index fac40d97..87522389 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -11,6 +11,7 @@ ConnectionPool, SSLConnection, ) +from redis.exceptions import ResponseError from redisvl.redis.constants import REDIS_REQUIRED_MODULES from redisvl.redis.utils import convert_bytes @@ -140,7 +141,6 @@ def validate_redis( ValueError: If required Redis modules are not installed. """ if isinstance(client, AsyncRedis): - print("VALIDATING ASYNC CLIENT", flush=True) RedisConnectionFactory._run_async( RedisConnectionFactory._validate_async_redis, client, @@ -160,7 +160,11 @@ def _validate_sync_redis( ) -> None: """Validates the sync client.""" # Set client library name - client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore + _lib_name = make_lib_name(lib_name) + try: + client.client_setinfo("LIB-NAME", _lib_name) # type: ignore + except ResponseError: + client.echo(_lib_name) # Get list of modules modules_list = convert_bytes(client.module_list()) @@ -176,8 +180,11 @@ async def _validate_async_redis( ) -> None: """Validates the async client.""" # Set client library name - res = await client.client_setinfo("LIB-NAME", make_lib_name(lib_name)) # type: ignore - print("SET ASYNC CLIENT NAME", res, flush=True) + _lib_name = make_lib_name(lib_name) + try: + await client.client_setinfo("LIB-NAME", _lib_name) # type: ignore + except ResponseError: + await client.echo(_lib_name) # Get list of modules modules_list = convert_bytes(await client.module_list()) From e5f96adfe22bd281232aa46472260be069ec5ea9 Mon Sep 17 00:00:00 2001 From: Tyler Hutcherson Date: Wed, 5 Jun 2024 16:32:15 -0400 Subject: [PATCH 10/10] improve the connection client handling --- redisvl/redis/connection.py | 2 ++ tests/integration/test_connection.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/redisvl/redis/connection.py b/redisvl/redis/connection.py index 87522389..d747b7ca 100644 --- a/redisvl/redis/connection.py +++ b/redisvl/redis/connection.py @@ -164,6 +164,7 @@ def _validate_sync_redis( try: client.client_setinfo("LIB-NAME", _lib_name) # type: ignore except ResponseError: + # Fall back to a simple log echo client.echo(_lib_name) # Get list of modules @@ -184,6 +185,7 @@ async def _validate_async_redis( try: await client.client_setinfo("LIB-NAME", _lib_name) # type: ignore except ResponseError: + # Fall back to a simple log echo await client.echo(_lib_name) # Get list of modules diff --git a/tests/integration/test_connection.py b/tests/integration/test_connection.py index dbc796ad..cfe6e45b 100644 --- a/tests/integration/test_connection.py +++ b/tests/integration/test_connection.py @@ -11,6 +11,35 @@ EXPECTED_LIB_NAME = f"redis-py(redisvl_v{__version__})" +def compare_versions(version1, version2): + """ + Compare two Redis version strings numerically. + + Parameters: + version1 (str): The first version string (e.g., "7.2.4"). + version2 (str): The second version string (e.g., "6.2.1"). + + Returns: + int: -1 if version1 < version2, 0 if version1 == version2, 1 if version1 > version2. + """ + v1_parts = list(map(int, version1.split("."))) + v2_parts = list(map(int, version2.split("."))) + + for v1, v2 in zip(v1_parts, v2_parts): + if v1 < v2: + return False + elif v1 > v2: + return True + + # If the versions are equal so far, compare the lengths of the version parts + if len(v1_parts) < len(v2_parts): + return False + elif len(v1_parts) > len(v2_parts): + return True + + return True + + def test_get_address_from_env(redis_url): assert get_address_from_env() == redis_url @@ -53,12 +82,18 @@ def test_unknown_redis(): def test_validate_redis(client): + redis_version = client.info()["redis_version"] + if not compare_versions(redis_version, "7.2.0"): + pytest.skip("Not using a late enough version of Redis") RedisConnectionFactory.validate_redis(client) lib_name = client.client_info() assert lib_name["lib-name"] == EXPECTED_LIB_NAME def test_validate_redis_custom_lib_name(client): + redis_version = client.info()["redis_version"] + if not compare_versions(redis_version, "7.2.0"): + pytest.skip("Not using a late enough version of Redis") RedisConnectionFactory.validate_redis(client, "langchain_v0.1.0") lib_name = client.client_info() assert lib_name["lib-name"] == f"redis-py(redisvl_v{__version__};langchain_v0.1.0)"