diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b72da87b..69068809 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,12 +83,12 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | - poetry run test-verbose + make test-all - name: Run tests if: matrix.connection != 'plain' || matrix.redis-stack-version != 'latest' run: | - SKIP_VECTORIZERS=True SKIP_RERANKERS=True poetry run test-verbose + make test - name: Run notebooks if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest' @@ -106,7 +106,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | - cd docs/ && poetry run pytest --nbval-lax ./user_guide -vv + make test-notebooks docs: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b35b96b..52db3032 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,8 +50,9 @@ If you use `make`, we've created shortcuts for running the commands in this docu | make format | Runs code formatting and import sorting | | make check-types | Runs mypy type checking | | make lint | Runs formatting, import sorting, and type checking | -| make test | Runs tests, excluding those that require API keys and/or remote network calls)| -| make test-all | Runs all tests, including those that require API keys and/or remote network calls)| +| make test | Runs tests, excluding those that require API keys and/or remote network calls| +| make test-all | Runs all tests, including those that require API keys and/or remote network calls| +| make test-notebooks | Runs all notebook tests| | make check | Runs all linting targets and a subset of tests | | make docs-build | Builds the documentation | | make docs-serve | Serves the documentation locally | @@ -76,19 +77,19 @@ To run Testcontainers-based tests you need a local Docker installation such as: #### Running the Tests -Tests w/ vectorizers: +Tests w/ external APIs: ```bash -poetry run test-verbose +poetry run test-verbose --run-api-tests ``` -Tests w/out vectorizers: +Tests w/out external APIs: ```bash -SKIP_VECTORIZERS=true poetry run test-verbose +poetry run test-verbose ``` -Tests w/out rerankers: +Run a test on a specific file: ```bash -SKIP_RERANKERS=true poetry run test-verbose +poetry run test-verbose tests/unit/test_fields.py ``` ### Documentation @@ -112,6 +113,17 @@ In order for your applications to use RedisVL, you must have [Redis](https://red docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest ``` +Or from your makefile simply run: + +```bash +make redis-start +``` + +And then: +```bash +make redis-stop +``` + This will also spin up the [FREE RedisInsight GUI](https://redis.io/insight/) at `http://localhost:8001`. ## How to Report a Bug diff --git a/Makefile b/Makefile index 1451a2f5..a71f8d22 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install format lint test clean redis-start redis-stop check-types integration-test docs-build docs-serve check +.PHONY: install format lint test test-all test-notebooks clean redis-start redis-stop check-types docs-build docs-serve check install: poetry install --all-extras @@ -19,10 +19,13 @@ check-types: lint: format check-types test: - SKIP_RERANKERS=true SKIP_VECTORIZERS=true poetry run test-verbose + poetry run test-verbose test-all: - poetry run test-verbose + poetry run test-verbose --run-api-tests + +test-notebooks: + poetry run test-notebooks check: lint test diff --git a/scripts.py b/scripts.py index 0bfbac01..77020aad 100644 --- a/scripts.py +++ b/scripts.py @@ -1,4 +1,5 @@ import subprocess +import sys def format(): @@ -29,17 +30,25 @@ def check_mypy(): def test(): - subprocess.run(["python", "-m", "pytest", "-n", "auto", "--log-level=CRITICAL"], check=True) + test_cmd = ["python", "-m", "pytest", "-n", "auto", "--log-level=CRITICAL"] + # Get any extra arguments passed to the script + extra_args = sys.argv[1:] + if extra_args: + test_cmd.extend(extra_args) + subprocess.run(test_cmd, check=True) def test_verbose(): - subprocess.run( - ["python", "-m", "pytest", "-n", "auto", "-vv", "-s", "--log-level=CRITICAL"], check=True - ) + test_cmd = ["python", "-m", "pytest", "-n", "auto", "-vv", "-s", "--log-level=CRITICAL"] + # Get any extra arguments passed to the script + extra_args = sys.argv[1:] + if extra_args: + test_cmd.extend(extra_args) + subprocess.run(test_cmd, check=True) def test_notebooks(): - subprocess.run(["cd", "docs/", "&&", "poetry run treon", "-v"], check=True) + subprocess.run("cd docs/ && python -m pytest --nbval-lax ./user_guide -vv", shell=True, check=True) def build_docs(): diff --git a/tests/conftest.py b/tests/conftest.py index 28ea7735..de41b4b2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -54,13 +54,10 @@ async def async_client(redis_url): """ An async Redis client that uses the dynamic `redis_url`. """ - client = await RedisConnectionFactory.get_async_redis_connection(redis_url) - yield client - try: - await client.aclose() - except RuntimeError as e: - if "Event loop is closed" not in str(e): - raise + async with await RedisConnectionFactory.get_async_redis_connection( + redis_url + ) as client: + yield client @pytest.fixture @@ -70,51 +67,6 @@ def client(redis_url): """ conn = RedisConnectionFactory.get_redis_connection(redis_url) yield conn - conn.close() - - -@pytest.fixture -def openai_key(): - return os.getenv("OPENAI_API_KEY") - - -@pytest.fixture -def openai_version(): - return os.getenv("OPENAI_API_VERSION") - - -@pytest.fixture -def azure_endpoint(): - return os.getenv("AZURE_OPENAI_ENDPOINT") - - -@pytest.fixture -def cohere_key(): - return os.getenv("COHERE_API_KEY") - - -@pytest.fixture -def mistral_key(): - return os.getenv("MISTRAL_API_KEY") - - -@pytest.fixture -def gcp_location(): - return os.getenv("GCP_LOCATION") - - -@pytest.fixture -def gcp_project_id(): - return os.getenv("GCP_PROJECT_ID") - - -@pytest.fixture -def aws_credentials(): - return { - "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"), - "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"), - "aws_region": os.getenv("AWS_REGION", "us-east-1"), - } @pytest.fixture @@ -179,13 +131,31 @@ def sample_data(): ] -@pytest.fixture -def clear_db(redis): - redis.flushall() - yield - redis.flushall() +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--run-api-tests", + action="store_true", + default=False, + help="Run tests that require API keys", + ) -@pytest.fixture -def app_name(): - return "test_app" +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", "requires_api_keys: mark test as requiring API keys" + ) + + +def pytest_collection_modifyitems( + config: pytest.Config, items: list[pytest.Item] +) -> None: + if config.getoption("--run-api-tests"): + return + + # Otherwise skip all tests requiring an API key + skip_api = pytest.mark.skip( + reason="Skipping test because API keys are not provided. Use --run-api-tests to run these tests." + ) + for item in items: + if item.get_closest_marker("requires_api_keys"): + item.add_marker(skip_api) diff --git a/tests/integration/test_rerankers.py b/tests/integration/test_rerankers.py index 65aad333..caee5a47 100644 --- a/tests/integration/test_rerankers.py +++ b/tests/integration/test_rerankers.py @@ -9,13 +9,6 @@ ) -@pytest.fixture -def skip_reranker() -> bool: - # os.getenv returns a string - v = os.getenv("SKIP_RERANKERS", "False").lower() == "true" - return v - - # Fixture for the reranker instance @pytest.fixture( params=[ @@ -23,10 +16,7 @@ def skip_reranker() -> bool: VoyageAIReranker, ] ) -def reranker(request, skip_reranker): - if skip_reranker: - pytest.skip("Skipping reranker instantiation...") - +def reranker(request): if request.param == CohereReranker: return request.param() elif request.param == VoyageAIReranker: @@ -43,7 +33,7 @@ def hfCrossEncoderRerankerWithCustomModel(): return HFCrossEncoderReranker("cross-encoder/stsb-distilroberta-base") -# Test for basic ranking functionality +@pytest.mark.requires_api_keys def test_rank_documents(reranker): docs = ["document one", "document two", "document three"] query = "search query" @@ -55,7 +45,7 @@ def test_rank_documents(reranker): assert all(isinstance(score, float) for score in scores) # Scores should be floats -# Test for asynchronous ranking functionality +@pytest.mark.requires_api_keys @pytest.mark.asyncio async def test_async_rank_documents(reranker): docs = ["document one", "document two", "document three"] @@ -68,7 +58,7 @@ async def test_async_rank_documents(reranker): assert all(isinstance(score, float) for score in scores) # Scores should be floats -# Test handling of bad input +@pytest.mark.requires_api_keys def test_bad_input(reranker): with pytest.raises(Exception): reranker.rank("", []) # Empty query or documents diff --git a/tests/integration/test_session_manager.py b/tests/integration/test_session_manager.py index 05188db5..59d64b97 100644 --- a/tests/integration/test_session_manager.py +++ b/tests/integration/test_session_manager.py @@ -12,6 +12,11 @@ from redisvl.utils.vectorize.text.huggingface import HFTextVectorizer +@pytest.fixture +def app_name(): + return "test_app" + + @pytest.fixture def standard_session(app_name, client): session = StandardSessionManager(app_name, redis_client=client) diff --git a/tests/integration/test_vectorizers.py b/tests/integration/test_vectorizers.py index 52a32eca..65a07f3b 100644 --- a/tests/integration/test_vectorizers.py +++ b/tests/integration/test_vectorizers.py @@ -15,12 +15,6 @@ ) -@pytest.fixture -def skip_vectorizer() -> bool: - v = os.getenv("SKIP_VECTORIZERS", "False").lower() == "true" - return v - - @pytest.fixture( params=[ HFTextVectorizer, @@ -34,10 +28,7 @@ def skip_vectorizer() -> bool: VoyageAITextVectorizer, ] ) -def vectorizer(request, skip_vectorizer): - if skip_vectorizer: - pytest.skip("Skipping vectorizer instantiation...") - +def vectorizer(request): if request.param == HFTextVectorizer: return request.param() elif request.param == OpenAITextVectorizer: @@ -70,10 +61,7 @@ def embed_many(texts): @pytest.fixture -def bedrock_vectorizer(skip_vectorizer): - if skip_vectorizer: - pytest.skip("Skipping Bedrock vectorizer tests...") - +def bedrock_vectorizer(): return BedrockTextVectorizer( model=os.getenv("BEDROCK_MODEL_ID", "amazon.titan-embed-text-v2:0") ) @@ -108,6 +96,7 @@ def embed_many_with_args(self, texts, param=True): return MyEmbedder +@pytest.mark.requires_api_keys def test_vectorizer_embed(vectorizer): text = "This is a test sentence." if isinstance(vectorizer, CohereTextVectorizer): @@ -121,6 +110,7 @@ def test_vectorizer_embed(vectorizer): assert len(embedding) == vectorizer.dims +@pytest.mark.requires_api_keys def test_vectorizer_embed_many(vectorizer): texts = ["This is the first test sentence.", "This is the second test sentence."] if isinstance(vectorizer, CohereTextVectorizer): @@ -137,6 +127,7 @@ def test_vectorizer_embed_many(vectorizer): ) +@pytest.mark.requires_api_keys def test_vectorizer_bad_input(vectorizer): with pytest.raises(TypeError): vectorizer.embed(1) @@ -148,6 +139,7 @@ def test_vectorizer_bad_input(vectorizer): vectorizer.embed_many(42) +@pytest.mark.requires_api_keys def test_bedrock_bad_credentials(): with pytest.raises(ValueError): BedrockTextVectorizer( @@ -158,6 +150,7 @@ def test_bedrock_bad_credentials(): ) +@pytest.mark.requires_api_keys def test_bedrock_invalid_model(bedrock_vectorizer): with pytest.raises(ValueError): bedrock = BedrockTextVectorizer(model="invalid-model") @@ -250,8 +243,9 @@ def bad_return_type(text: str) -> str: ) +@pytest.mark.requires_api_keys @pytest.mark.parametrize( - "vector_class", + "vectorizer_", [ AzureOpenAITextVectorizer, BedrockTextVectorizer, @@ -264,50 +258,76 @@ def bad_return_type(text: str) -> str: VoyageAITextVectorizer, ], ) -def test_dtypes(vector_class, skip_vectorizer): - if skip_vectorizer: - pytest.skip("Skipping vectorizer instantiation...") - +def test_default_dtype(vectorizer_): # test dtype defaults to float32 - if issubclass(vector_class, CustomTextVectorizer): - vectorizer = vector_class(embed=lambda x, input_type=None: [1.0, 2.0, 3.0]) - elif issubclass(vector_class, AzureOpenAITextVectorizer): - vectorizer = vector_class( + if issubclass(vectorizer_, CustomTextVectorizer): + vectorizer = vectorizer_(embed=lambda x, input_type=None: [1.0, 2.0, 3.0]) + elif issubclass(vectorizer_, AzureOpenAITextVectorizer): + vectorizer = vectorizer_( model=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "text-embedding-ada-002") ) else: - vectorizer = vector_class() + vectorizer = vectorizer_() assert vectorizer.dtype == "float32" + +@pytest.mark.requires_api_keys +@pytest.mark.parametrize( + "vectorizer_", + [ + AzureOpenAITextVectorizer, + BedrockTextVectorizer, + CohereTextVectorizer, + CustomTextVectorizer, + HFTextVectorizer, + MistralAITextVectorizer, + OpenAITextVectorizer, + VertexAITextVectorizer, + VoyageAITextVectorizer, + ], +) +def test_other_dtypes(vectorizer_): # test initializing dtype in constructor for dtype in ["float16", "float32", "float64", "bfloat16"]: - if issubclass(vector_class, CustomTextVectorizer): - vectorizer = vector_class(embed=lambda x: [1.0, 2.0, 3.0], dtype=dtype) - elif issubclass(vector_class, AzureOpenAITextVectorizer): - vectorizer = vector_class( + if issubclass(vectorizer_, CustomTextVectorizer): + vectorizer = vectorizer_(embed=lambda x: [1.0, 2.0, 3.0], dtype=dtype) + elif issubclass(vectorizer_, AzureOpenAITextVectorizer): + vectorizer = vectorizer_( model=os.getenv( "AZURE_OPENAI_DEPLOYMENT_NAME", "text-embedding-ada-002" ), dtype=dtype, ) else: - vectorizer = vector_class(dtype=dtype) + vectorizer = vectorizer_(dtype=dtype) assert vectorizer.dtype == dtype - # test validation of dtype on init - if issubclass(vector_class, CustomTextVectorizer): - pytest.skip("skipping custom text vectorizer") +@pytest.mark.requires_api_keys +@pytest.mark.parametrize( + "vectorizer_", + [ + AzureOpenAITextVectorizer, + BedrockTextVectorizer, + CohereTextVectorizer, + HFTextVectorizer, + MistralAITextVectorizer, + OpenAITextVectorizer, + VertexAITextVectorizer, + VoyageAITextVectorizer, + ], +) +def test_bad_dtypes(vectorizer_): with pytest.raises(ValueError): - vectorizer = vector_class(dtype="float25") + vectorizer_(dtype="float25") with pytest.raises(ValueError): - vectorizer = vector_class(dtype=7) + vectorizer_(dtype=7) with pytest.raises(ValueError): - vectorizer = vector_class(dtype=None) + vectorizer_(dtype=None) @pytest.fixture( @@ -319,10 +339,7 @@ def test_dtypes(vector_class, skip_vectorizer): VoyageAITextVectorizer, ] ) -def avectorizer(request, skip_vectorizer): - if skip_vectorizer: - pytest.skip("Skipping vectorizer instantiation...") - +def avectorizer(request): if request.param == CustomTextVectorizer: def embed_func(text): @@ -341,6 +358,7 @@ async def aembed_many_func(texts): return request.param() +@pytest.mark.requires_api_keys @pytest.mark.asyncio async def test_vectorizer_aembed(avectorizer): text = "This is a test sentence." @@ -350,6 +368,7 @@ async def test_vectorizer_aembed(avectorizer): assert len(embedding) == avectorizer.dims +@pytest.mark.requires_api_keys @pytest.mark.asyncio async def test_vectorizer_aembed_many(avectorizer): texts = ["This is the first test sentence.", "This is the second test sentence."] @@ -362,6 +381,7 @@ async def test_vectorizer_aembed_many(avectorizer): ) +@pytest.mark.requires_api_keys @pytest.mark.asyncio async def test_avectorizer_bad_input(avectorizer): with pytest.raises(TypeError):