Modularize testing suite (#278)

This PR introduces a few improvements to our test suite to make it more
modular.

1) Adds the ability to run tests against specific file(s) using the
poetry scripting:
```bash
poetry run test tests/unit/test_fields.py
```

2) Adds the ability to toggle the API-enabled tests via command line arg
using the poetry scripting
```bash
poetry run test-verbose --run-api-tests
```
OR
```bash
make test-all
```
*This removes the need to set those ugly `SKIP_VECTORIZER=true` flags.*

And run without the API dependencies:
```bash
poetry run test-verbose
```
```bash
make test
```

3) Wraps notebook tests into a convenient poetry script and makefile
wrapper:
```bash
poetry run test-notebooks
```
OR
```bash
make test-notebooks
```

Author: tylerhutcherson

.github/workflows/test.yml

@@ -83,12 +83,12 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          poetry run test-verbose
+          make test-all
 
       - name: Run tests
         if: matrix.connection != 'plain' || matrix.redis-stack-version != 'latest'
         run: |
-          SKIP_VECTORIZERS=True SKIP_RERANKERS=True poetry run test-verbose
+          make test
 
       - name: Run notebooks
         if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
@@ -106,7 +106,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          cd docs/ && poetry run pytest --nbval-lax ./user_guide -vv 
+          make test-notebooks 
 
   docs:
     runs-on: ubuntu-latest

CONTRIBUTING.md

@@ -50,8 +50,9 @@ If you use `make`, we've created shortcuts for running the commands in this docu
 | make format | Runs code formatting and import sorting |
 | make check-types | Runs mypy type checking |
 | make lint | Runs formatting, import sorting, and type checking |
-| make test | Runs tests, excluding those that require API keys and/or remote network calls)|
-| make test-all | Runs all tests, including those that require API keys and/or remote network calls)|
+| make test | Runs tests, excluding those that require API keys and/or remote network calls|
+| make test-all | Runs all tests, including those that require API keys and/or remote network calls|
+| make test-notebooks | Runs all notebook tests|
 | make check | Runs all linting targets and a subset of tests |
 | make docs-build | Builds the documentation |
 | make docs-serve | Serves the documentation locally |
@@ -76,19 +77,19 @@ To run Testcontainers-based tests you need a local Docker installation such as:
 
 #### Running the Tests
 
-Tests w/ vectorizers:
+Tests w/ external APIs:
 ```bash
-poetry run test-verbose
+poetry run test-verbose --run-api-tests
 ```
 
-Tests w/out vectorizers:
+Tests w/out external APIs:
 ```bash
-SKIP_VECTORIZERS=true poetry run test-verbose
+poetry run test-verbose
 ```
 
-Tests w/out rerankers:
+Run a test on a specific file:
 ```bash
-SKIP_RERANKERS=true poetry run test-verbose
+poetry run test-verbose tests/unit/test_fields.py
 ```
 
 ### Documentation
@@ -112,6 +113,17 @@ In order for your applications to use RedisVL, you must have [Redis](https://red
 docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
 ```
 
+Or from your makefile simply run:
+
+```bash
+make redis-start
+```
+
+And then:
+```bash
+make redis-stop
+```
+
 This will also spin up the [FREE RedisInsight GUI](https://redis.io/insight/) at `http://localhost:8001`.
 
 ## How to Report a Bug

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install format lint test clean redis-start redis-stop check-types integration-test docs-build docs-serve check
+.PHONY: install format lint test test-all test-notebooks clean redis-start redis-stop check-types docs-build docs-serve check
 
 install:
 	poetry install --all-extras
@@ -19,10 +19,13 @@ check-types:
 lint: format check-types
 
 test:
-	SKIP_RERANKERS=true SKIP_VECTORIZERS=true poetry run test-verbose
+	poetry run test-verbose 
 
 test-all:
-	poetry run test-verbose
+	poetry run test-verbose --run-api-tests
+
+test-notebooks:
+	poetry run test-notebooks
 
 check: lint test
 

scripts.py

@@ -1,4 +1,5 @@
 import subprocess
+import sys
 
 
 def format():
@@ -29,17 +30,25 @@ def check_mypy():
 
 
 def test():
-    subprocess.run(["python", "-m", "pytest", "-n", "auto", "--log-level=CRITICAL"], check=True)
+    test_cmd = ["python", "-m", "pytest", "-n", "auto", "--log-level=CRITICAL"]
+    # Get any extra arguments passed to the script
+    extra_args = sys.argv[1:]
+    if extra_args:
+        test_cmd.extend(extra_args)
+    subprocess.run(test_cmd, check=True)
 
 
 def test_verbose():
-    subprocess.run(
-        ["python", "-m", "pytest", "-n", "auto", "-vv", "-s", "--log-level=CRITICAL"], check=True
-    )
+    test_cmd = ["python", "-m", "pytest", "-n", "auto", "-vv", "-s", "--log-level=CRITICAL"]
+    # Get any extra arguments passed to the script
+    extra_args = sys.argv[1:]
+    if extra_args:
+        test_cmd.extend(extra_args)
+    subprocess.run(test_cmd, check=True)
 
 
 def test_notebooks():
-    subprocess.run(["cd", "docs/", "&&", "poetry run treon", "-v"], check=True)
+    subprocess.run("cd docs/ && python -m pytest --nbval-lax ./user_guide -vv", shell=True, check=True)
 
 
 def build_docs():

tests/conftest.py

@@ -54,13 +54,10 @@ async def async_client(redis_url):
     """
     An async Redis client that uses the dynamic `redis_url`.
     """
-    client = await RedisConnectionFactory.get_async_redis_connection(redis_url)
-    yield client
-    try:
-        await client.aclose()
-    except RuntimeError as e:
-        if "Event loop is closed" not in str(e):
-            raise
+    async with await RedisConnectionFactory.get_async_redis_connection(
+        redis_url
+    ) as client:
+        yield client
 
 
 @pytest.fixture
@@ -70,51 +67,6 @@ def client(redis_url):
     """
     conn = RedisConnectionFactory.get_redis_connection(redis_url)
     yield conn
-    conn.close()
-
-
-@pytest.fixture
-def openai_key():
-    return os.getenv("OPENAI_API_KEY")
-
-
-@pytest.fixture
-def openai_version():
-    return os.getenv("OPENAI_API_VERSION")
-
-
-@pytest.fixture
-def azure_endpoint():
-    return os.getenv("AZURE_OPENAI_ENDPOINT")
-
-
-@pytest.fixture
-def cohere_key():
-    return os.getenv("COHERE_API_KEY")
-
-
-@pytest.fixture
-def mistral_key():
-    return os.getenv("MISTRAL_API_KEY")
-
-
-@pytest.fixture
-def gcp_location():
-    return os.getenv("GCP_LOCATION")
-
-
-@pytest.fixture
-def gcp_project_id():
-    return os.getenv("GCP_PROJECT_ID")
-
-
-@pytest.fixture
-def aws_credentials():
-    return {
-        "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
-        "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
-        "aws_region": os.getenv("AWS_REGION", "us-east-1"),
-    }
 
 
 @pytest.fixture
@@ -179,13 +131,31 @@ def sample_data():
     ]
 
 
-@pytest.fixture
-def clear_db(redis):
-    redis.flushall()
-    yield
-    redis.flushall()
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--run-api-tests",
+        action="store_true",
+        default=False,
+        help="Run tests that require API keys",
+    )
 
 
-@pytest.fixture
-def app_name():
-    return "test_app"
+def pytest_configure(config: pytest.Config) -> None:
+    config.addinivalue_line(
+        "markers", "requires_api_keys: mark test as requiring API keys"
+    )
+
+
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: list[pytest.Item]
+) -> None:
+    if config.getoption("--run-api-tests"):
+        return
+
+    # Otherwise skip all tests requiring an API key
+    skip_api = pytest.mark.skip(
+        reason="Skipping test because API keys are not provided. Use --run-api-tests to run these tests."
+    )
+    for item in items:
+        if item.get_closest_marker("requires_api_keys"):
+            item.add_marker(skip_api)

tests/integration/test_rerankers.py

@@ -9,24 +9,14 @@
 )
 
 
-@pytest.fixture
-def skip_reranker() -> bool:
-    # os.getenv returns a string
-    v = os.getenv("SKIP_RERANKERS", "False").lower() == "true"
-    return v
-
-
 # Fixture for the reranker instance
 @pytest.fixture(
     params=[
         CohereReranker,
         VoyageAIReranker,
     ]
 )
-def reranker(request, skip_reranker):
-    if skip_reranker:
-        pytest.skip("Skipping reranker instantiation...")
-
+def reranker(request):
     if request.param == CohereReranker:
         return request.param()
     elif request.param == VoyageAIReranker:
@@ -43,7 +33,7 @@ def hfCrossEncoderRerankerWithCustomModel():
     return HFCrossEncoderReranker("cross-encoder/stsb-distilroberta-base")
 
 
-# Test for basic ranking functionality
+@pytest.mark.requires_api_keys
 def test_rank_documents(reranker):
     docs = ["document one", "document two", "document three"]
     query = "search query"
@@ -55,7 +45,7 @@ def test_rank_documents(reranker):
     assert all(isinstance(score, float) for score in scores)  # Scores should be floats
 
 
-# Test for asynchronous ranking functionality
+@pytest.mark.requires_api_keys
 @pytest.mark.asyncio
 async def test_async_rank_documents(reranker):
     docs = ["document one", "document two", "document three"]
@@ -68,7 +58,7 @@ async def test_async_rank_documents(reranker):
     assert all(isinstance(score, float) for score in scores)  # Scores should be floats
 
 
-# Test handling of bad input
+@pytest.mark.requires_api_keys
 def test_bad_input(reranker):
     with pytest.raises(Exception):
         reranker.rank("", [])  # Empty query or documents

tests/integration/test_session_manager.py

@@ -12,6 +12,11 @@
 from redisvl.utils.vectorize.text.huggingface import HFTextVectorizer
 
 
+@pytest.fixture
+def app_name():
+    return "test_app"
+
+
 @pytest.fixture
 def standard_session(app_name, client):
     session = StandardSessionManager(app_name, redis_client=client)