feat: redis-separator

#192
#277
This adds the ability to customize the redis separator and changes the
default separator from _%&_ to :.

Additionally, it adds a way for a user to add a prefix to the key space
we use, to further differentiate their keys.

Finally, it fixes a bug with testing where redislite would not shutdown
properly and updates the fixtures to work with the newest version of
pytest_asyncio.

BREAKING CHANGE: This will result in "data loss" for existing models
stored in redis due to the change in default separator. To maintain
backwards compatbility with 0.7.0 and below, you will need to modify
your existing models to set _redis_separator = "_%&_" as a field on
them.

Author: andrewthetechie

.github/workflows/constraints.txt

@@ -4,3 +4,4 @@ nox-poetry==1.0.1
 poetry==1.2.0
 virtualenv==20.16.5
 poetry-dynamic-versioning==0.18.0
+toml

README.md

@@ -113,6 +113,15 @@ loop = asyncio.get_event_loop()
 loop.run_until_complete(work_with_orm())
 ```
 
+#### Custom Fields in Model
+
+| Field Name          | Required | Default      | Description                                                          |
+| ------------------- | -------- | ------------ | -------------------------------------------------------------------- |
+| \_primary_key_field | Yes      | None         | The field of your model that is the primary key                      |
+| \_redis_prefix      | No       | None         | If set, will be added to the beginning of the keys we store in redis |
+| \_redis_separator   | No       | :            | Defaults to :, used to separate prefix, table_name, and primary_key  |
+| \_table_name        | NO       | cls.**name** | Defaults to the model's name, can set a custom name in redis         |
+
 ## Contributing
 
 Contributions are very welcome.

noxfile.py

@@ -7,6 +7,7 @@
 from textwrap import dedent
 
 import nox
+import toml
 
 try:
     from nox_poetry import Session
@@ -35,24 +36,8 @@
     "docs-build",
 )
 mypy_type_packages = ()
-test_requirements = (
-    "coverage[toml]",
-    "pytest",
-    "pygments",
-    "fastapi>=0.6.3",
-    "fastapi-crudrouter>=0.8.4",
-    "httpx",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-env",
-    "pytest-lazy-fixture",
-    "pytest-mock",
-    "pytest-mockservers",
-    "pytest-xdist",
-    "redislite",
-    "pytest-asyncio",
-    "pytest-lazy-fixture",
-)
+pyproject = toml.load("pyproject.toml")
+test_requirements = pyproject["tool"]["poetry"]["dev-dependencies"].keys()
 
 
 def activate_virtualenv_in_precommit_hooks(session: Session) -> None:

pydantic_aioredis/model.py

@@ -13,25 +13,54 @@
 class Model(_AbstractModel):
     """
     The section in the store that saves rows of the same kind
+
+    Model has some custom fields you can set in your models that alter the behavior of how this is stored in redis
+
+    _primary_key_field -- The field of your model that is the primary key
+    _redis_prefix -- If set, will be added to the beginning of the keys we store in redis
+    _redis_separator -- Defaults to :, used to separate prefix, table_name, and primary_key
+    _table_name -- Defaults to the model's name, can set a custom name in redis
+
+
+    If your model was named ThisModel, the primary key was "key", and prefix and separator were left at default (not set), the
+    keys stored in redis would be
+    thismodel:key
     """
 
+    @classmethod
+    @property
+    def _prefix(cls):
+        return getattr(cls, "_redis_prefix", "").lower()
+
+    @classmethod
+    @property
+    def _separator(cls):
+        return getattr(cls, "_redis_separator", ":").lower()
+
+    @classmethod
+    @property
+    def _tablename(cls):
+        return cls.__name__.lower() if cls._table_name is None else cls._table_name
+
     @classmethod
     def __get_primary_key(cls, primary_key_value: Any):
         """
-        Returns the primary key value concatenated to the table name for uniqueness
+        Uses _table_name, _table_refix, and _redis_separator from the model to build our primary key.
+
+        _table_name defaults to the name of the model class if it is not set
+        _redis_separator defualts to : if it is not set
+        _prefix defaults to nothing if it is not set
+
+        The key is contructed as {_prefix}{_redis_separator}{_table_name}{_redis_separator}{primary_key_value}
+        So a model named ThisModel with a primary key of id, by default would result in a key of thismodel:id
         """
-        table_name = (
-            cls.__name__.lower() if cls._table_name is None else cls._table_name
-        )
-        return f"{table_name}_%&_{primary_key_value}"
+        prefix = f"{cls._prefix}{cls._separator}" if cls._prefix != "" else ""
+        return f"{prefix}{cls._tablename}{cls._separator}{primary_key_value}"
 
     @classmethod
     def get_table_index_key(cls):
         """Returns the key in which the primary keys of the given table have been saved"""
-        table_name = (
-            cls.__name__.lower() if cls._table_name is None else cls._table_name
-        )
-        return f"{table_name}__index"
+        return f"{cls._tablename}{cls._separator}__index"
 
     @classmethod
     async def _ids_to_primary_keys(

test/conftest.py

@@ -1,13 +1,17 @@
 import pytest
+import pytest_asyncio
 import redislite
 from pydantic_aioredis.config import RedisConfig
 from pydantic_aioredis.model import Model
 from pydantic_aioredis.store import Store
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 def redis_server(unused_tcp_port):
     """Sets up a fake redis server we can use for tests"""
-    instance = redislite.Redis(serverconfig={"port": unused_tcp_port})
-    yield unused_tcp_port
-    instance.close()
+    try:
+        instance = redislite.Redis(serverconfig={"port": unused_tcp_port})
+        yield unused_tcp_port
+    finally:
+        instance.close()
+        instance.shutdown()

test/ext/FastAPI/test_ext_fastapi.py

@@ -1,6 +1,7 @@
 from typing import List
 
 import pytest
+import pytest_asyncio
 from fastapi import FastAPI
 from httpx import AsyncClient
 from pydantic_aioredis.config import RedisConfig
@@ -13,7 +14,7 @@ class Model(FastAPIModel):
     name: str
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 async def test_app(redis_server):
     store = Store(
         name="sample",

test/ext/FastAPI/test_ext_fastapi_crudrouter.py

@@ -1,6 +1,7 @@
 from typing import List
 
 import pytest
+import pytest_asyncio
 from fastapi import FastAPI
 from httpx import AsyncClient
 from pydantic_aioredis import Model as PAModel
@@ -15,7 +16,7 @@ class Model(PAModel):
     value: int
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 async def test_app(redis_server):
     store = Store(
         name="sample",
@@ -31,7 +32,7 @@ async def test_app(redis_server):
     yield store, app, Model
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 def test_models():
     return [Model(name=f"test{i}", value=i) for i in range(1, 10)]
 

test/test_pydantic_aioredis.py

@@ -11,6 +11,7 @@
 from uuid import uuid4
 
 import pytest
+import pytest_asyncio
 from pydantic_aioredis.config import RedisConfig
 from pydantic_aioredis.model import Model
 from pydantic_aioredis.store import Store
@@ -85,7 +86,7 @@ class ModelWithIP(Model):
 ]
 
 
-@pytest.fixture()
+@pytest_asyncio.fixture()
 async def redis_store(redis_server):
     """Sets up a redis store using the redis_server fixture and adds the book model to it"""
     store = Store(
@@ -150,10 +151,10 @@ def test_store_model(redis_store):
 async def test_bulk_insert(store, models, model_class):
     """Providing a list of Model instances to the insert method inserts the records in redis"""
     keys = [
-        f"{type(model).__name__.lower()}_%&_{getattr(model, type(model)._primary_key_field)}"
+        f"{type(model).__name__.lower()}:{getattr(model, type(model)._primary_key_field)}"
         for model in models
     ]
-    # keys = [f"book_%&_{book.title}" for book in models]
+    # keys = [f"book:{book.title}" for book in models]
     await store.redis_store.delete(*keys)
 
     for key in keys:
@@ -179,7 +180,7 @@ async def test_insert_single(store, models, model_class):
     """
     Providing a single Model instance
     """
-    key = f"{type(models[0]).__name__.lower()}_%&_{getattr(models[0], type(models[0])._primary_key_field)}"
+    key = f"{type(models[0]).__name__.lower()}:{getattr(models[0], type(models[0])._primary_key_field)}"
     model = await store.redis_store.hgetall(name=key)
     assert model == {}
 
@@ -196,7 +197,7 @@ async def test_insert_single_lifespan(store, models, model_class):
     """
     Providing a single Model instance with a lifespam
     """
-    key = f"{type(models[0]).__name__.lower()}_%&_{getattr(models[0], type(models[0])._primary_key_field)}"
+    key = f"{type(models[0]).__name__.lower()}:{getattr(models[0], type(models[0])._primary_key_field)}"
     model = await store.redis_store.hgetall(name=key)
     assert model == {}
 
@@ -317,7 +318,7 @@ async def test_update(redis_store):
     await Book.insert(books)
     title = books[0].title
     new_author = "John Doe"
-    key = f"book_%&_{title}"
+    key = f"book:{title}"
     old_book_data = await redis_store.redis_store.hgetall(name=key)
     old_book = Book(**Book.deserialize_partially(old_book_data))
     assert old_book == books[0]
@@ -355,8 +356,8 @@ async def test_delete_multiple(redis_store):
     ids_to_delete = [book.title for book in books_to_delete]
     ids_to_leave_intact = [book.title for book in books_left_in_db]
 
-    keys_to_delete = [f"book_%&_{_id}" for _id in ids_to_delete]
-    keys_to_leave_intact = [f"book_%&_{_id}" for _id in ids_to_leave_intact]
+    keys_to_delete = [f"book:{_id}" for _id in ids_to_delete]
+    keys_to_leave_intact = [f"book:{_id}" for _id in ids_to_leave_intact]
 
     await Book.delete(ids=ids_to_delete)