fix: cleanup serializer (#399)

serialize_partially and deserialize_partially had a dead code path,
using "json dump shapes" to determine to json dump something. However,
the first if checking for str, float, or int already handles that.

This dead code was found using hypothesis testing, included in the new
tests/test_abstract.py. In the future, I'd like to augment the
hardcoded test data in test_pydantic_aioredis with hypothesis data

Author: andrewthetechie

.github/workflows/tests.yml

@@ -92,6 +92,8 @@ jobs:
             ${{ steps.pre-commit-cache.outputs.result }}-
 
       - name: Run Nox
+        env:
+          HYPOTHESIS_PROFILE: ci
         run: |
           nox --force-color --python=${{ matrix.python }}
 

.gitignore

@@ -11,3 +11,4 @@ coverage.xml
 __pycache__/
 examples/benchmarks/.benchmarks
 examples/benchmarks/.coverage
+.hypothesis

poetry.lock

@@ -2,7 +2,6 @@
 import json
 from datetime import date
 from datetime import datetime
-from enum import Enum
 from ipaddress import IPv4Address
 from ipaddress import IPv4Network
 from ipaddress import IPv6Address
@@ -15,32 +14,9 @@
 from uuid import UUID
 
 from pydantic import BaseModel
-from pydantic.fields import SHAPE_DEFAULTDICT
-from pydantic.fields import SHAPE_DICT
-from pydantic.fields import SHAPE_FROZENSET
-from pydantic.fields import SHAPE_LIST
-from pydantic.fields import SHAPE_MAPPING
-from pydantic.fields import SHAPE_SEQUENCE
-from pydantic.fields import SHAPE_SET
-from pydantic.fields import SHAPE_TUPLE
-from pydantic.fields import SHAPE_TUPLE_ELLIPSIS
 from pydantic_aioredis.config import RedisConfig
 from redis import asyncio as aioredis
 
-# JSON_DUMP_SHAPES are object types that are serialized to JSON using json.dumps
-JSON_DUMP_SHAPES = (
-    SHAPE_LIST,
-    SHAPE_SET,
-    SHAPE_MAPPING,
-    SHAPE_TUPLE,
-    SHAPE_TUPLE_ELLIPSIS,
-    SHAPE_SEQUENCE,
-    SHAPE_FROZENSET,
-    SHAPE_DICT,
-    SHAPE_DEFAULTDICT,
-    Enum,
-)
-
 # STR_DUMP_SHAPES are object types that are serialized to strings using str(obj)
 # They are stored in redis as strings and rely on pydantic to deserialize them
 STR_DUMP_SHAPES = (IPv4Address, IPv4Network, IPv6Address, IPv6Network, UUID)
@@ -113,8 +89,6 @@ def serialize_partially(cls, data: Dict[str, Any]):
                 continue
             if cls.__fields__[field].type_ not in [str, float, int]:
                 data[field] = json.dumps(data[field], default=cls.json_default)
-            if getattr(cls.__fields__[field], "shape", None) in JSON_DUMP_SHAPES:
-                data[field] = json.dumps(data[field], default=cls.json_default)
             if getattr(cls.__fields__[field], "allow_none", False):
                 if data[field] is None:
                     data[field] = "None"
@@ -133,8 +107,6 @@ def deserialize_partially(cls, data: Dict[bytes, Any]):
                 continue
             if cls.__fields__[field].type_ not in [str, float, int]:
                 data[field] = json.loads(data[field], object_hook=cls.json_object_hook)
-            if getattr(cls.__fields__[field], "shape", None) in JSON_DUMP_SHAPES:
-                data[field] = json.loads(data[field], object_hook=cls.json_object_hook)
             if getattr(cls.__fields__[field], "allow_none", False):
                 if data[field] == "None":
                     data[field] = None

pydantic_aioredis/abstract.py

@@ -75,6 +75,7 @@ pylint = "^2.13.9"
 setuptools-git-versioning = "^1.13.1"
 bandit = "^1.7.4"
 fakeredis = {extras = ["json"], version = "2.2.0"}
+hypothesis = "^6.61.0"
 
 [tool.coverage.paths]
 source = ["pydantic_aioredis", "*/site-packages"]

pyproject.toml

@@ -24,6 +24,7 @@ async def redis_store():
 def pytest_configure(config):
     """Configure our markers"""
     config.addinivalue_line("markers", "union_test: Tests for union types")
+    config.addinivalue_line("markers", "hypothesis: Tests that use hypothesis")
 
 
 @pytest.hookimpl(trylast=True)
@@ -32,3 +33,12 @@ def pytest_collection_modifyitems(config, items):
     for item in items:
         if inspect.iscoroutinefunction(item.function):
             item.add_marker(pytest.mark.asyncio)
+
+
+import os
+from hypothesis import settings, Verbosity
+
+settings.register_profile("ci", max_examples=5000)
+settings.register_profile("dev", max_examples=100)
+settings.register_profile("debug", max_examples=10, verbosity=Verbosity.verbose)
+settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "dev"))

test/conftest.py

@@ -0,0 +1,76 @@
+"""Test methods in abstract.py. Uses hypothesis"""
+import json
+from datetime import date
+from datetime import datetime
+from ipaddress import IPv4Address
+from ipaddress import IPv6Address
+from typing import Dict
+from typing import List
+from typing import Tuple
+from typing import Union
+
+import pytest
+from hypothesis import given
+from hypothesis import strategies as st
+from pydantic_aioredis.model import Model
+
+
+class SimpleModel(Model):
+    _primary_key_field: str = "test_str"
+    test_str: str
+    test_int: int
+    test_float: float
+    test_date: date
+    test_datetime: datetime
+    test_ip_v4: IPv4Address
+    test_ip_v6: IPv6Address
+    test_list: List
+    test_dict: Dict[str, Union[int, float]]
+    test_tuple: Tuple[str]
+
+
+parameters = [
+    (st.text, [], {}, "test_str", None),
+    (st.integers, [], {}, "test_int", None),
+    (st.floats, [], {"allow_nan": False}, "test_float", None),
+    (st.dates, [], {}, "test_date", lambda x: json.dumps(x.isoformat())),
+    (st.datetimes, [], {}, "test_datetime", lambda x: json.dumps(x.isoformat())),
+    (st.ip_addresses, [], {"v": 4}, "test_ip_v4", lambda x: json.dumps(str(x))),
+    (st.ip_addresses, [], {"v": 6}, "test_ip_v4", lambda x: json.dumps(str(x))),
+    (
+        st.lists,
+        [st.tuples(st.integers(), st.floats())],
+        {},
+        "test_list",
+        lambda x: json.dumps(x),
+    ),
+    (
+        st.dictionaries,
+        [st.text(), st.tuples(st.integers(), st.floats())],
+        {},
+        "test_dict",
+        lambda x: json.dumps(x),
+    ),
+    (st.tuples, [st.text()], {}, "test_tuple", lambda x: json.dumps(x)),
+]
+
+
+@pytest.mark.parametrize(
+    "strategy, strategy_args, strategy_kwargs, model_field, serialize_callable",
+    parameters,
+)
+@given(st.data())
+def test_serialize_partially(
+    strategy, strategy_args, strategy_kwargs, model_field, serialize_callable, data
+):
+    value = data.draw(strategy(*strategy_args, **strategy_kwargs))
+    serialized = SimpleModel.serialize_partially({model_field: value})
+    if serialize_callable is None:
+        assert serialized[model_field] == value
+    else:
+        assert serialized[model_field] == serialize_callable(value)
+
+
+def test_serialize_partially_skip_missing_filed():
+    serialized = SimpleModel.serialize_partially({"unknown": "test"})
+    assert serialized["unknown"] == "test"

test/test_abstract.py

@@ -13,6 +13,7 @@
 import pytest
 import pytest_asyncio
 from fakeredis.aioredis import FakeRedis
+from pydantic_aioredis.abstract import _AbstractModel
 from pydantic_aioredis.config import RedisConfig
 from pydantic_aioredis.model import Model
 from pydantic_aioredis.store import Store
@@ -191,6 +192,15 @@ def test_store_model(redis_store):
         redis_store.model("Notabook")
 
 
+def test_json_object_hook():
+    class TestObj:
+        def __init__(self, value: str):
+            self.value = value
+
+    test_obj = TestObj("test")
+    assert test_obj.value == _AbstractModel.json_object_hook(test_obj).value
+
+
 parameters = [
     (pytest.lazy_fixture("redis_store"), books, Book, "book:"),
     (pytest.lazy_fixture("redis_store"), extended_books, ExtendedBook, "extendedbook:"),