Skip to content

Commit 36aa82e

Browse files
refactor: move serialization adapters into store implementations
Move store-specific serialization adapters from shared serialization.py into their respective store implementations: - FullJsonAdapter → Redis store - StringifiedDictAdapter → MongoDB store - ElasticsearchAdapter → Elasticsearch store This improves code organization by keeping serialization logic with the stores that use them, reducing coupling and making the codebase more maintainable. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: William Easton <[email protected]>
1 parent d22a52d commit 36aa82e

File tree

13 files changed

+471
-184
lines changed

13 files changed

+471
-184
lines changed

key-value/key-value-aio/src/key_value/aio/stores/elasticsearch/store.py

Lines changed: 119 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
import logging
2+
from abc import ABC, abstractmethod
23
from collections.abc import Sequence
34
from datetime import datetime
45
from typing import Any, overload
56

67
from elastic_transport import ObjectApiResponse
78
from elastic_transport import SerializationError as ElasticsearchSerializationError
89
from key_value.shared.errors import DeserializationError, SerializationError
9-
from key_value.shared.utils.managed_entry import ManagedEntry
10+
from key_value.shared.utils.managed_entry import ManagedEntry, load_from_json, verify_dict
1011
from key_value.shared.utils.sanitize import (
1112
ALPHANUMERIC_CHARACTERS,
1213
LOWERCASE_ALPHABET,
1314
NUMBERS,
1415
sanitize_string,
1516
)
16-
from key_value.shared.utils.serialization import ElasticsearchAdapter, SerializationAdapter
1717
from key_value.shared.utils.time_to_live import now_as_epoch
1818
from typing_extensions import override
1919

@@ -85,6 +85,123 @@
8585
ALLOWED_INDEX_CHARACTERS: str = LOWERCASE_ALPHABET + NUMBERS + "_" + "-" + "."
8686

8787

88+
class SerializationAdapter(ABC):
89+
"""Base class for ManagedEntry serialization adapters.
90+
91+
Adapters convert ManagedEntry objects to/from store-specific formats,
92+
encapsulating the serialization logic and keeping it separate from the
93+
ManagedEntry class and store implementations.
94+
"""
95+
96+
@abstractmethod
97+
def to_storage(self, key: str, entry: ManagedEntry, collection: str | None = None) -> dict[str, Any] | str:
98+
"""Convert ManagedEntry to store-specific format.
99+
100+
Args:
101+
key: The key associated with this entry.
102+
entry: The ManagedEntry to serialize.
103+
collection: Optional collection name for stores that include it in documents.
104+
105+
Returns:
106+
Store-specific format (dict or string depending on store requirements).
107+
"""
108+
109+
@abstractmethod
110+
def from_storage(self, data: dict[str, Any] | str) -> ManagedEntry:
111+
"""Convert store-specific format back to ManagedEntry.
112+
113+
Args:
114+
data: The store-specific data to deserialize.
115+
116+
Returns:
117+
A reconstructed ManagedEntry object.
118+
119+
Raises:
120+
DeserializationError: If the data cannot be deserialized.
121+
"""
122+
123+
124+
class ElasticsearchAdapter(SerializationAdapter):
125+
"""Adapter for Elasticsearch with support for both native and stringified storage.
126+
127+
Elasticsearch supports two storage modes:
128+
- Native (flattened field type): Stores value as native dict for querying
129+
- Stringified (disabled object field): Stores value as JSON string for compatibility
130+
131+
The adapter can read from both fields for backward compatibility but writes to
132+
only one based on the native_storage parameter.
133+
"""
134+
135+
def __init__(self, native_storage: bool = True) -> None:
136+
"""Initialize the Elasticsearch adapter.
137+
138+
Args:
139+
native_storage: If True, use flattened field for native dict storage.
140+
If False, use string field for JSON string storage.
141+
"""
142+
self.native_storage = native_storage
143+
144+
def to_storage(self, key: str, entry: ManagedEntry, collection: str | None = None) -> dict[str, Any]:
145+
"""Serialize to Elasticsearch document format."""
146+
document: dict[str, Any] = {"collection": collection or "", "key": key, "value": {}}
147+
148+
# Store in appropriate field based on mode
149+
if self.native_storage:
150+
document["value"]["flattened"] = entry.value_as_dict
151+
else:
152+
document["value"]["string"] = entry.value_as_json
153+
154+
if entry.created_at:
155+
document["created_at"] = entry.created_at.isoformat()
156+
if entry.expires_at:
157+
document["expires_at"] = entry.expires_at.isoformat()
158+
159+
return document
160+
161+
def from_storage(self, data: dict[str, Any] | str) -> ManagedEntry:
162+
"""Deserialize from Elasticsearch document format.
163+
164+
Supports reading from both flattened and string fields for backward compatibility.
165+
"""
166+
if not isinstance(data, dict):
167+
msg = "Expected dict data for ElasticsearchAdapter"
168+
raise DeserializationError(msg)
169+
170+
value: dict[str, Any] = {}
171+
raw_value = data.get("value")
172+
173+
# Try flattened field first, fall back to string field
174+
if not raw_value or not isinstance(raw_value, dict):
175+
msg = "Value field not found or invalid type"
176+
raise DeserializationError(msg)
177+
178+
value_flattened = raw_value.get("flattened")
179+
if value_flattened:
180+
value = verify_dict(obj=value_flattened)
181+
else:
182+
value_str = raw_value.get("string")
183+
if value_str:
184+
if not isinstance(value_str, str):
185+
msg = "Value in `value` field is not a string"
186+
raise DeserializationError(msg)
187+
value = load_from_json(value_str)
188+
else:
189+
msg = "Neither flattened nor string field found in value"
190+
raise DeserializationError(msg)
191+
192+
# Import here to avoid circular dependency
193+
from key_value.shared.utils.time_to_live import try_parse_datetime_str
194+
195+
created_at = try_parse_datetime_str(value=data.get("created_at"))
196+
expires_at = try_parse_datetime_str(value=data.get("expires_at"))
197+
198+
return ManagedEntry(
199+
value=value,
200+
created_at=created_at,
201+
expires_at=expires_at,
202+
)
203+
204+
88205
class ElasticsearchStore(
89206
BaseEnumerateCollectionsStore, BaseEnumerateKeysStore, BaseDestroyCollectionStore, BaseCullStore, BaseContextManagerStore, BaseStore
90207
):

key-value/key-value-aio/src/key_value/aio/stores/mongodb/store.py

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1+
from abc import ABC, abstractmethod
12
from collections.abc import Sequence
23
from datetime import datetime
34
from typing import Any, overload
45

6+
from key_value.shared.errors import DeserializationError
57
from key_value.shared.utils.managed_entry import ManagedEntry
68
from key_value.shared.utils.sanitize import ALPHANUMERIC_CHARACTERS, sanitize_string
7-
from key_value.shared.utils.serialization import SerializationAdapter, StringifiedDictAdapter
89
from typing_extensions import Self, override
910

1011
from key_value.aio.stores.base import BaseContextManagerStore, BaseDestroyCollectionStore, BaseEnumerateCollectionsStore, BaseStore
@@ -34,6 +35,64 @@
3435
COLLECTION_ALLOWED_CHARACTERS = ALPHANUMERIC_CHARACTERS + "_"
3536

3637

38+
class SerializationAdapter(ABC):
39+
"""Base class for ManagedEntry serialization adapters.
40+
41+
Adapters convert ManagedEntry objects to/from store-specific formats,
42+
encapsulating the serialization logic and keeping it separate from the
43+
ManagedEntry class and store implementations.
44+
"""
45+
46+
@abstractmethod
47+
def to_storage(self, key: str, entry: ManagedEntry, collection: str | None = None) -> dict[str, Any] | str:
48+
"""Convert ManagedEntry to store-specific format.
49+
50+
Args:
51+
key: The key associated with this entry.
52+
entry: The ManagedEntry to serialize.
53+
collection: Optional collection name for stores that include it in documents.
54+
55+
Returns:
56+
Store-specific format (dict or string depending on store requirements).
57+
"""
58+
59+
@abstractmethod
60+
def from_storage(self, data: dict[str, Any] | str) -> ManagedEntry:
61+
"""Convert store-specific format back to ManagedEntry.
62+
63+
Args:
64+
data: The store-specific data to deserialize.
65+
66+
Returns:
67+
A reconstructed ManagedEntry object.
68+
69+
Raises:
70+
DeserializationError: If the data cannot be deserialized.
71+
"""
72+
73+
74+
class StringifiedDictAdapter(SerializationAdapter):
75+
"""Adapter that stores metadata as dict fields but stringifies the value field.
76+
77+
Used by MongoDB and other document databases that prefer stringified values
78+
for consistency or to avoid schema issues with dynamic value structures.
79+
"""
80+
81+
def to_storage(self, key: str, entry: ManagedEntry, collection: str | None = None) -> dict[str, Any]: # noqa: ARG002
82+
"""Serialize to dict with stringified value field."""
83+
return {
84+
"key": key,
85+
**entry.to_dict(include_metadata=True, include_expiration=True, include_creation=True, stringify_value=True),
86+
}
87+
88+
def from_storage(self, data: dict[str, Any] | str) -> ManagedEntry:
89+
"""Deserialize from dict with stringified value field."""
90+
if not isinstance(data, dict):
91+
msg = "Expected dict data for StringifiedDictAdapter"
92+
raise DeserializationError(msg)
93+
return ManagedEntry.from_dict(data=data, stringified_value=True)
94+
95+
3796
class MongoDBStore(BaseEnumerateCollectionsStore, BaseDestroyCollectionStore, BaseContextManagerStore, BaseStore):
3897
"""MongoDB-based key-value store using Motor (async MongoDB driver)."""
3998

key-value/key-value-aio/src/key_value/aio/stores/redis/store.py

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
1+
from abc import ABC, abstractmethod
12
from collections.abc import Sequence
23
from datetime import datetime
34
from typing import Any, overload
45
from urllib.parse import urlparse
56

7+
from key_value.shared.errors import DeserializationError
68
from key_value.shared.type_checking.bear_spray import bear_spray
79
from key_value.shared.utils.compound import compound_key, get_keys_from_compound_keys
810
from key_value.shared.utils.managed_entry import ManagedEntry
9-
from key_value.shared.utils.serialization import FullJsonAdapter, SerializationAdapter
1011
from typing_extensions import override
1112

1213
from key_value.aio.stores.base import BaseContextManagerStore, BaseDestroyStore, BaseEnumerateKeysStore, BaseStore
@@ -21,6 +22,61 @@
2122
PAGE_LIMIT = 10000
2223

2324

25+
class SerializationAdapter(ABC):
26+
"""Base class for ManagedEntry serialization adapters.
27+
28+
Adapters convert ManagedEntry objects to/from store-specific formats,
29+
encapsulating the serialization logic and keeping it separate from the
30+
ManagedEntry class and store implementations.
31+
"""
32+
33+
@abstractmethod
34+
def to_storage(self, key: str, entry: ManagedEntry, collection: str | None = None) -> dict[str, Any] | str:
35+
"""Convert ManagedEntry to store-specific format.
36+
37+
Args:
38+
key: The key associated with this entry.
39+
entry: The ManagedEntry to serialize.
40+
collection: Optional collection name for stores that include it in documents.
41+
42+
Returns:
43+
Store-specific format (dict or string depending on store requirements).
44+
"""
45+
46+
@abstractmethod
47+
def from_storage(self, data: dict[str, Any] | str) -> ManagedEntry:
48+
"""Convert store-specific format back to ManagedEntry.
49+
50+
Args:
51+
data: The store-specific data to deserialize.
52+
53+
Returns:
54+
A reconstructed ManagedEntry object.
55+
56+
Raises:
57+
DeserializationError: If the data cannot be deserialized.
58+
"""
59+
60+
61+
class FullJsonAdapter(SerializationAdapter):
62+
"""Adapter that serializes to full JSON string with all metadata.
63+
64+
Used by stores that cannot store native dict structures (e.g., Redis, Valkey, Memcached).
65+
The entire entry is serialized to a JSON string including value, TTL, and timestamps.
66+
"""
67+
68+
def to_storage(self, key: str, entry: ManagedEntry, collection: str | None = None) -> str: # noqa: ARG002
69+
"""Serialize to JSON string with full metadata."""
70+
return entry.to_json(include_metadata=True, include_expiration=True, include_creation=True)
71+
72+
def from_storage(self, data: dict[str, Any] | str) -> ManagedEntry:
73+
"""Deserialize from JSON string."""
74+
if not isinstance(data, str):
75+
msg = "Expected string data for FullJsonAdapter"
76+
raise DeserializationError(msg)
77+
return ManagedEntry.from_json(json_str=data, includes_metadata=True)
78+
79+
2480
class RedisStore(BaseDestroyStore, BaseEnumerateKeysStore, BaseContextManagerStore, BaseStore):
2581
"""Redis-based key-value store."""
2682

key-value/key-value-aio/tests/stores/elasticsearch/test_elasticsearch.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from key_value.aio.stores.base import BaseStore
1313
from key_value.aio.stores.elasticsearch import ElasticsearchStore
14-
from key_value.shared.utils.serialization import ElasticsearchAdapter
14+
from key_value.aio.stores.elasticsearch.store import ElasticsearchAdapter
1515
from tests.conftest import docker_container, should_skip_docker_tests
1616
from tests.stores.base import BaseStoreTests, ContextManagerStoreTestMixin
1717

key-value/key-value-aio/tests/stores/mongodb/test_mongodb.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
from key_value.aio.stores.base import BaseStore
1515
from key_value.aio.stores.mongodb import MongoDBStore
16-
from key_value.shared.utils.serialization import StringifiedDictAdapter
16+
from key_value.aio.stores.mongodb.store import StringifiedDictAdapter
1717
from tests.conftest import docker_container, should_skip_docker_tests
1818
from tests.stores.base import BaseStoreTests, ContextManagerStoreTestMixin
1919

key-value/key-value-aio/tests/stores/redis/test_redis.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from key_value.aio.stores.base import BaseStore
1313
from key_value.aio.stores.redis import RedisStore
14-
from key_value.shared.utils.serialization import FullJsonAdapter
14+
from key_value.aio.stores.redis.store import FullJsonAdapter
1515
from tests.conftest import docker_container, should_skip_docker_tests
1616
from tests.stores.base import BaseStoreTests, ContextManagerStoreTestMixin
1717

0 commit comments

Comments
 (0)