Plomin deser (#420)

* Add OrderedSet as a serialization type

* Minor fixes

* All tests passing

* Fix qa

* Fix failed tests in ^3.11

Author: cffls

.github/workflows/main.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
 
     steps:
       - uses: actions/checkout@v4

Makefile

@@ -57,6 +57,9 @@ clean-test: ## remove test and coverage artifacts
 test: ## runs tests
 	poetry run pytest -vv -n 4
 
+test-integration: ## runs integration tests
+	cd integration-test && ./run_tests.sh
+
 test-single: ## runs tests with "single" markers
 	poetry run pytest -s -vv -m single
 

integration-test/configs/local-chang/shelley-genesis.json

@@ -21,7 +21,7 @@
         "keyDeposit": 1000000,
         "protocolVersion": {
             "minor": 0,
-            "major": 9
+            "major": 10
         },
         "poolDeposit": 1000000,
         "a0": 0.0,

integration-test/docker-compose-chang.yml

@@ -9,7 +9,7 @@ networks:
 services:
 
   cardano-node:
-    image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_VERSION:-10.1.3}
+    image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_VERSION:-10.1.4}
     platform: linux/amd64
     entrypoint: bash
     environment:
@@ -35,7 +35,7 @@ services:
         max-file: "10"
 
   cardano-pool:
-    image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_VERSION:-10.1.3}
+    image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_VERSION:-10.1.4}
     platform: linux/amd64
     entrypoint: bash
     environment:
@@ -56,7 +56,7 @@ services:
         max-file: "10"
 
   ogmios:
-    image: cardanosolutions/ogmios:v6.9.0
+    image: cardanosolutions/ogmios:v6.11.0
     platform: linux/amd64
     environment:
       NETWORK: "${NETWORK:-local-alonzo}"

integration-test/run_tests.sh

@@ -5,7 +5,7 @@ set -o pipefail
 
 ROOT=$(pwd)
 
-poetry install
+poetry install -C ..
 #poetry run pip install ogmios
 
 ##########

poetry.lock

@@ -45,7 +45,7 @@ def __post_init__(self):
             self._CODE = 1
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[StakeCredential], values: Union[list, tuple]
     ) -> StakeCredential:
@@ -67,7 +67,7 @@ def __post_init__(self):
         self._CODE = 0
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[StakeRegistration], values: Union[list, tuple]
     ) -> StakeRegistration:
@@ -87,7 +87,7 @@ def __post_init__(self):
         self._CODE = 1
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[StakeDeregistration], values: Union[list, tuple]
     ) -> StakeDeregistration:
@@ -109,7 +109,7 @@ def __post_init__(self):
         self._CODE = 2
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[StakeDelegation], values: Union[list, tuple]
     ) -> StakeDelegation:
@@ -138,7 +138,7 @@ def to_primitive(self):
         return super().to_primitive()
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[PoolRegistration], values: Union[list, tuple]
     ) -> PoolRegistration:
@@ -166,7 +166,7 @@ def __post_init__(self):
         self._CODE = 4
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[PoolRetirement], values: Union[list, tuple]
     ) -> PoolRetirement:

pycardano/certificate.py

@@ -123,7 +123,9 @@ def to_primitive(self) -> Primitive:
         return self.data.to_primitive()
 
     @classmethod
-    def from_primitive(cls: Type[AuxiliaryData], value: Primitive) -> AuxiliaryData:
+    def from_primitive(
+        cls: Type[AuxiliaryData], value: Primitive, type_args: Optional[tuple] = None
+    ) -> AuxiliaryData:
         for t in [AlonzoMetadata, ShelleyMarryMetadata, Metadata]:
             # The schema of metadata in different eras are mutually exclusive, so we can try deserializing
             # them one by one without worrying about mismatch.

pycardano/metadata.py

@@ -35,7 +35,7 @@ class NativeScript(ArrayCBORSerializable):
     json_field: ClassVar[str]
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[NativeScript], value: list
     ) -> Union[

pycardano/nativescript.py

@@ -989,7 +989,7 @@ class Redeemer(ArrayCBORSerializable):
     ex_units: Optional[ExecutionUnits] = None
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(cls: Type[Redeemer], values: list) -> Redeemer:
         if isinstance(values[2], CBORTag) and cls is Redeemer:
             values[2] = RawPlutusData.from_primitive(values[2])
@@ -1028,7 +1028,7 @@ class RedeemerValue(ArrayCBORSerializable):
     ex_units: ExecutionUnits
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(cls: Type[RedeemerValue], values: list) -> RedeemerValue:
         if isinstance(values[0], CBORTag) and cls is RedeemerValue:
             values[0] = RawPlutusData.from_primitive(values[0])

pycardano/plutus.py

@@ -165,7 +165,7 @@ def to_primitive(self) -> list:
         ]
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[SingleHostAddr], values: Union[list, tuple]
     ) -> SingleHostAddr:
@@ -190,7 +190,7 @@ def __post_init__(self):
         self._CODE = 1
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[SingleHostName], values: Union[list, tuple]
     ) -> SingleHostName:
@@ -213,7 +213,7 @@ def __post_init__(self):
         self._CODE = 2
 
     @classmethod
-    @limit_primitive_type(list)
+    @limit_primitive_type(list, tuple)
     def from_primitive(
         cls: Type[MultiHostName], values: Union[list, tuple]
     ) -> MultiHostName:

pycardano/pool_params.py

@@ -10,20 +10,35 @@
 from datetime import datetime
 from decimal import Decimal
 from functools import wraps
-from inspect import isclass
+from inspect import getfullargspec, isclass
 from typing import (
     Any,
     Callable,
     ClassVar,
     Dict,
+    Generic,
+    Iterable,
     List,
     Optional,
+    Set,
     Type,
     TypeVar,
     Union,
+    cast,
     get_type_hints,
 )
 
+import cbor2
+
+from pycardano.logging import logger
+
+# Remove the semantic decoder for 258 (CBOR tag for set) as we care about the order of elements
+try:
+    cbor2._decoder.semantic_decoders.pop(258)
+except Exception as e:
+    logger.warning("Failed to remove semantic decoder for CBOR tag 258", e)
+    pass
+
 from cbor2 import CBOREncoder, CBORSimpleValue, CBORTag, dumps, loads, undefined
 from frozendict import frozendict
 from frozenlist import FrozenList
@@ -44,8 +59,12 @@
     "RawCBOR",
     "list_hook",
     "limit_primitive_type",
+    "OrderedSet",
+    "NonEmptyOrderedSet",
 ]
 
+T = TypeVar("T")
+
 
 def _identity(x):
     return x
@@ -314,10 +333,12 @@ def validate(self):
         def _check_recursive(value, type_hint):
             if type_hint is Any:
                 return True
+
+            if isinstance(value, CBORSerializable):
+                value.validate()
+
             origin = getattr(type_hint, "__origin__", None)
             if origin is None:
-                if isinstance(value, CBORSerializable):
-                    value.validate()
                 return isinstance(value, type_hint)
             elif origin is ClassVar:
                 return _check_recursive(value, type_hint.__args__[0])
@@ -329,7 +350,7 @@ def _check_recursive(value, type_hint):
                     _check_recursive(k, key_type) and _check_recursive(v, value_type)
                     for k, v in value.items()
                 )
-            elif origin in (list, set, tuple, frozenset):
+            elif origin in (list, set, tuple, frozenset, OrderedSet):
                 if value is None:
                     return True
                 args = type_hint.__args__
@@ -364,12 +385,15 @@ def to_validated_primitive(self) -> Primitive:
         return self.to_primitive()
 
     @classmethod
-    def from_primitive(cls: Type[CBORBase], value: Any) -> CBORBase:
+    def from_primitive(
+        cls: Type[CBORBase], value: Any, type_args: Optional[tuple] = None
+    ) -> CBORBase:
         """Turn a CBOR primitive to its original class type.
 
         Args:
             cls (CBORBase): The original class type.
             value (:const:`Primitive`): A CBOR primitive.
+            type_args (Optional[tuple]): Type arguments for the class.
 
         Returns:
             CBORBase: A CBOR serializable object.
@@ -519,10 +543,26 @@ def _restore_typed_primitive(
         Union[:const:`Primitive`, CBORSerializable]: A CBOR primitive or a CBORSerializable.
     """
 
+    is_cbor_serializable = False
+    try:
+        is_cbor_serializable = issubclass(t, CBORSerializable)
+    except TypeError:
+        # Handle the case when t is a generic alias
+        origin = typing.get_origin(t)
+        if origin is not None:
+            try:
+                is_cbor_serializable = issubclass(origin, CBORSerializable)
+            except TypeError:
+                pass
+
     if t is Any or (t in PRIMITIVE_TYPES and isinstance(v, t)):
         return v
-    elif isclass(t) and issubclass(t, CBORSerializable):
-        return t.from_primitive(v)
+    elif is_cbor_serializable:
+        if "type_args" in getfullargspec(t.from_primitive).args:
+            args = typing.get_args(t)
+            return t.from_primitive(v, type_args=args)
+        else:
+            return t.from_primitive(v)
     elif hasattr(t, "__origin__") and (t.__origin__ is list):
         t_args = t.__args__
         if len(t_args) != 1:
@@ -941,3 +981,82 @@ def list_hook(
             CBORSerializables.
     """
     return lambda vals: [cls.from_primitive(v) for v in vals]
+
+
+class OrderedSet(list, Generic[T], CBORSerializable):
+    def __init__(self, iterable: Optional[List[T]] = None, use_tag: bool = True):
+        super().__init__()
+        self._set: Set[str] = set()
+        self._use_tag = use_tag
+        if iterable:
+            self.extend(iterable)
+
+    def append(self, item: T) -> None:
+        item_key = str(item)
+        if item_key not in self._set:
+            super().append(item)
+            self._set.add(item_key)
+
+    def extend(self, items: Iterable[T]) -> None:
+        for item in items:
+            self.append(item)
+
+    def __contains__(self, item: object) -> bool:
+        return str(item) in self._set
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, OrderedSet):
+            if isinstance(other, list):
+                return list(self) == other
+            return False
+        return list(self) == list(other)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({list(self)})"
+
+    def to_shallow_primitive(self) -> Union[CBORTag, List[T]]:
+        if self._use_tag:
+            return CBORTag(258, list(self))
+        return list(self)
+
+    @classmethod
+    def from_primitive(
+        cls: Type[OrderedSet[T]], value: Primitive, type_args: Optional[tuple] = None
+    ) -> OrderedSet[T]:
+        assert (
+            type_args is None or len(type_args) == 1
+        ), "OrderedSet should have exactly one type argument"
+        # Retrieve the type arguments from the class
+        type_arg = type_args[0] if type_args else None
+
+        if isinstance(value, CBORTag) and value.tag == 258:
+            if isclass(type_arg) and issubclass(type_arg, CBORSerializable):
+                value.value = [type_arg.from_primitive(v) for v in value.value]
+            return cls(value.value, use_tag=True)
+
+        if isinstance(value, (list, tuple, set)):
+            if isclass(type_arg) and issubclass(type_arg, CBORSerializable):
+                value = [type_arg.from_primitive(v) for v in value]
+            return cls(list(value), use_tag=False)
+
+        raise ValueError(f"Cannot deserialize {value} to {cls}")
+
+
+class NonEmptyOrderedSet(OrderedSet[T]):
+    def __init__(self, iterable: Optional[List[T]] = None, use_tag: bool = True):
+        super().__init__(iterable, use_tag)
+
+    def validate(self):
+        if not self:
+            raise ValueError("NonEmptyOrderedSet cannot be empty")
+
+    @classmethod
+    def from_primitive(
+        cls: Type[NonEmptyOrderedSet[T]],
+        value: Primitive,
+        type_args: Optional[tuple] = None,
+    ) -> NonEmptyOrderedSet[T]:
+        result = cast(NonEmptyOrderedSet[T], super().from_primitive(value, type_args))
+        if not result:
+            raise ValueError("NonEmptyOrderedSet cannot be empty")
+        return result