Automatically reduce 0 and empty multiassets (#372)

* Add tests where we try to submit empty or zero multi-assets

* Add tests for empty multiassets

* Fix min_utxo could be none

* Automatically reduce multiassets and assets

* Formatting

* Fix failing test due to now empty multiasset

* Formatting

* A bit more comprehensive on testing

* formatting

* Remove policy id where it is not needed

* Reduce code duplication, include normalization for any serialization

* Fix

* Remove redundant line

* Fix fee too low

User can potentially provide signing keys that are not required by the finalized transaction.
For example, if a transaction include a minting script and signing key for the script, but the minted value is 0, the signature of the minting script isn't required.
This commit will auto detect such cases and skip the signing of this key.

* Fix test

---------

Co-authored-by: Jerry <[email protected]>

Author: nielstron

integration-test/test/test_zero_empty_asset.py

@@ -0,0 +1,205 @@
+import pathlib
+import tempfile
+
+import pytest
+from retry import retry
+
+from pycardano import *
+
+from .base import TEST_RETRIES, TestBase
+
+
+class TestZeroEmptyAsset(TestBase):
+    @retry(tries=TEST_RETRIES, backoff=1.5, delay=6, jitter=(0, 4))
+    @pytest.mark.post_chang
+    def test_submit_zero_and_empty(self):
+        address = Address(self.payment_vkey.hash(), network=self.NETWORK)
+
+        # Load payment keys or create them if they don't exist
+        def load_or_create_key_pair(base_dir, base_name):
+            skey_path = base_dir / f"{base_name}.skey"
+            vkey_path = base_dir / f"{base_name}.vkey"
+
+            if skey_path.exists():
+                skey = PaymentSigningKey.load(str(skey_path))
+                vkey = PaymentVerificationKey.from_signing_key(skey)
+            else:
+                key_pair = PaymentKeyPair.generate()
+                key_pair.signing_key.save(str(skey_path))
+                key_pair.verification_key.save(str(vkey_path))
+                skey = key_pair.signing_key
+                vkey = key_pair.verification_key
+            return skey, vkey
+
+        tempdir = tempfile.TemporaryDirectory()
+        PROJECT_ROOT = tempdir.name
+
+        root = pathlib.Path(PROJECT_ROOT)
+        # Create the directory if it doesn't exist
+        root.mkdir(parents=True, exist_ok=True)
+        """Generate keys"""
+        key_dir = root / "keys"
+        key_dir.mkdir(exist_ok=True)
+
+        # Generate policy keys, which will be used when minting NFT
+        policy_skey, policy_vkey = load_or_create_key_pair(key_dir, "policy")
+
+        """Create policy"""
+        # A policy that requires a signature from the policy key we generated above
+        pub_key_policy_1 = ScriptPubkey(policy_vkey.hash())
+
+        # A policy that requires a signature from the extended payment key
+        pub_key_policy_2 = ScriptPubkey(self.extended_payment_vkey.hash())
+
+        # Combine two policies using ScriptAll policy
+        policy = ScriptAll([pub_key_policy_1, pub_key_policy_2])
+
+        # Calculate policy ID, which is the hash of the policy
+        policy_id = policy.hash()
+
+        """Define NFT"""
+        my_nft = MultiAsset.from_primitive(
+            {
+                policy_id.payload: {
+                    b"MY_NFT_1": 1,  # Name of our NFT1  # Quantity of this NFT
+                    b"MY_NFT_2": 1,  # Name of our NFT2  # Quantity of this NFT
+                }
+            }
+        )
+
+        native_scripts = [policy]
+
+        """Create metadata"""
+        # We need to create a metadata for our NFTs, so they could be displayed correctly by blockchain explorer
+        metadata = {
+            721: {  # 721 refers to the metadata label registered for NFT standard here:
+                # https://github.com/cardano-foundation/CIPs/blob/master/CIP-0010/registry.json#L14-L17
+                policy_id.payload.hex(): {
+                    "MY_NFT_1": {
+                        "description": "This is my first NFT thanks to PyCardano",
+                        "name": "PyCardano NFT example token 1",
+                        "id": 1,
+                        "image": "ipfs://QmRhTTbUrPYEw3mJGGhQqQST9k86v1DPBiTTWJGKDJsVFw",
+                    },
+                    "MY_NFT_2": {
+                        "description": "This is my second NFT thanks to PyCardano",
+                        "name": "PyCardano NFT example token 2",
+                        "id": 2,
+                        "image": "ipfs://QmRhTTbUrPYEw3mJGGhQqQST9k86v1DPBiTTWJGKDJsVFw",
+                    },
+                }
+            }
+        }
+
+        # Place metadata in AuxiliaryData, the format acceptable by a transaction.
+        auxiliary_data = AuxiliaryData(AlonzoMetadata(metadata=Metadata(metadata)))
+
+        """Build mint transaction"""
+
+        # Create a transaction builder
+        builder = TransactionBuilder(self.chain_context)
+
+        # Add our own address as the input address
+        builder.add_input_address(address)
+
+        # Set nft we want to mint
+        builder.mint = my_nft
+
+        # Set native script
+        builder.native_scripts = native_scripts
+
+        # Set transaction metadata
+        builder.auxiliary_data = auxiliary_data
+
+        # Calculate the minimum amount of lovelace that need to hold the NFT we are going to mint
+        min_val = min_lovelace_pre_alonzo(Value(0, my_nft), self.chain_context)
+
+        # Send the NFT to our own address
+        nft_output = TransactionOutput(address, Value(min_val, my_nft))
+        builder.add_output(nft_output)
+
+        # Build and sign transaction
+        signed_tx = builder.build_and_sign(
+            [self.payment_skey, self.extended_payment_skey, policy_skey], address
+        )
+
+        print("############### Transaction created ###############")
+        print(signed_tx)
+        print(signed_tx.to_cbor_hex())
+
+        # Submit signed transaction to the network
+        print("############### Submitting transaction ###############")
+        self.chain_context.submit_tx(signed_tx)
+
+        self.assert_output(address, nft_output)
+
+        """Build transaction with 0 nft"""
+
+        # Create a transaction builder
+        builder = TransactionBuilder(self.chain_context)
+
+        # Add our own address as the input address
+        builder.add_input_address(address)
+
+        # Calculate the minimum amount of lovelace that need to hold the NFT we are going to mint
+        min_val = min_lovelace_pre_alonzo(Value(0), self.chain_context)
+
+        # Send the NFT to our own address
+        nft_output = TransactionOutput(
+            address,
+            Value(
+                min_val,
+                MultiAsset.from_primitive(
+                    {policy_vkey.hash().payload: {b"MY_NFT_1": 0}}
+                ),
+            ),
+        )
+        builder.add_output(nft_output)
+
+        # Build and sign transaction
+        signed_tx = builder.build_and_sign(
+            [self.payment_skey, self.extended_payment_skey], address
+        )
+
+        print("############### Transaction created ###############")
+        print(signed_tx)
+        print(signed_tx.to_cbor_hex())
+
+        # Submit signed transaction to the network
+        print("############### Submitting transaction ###############")
+        self.chain_context.submit_tx(signed_tx)
+
+        self.assert_output(address, nft_output)
+
+        """Build transaction with empty multi-asset"""
+
+        # Create a transaction builder
+        builder = TransactionBuilder(self.chain_context)
+
+        # Add our own address as the input address
+        builder.add_input_address(address)
+
+        # Calculate the minimum amount of lovelace that need to hold the NFT we are going to mint
+        min_val = min_lovelace_pre_alonzo(Value(0), self.chain_context)
+
+        # Send the NFT to our own address
+        nft_output = TransactionOutput(
+            address,
+            Value(min_val, MultiAsset.from_primitive({policy_vkey.hash().payload: {}})),
+        )
+        builder.add_output(nft_output)
+
+        # Build and sign transaction
+        signed_tx = builder.build_and_sign(
+            [self.payment_skey, self.extended_payment_skey], address
+        )
+
+        print("############### Transaction created ###############")
+        print(signed_tx)
+        print(signed_tx.to_cbor_hex())
+
+        # Submit signed transaction to the network
+        print("############### Submitting transaction ###############")
+        self.chain_context.submit_tx(signed_tx)
+
+        self.assert_output(address, nft_output)

pycardano/transaction.py

@@ -38,10 +38,12 @@
 from pycardano.serialization import (
     ArrayCBORSerializable,
     CBORSerializable,
+    DictBase,
     DictCBORSerializable,
     MapCBORSerializable,
     Primitive,
     default_encoder,
+    limit_primitive_type,
     list_hook,
 )
 from pycardano.types import typechecked
@@ -87,25 +89,32 @@ class Asset(DictCBORSerializable):
 
     VALUE_TYPE = int
 
+    def normalize(self) -> Asset:
+        """Normalize the Asset by removing zero values."""
+        for k, v in list(self.items()):
+            if v == 0:
+                self.pop(k)
+        return self
+
     def union(self, other: Asset) -> Asset:
         return self + other
 
     def __add__(self, other: Asset) -> Asset:
         new_asset = deepcopy(self)
         for n in other:
             new_asset[n] = new_asset.get(n, 0) + other[n]
-        return new_asset
+        return new_asset.normalize()
 
     def __iadd__(self, other: Asset) -> Asset:
         new_item = self + other
         self.update(new_item)
-        return self
+        return self.normalize()
 
     def __sub__(self, other: Asset) -> Asset:
         new_asset = deepcopy(self)
         for n in other:
             new_asset[n] = new_asset.get(n, 0) - other[n]
-        return new_asset
+        return new_asset.normalize()
 
     def __eq__(self, other):
         if not isinstance(other, Asset):
@@ -124,6 +133,20 @@ def __le__(self, other: Asset) -> bool:
                 return False
         return True
 
+    @classmethod
+    @limit_primitive_type(dict)
+    def from_primitive(cls: Type[DictBase], value: dict) -> DictBase:
+        res = super().from_primitive(value)
+        # pop zero values
+        for n, v in list(res.items()):
+            if v == 0:
+                res.pop(n)
+        return res
+
+    def to_shallow_primitive(self) -> dict:
+        x = deepcopy(self).normalize()
+        return super(self.__class__, x).to_shallow_primitive()
+
 
 @typechecked
 class MultiAsset(DictCBORSerializable):
@@ -134,22 +157,30 @@ class MultiAsset(DictCBORSerializable):
     def union(self, other: MultiAsset) -> MultiAsset:
         return self + other
 
+    def normalize(self) -> MultiAsset:
+        """Normalize the MultiAsset by removing zero values."""
+        for k, v in list(self.items()):
+            v.normalize()
+            if len(v) == 0:
+                self.pop(k)
+        return self
+
     def __add__(self, other):
         new_multi_asset = deepcopy(self)
         for p in other:
             new_multi_asset[p] = new_multi_asset.get(p, Asset()) + other[p]
-        return new_multi_asset
+        return new_multi_asset.normalize()
 
     def __iadd__(self, other):
         new_item = self + other
         self.update(new_item)
-        return self
+        return self.normalize()
 
     def __sub__(self, other: MultiAsset) -> MultiAsset:
         new_multi_asset = deepcopy(self)
         for p in other:
             new_multi_asset[p] = new_multi_asset.get(p, Asset()) - other[p]
-        return new_multi_asset
+        return new_multi_asset.normalize()
 
     def __eq__(self, other):
         if not isinstance(other, MultiAsset):
@@ -209,6 +240,20 @@ def count(self, criteria=Callable[[ScriptHash, AssetName, int], bool]) -> int:
 
         return count
 
+    @classmethod
+    @limit_primitive_type(dict)
+    def from_primitive(cls: Type[DictBase], value: dict) -> DictBase:
+        res = super().from_primitive(value)
+        # pop empty values
+        for n, v in list(res.items()):
+            if not v:
+                res.pop(n)
+        return res
+
+    def to_shallow_primitive(self) -> dict:
+        x = deepcopy(self).normalize()
+        return super(self.__class__, x).to_shallow_primitive()
+
 
 @typechecked
 @dataclass(repr=False)

pycardano/txbuilder.py

@@ -928,12 +928,16 @@ def _build_tx_body(self) -> TransactionBody:
         )
         return tx_body
 
-    def _build_fake_vkey_witnesses(self) -> List[VerificationKeyWitness]:
+    def _build_required_vkeys(self) -> Set[VerificationKeyHash]:
         vkey_hashes = self._input_vkey_hashes()
         vkey_hashes.update(self._required_signer_vkey_hashes())
         vkey_hashes.update(self._native_scripts_vkey_hashes())
         vkey_hashes.update(self._certificate_vkey_hashes())
         vkey_hashes.update(self._withdrawal_vkey_hashes())
+        return vkey_hashes
+
+    def _build_fake_vkey_witnesses(self) -> List[VerificationKeyWitness]:
+        vkey_hashes = self._build_required_vkeys()
 
         witness_count = self.witness_override or len(vkey_hashes)
 
@@ -1441,6 +1445,7 @@ def build_and_sign(
         auto_validity_start_offset: Optional[int] = None,
         auto_ttl_offset: Optional[int] = None,
         auto_required_signers: Optional[bool] = None,
+        force_skeys: Optional[bool] = False,
     ) -> Transaction:
         """Build a transaction body from all constraints set through the builder and sign the transaction with
         provided signing keys.
@@ -1462,6 +1467,10 @@ def build_and_sign(
             auto_required_signers (Optional[bool]): Automatically add all pubkeyhashes of transaction inputs
                 and the given signers to required signatories (default only for Smart Contract transactions).
                 Manually set required signers will always take precedence.
+            force_skeys (Optional[bool]): Whether to force the use of signing keys for signing the transaction.
+                Default is False, which means that provided signing keys will only be used to sign the transaction if
+                they are actually required by the transaction. This is useful to reduce tx fees by not including
+                unnecessary signatures. If set to True, all provided signing keys will be used to sign the transaction.
 
         Returns:
             Transaction: A signed transaction.
@@ -1483,7 +1492,15 @@ def build_and_sign(
         witness_set = self.build_witness_set(True)
         witness_set.vkey_witnesses = []
 
+        required_vkeys = self._build_required_vkeys()
+
         for signing_key in set(signing_keys):
+            vkey_hash = signing_key.to_verification_key().hash()
+            if not force_skeys and vkey_hash not in required_vkeys:
+                logger.warn(
+                    f"Verification key hash {vkey_hash} is not required for this tx."
+                )
+                continue
             signature = signing_key.sign(tx_body.hash())
             witness_set.vkey_witnesses.append(
                 VerificationKeyWitness(signing_key.to_verification_key(), signature)

pycardano/utils.py

@@ -184,7 +184,7 @@ def min_lovelace_pre_alonzo(
         int: Minimum required lovelace amount for this transaction output.
     """
     if amount is None or isinstance(amount, int) or not amount.multi_asset:
-        return context.protocol_param.min_utxo
+        return context.protocol_param.min_utxo or 1_000_000
 
     b_size = bundle_size(amount.multi_asset)
     utxo_entry_size = 27