Merge #774: [0.18] HF: Dyanfed pruned header now has extra root

9d5eb6a HF: Simplify legacy->dynafed transition max sig size (Gregory Sanders)
3751e14 HF: Publish full dynafed block no matter epoch age (Gregory Sanders)
aec8b6c HF: Dyanfed pruned header now has extra root (Gregory Sanders)

Pull request description:

  Backport of #773

Tree-SHA512: 20085b52d7d7ef0595611a5346ba1af6ecad947aafe84ea5706bd16dae6462239153e94078bfef9e7a52cbc443592fb702902812466b14cd708bbf026e8b1cef

Author: stevenroose

src/chainparams.cpp

@@ -720,7 +720,8 @@ class CLiquidV1Params : public CChainParams {
         // Block signing encumberance script, default of 51 aka OP_TRUE
         std::vector<unsigned char> sign_bytes = ParseHex("5b21026a2a106ec32c8a1e8052e5d02a7b0a150423dbd9b116fc48d46630ff6e6a05b92102791646a8b49c2740352b4495c118d876347bf47d0551c01c4332fdc2df526f1a2102888bda53a424466b0451627df22090143bbf7c060e9eacb1e38426f6b07f2ae12102aee8967150dee220f613de3b239320355a498808084a93eaf39a34dcd62024852102d46e9259d0a0bb2bcbc461a3e68f34adca27b8d08fbe985853992b4b104e27412102e9944e35e5750ab621e098145b8e6cf373c273b7c04747d1aa020be0af40ccd62102f9a9d4b10a6d6c56d8c955c547330c589bb45e774551d46d415e51cd9ad5116321033b421566c124dfde4db9defe4084b7aa4e7f36744758d92806b8f72c2e943309210353dcc6b4cf6ad28aceb7f7b2db92a4bf07ac42d357adf756f3eca790664314b621037f55980af0455e4fb55aad9b85a55068bb6dc4740ea87276dc693f4598db45fa210384001daa88dabd23db878dbb1ce5b4c2a5fa72c3113e3514bf602325d0c37b8e21039056d089f2fe72dbc0a14780b4635b0dc8a1b40b7a59106325dd1bc45cc70493210397ab8ea7b0bf85bc7fc56bb27bf85e75502e94e76a6781c409f3f2ec3d1122192103b00e3b5b77884bf3cae204c4b4eac003601da75f96982ffcb3dcb29c5ee419b92103c1f3c0874cfe34b8131af34699589aacec4093399739ae352e8a46f80a6f68375fae");
         consensus.signblockscript = CScript(sign_bytes.begin(), sign_bytes.end());
-        consensus.max_block_signature_size = 12*74; // 11 signatures plus wiggle room
+        // 11 signatures, 15 pubkeys, plus wiggle room
+        consensus.max_block_signature_size = 12*74+16*33;
         g_signed_blocks = true;
 
         g_con_blockheightinheader = true;

src/dynafed.cpp

@@ -73,7 +73,7 @@ DynaFedParamEntry ComputeNextBlockFullCurrentParameters(const CBlockIndex* pinde
         CScript sh_wsh_fedpeg_program = CScript() << OP_HASH160 << ToByteVector(fedpeg_p2sh) << OP_EQUAL;
 
         // Put them in winning proposal
-        winning_proposal = DynaFedParamEntry(p2wsh_signblock_script, consensus.max_block_signature_size+consensus.signblockscript.size(), sh_wsh_fedpeg_program, consensus.fedpegScript, consensus.first_extension_space);
+        winning_proposal = DynaFedParamEntry(p2wsh_signblock_script, consensus.max_block_signature_size, sh_wsh_fedpeg_program, consensus.fedpegScript, consensus.first_extension_space);
     } else {
         winning_proposal = p_epoch_start->dynafed_params.m_current;
     }
@@ -91,12 +91,13 @@ DynaFedParamEntry ComputeNextBlockCurrentParameters(const CBlockIndex* pindexPre
     const uint32_t epoch_length = consensus.dynamic_epoch_length;
     uint32_t epoch_age = next_height % epoch_length;
 
-    // Return appropriate format based on epoch age
-    if (epoch_age > 0) {
-        // TODO implement "prune" function to remove fields in place and change serialize type
-        return DynaFedParamEntry(entry.m_signblockscript, entry.m_signblock_witness_limit);
-    } else {
+    // Return appropriate format based on epoch age or if we *just* activated
+    // dynafed via BIP9
+    if (epoch_age == 0 || pindexPrev->dynafed_params.IsNull()) {
         return entry;
+    } else {
+        return DynaFedParamEntry(entry.m_signblockscript, entry.m_signblock_witness_limit, entry.CalculateExtraRoot());
+
     }
 }
 

src/miner.cpp

@@ -151,8 +151,8 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc
     fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus());
 
     if (IsDynaFedEnabled(pindexPrev, chainparams.GetConsensus())) {
-        DynaFedParamEntry current_params = ComputeNextBlockCurrentParameters(chainActive.Tip(), chainparams.GetConsensus());
-        DynaFedParams block_params(current_params, proposed_entry ? *proposed_entry : DynaFedParamEntry());
+        const DynaFedParamEntry current_params = ComputeNextBlockCurrentParameters(chainActive.Tip(), chainparams.GetConsensus());
+        const DynaFedParams block_params(current_params, proposed_entry ? *proposed_entry : DynaFedParamEntry());
         pblock->m_dynafed_params = block_params;
         nBlockWeight += ::GetSerializeSize(block_params, PROTOCOL_VERSION)*WITNESS_SCALE_FACTOR;
         nBlockWeight += current_params.m_signblock_witness_limit; // Note witness discount

src/primitives/block.cpp

@@ -43,19 +43,39 @@ std::string CBlock::ToString() const
 
 uint256 DynaFedParamEntry::CalculateRoot() const
 {
-    if (IsNull()) {
+    if (m_serialize_type == 0) {
         return uint256();
     }
 
+    std::vector<uint256> compact_leaves;
+    compact_leaves.push_back(SerializeHash(m_signblockscript, SER_GETHASH, 0));
+    compact_leaves.push_back(SerializeHash(m_signblock_witness_limit, SER_GETHASH, 0));
+    uint256 compact_root(ComputeFastMerkleRoot(compact_leaves));
+
+    uint256 extra_root;
+    if (m_serialize_type ==1 ) {
+        // It's pruned, take the stored value
+        extra_root = m_elided_root;
+    } else if (m_serialize_type == 2) {
+        // It's unpruned, compute the node value
+        extra_root = CalculateExtraRoot();
+    }
+
     std::vector<uint256> leaves;
-    leaves.push_back(SerializeHash(m_signblockscript, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_signblock_witness_limit, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_fedpeg_program, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_fedpegscript, SER_GETHASH, 0));
-    leaves.push_back(SerializeHash(m_extension_space, SER_GETHASH, 0));
+    leaves.push_back(compact_root);
+    leaves.push_back(extra_root);
     return ComputeFastMerkleRoot(leaves);
 }
 
+uint256 DynaFedParamEntry::CalculateExtraRoot() const
+{
+    std::vector<uint256> extra_leaves;
+    extra_leaves.push_back(SerializeHash(m_fedpeg_program, SER_GETHASH, 0));
+    extra_leaves.push_back(SerializeHash(m_fedpegscript, SER_GETHASH, 0));
+    extra_leaves.push_back(SerializeHash(m_extension_space, SER_GETHASH, 0));
+    return ComputeFastMerkleRoot(extra_leaves);
+}
+
 uint256 DynaFedParams::CalculateRoot() const
 {
     if (IsNull()) {

src/primitives/block.h

@@ -58,18 +58,25 @@ class CProof
 class DynaFedParamEntry
 {
 public:
-    unsigned char m_serialize_type; // Determines how it is serialized, defaults to null
+    // Determines how these entries are serialized and stored
+    // 0 -> Null. Only used for proposed parameter "null votes"
+    // 1 -> Pruned. Doesn't have non-signblockscript data. That elided data
+    // is committed to in m_elided_root, and validated against chainstate.
+    // 2 -> Full. Typically only consensus-legal at epoch start.
+    unsigned char m_serialize_type;
+
     CScript m_signblockscript;
     uint32_t m_signblock_witness_limit; // Max block signature witness serialized size
     CScript m_fedpeg_program; // The "scriptPubKey" of the fedpegscript
     CScript m_fedpegscript; // The witnessScript for witness v0 or undefined otherwise.
     // No consensus meaning to the particular bytes, currently we interpret as PAK keys, details in pak.h
     std::vector<std::vector<unsigned char>> m_extension_space;
+    uint256 m_elided_root; // non-zero only when m_serialize_type == 1
 
     // Each constructor sets its own serialization type implicitly based on which
     // arguments are given
     DynaFedParamEntry() { m_signblock_witness_limit = 0; m_serialize_type = 0; };
-    DynaFedParamEntry(const CScript& signblockscript_in, const uint32_t sbs_wit_limit_in) : m_signblockscript(signblockscript_in), m_signblock_witness_limit(sbs_wit_limit_in) { m_serialize_type = 1; };
+    DynaFedParamEntry(const CScript& signblockscript_in, const uint32_t sbs_wit_limit_in, const uint256 elided_root_in) : m_signblockscript(signblockscript_in), m_signblock_witness_limit(sbs_wit_limit_in), m_elided_root(elided_root_in) { m_serialize_type = 1; };
     DynaFedParamEntry(const CScript& signblockscript_in, const uint32_t sbs_wit_limit_in, const CScript& fedpeg_program_in, const CScript& fedpegscript_in, const std::vector<std::vector<unsigned char>> extension_space_in) : m_signblockscript(signblockscript_in), m_signblock_witness_limit(sbs_wit_limit_in), m_fedpeg_program(fedpeg_program_in), m_fedpegscript(fedpegscript_in), m_extension_space(extension_space_in) { m_serialize_type = 2; };
 
     ADD_SERIALIZE_METHODS;
@@ -84,6 +91,7 @@ class DynaFedParamEntry
             case 1:
                 READWRITE(m_signblockscript);
                 READWRITE(m_signblock_witness_limit);
+                READWRITE(m_elided_root);
                 break;
             case 2:
                 READWRITE(m_signblockscript);
@@ -98,6 +106,8 @@ class DynaFedParamEntry
     }
 
     uint256 CalculateRoot() const;
+    // Calculates root for the non-blocksigning merkle fields
+    uint256 CalculateExtraRoot() const;
 
     bool IsNull() const
     {

test/functional/feature_blocksign.py

@@ -82,7 +82,7 @@ def set_test_params(self):
         self.witnessScript = signblockscript # post-dynafed this becomes witnessScript
         self.extra_args = [[
             "-signblockscript={}".format(signblockscript),
-            "-con_max_block_sig_size={}".format(self.required_signers*74),
+            "-con_max_block_sig_size={}".format(self.required_signers*74+self.num_nodes*33),
             "-anyonecanspendaremine=1",
             "-con_dyna_deploy_start=0",
         ]] * self.num_nodes
@@ -216,14 +216,16 @@ def run_test(self):
 
         # Next let's activate dynafed
         blocks_til_dynafed = 431 - self.nodes[0].getblockcount()
+        self.log.info("Activating dynafed")
         self.mine_blocks(blocks_til_dynafed, False)
         self.check_height(111+blocks_til_dynafed)
 
         assert_equal(self.nodes[0].getblockchaininfo()['bip9_softforks']['dynafed']['status'], "active")
 
-        self.log.info("Mine some dynamic federation blocks without and with txns")
-        self.mine_blocks(50, False)
-        self.mine_blocks(50, True)
+        self.log.info("Mine some dynamic federation blocks without txns")
+        self.mine_blocks(10, False)
+        self.log.info("Mine some dynamic federation blocks with txns")
+        self.mine_blocks(10, True)
 
 if __name__ == '__main__':
     BlockSignTest().main()

test/functional/feature_dynafed.py

@@ -34,7 +34,7 @@ def go_to_epoch_end(node):
     blocks_to_mine = epoch_info["epoch_length"] - epoch_info["epoch_age"] - 1
     node.generatetoaddress(blocks_to_mine, node.getnewaddress())
 
-def validate_no_vote_op_true(node, block):
+def validate_no_vote_op_true(node, block, first_dynafed_active_block):
 
     block_info = node.getblock(block)
     dynamic_parameters = block_info["dynamic_parameters"]
@@ -44,14 +44,13 @@ def validate_no_vote_op_true(node, block):
     # signblockscript is now the P2WSH-ification of OP_TRUE
     WSH_OP_TRUE = node.decodescript("51")["segwit"]["hex"]
     assert_equal(dynamic_parameters["current"]["signblockscript"], WSH_OP_TRUE)
-    if block_height % 10 == 0:
+    if block_height % 10 == 0 or first_dynafed_active_block:
         assert_equal(dynamic_parameters["current"]["fedpegscript"], "51")
         assert_equal(dynamic_parameters["current"]["extension_space"], initial_extension)
     else:
         assert_equal(dynamic_parameters["current"]["fedpegscript"], "")
         assert_equal(dynamic_parameters["current"]["extension_space"], [])
-    # TODO workshop this bump, or commit to new value in chainparams instead
-    assert_equal(dynamic_parameters["current"]["max_block_witness"], 75)
+    assert_equal(dynamic_parameters["current"]["max_block_witness"], 74)
     # nothing was proposed, null fields make impossible to be valid blockheader
     # due to script rules requiring bool true on stack
     assert_equal(dynamic_parameters["proposed"]["signblockscript"], "")
@@ -118,8 +117,9 @@ def test_dynafed_activation(self):
         # Next block is first dynamic federation block
         block = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0]
         self.sync_all()
+        # We publish full block on BIP9 transition
         for i in range(self.num_nodes):
-            validate_no_vote_op_true(self.nodes[i], block)
+            validate_no_vote_op_true(self.nodes[i], block, True)
 
     def test_illegal_proposals(self):
 
@@ -148,14 +148,14 @@ def test_no_vote(self):
 
         for i in range(self.num_nodes):
             for block in blocks:
-                validate_no_vote_op_true(self.nodes[i], block)
+                validate_no_vote_op_true(self.nodes[i], block, False)
 
         # Now transition using vanilla getnewblockhex, nothing changed
         block = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0]
         self.sync_all()
 
         for i in range(self.num_nodes):
-            validate_no_vote_op_true(self.nodes[i], block)
+            validate_no_vote_op_true(self.nodes[i], block, False)
 
     def test_under_vote(self):
         self.log.info("Testing failed voting epoch...")
@@ -176,7 +176,7 @@ def test_under_vote(self):
         self.sync_all()
 
         for i in range(self.num_nodes):
-            validate_no_vote_op_true(self.nodes[i], block)
+            validate_no_vote_op_true(self.nodes[i], block, False)
 
     def test_four_fifth_vote(self):
         self.log.info("Testing just-successful transition epoch...")
@@ -192,7 +192,7 @@ def test_four_fifth_vote(self):
                 chain_info = self.nodes[i].getblockchaininfo()
                 fedpeg_info = self.nodes[i].getsidechaininfo()
                 assert_equal(chain_info["current_signblock_hex"], WSH_OP_TRUE)
-                assert_equal(chain_info["max_block_witness"], 75)
+                assert_equal(chain_info["max_block_witness"], 74)
                 assert_equal(chain_info["extension_space"], initial_extension)
                 assert_equal(fedpeg_info["current_fedpegscripts"], ["51", "51"])
 
@@ -210,7 +210,7 @@ def test_four_fifth_vote(self):
             chain_info = self.nodes[i].getblockchaininfo()
             fedpeg_info = self.nodes[i].getsidechaininfo()
             assert_equal(chain_info["current_signblock_hex"], WSH_OP_TRUE)
-            assert_equal(chain_info["max_block_witness"], 75)
+            assert_equal(chain_info["max_block_witness"], 74)
             assert_equal(chain_info["extension_space"], initial_extension)
             assert_equal(fedpeg_info["current_fedpegscripts"], ["51", "51"])
 

test/functional/test_framework/messages.py

@@ -836,22 +836,24 @@ def __repr__(self):
             % (self.challenge, self.solution)
 
 class DynaFedParamEntry:
-    __slots__ = ("m_serialize_type", "m_signblockscript", "m_signblock_witness_limit", "m_fedpeg_program", "m_fedpegscript", "m_extension_space")
+    __slots__ = ("m_serialize_type", "m_signblockscript", "m_signblock_witness_limit", "m_fedpeg_program", "m_fedpegscript", "m_extension_space", "m_elided_root")
 
     # Constructor args will define serialization type:
     # null = 0
     # signblock-related fields = 1, required for m_current on non-epoch-starts
     # all fields = 2, required for epoch starts
-    def __init__(self, m_signblockscript=b"", m_signblock_witness_limit=0, m_fedpeg_program=b"", m_fedpegscript=b"", m_extension_space=[]):
+    def __init__(self, m_signblockscript=b"", m_signblock_witness_limit=0, m_fedpeg_program=b"", m_fedpegscript=b"", m_extension_space=[], m_elided_root=0):
         self.m_signblockscript = m_signblockscript
         self.m_signblock_witness_limit = m_signblock_witness_limit
         self.m_fedpeg_program = m_fedpeg_program
         self.m_fedpegscript = m_fedpegscript
         self.m_extension_space = m_extension_space
         if self.is_null():
             self.m_serialize_type = 0
-        elif m_fedpegscript==b"" and m_extension_space == []:
+        elif m_fedpegscript==b"" and m_fedpeg_program==b"" and m_extension_space == []:
             self.m_serialize_type = 1
+            # We also set the "extra root" in this case
+            self.m_elided_root = m_elided_root
         else:
             self.m_serialize_type = 2
 
@@ -862,6 +864,7 @@ def set_null(self):
         self.m_fedpegscript = b""
         self.m_extension_space = []
         self.m_serialize_type = 0
+        self.m_elided_root = 0
 
     def is_null(self):
         return self.m_signblockscript == b"" and self.m_signblock_witness_limit == 0 and \
@@ -874,6 +877,7 @@ def serialize(self):
         if self.m_serialize_type == 1:
             r += ser_string(self.m_signblockscript)
             r += struct.pack("<I", self.m_signblock_witness_limit)
+            r += ser_uint256(self.m_elided_root)
         elif self.m_serialize_type == 2:
             r += ser_string(self.m_signblockscript)
             r += struct.pack("<I", self.m_signblock_witness_limit)
@@ -889,6 +893,7 @@ def deserialize(self, f):
         if self.m_serialize_type == 1:
             self.m_signblockscript = deser_string(f)
             self.m_signblock_witness_limit = struct.unpack("<I", f.read(4))[0]
+            self.m_elided_root = deser_uint256(f)
         elif self.m_serialize_type == 2:
             self.m_signblockscript = deser_string(f)
             self.m_signblock_witness_limit = struct.unpack("<I", f.read(4))[0]