From b578fcee9b6d15d2b9fddf8a79de97fdbbabd799 Mon Sep 17 00:00:00 2001 From: Chris Pacia Date: Fri, 14 Oct 2016 23:53:43 -0400 Subject: [PATCH 1/3] Add CMerkleBlock class --- bitcoin/core/__init__.py | 118 +++++++++++++++++++++++++++++++++++++ bitcoin/tests/test_core.py | 24 ++++++++ 2 files changed, 142 insertions(+) diff --git a/bitcoin/core/__init__.py b/bitcoin/core/__init__.py index a3d55809..2af90c9c 100644 --- a/bitcoin/core/__init__.py +++ b/bitcoin/core/__init__.py @@ -539,6 +539,123 @@ def GetHash(self): object.__setattr__(self, '_cached_GetHash', _cached_GetHash) return _cached_GetHash +class CMerkleBlock(CBlockHeader): + """ + The merkle block returned to spv clients when a filter is set on the remote peer. + """ + + __slots__ = ['nTX', 'vHashes', 'vFlags'] + + def __init__(self, nVersion=3, hashPrevBlock=b'\x00'*32, hashMerkleRoot=b'\x00'*32, nTime=0, nBits=0, nNonce=0, nTX=0, vHashes=(), vFlags=()): + """Create a new block""" + super(CMerkleBlock, self).__init__(nVersion, hashPrevBlock, hashMerkleRoot, nTime, nBits, nNonce) + + object.__setattr__(self, 'nTX', nTX) + object.__setattr__(self, 'vHashes', vHashes) + object.__setattr__(self, 'vFlags', vFlags) + + @classmethod + def stream_deserialize(cls, f): + + def bits(f, n): + ret = [] + bytes = (ord(b) for b in f.read(n)) + for b in bytes: + for i in xrange(8): + ret.append((b >> i) & 1) + return ret + + self = super(CMerkleBlock, cls).stream_deserialize(f) + + nTX = struct.unpack('> height + + matched_hashes = [] + + def recursive_extract_hashes(height, pos): + parent_of_match = bool(self.vFlags.pop(0)) + if height == 0 or not parent_of_match: + hash = self.vHashes.pop(0) + if height == 0 and parent_of_match: + matched_hashes.append(hash) + return hash + else: + left = recursive_extract_hashes(height - 1, pos * 2) + if pos * 2 + 1 < getTreeWidth(self.nTX, height-1): + right = recursive_extract_hashes(height - 1, pos * 2 + 1) + if left == right: + raise Exception("Invalid Merkle Tree") + else: + right = left + return sha256(sha256(left+right).digest()).digest() + + height = 0 + while getTreeWidth(self.nTX, height) > 1: + height += 1 + + calculated_root = recursive_extract_hashes(height, 0) + if calculated_root == self.get_header().hashMerkleRoot: + return matched_hashes + else: + return None + + def get_header(self): + """Return the block header + Returned header is a new object. + """ + return CBlockHeader(nVersion=self.nVersion, + hashPrevBlock=self.hashPrevBlock, + hashMerkleRoot=self.hashMerkleRoot, + nTime=self.nTime, + nBits=self.nBits, + nNonce=self.nNonce) + + def GetHash(self): + """Return the block hash + Note that this is the hash of the header, not the entire serialized + block. + """ + try: + return self._cached_GetHash + except AttributeError: + _cached_GetHash = self.get_header().GetHash() + object.__setattr__(self, '_cached_GetHash', _cached_GetHash) + return _cached_GetHash + class CoreChainParams(object): """Define consensus-critical parameters of a given instance of the Bitcoin system""" MAX_MONEY = None @@ -766,6 +883,7 @@ def CheckBlock(block, fCheckPoW = True, fCheckMerkleRoot = True, cur_time=None): 'CMutableTransaction', 'CBlockHeader', 'CBlock', + 'CMerkleBlock', 'CoreChainParams', 'CoreMainParams', 'CoreTestNetParams', diff --git a/bitcoin/tests/test_core.py b/bitcoin/tests/test_core.py index 1d612afd..0f95a6d9 100644 --- a/bitcoin/tests/test_core.py +++ b/bitcoin/tests/test_core.py @@ -120,3 +120,27 @@ def test_calc_merkle_root(self): # 99993 four transactions block = CBlock.deserialize(x('01000000acda3db591d5c2c63e8c09e7523a5b0581707ef3e3520d6ca180000000000000701179cb9a9e0fe709cc96261b6b943b31362b61dacba94b03f9b71a06cc2eff7d1c1b4d4c86041b75962f880401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0152ffffffff014034152a01000000434104216220ab283b5e2871c332de670d163fb1b7e509fd67db77997c5568e7c25afd988f19cd5cc5aec6430866ec64b5214826b28e0f7a86458073ff933994b47a5cac0000000001000000042a40ae58b06c3a61ae55dbee05cab546e80c508f71f24ef0cdc9749dac91ea5f000000004a49304602210089c685b37903c4aa62d984929afeaca554d1641f9a668398cd228fb54588f06b0221008a5cfbc5b0a38ba78c4f4341e53272b9cd0e377b2fb740106009b8d7fa693f0b01ffffffff7b999491e30af112b11105cb053bc3633a8a87f44740eb158849a76891ff228b00000000494830450221009a4aa8663ff4017063d2020519f2eade5b4e3e30be69bf9a62b4e6472d1747b2022021ee3b3090b8ce439dbf08a5df31e2dc23d68073ebda45dc573e8a4f74f5cdfc01ffffffffdea82ec2f9e88e0241faa676c13d093030b17c479770c6cc83239436a4327d49000000004a493046022100c29d9de71a34707c52578e355fa0fdc2bb69ce0a957e6b591658a02b1e039d69022100f82c8af79c166a822d305f0832fb800786d831aea419069b3aed97a6edf8f02101fffffffff3e7987da9981c2ae099f97a551783e1b21669ba0bf3aca8fe12896add91a11a0000000049483045022100e332c81781b281a3b35cf75a5a204a2be451746dad8147831255291ebac2604d02205f889a2935270d1bf1ef47db773d68c4d5c6a51bb51f082d3e1c491de63c345601ffffffff0100c817a8040000001976a91420420e56079150b50fb0617dce4c374bd61eccea88ac00000000010000000265a7293b2d69ba51d554cd32ac7586f7fbeaeea06835f26e03a2feab6aec375f000000004a493046022100922361eaafe316003087d355dd3c0ef3d9f44edae661c212a28a91e020408008022100c9b9c84d53d82c0ba9208f695c79eb42a453faea4d19706a8440e1d05e6cff7501fffffffff6971f00725d17c1c531088144b45ed795a307a22d51ca377c6f7f93675bb03a000000008b483045022100d060f2b2f4122edac61a25ea06396fe9135affdabc66d350b5ae1813bc6bf3f302205d8363deef2101fc9f3d528a8b3907e9d29c40772e587dcea12838c574cb80f801410449fce4a25c972a43a6bc67456407a0d4ced782d4cf8c0a35a130d5f65f0561e9f35198349a7c0b4ec79a15fead66bd7642f17cc8c40c5df95f15ac7190c76442ffffffff0200f2052a010000001976a914c3f537bc307c7eda43d86b55695e46047b770ea388ac00cf7b05000000001976a91407bef290008c089a60321b21b1df2d7f2202f40388ac0000000001000000014ab7418ecda2b2531eef0145d4644a4c82a7da1edd285d1aab1ec0595ac06b69000000008c493046022100a796490f89e0ef0326e8460edebff9161da19c36e00c7408608135f72ef0e03e0221009e01ef7bc17cddce8dfda1f1a6d3805c51f9ab2f8f2145793d8e85e0dd6e55300141043e6d26812f24a5a9485c9d40b8712215f0c3a37b0334d76b2c24fcafa587ae5258853b6f49ceeb29cd13ebb76aa79099fad84f516bbba47bd170576b121052f1ffffffff0200a24a04000000001976a9143542e17b6229a25d5b76909f9d28dd6ed9295b2088ac003fab01000000001976a9149cea2b6e3e64ad982c99ebba56a882b9e8a816fe88ac00000000')) self.assertEqual(block.calc_merkle_root(), lx('ff2ecc061ab7f9034ba9cbda612b36313b946b1b2696cc09e70f9e9acb791170')) + +class Test_CMerkleBlock(unittest.TestCase): + def test_serialization(self): + initial_serialized = x('0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b852907000000043612262624047ee87660be1a707519a443b1c1ce3d248cbfc6c15870f6c5daa2019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b6541ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d06820d2a7bc994987302e5b1ac80fc425fe25f8b63169ea78e68fbaaefa59379bbf011d') + block = CMerkleBlock.deserialize(initial_serialized) + self.assertEqual(block.nTX, 7) + self.assertEqual(len(block.vHashes), 4) + self.assertEqual(len(block.vFlags), 8) + serialized = block.serialize() + self.assertEqual(Hash(serialized[0:80]), Hash(initial_serialized[:80])) + self.assertEqual(serialized, initial_serialized) + + def test_GetHash(self): + block = CMerkleBlock.deserialize(x('0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b852907000000043612262624047ee87660be1a707519a443b1c1ce3d248cbfc6c15870f6c5daa2019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b6541ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d06820d2a7bc994987302e5b1ac80fc425fe25f8b63169ea78e68fbaaefa59379bbf011d')) + self.assertEqual(block.GetHash(), lx('000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4')) + + def test_extract_hashes(self): + initial_serialized = x('0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b852907000000043612262624047ee87660be1a707519a443b1c1ce3d248cbfc6c15870f6c5daa2019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b6541ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d06820d2a7bc994987302e5b1ac80fc425fe25f8b63169ea78e68fbaaefa59379bbf011d') + block = CMerkleBlock.deserialize(initial_serialized) + matches = block.get_matched_txs() + self.assertEqual(len(matches), 1) + self.assertEqual(matches[0], x("019f5b01d4195ecbc9398fbf3c3b1fa9bb3183301d7a1fb3bd174fcfa40a2b65")) + + From bcf3448bbd0d8db3e1f65ec0ff45525c797cadcc Mon Sep 17 00:00:00 2001 From: Chris Pacia Date: Sat, 15 Oct 2016 00:01:24 -0400 Subject: [PATCH 2/3] Add merkleblock wire message --- bitcoin/messages.py | 21 +++++++++++++++++++++ bitcoin/tests/test_messages.py | 7 ++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/bitcoin/messages.py b/bitcoin/messages.py index 27e9aafb..2a85caed 100644 --- a/bitcoin/messages.py +++ b/bitcoin/messages.py @@ -386,6 +386,26 @@ def __repr__(self): return "msg_block(block=%s)" % (repr(self.block)) +class msg_merkleblock(MsgSerializable): + command = b"merkleblock" + + def __init__(self, protover=PROTO_VERSION): + super(msg_merkleblock, self).__init__(protover) + self.block = CMerkleBlock() + + @classmethod + def msg_deser(cls, f, protover=PROTO_VERSION): + c = cls() + c.block = CMerkleBlock.stream_deserialize(f) + return c + + def msg_ser(self, f): + self.block.stream_serialize(f) + + def __repr__(self): + return "msg_merkleblock(header=%s)" % (repr(self.block.get_header())) + + class msg_getaddr(MsgSerializable): command = b"getaddr" @@ -511,6 +531,7 @@ def __repr__(self): 'msg_headers', 'msg_tx', 'msg_block', + 'msg_merkleblock', 'msg_getaddr', 'msg_ping', 'msg_pong', diff --git a/bitcoin/tests/test_messages.py b/bitcoin/tests/test_messages.py index 8b51a126..ed958579 100644 --- a/bitcoin/tests/test_messages.py +++ b/bitcoin/tests/test_messages.py @@ -13,7 +13,7 @@ from bitcoin.messages import msg_version, msg_verack, msg_addr, msg_alert, \ msg_inv, msg_getdata, msg_getblocks, msg_getheaders, msg_headers, msg_tx, \ - msg_block, msg_getaddr, msg_ping, msg_pong, msg_mempool, MsgSerializable, \ + msg_block, msg_merkleblock, msg_getaddr, msg_ping, msg_pong, msg_mempool, MsgSerializable, \ msg_notfound, msg_reject import sys @@ -92,6 +92,11 @@ def test_serialization(self): super(Test_msg_block, self).serialization_test(msg_block) +class Test_msg_merkleblock(MessageTestCase): + def test_serialization(self): + super(Test_msg_merkleblock, self).serialization_test(msg_merkleblock) + + class Test_msg_getaddr(MessageTestCase): def test_serialization(self): super(Test_msg_getaddr, self).serialization_test(msg_getaddr) From aedaea20b5d43d639bd5badac32f676add61b492 Mon Sep 17 00:00:00 2001 From: Chris Pacia Date: Sat, 15 Oct 2016 00:04:46 -0400 Subject: [PATCH 3/3] Add filterload wire message --- bitcoin/messages.py | 24 ++++++++++++++++++++++++ bitcoin/tests/test_messages.py | 9 +++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bitcoin/messages.py b/bitcoin/messages.py index 2a85caed..2169396d 100644 --- a/bitcoin/messages.py +++ b/bitcoin/messages.py @@ -33,6 +33,7 @@ from bitcoin.core import * from bitcoin.core.serialize import * from bitcoin.net import * +from bitcoin.bloom import CBloomFilter import bitcoin MSG_TX = 1 @@ -505,6 +506,28 @@ def msg_ser(self, f): def __repr__(self): return "msg_mempool()" + +class msg_filterload(MsgSerializable): + command = b"filterload" + + def __init__(self, protover=PROTO_VERSION, filter=None): + super(msg_filterload, self).__init__(protover) + self.protover = protover + self.filter = filter + + @classmethod + def msg_deser(cls, f, protover=PROTO_VERSION): + c = cls() + c.filter = CBloomFilter.stream_deserialize(f) + return c + + def msg_ser(self, f): + self.filter.stream_serialize(f) + + def __repr__(self): + return "msg_filterload(vData=%i nHashFunctions=%i nTweak=%i nFlags=%i" % (self.filter.vData, self.filter.nHashFunctions, self.filter.nTweak, self.filter.nFlags) + + msg_classes = [msg_version, msg_verack, msg_addr, msg_alert, msg_inv, msg_getdata, msg_notfound, msg_getblocks, msg_getheaders, msg_headers, msg_tx, msg_block, msg_getaddr, msg_ping, @@ -536,6 +559,7 @@ def __repr__(self): 'msg_ping', 'msg_pong', 'msg_mempool', + 'msg_filterload', 'msg_classes', 'messagemap', ) diff --git a/bitcoin/tests/test_messages.py b/bitcoin/tests/test_messages.py index ed958579..265a7230 100644 --- a/bitcoin/tests/test_messages.py +++ b/bitcoin/tests/test_messages.py @@ -13,8 +13,8 @@ from bitcoin.messages import msg_version, msg_verack, msg_addr, msg_alert, \ msg_inv, msg_getdata, msg_getblocks, msg_getheaders, msg_headers, msg_tx, \ - msg_block, msg_merkleblock, msg_getaddr, msg_ping, msg_pong, msg_mempool, MsgSerializable, \ - msg_notfound, msg_reject + msg_block, msg_merkleblock, msg_getaddr, msg_ping, msg_pong, msg_mempool, msg_filterload, \ + MsgSerializable, msg_notfound, msg_reject import sys if sys.version > '3': @@ -122,6 +122,11 @@ def test_serialization(self): super(Test_msg_mempool, self).serialization_test(msg_mempool) +class Test_msg_filterload(MessageTestCase): + def test_serialization(self): + super(Test_msg_filterload, self).serialization_test(msg_filterload) + + class Test_messages(unittest.TestCase): verackbytes = b'\xf9\xbe\xb4\xd9verack\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00]\xf6\xe0\xe2'