Add a --descriptors option to various tests

Adds a --descriptors option globally to the test framework. This will
make the test create and use descriptor wallets. However some tests may
not work with this.

Some tests are modified to work with --descriptors and run with that
option in test_runer:
* wallet_basic.py
* wallet_encryption.py
* wallet_keypool.py
* wallet_keypool_topup.py
* wallet_labels.py
* wallet_avoidreuse.py

Author: achow101

test/functional/rpc_createmultisig.py

@@ -4,13 +4,14 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test multisig RPCs"""
 
+from test_framework.authproxy import JSONRPCException
 from test_framework.descriptors import descsum_create, drop_origins
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_raises_rpc_error,
     assert_equal,
 )
-from test_framework.key import ECPubKey
+from test_framework.key import ECPubKey, ECKey, bytes_to_wif
 
 import binascii
 import decimal
@@ -28,10 +29,14 @@ def skip_test_if_missing_module(self):
         self.skip_if_no_wallet()
 
     def get_keys(self):
+        self.pub = []
+        self.priv = []
         node0, node1, node2 = self.nodes
-        add = [node1.getnewaddress() for _ in range(self.nkeys)]
-        self.pub = [node1.getaddressinfo(a)["pubkey"] for a in add]
-        self.priv = [node1.dumpprivkey(a) for a in add]
+        for _ in range(self.nkeys):
+            k = ECKey()
+            k.generate()
+            self.pub.append(k.get_pubkey().get_bytes().hex())
+            self.priv.append(bytes_to_wif(k.get_bytes(), k.is_compressed))
         self.final = node2.getnewaddress()
 
     def run_test(self):
@@ -64,17 +69,20 @@ def run_test(self):
         pk_obj.compressed = False
         pk2 = binascii.hexlify(pk_obj.get_bytes()).decode()
 
+        node0.createwallet(wallet_name='wmulti0', disable_private_keys=True)
+        wmulti0 = node0.get_wallet_rpc('wmulti0')
+
         # Check all permutations of keys because order matters apparently
         for keys in itertools.permutations([pk0, pk1, pk2]):
             # Results should be the same as this legacy one
             legacy_addr = node0.createmultisig(2, keys, 'legacy')['address']
-            assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'legacy')['address'])
+            assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'legacy')['address'])
 
             # Generate addresses with the segwit types. These should all make legacy addresses
-            assert_equal(legacy_addr, node0.createmultisig(2, keys, 'bech32')['address'])
-            assert_equal(legacy_addr, node0.createmultisig(2, keys, 'p2sh-segwit')['address'])
-            assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'bech32')['address'])
-            assert_equal(legacy_addr, node0.addmultisigaddress(2, keys, '', 'p2sh-segwit')['address'])
+            assert_equal(legacy_addr, wmulti0.createmultisig(2, keys, 'bech32')['address'])
+            assert_equal(legacy_addr, wmulti0.createmultisig(2, keys, 'p2sh-segwit')['address'])
+            assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'bech32')['address'])
+            assert_equal(legacy_addr, wmulti0.addmultisigaddress(2, keys, '', 'p2sh-segwit')['address'])
 
         self.log.info('Testing sortedmulti descriptors with BIP 67 test vectors')
         with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_bip67.json'), encoding='utf-8') as f:
@@ -89,6 +97,8 @@ def run_test(self):
             assert_equal(self.nodes[0].deriveaddresses(sorted_key_desc)[0], t['address'])
 
     def check_addmultisigaddress_errors(self):
+        if self.options.descriptors:
+            return
         self.log.info('Check that addmultisigaddress fails when the private keys are missing')
         addresses = [self.nodes[1].getnewaddress(address_type='legacy') for _ in range(2)]
         assert_raises_rpc_error(-5, 'no full public key for address', lambda: self.nodes[0].addmultisigaddress(nrequired=1, keys=addresses))
@@ -115,6 +125,15 @@ def checkbalances(self):
 
     def do_multisig(self):
         node0, node1, node2 = self.nodes
+        if 'wmulti' not in node1.listwallets():
+            try:
+                node1.loadwallet('wmulti')
+            except JSONRPCException as e:
+                if e.error['code'] == -18 and 'Wallet wmulti not found' in e.error['message']:
+                    node1.createwallet(wallet_name='wmulti', disable_private_keys=True)
+                else:
+                    raise
+        wmulti = node1.get_wallet_rpc('wmulti')
 
         # Construct the expected descriptor
         desc = 'multi({},{})'.format(self.nsigs, ','.join(self.pub))
@@ -134,7 +153,7 @@ def do_multisig(self):
             assert madd[0:4] == "bcrt"  # actually a bech32 address
 
         # compare against addmultisigaddress
-        msigw = node1.addmultisigaddress(self.nsigs, self.pub, None, self.output_type)
+        msigw = wmulti.addmultisigaddress(self.nsigs, self.pub, None, self.output_type)
         maddw = msigw["address"]
         mredeemw = msigw["redeemScript"]
         assert_equal(desc, drop_origins(msigw['descriptor']))
@@ -194,6 +213,8 @@ def do_multisig(self):
         txinfo = node0.getrawtransaction(tx, True, blk)
         self.log.info("n/m=%d/%d %s size=%d vsize=%d weight=%d" % (self.nsigs, self.nkeys, self.output_type, txinfo["size"], txinfo["vsize"], txinfo["weight"]))
 
+        wmulti.unloadwallet()
+
 
 if __name__ == '__main__':
     RpcCreateMultiSigTest().main()

test/functional/rpc_psbt.py

@@ -48,18 +48,23 @@ def test_utxo_conversion(self):
         disconnect_nodes(offline_node, 2)
         disconnect_nodes(mining_node, 0)
 
+        # Create watchonly on online_node
+        online_node.createwallet(wallet_name='wonline', disable_private_keys=True)
+        wonline = online_node.get_wallet_rpc('wonline')
+        w2 = online_node.get_wallet_rpc('')
+
         # Mine a transaction that credits the offline address
         offline_addr = offline_node.getnewaddress(address_type="p2sh-segwit")
-        online_addr = online_node.getnewaddress(address_type="p2sh-segwit")
-        online_node.importaddress(offline_addr, "", False)
+        online_addr = w2.getnewaddress(address_type="p2sh-segwit")
+        wonline.importaddress(offline_addr, "", False)
         mining_node.sendtoaddress(address=offline_addr, amount=1.0)
         mining_node.generate(nblocks=1)
         self.sync_blocks([mining_node, online_node])
 
         # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO)
-        utxos = online_node.listunspent(addresses=[offline_addr])
-        raw = online_node.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}])
-        psbt = online_node.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"]
+        utxos = wonline.listunspent(addresses=[offline_addr])
+        raw = wonline.createrawtransaction([{"txid":utxos[0]["txid"], "vout":utxos[0]["vout"]}],[{online_addr:0.9999}])
+        psbt = wonline.walletprocesspsbt(online_node.converttopsbt(raw))["psbt"]
         assert "non_witness_utxo" in mining_node.decodepsbt(psbt)["inputs"][0]
 
         # Have the offline node sign the PSBT (which will update the UTXO to segwit)
@@ -72,6 +77,8 @@ def test_utxo_conversion(self):
         self.sync_blocks([mining_node, online_node])
         assert_equal(online_node.gettxout(txid,0)["confirmations"], 1)
 
+        wonline.unloadwallet()
+
         # Reconnect
         connect_nodes(self.nodes[0], 1)
         connect_nodes(self.nodes[0], 2)
@@ -89,13 +96,23 @@ def run_test(self):
         final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex']
         self.nodes[0].sendrawtransaction(final_tx)
 
-        # Create p2sh, p2wpkh, and p2wsh addresses
+        # Get pubkeys
         pubkey0 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress())['pubkey']
         pubkey1 = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey']
         pubkey2 = self.nodes[2].getaddressinfo(self.nodes[2].getnewaddress())['pubkey']
-        p2sh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "legacy")['address']
-        p2wsh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "bech32")['address']
-        p2sh_p2wsh = self.nodes[1].addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "p2sh-segwit")['address']
+
+        # Setup watchonly wallets
+        self.nodes[2].createwallet(wallet_name='wmulti', disable_private_keys=True)
+        wmulti = self.nodes[2].get_wallet_rpc('wmulti')
+
+        # Create all the addresses
+        p2sh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "legacy")['address']
+        p2wsh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "bech32")['address']
+        p2sh_p2wsh = wmulti.addmultisigaddress(2, [pubkey0, pubkey1, pubkey2], "", "p2sh-segwit")['address']
+        if not self.options.descriptors:
+            wmulti.importaddress(p2sh)
+            wmulti.importaddress(p2wsh)
+            wmulti.importaddress(p2sh_p2wsh)
         p2wpkh = self.nodes[1].getnewaddress("", "bech32")
         p2pkh = self.nodes[1].getnewaddress("", "legacy")
         p2sh_p2wpkh = self.nodes[1].getnewaddress("", "p2sh-segwit")
@@ -146,11 +163,14 @@ def run_test(self):
         assert_raises_rpc_error(-4, "Fee exceeds maximum configured by -maxtxfee", self.nodes[1].walletcreatefundedpsbt, [{"txid":txid,"vout":p2wpkh_pos},{"txid":txid,"vout":p2sh_p2wpkh_pos},{"txid":txid,"vout":p2pkh_pos}], {self.nodes[1].getnewaddress():29.99}, 0, {"feeRate": 10})
 
         # partially sign multisig things with node 1
-        psbtx = self.nodes[1].walletcreatefundedpsbt([{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], {self.nodes[1].getnewaddress():29.99})['psbt']
+        psbtx = wmulti.walletcreatefundedpsbt(inputs=[{"txid":txid,"vout":p2wsh_pos},{"txid":txid,"vout":p2sh_pos},{"txid":txid,"vout":p2sh_p2wsh_pos}], outputs={self.nodes[1].getnewaddress():29.99}, options={'changeAddress': self.nodes[1].getrawchangeaddress()})['psbt']
         walletprocesspsbt_out = self.nodes[1].walletprocesspsbt(psbtx)
         psbtx = walletprocesspsbt_out['psbt']
         assert_equal(walletprocesspsbt_out['complete'], False)
 
+        # Unload wmulti, we don't need it anymore
+        wmulti.unloadwallet()
+
         # partially sign with node 2. This should be complete and sendable
         walletprocesspsbt_out = self.nodes[2].walletprocesspsbt(psbtx)
         assert_equal(walletprocesspsbt_out['complete'], True)
@@ -297,7 +317,7 @@ def run_test(self):
 
         # Signer tests
         for i, signer in enumerate(signers):
-            self.nodes[2].createwallet("wallet{}".format(i))
+            self.nodes[2].createwallet(wallet_name="wallet{}".format(i))
             wrpc = self.nodes[2].get_wallet_rpc("wallet{}".format(i))
             for key in signer['privkeys']:
                 wrpc.importprivkey(key)

test/functional/test_framework/key.py

@@ -8,6 +8,8 @@
 anything but tests."""
 import random
 
+from .address import byte_to_base58
+
 def modinv(a, n):
     """Compute the modular inverse of a modulo n
 
@@ -384,3 +386,14 @@ def sign_ecdsa(self, msg, low_s=True):
         rb = r.to_bytes((r.bit_length() + 8) // 8, 'big')
         sb = s.to_bytes((s.bit_length() + 8) // 8, 'big')
         return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb
+
+def bytes_to_wif(b, compressed=True):
+    if compressed:
+        b += b'\x01'
+    return byte_to_base58(b, 239)
+
+def generate_wif_key():
+    # Makes a WIF privkey for imports
+    k = ECKey()
+    k.generate()
+    return bytes_to_wif(k.get_bytes(), k.is_compressed)

test/functional/test_framework/test_framework.py

@@ -165,6 +165,8 @@ def parse_args(self):
                             help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown, valgrind 3.14 or later required")
         parser.add_argument("--randomseed", type=int,
                             help="set a random seed for deterministically reproducing a previous test run")
+        parser.add_argument("--descriptors", default=False, action="store_true",
+                            help="Run test using a descriptor wallet")
         self.add_options(parser)
         self.options = parser.parse_args()
 
@@ -333,11 +335,23 @@ def setup_network(self):
 
     def setup_nodes(self):
         """Override this method to customize test node setup"""
-        extra_args = None
+        extra_args = [[]] * self.num_nodes
+        wallets = [[]] * self.num_nodes
         if hasattr(self, "extra_args"):
             extra_args = self.extra_args
+            wallets = [[x for x in eargs if x.startswith('-wallet=')] for eargs in extra_args]
+        extra_args = [x + ['-nowallet'] for x in extra_args]
         self.add_nodes(self.num_nodes, extra_args)
         self.start_nodes()
+        for i, n in enumerate(self.nodes):
+            n.extra_args.pop()
+            if '-wallet=0' in n.extra_args or '-nowallet' in n.extra_args or '-disablewallet' in n.extra_args or not self.is_wallet_compiled():
+                continue
+            if '-wallet=' not in wallets[i] and not any([x.startswith('-wallet=') for x in wallets[i]]):
+                wallets[i].append('-wallet=')
+            for w in wallets[i]:
+                wallet_name = w.split('=', 1)[1]
+                n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors)
         self.import_deterministic_coinbase_privkeys()
         if not self.setup_clean_chain:
             for n in self.nodes:
@@ -408,6 +422,7 @@ def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None, bi
                 use_cli=self.options.usecli,
                 start_perf=self.options.perf,
                 use_valgrind=self.options.valgrind,
+                descriptors=self.options.descriptors,
             ))
 
     def start_node(self, i, *args, **kwargs):
@@ -547,6 +562,7 @@ def _initialize_chain(self):
                     bitcoin_cli=self.options.bitcoincli,
                     coverage_dir=None,
                     cwd=self.options.tmpdir,
+                    descriptors=self.options.descriptors,
                 ))
             self.start_node(CACHE_NODE_ID)
             cache_node = self.nodes[CACHE_NODE_ID]

test/functional/test_framework/test_node.py

@@ -62,7 +62,7 @@ class TestNode():
     To make things easier for the test writer, any unrecognised messages will
     be dispatched to the RPC connection."""
 
-    def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None):
+    def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cli, coverage_dir, cwd, extra_conf=None, extra_args=None, use_cli=False, start_perf=False, use_valgrind=False, version=None, descriptors=False):
         """
         Kwargs:
             start_perf (bool): If True, begin profiling the node with `perf` as soon as
@@ -80,6 +80,7 @@ def __init__(self, i, datadir, *, chain, rpchost, timewait, bitcoind, bitcoin_cl
         self.binary = bitcoind
         self.coverage_dir = coverage_dir
         self.cwd = cwd
+        self.descriptors = descriptors
         if extra_conf is not None:
             append_config(datadir, extra_conf)
         # Most callers will just need to add extra args to the standard list below.
@@ -171,10 +172,10 @@ def __del__(self):
     def __getattr__(self, name):
         """Dispatches any unrecognised messages to the RPC connection or a CLI instance."""
         if self.use_cli:
-            return getattr(RPCOverloadWrapper(self.cli, True), name)
+            return getattr(RPCOverloadWrapper(self.cli, True, self.descriptors), name)
         else:
             assert self.rpc_connected and self.rpc is not None, self._node_msg("Error: no RPC connection")
-            return getattr(RPCOverloadWrapper(self.rpc), name)
+            return getattr(RPCOverloadWrapper(self.rpc, descriptors=self.descriptors), name)
 
     def start(self, extra_args=None, *, cwd=None, stdout=None, stderr=None, **kwargs):
         """Start the node."""
@@ -266,11 +267,11 @@ def generate(self, nblocks, maxtries=1000000):
 
     def get_wallet_rpc(self, wallet_name):
         if self.use_cli:
-            return RPCOverloadWrapper(self.cli("-rpcwallet={}".format(wallet_name)), True)
+            return RPCOverloadWrapper(self.cli("-rpcwallet={}".format(wallet_name)), True, self.descriptors)
         else:
             assert self.rpc_connected and self.rpc, self._node_msg("RPC not connected")
             wallet_path = "wallet/{}".format(urllib.parse.quote(wallet_name))
-            return RPCOverloadWrapper(self.rpc / wallet_path)
+            return RPCOverloadWrapper(self.rpc / wallet_path, descriptors=self.descriptors)
 
     def stop_node(self, expected_stderr='', wait=0):
         """Stop the node."""
@@ -598,13 +599,28 @@ def send_cli(self, command=None, *args, **kwargs):
             return cli_stdout.rstrip("\n")
 
 class RPCOverloadWrapper():
-    def __init__(self, rpc, cli=False):
+    def __init__(self, rpc, cli=False, descriptors=False):
         self.rpc = rpc
         self.is_cli = cli
+        self.descriptors = descriptors
 
     def __getattr__(self, name):
         return getattr(self.rpc, name)
 
+    def createwallet(self, wallet_name, disable_private_keys=None, blank=None, passphrase=None, avoid_reuse=None, descriptors=None):
+        if self.is_cli:
+            if disable_private_keys is None:
+                disable_private_keys = 'null'
+            if blank is None:
+                blank = 'null'
+            if passphrase is None:
+                passphrase = ''
+            if avoid_reuse is None:
+                avoid_reuse = 'null'
+        if descriptors is None:
+            descriptors = self.descriptors
+        return self.__getattr__('createwallet')(wallet_name, disable_private_keys, blank, passphrase, avoid_reuse, descriptors)
+
     def importprivkey(self, privkey, label=None, rescan=None):
         wallet_info = self.getwalletinfo()
         if self.is_cli: