From ef72e9bd4124645fe2d00521a71c1c298d760225 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 21 Oct 2019 18:11:27 +0200 Subject: [PATCH 01/96] doc: nChainTx needs to become a 64-bit earlier due to SegWit --- src/chain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chain.h b/src/chain.h index 04a5db5a17..e452e0ffff 100644 --- a/src/chain.h +++ b/src/chain.h @@ -170,7 +170,7 @@ class CBlockIndex //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. - //! Change to 64-bit type when necessary; won't happen before 2030 + //! Change to 64-bit type before 2024 (assuming worst case of 60 byte transactions). //! //! Note: this value is faked during use of a UTXO snapshot because we don't //! have the underlying block data available during snapshot load. From 1f20501efce041d34e63ab9a11359bedf4a82cd5 Mon Sep 17 00:00:00 2001 From: Michael Dietz Date: Wed, 26 May 2021 10:36:02 -0400 Subject: [PATCH 02/96] test: add functional test for multisig flow with descriptor wallets and PSBTs --- test/functional/test_runner.py | 1 + .../wallet_multisig_descriptor_psbt.py | 143 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100755 test/functional/wallet_multisig_descriptor_psbt.py diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 1a1a6a263a..ade8bf2aa9 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -204,6 +204,7 @@ 'feature_assumevalid.py', 'example_test.py', 'wallet_txn_doublespend.py --legacy-wallet', + 'wallet_multisig_descriptor_psbt.py', 'wallet_txn_doublespend.py --descriptors', 'feature_backwards_compatibility.py --legacy-wallet', 'feature_backwards_compatibility.py --descriptors', diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py new file mode 100755 index 0000000000..e8b9ad8f2a --- /dev/null +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test a basic M-of-N multisig setup between multiple people using descriptor wallets and PSBTs, as well as a signing flow. + +This is meant to be documentation as much as functional tests, so it is kept as simple and readable as possible. +""" + +from test_framework.address import base58_to_byte +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_approx, + assert_equal, +) + + +class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 3 + self.setup_clean_chain = True + self.wallet_names = [] + self.extra_args = [["-keypool=100"]] * self.num_nodes + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + def _get_xpub(self, wallet): + """Extract the wallet's xpubs using `listdescriptors` and pick the one from the `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses).""" + descriptor = next(filter(lambda d: d["desc"].startswith("pkh"), wallet.listdescriptors()["descriptors"])) + return descriptor["desc"].split("]")[-1].split("/")[0] + + def _check_psbt(self, psbt, to, value, multisig): + """Helper method for any of the N participants to check the psbt with decodepsbt and verify it is OK before signing.""" + tx = multisig.decodepsbt(psbt)["tx"] + amount = 0 + for vout in tx["vout"]: + address = vout["scriptPubKey"]["address"] + assert_equal(multisig.getaddressinfo(address)["ischange"], address != to) + if address == to: + amount += vout["value"] + assert_approx(amount, float(value), vspan=0.001) + + def generate_and_exchange_xpubs(self, participants): + """Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet. Avoid reusing this wallet for any other purpose..""" + for i, node in enumerate(participants): + node.createwallet(wallet_name=f"participant_{i}", descriptors=True) + yield self._get_xpub(node.get_wallet_rpc(f"participant_{i}")) + + def participants_import_descriptors(self, participants, xpubs): + """The multisig is created by importing the following descriptors. The resulting wallet is watch-only and every participant can do this.""" + # some simple validation + assert_equal(len(xpubs), self.N) + # a sanity-check/assertion, this will throw if the base58 checksum of any of the provided xpubs are invalid + for xpub in xpubs: + base58_to_byte(xpub) + + for i, node in enumerate(participants): + node.createwallet(wallet_name=f"{self.name}_{i}", blank=True, descriptors=True, disable_private_keys=True) + multisig = node.get_wallet_rpc(f"{self.name}_{i}") + external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/{0}/*,'.join(xpubs)}/{0}/*))") + internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/{1}/*,'.join(xpubs)}/{1}/*))") + result = multisig.importdescriptors([ + { # receiving addresses (internal: False) + "desc": external["descriptor"], + "active": True, + "internal": False, + "timestamp": "now", + }, + { # change addresses (internal: True) + "desc": internal["descriptor"], + "active": True, + "internal": True, + "timestamp": "now", + }, + ]) + assert all(r["success"] for r in result) + + def get_multisig_receiving_address(self): + """We will send funds to the resulting address (every participant should get the same addresses).""" + multisig = self.nodes[0].get_wallet_rpc(f"{self.name}_{0}") + receiving_address = multisig.getnewaddress() + for i in range(1, self.N): + assert_equal(receiving_address, self.nodes[i].get_wallet_rpc(f"{self.name}_{i}").getnewaddress()) + return receiving_address + + def make_sending_transaction(self, to, value): + """Make a sending transaction, created using walletcreatefundedpsbt (anyone can initiate this).""" + return self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + + def run_test(self): + self.M = 2 + self.N = self.num_nodes + self.name = f"{self.M}_of_{self.N}_multisig" + self.log.info(f"Testing {self.name}...") + + self.log.info("Generate and exchange xpubs...") + xpubs = list(self.generate_and_exchange_xpubs(self.nodes)) + + self.log.info("Every participant imports the following descriptors to create the watch-only multisig...") + self.participants_import_descriptors(self.nodes, xpubs) + + self.log.info("Get a mature utxo to send to the multisig...") + coordinator_wallet = self.nodes[0].get_wallet_rpc(f"participant_{0}") + coordinator_wallet.generatetoaddress(101, coordinator_wallet.getnewaddress()) + + deposit_amount = 6.15 + multisig_receiving_address = self.get_multisig_receiving_address() + self.log.info("Send funds to the resulting multisig receiving address...") + coordinator_wallet.sendtoaddress(multisig_receiving_address, deposit_amount) + self.nodes[0].generate(1) + self.sync_all() + for n in range(self.N): + assert_approx(self.nodes[n].get_wallet_rpc(f"{self.name}_{n}").getbalance(), deposit_amount, vspan=0.001) + + self.log.info("Send a transaction from the multisig!") + to = self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getnewaddress() + value = 1 + psbt = self.make_sending_transaction(to, value) + + psbts = [] + self.log.info("At least M users check the psbt with decodepsbt and (if OK) signs it with walletprocesspsbt...") + for m in range(self.M): + signers_multisig = self.nodes[m].get_wallet_rpc(f"{self.name}_{m}") + self._check_psbt(psbt["psbt"], to, value, signers_multisig) + signing_wallet = self.nodes[m].get_wallet_rpc(f"participant_{m}") + partially_signed_psbt = signing_wallet.walletprocesspsbt(psbt["psbt"]) + psbts.append(partially_signed_psbt["psbt"]) + + self.log.info("Collect the signed PSBTs with combinepsbt, finalizepsbt, then broadcast the resulting transaction...") + combined = coordinator_wallet.combinepsbt(psbts) + finalized = coordinator_wallet.finalizepsbt(combined) + coordinator_wallet.sendrawtransaction(finalized["hex"]) + + self.log.info("Check that balances are correct after the transaction has been included in a block.") + self.nodes[0].generate(1) + self.sync_all() + assert_approx(self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").getbalance(), deposit_amount - value, vspan=0.001) + assert_equal(self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getbalance(), value) + +if __name__ == "__main__": + WalletMultisigDescriptorPSBTTest().main() From 17dd6573008c8aca9fc0da9419225c85a4f94330 Mon Sep 17 00:00:00 2001 From: Michael Dietz Date: Wed, 26 May 2021 10:37:24 -0400 Subject: [PATCH 03/96] doc: M-of-N multisig using descriptor wallets and PSBTs, as well as a signing flow --- doc/descriptors.md | 26 ++++++++++++++++++++++++++ doc/psbt.md | 3 +++ 2 files changed, 29 insertions(+) diff --git a/doc/descriptors.md b/doc/descriptors.md index e27ff87546..ab04f64b6b 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -139,6 +139,32 @@ Key order does not matter for `sortedmulti()`. `sortedmulti()` behaves in the sa as `multi()` does but the keys are reordered in the resulting script such that they are lexicographically ordered as described in BIP67. +#### Basic multisig example + +For a good example of a basic M-of-N multisig between multiple participants using descriptor +wallets and PSBTs, as well as a signing flow, see [this functional test](/test/functional/wallet_multisig_descriptor_psbt.py). +The basic steps are: + + 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet. + Avoid reusing this wallet for any other purpose. Hint: extract the wallet's xpubs using `listdescriptors` + and pick the one from the `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses) + 2. Create a watch-only descriptor wallet (blank, private keys disabled). Now the multisig is created by importing the two descriptors: + `wsh(sortedmulti(,XPUB1/0/*,XPUB2/0/*,…,XPUBN/0/*))` and `wsh(sortedmulti(,XPUB1/1/*,XPUB2/1/*,…,XPUBN/1/*))` + (one descriptor w/ `0` for receiving addresses and another w/ `1` for change). Every participant does this + 3. A receiving address is generated for the multisig. As a check to ensure step 2 was done correctly, every participant + should verify they get the same addresses + 4. Funds are sent to the resulting address + 5. A sending transaction is created using `walletcreatefundedpsbt` (anyone can initiate this). It is simple to do this in + the GUI by going to the `Send` tab in the multisig wallet and creating an unsigned transaction (PSBT) + 6. At least `M` users check the PSBT with `decodepsbt` and (if OK) signs it with `walletprocesspsbt`. It is simple to do + this in the GUI by Loading the PSBT from file and signing it + 7. The signed PSBTs are collected with `combinepsbt`, finalized w/ `finalizepsbt`, and + then the resulting transaction is broadcasted to the network + 8. Checks that balances are correct after the transaction has been included in a block + +[The test](/test/functional/wallet_multisig_descriptor_psbt.py) is meant to be documentation as much as it is a functional test, so +it is kept as simple and readable as possible. + ### BIP32 derived keys and chains Most modern wallet software and hardware uses keys that are derived using diff --git a/doc/psbt.md b/doc/psbt.md index c411b31d5d..0f31cb8eba 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -92,6 +92,9 @@ hardware implementations will typically implement multiple roles simultaneously. #### Multisig with multiple Bitcoin Core instances +For a quick start see [Basic M-of-N multisig example using descriptor wallets and PSBTs](./descriptors.md#basic-multisig-example). +If you are using legacy wallets feel free to continue with the example provided here. + Alice, Bob, and Carol want to create a 2-of-3 multisig address. They're all using Bitcoin Core. We assume their wallets only contain the multisig funds. In case they also have a personal wallet, this can be accomplished through the From e05cd0546a155afcd45c43ce730c4abecd40dfed Mon Sep 17 00:00:00 2001 From: Michael Dietz Date: Wed, 26 May 2021 10:38:12 -0400 Subject: [PATCH 04/96] doc: add another signing flow for multisig with descriptor wallets and PSBTs --- doc/descriptors.md | 5 +++++ .../wallet_multisig_descriptor_psbt.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/doc/descriptors.md b/doc/descriptors.md index ab04f64b6b..75b9582343 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -162,6 +162,11 @@ The basic steps are: then the resulting transaction is broadcasted to the network 8. Checks that balances are correct after the transaction has been included in a block +You may prefer a daisy chained signing flow where each participant signs the PSBT one after another until +the PSBT has been signed `M` times and is "complete." For the most part, the steps above remain the same, except (6, 7) +change slightly from signing the original PSBT in parallel to signing it in series. `combinepsbt` is not necessary with +this signing flow and the last (`m`th) signer can just broadcast the PSBT after signing. Note that a parallel signing flow may be +preferable in cases where there are more signers. This signing flow is also included in the test / Python example. [The test](/test/functional/wallet_multisig_descriptor_psbt.py) is meant to be documentation as much as it is a functional test, so it is kept as simple and readable as possible. diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index e8b9ad8f2a..59a0b94a9f 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -139,5 +139,21 @@ def run_test(self): assert_approx(self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").getbalance(), deposit_amount - value, vspan=0.001) assert_equal(self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getbalance(), value) + self.log.info("Send another transaction from the multisig, this time with a daisy chained signing flow (one after another in series)!") + psbt = self.make_sending_transaction(to, value) + for m in range(self.M): + signing_wallet = self.nodes[m].get_wallet_rpc(f"participant_{m}") + psbt = signing_wallet.walletprocesspsbt(psbt["psbt"]) + assert_equal(psbt["complete"], m == self.M - 1) + finalized = coordinator_wallet.finalizepsbt(psbt["psbt"]) + coordinator_wallet.sendrawtransaction(finalized["hex"]) + + self.log.info("Check that balances are correct after the transaction has been included in a block.") + self.nodes[0].generate(1) + self.sync_all() + assert_approx(self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").getbalance(), deposit_amount - (value * 2), vspan=0.001) + assert_equal(self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getbalance(), value * 2) + + if __name__ == "__main__": WalletMultisigDescriptorPSBTTest().main() From d047ed729f1d4732d23324fc76849f3c657cdbe4 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Tue, 24 Aug 2021 11:19:43 +0200 Subject: [PATCH 05/96] external_signer: improve fingerprint matching logic (stop on first match) --- src/external_signer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/external_signer.cpp b/src/external_signer.cpp index d6388b759a..75070899c6 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -75,15 +76,14 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str ssTx << psbtx; // Check if signer fingerprint matches any input master key fingerprint - bool match = false; - for (unsigned int i = 0; i < psbtx.inputs.size(); ++i) { - const PSBTInput& input = psbtx.inputs[i]; + auto matches_signer_fingerprint = [&](const PSBTInput& input) { for (const auto& entry : input.hd_keypaths) { - if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) match = true; + if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) return true; } - } + return false; + }; - if (!match) { + if (!std::any_of(psbtx.inputs.begin(), psbtx.inputs.end(), matches_signer_fingerprint)) { error = "Signer fingerprint " + m_fingerprint + " does not match any of the inputs:\n" + EncodeBase64(ssTx.str()); return false; } From f9479e4626f6b5126ff8cdab3a7e718c609429ef Mon Sep 17 00:00:00 2001 From: Michael Dietz Date: Fri, 3 Sep 2021 13:36:07 -0500 Subject: [PATCH 06/96] test, doc: basic M-of-N multisig minor cleanup and clarifications wallet_multisig_descriptor_psbt.py is refactored in this commit. While behavior doesn't change we do cleanup the way wallets are accessed throughout the test as this is done a lot for the various signers and their multisigs. We also get rid of some shallow methods and instead inline them for improved readability. descriptors.md is improved to be more explicit about which wallet (ie the signer or multisig) is required for each step. --- doc/descriptors.md | 22 ++--- .../wallet_multisig_descriptor_psbt.py | 90 ++++++++++--------- 2 files changed, 59 insertions(+), 53 deletions(-) diff --git a/doc/descriptors.md b/doc/descriptors.md index 75b9582343..f38caa81cb 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -145,22 +145,24 @@ For a good example of a basic M-of-N multisig between multiple participants usin wallets and PSBTs, as well as a signing flow, see [this functional test](/test/functional/wallet_multisig_descriptor_psbt.py). The basic steps are: - 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet. - Avoid reusing this wallet for any other purpose. Hint: extract the wallet's xpubs using `listdescriptors` - and pick the one from the `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses) + 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet which we will refer to as + the participant's signer wallet. Avoid reusing this wallet for any purpose other than signing transactions from the + corresponding multisig we are about to create. Hint: extract the wallet's xpubs using `listdescriptors` and pick the one from the + `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses) 2. Create a watch-only descriptor wallet (blank, private keys disabled). Now the multisig is created by importing the two descriptors: `wsh(sortedmulti(,XPUB1/0/*,XPUB2/0/*,…,XPUBN/0/*))` and `wsh(sortedmulti(,XPUB1/1/*,XPUB2/1/*,…,XPUBN/1/*))` (one descriptor w/ `0` for receiving addresses and another w/ `1` for change). Every participant does this 3. A receiving address is generated for the multisig. As a check to ensure step 2 was done correctly, every participant should verify they get the same addresses 4. Funds are sent to the resulting address - 5. A sending transaction is created using `walletcreatefundedpsbt` (anyone can initiate this). It is simple to do this in - the GUI by going to the `Send` tab in the multisig wallet and creating an unsigned transaction (PSBT) - 6. At least `M` users check the PSBT with `decodepsbt` and (if OK) signs it with `walletprocesspsbt`. It is simple to do - this in the GUI by Loading the PSBT from file and signing it - 7. The signed PSBTs are collected with `combinepsbt`, finalized w/ `finalizepsbt`, and - then the resulting transaction is broadcasted to the network - 8. Checks that balances are correct after the transaction has been included in a block + 5. A sending transaction from the multisig is created using `walletcreatefundedpsbt` (anyone can initiate this). It is simple to do + this in the GUI by going to the `Send` tab in the multisig wallet and creating an unsigned transaction (PSBT) + 6. At least `M` participants check the PSBT with their multisig using `decodepsbt` to verify the transaction is OK before signing it. + 7. (If OK) the participant signs the PSBT with their signer wallet using `walletprocesspsbt`. It is simple to do this in the GUI by + loading the PSBT from file and signing it + 8. The signed PSBTs are collected with `combinepsbt`, finalized w/ `finalizepsbt`, and then the resulting transaction is broadcasted + to the network. Note that any wallet (eg one of the signers or multisig) is capable of doing this. + 9. Checks that balances are correct after the transaction has been included in a block You may prefer a daisy chained signing flow where each participant signs the PSBT one after another until the PSBT has been signed `M` times and is "complete." For the most part, the steps above remain the same, except (6, 7) diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index 59a0b94a9f..68c206b038 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -26,13 +26,15 @@ def skip_test_if_missing_module(self): self.skip_if_no_wallet() self.skip_if_no_sqlite() - def _get_xpub(self, wallet): + @staticmethod + def _get_xpub(wallet): """Extract the wallet's xpubs using `listdescriptors` and pick the one from the `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses).""" descriptor = next(filter(lambda d: d["desc"].startswith("pkh"), wallet.listdescriptors()["descriptors"])) return descriptor["desc"].split("]")[-1].split("/")[0] - def _check_psbt(self, psbt, to, value, multisig): - """Helper method for any of the N participants to check the psbt with decodepsbt and verify it is OK before signing.""" + @staticmethod + def _check_psbt(psbt, to, value, multisig): + """Helper function for any of the N participants to check the psbt with decodepsbt and verify it is OK before signing.""" tx = multisig.decodepsbt(psbt)["tx"] amount = 0 for vout in tx["vout"]: @@ -42,13 +44,7 @@ def _check_psbt(self, psbt, to, value, multisig): amount += vout["value"] assert_approx(amount, float(value), vspan=0.001) - def generate_and_exchange_xpubs(self, participants): - """Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet. Avoid reusing this wallet for any other purpose..""" - for i, node in enumerate(participants): - node.createwallet(wallet_name=f"participant_{i}", descriptors=True) - yield self._get_xpub(node.get_wallet_rpc(f"participant_{i}")) - - def participants_import_descriptors(self, participants, xpubs): + def participants_create_multisigs(self, xpubs): """The multisig is created by importing the following descriptors. The resulting wallet is watch-only and every participant can do this.""" # some simple validation assert_equal(len(xpubs), self.N) @@ -56,11 +52,11 @@ def participants_import_descriptors(self, participants, xpubs): for xpub in xpubs: base58_to_byte(xpub) - for i, node in enumerate(participants): + for i, node in enumerate(self.nodes): node.createwallet(wallet_name=f"{self.name}_{i}", blank=True, descriptors=True, disable_private_keys=True) multisig = node.get_wallet_rpc(f"{self.name}_{i}") - external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/{0}/*,'.join(xpubs)}/{0}/*))") - internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/{1}/*,'.join(xpubs)}/{1}/*))") + external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/0/*,'.join(xpubs)}/0/*))") + internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/1/*,'.join(xpubs)}/1/*))") result = multisig.importdescriptors([ { # receiving addresses (internal: False) "desc": external["descriptor"], @@ -76,18 +72,7 @@ def participants_import_descriptors(self, participants, xpubs): }, ]) assert all(r["success"] for r in result) - - def get_multisig_receiving_address(self): - """We will send funds to the resulting address (every participant should get the same addresses).""" - multisig = self.nodes[0].get_wallet_rpc(f"{self.name}_{0}") - receiving_address = multisig.getnewaddress() - for i in range(1, self.N): - assert_equal(receiving_address, self.nodes[i].get_wallet_rpc(f"{self.name}_{i}").getnewaddress()) - return receiving_address - - def make_sending_transaction(self, to, value): - """Make a sending transaction, created using walletcreatefundedpsbt (anyone can initiate this).""" - return self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + yield multisig def run_test(self): self.M = 2 @@ -95,40 +80,57 @@ def run_test(self): self.name = f"{self.M}_of_{self.N}_multisig" self.log.info(f"Testing {self.name}...") + participants = { + # Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet. + # This wallet will be the participant's `signer` for the resulting multisig. Avoid reusing this wallet for any other purpose (for privacy reasons). + "signers": [node.get_wallet_rpc(node.createwallet(wallet_name=f"participant_{self.nodes.index(node)}", descriptors=True)["name"]) for node in self.nodes], + # After participants generate and exchange their xpubs they will each create their own watch-only multisig. + # Note: these multisigs are all the same, this justs highlights that each participant can independently verify everything on their own node. + "multisigs": [] + } + self.log.info("Generate and exchange xpubs...") - xpubs = list(self.generate_and_exchange_xpubs(self.nodes)) + xpubs = [self._get_xpub(signer) for signer in participants["signers"]] self.log.info("Every participant imports the following descriptors to create the watch-only multisig...") - self.participants_import_descriptors(self.nodes, xpubs) + participants["multisigs"] = list(self.participants_create_multisigs(xpubs)) + + self.log.info("Check that every participant's multisig generates the same addresses...") + for _ in range(10): # we check that the first 10 generated addresses are the same for all participant's multisigs + receive_addresses = [multisig.getnewaddress() for multisig in participants["multisigs"]] + all(address == receive_addresses[0] for address in receive_addresses) + change_addresses = [multisig.getrawchangeaddress() for multisig in participants["multisigs"]] + all(address == change_addresses[0] for address in change_addresses) self.log.info("Get a mature utxo to send to the multisig...") - coordinator_wallet = self.nodes[0].get_wallet_rpc(f"participant_{0}") + coordinator_wallet = participants["signers"][0] coordinator_wallet.generatetoaddress(101, coordinator_wallet.getnewaddress()) deposit_amount = 6.15 - multisig_receiving_address = self.get_multisig_receiving_address() + multisig_receiving_address = participants["multisigs"][0].getnewaddress() self.log.info("Send funds to the resulting multisig receiving address...") coordinator_wallet.sendtoaddress(multisig_receiving_address, deposit_amount) self.nodes[0].generate(1) self.sync_all() - for n in range(self.N): - assert_approx(self.nodes[n].get_wallet_rpc(f"{self.name}_{n}").getbalance(), deposit_amount, vspan=0.001) + for participant in participants["multisigs"]: + assert_approx(participant.getbalance(), deposit_amount, vspan=0.001) self.log.info("Send a transaction from the multisig!") - to = self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getnewaddress() + to = participants["signers"][self.N - 1].getnewaddress() value = 1 - psbt = self.make_sending_transaction(to, value) + self.log.info("First, make a sending transaction, created using `walletcreatefundedpsbt` (anyone can initiate this)...") + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) psbts = [] - self.log.info("At least M users check the psbt with decodepsbt and (if OK) signs it with walletprocesspsbt...") + self.log.info("Now at least M users check the psbt with decodepsbt and (if OK) signs it with walletprocesspsbt...") for m in range(self.M): - signers_multisig = self.nodes[m].get_wallet_rpc(f"{self.name}_{m}") + signers_multisig = participants["multisigs"][m] self._check_psbt(psbt["psbt"], to, value, signers_multisig) - signing_wallet = self.nodes[m].get_wallet_rpc(f"participant_{m}") + signing_wallet = participants["signers"][m] partially_signed_psbt = signing_wallet.walletprocesspsbt(psbt["psbt"]) psbts.append(partially_signed_psbt["psbt"]) - self.log.info("Collect the signed PSBTs with combinepsbt, finalizepsbt, then broadcast the resulting transaction...") + self.log.info("Finally, collect the signed PSBTs with combinepsbt, finalizepsbt, then broadcast the resulting transaction...") combined = coordinator_wallet.combinepsbt(psbts) finalized = coordinator_wallet.finalizepsbt(combined) coordinator_wallet.sendrawtransaction(finalized["hex"]) @@ -136,13 +138,15 @@ def run_test(self): self.log.info("Check that balances are correct after the transaction has been included in a block.") self.nodes[0].generate(1) self.sync_all() - assert_approx(self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").getbalance(), deposit_amount - value, vspan=0.001) - assert_equal(self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getbalance(), value) + assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - value, vspan=0.001) + assert_equal(participants["signers"][self.N - 1].getbalance(), value) self.log.info("Send another transaction from the multisig, this time with a daisy chained signing flow (one after another in series)!") - psbt = self.make_sending_transaction(to, value) + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) for m in range(self.M): - signing_wallet = self.nodes[m].get_wallet_rpc(f"participant_{m}") + signers_multisig = participants["multisigs"][m] + self._check_psbt(psbt["psbt"], to, value, signers_multisig) + signing_wallet = participants["signers"][m] psbt = signing_wallet.walletprocesspsbt(psbt["psbt"]) assert_equal(psbt["complete"], m == self.M - 1) finalized = coordinator_wallet.finalizepsbt(psbt["psbt"]) @@ -151,8 +155,8 @@ def run_test(self): self.log.info("Check that balances are correct after the transaction has been included in a block.") self.nodes[0].generate(1) self.sync_all() - assert_approx(self.nodes[0].get_wallet_rpc(f"{self.name}_{0}").getbalance(), deposit_amount - (value * 2), vspan=0.001) - assert_equal(self.nodes[self.N - 1].get_wallet_rpc(f"participant_{self.N - 1}").getbalance(), value * 2) + assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - (value * 2), vspan=0.001) + assert_equal(participants["signers"][self.N - 1].getbalance(), value * 2) if __name__ == "__main__": From 9de0d94508828f5fdfaf688ccda5a91d38b32c58 Mon Sep 17 00:00:00 2001 From: Michael Dietz Date: Fri, 3 Sep 2021 13:42:05 -0500 Subject: [PATCH 07/96] doc: add disclaimer highlighting shortcomings of the basic multisig example --- doc/descriptors.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/descriptors.md b/doc/descriptors.md index f38caa81cb..70d0926a1b 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -143,6 +143,14 @@ are lexicographically ordered as described in BIP67. For a good example of a basic M-of-N multisig between multiple participants using descriptor wallets and PSBTs, as well as a signing flow, see [this functional test](/test/functional/wallet_multisig_descriptor_psbt.py). + +Disclaimers: It is important to note that this example serves as a quick-start and is kept basic for readability. A downside of the approach +outlined here is that each participant must maintain (and backup) two separate wallets: a signer and the corresponding multisig. +It should also be noted that privacy best-practices are not "by default" here - participants should take care to only use the signer to sign +transactions related to the multisig. Lastly, it is not recommended to use anything other than a Bitcoin Core descriptor wallet to serve as your +signer(s). Other wallets, whether hardware or software, likely impose additional checks and safeguards to prevent users from signing transactions that +could lead to loss of funds, or are deemed security hazards. Conforming to various 3rd-party checks and verifications is not in the scope of this example. + The basic steps are: 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet which we will refer to as From d5f985e51f2863fda0c0d632e836eba42a5c3e15 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 5 Dec 2017 15:57:12 -0500 Subject: [PATCH 08/96] multiprocess: Add new bitcoin-gui, bitcoin-qt, bitcoin-wallet init implementations Add separate init implementations instead of sharing existing bitcoind and bitcoin-node ones, so they can start to be differentiated in upcoming commits with node and wallet code no longer linked into the bitcoin-gui binary and wallet code no longer linked into the bitcoin-node binary. --- build_msvc/bitcoin-qt/bitcoin-qt.vcxproj | 2 +- .../bitcoin-wallet/bitcoin-wallet.vcxproj | 3 ++ .../test_bitcoin-qt/test_bitcoin-qt.vcxproj | 2 +- src/Makefile.am | 1 + src/Makefile.qt.include | 6 +-- src/Makefile.qttest.include | 2 +- src/bitcoin-wallet.cpp | 8 ++++ src/init/bitcoin-gui.cpp | 47 +++++++++++++++++++ src/init/bitcoin-qt.cpp | 42 +++++++++++++++++ src/init/bitcoin-wallet.cpp | 12 +++++ src/qt/bitcoin.cpp | 5 +- src/qt/test/test_main.cpp | 4 +- 12 files changed, 121 insertions(+), 13 deletions(-) create mode 100644 src/init/bitcoin-gui.cpp create mode 100644 src/init/bitcoin-qt.cpp create mode 100644 src/init/bitcoin-wallet.cpp diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index 724dae1969..2800a42767 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -9,7 +9,7 @@ - + diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index 40c5db5522..affb60425b 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -10,6 +10,9 @@ + + $(IntDir)init_bitcoin-wallet.obj + diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 08b12bdd85..f9948b6f13 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -8,7 +8,7 @@ $(SolutionDir)$(Platform)\$(Configuration)\ - + diff --git a/src/Makefile.am b/src/Makefile.am index 52c8b85357..28548c221d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -710,6 +710,7 @@ bitcoin_tx_LDADD += $(BOOST_LIBS) # bitcoin-wallet binary # bitcoin_wallet_SOURCES = bitcoin-wallet.cpp +bitcoin_wallet_SOURCES += init/bitcoin-wallet.cpp bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index f4b0b3adbe..86d397f163 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -338,15 +338,15 @@ bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) -qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp +qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoin-qt.cpp qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd) qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags) qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags) bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) -bitcoin_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp -bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) +bitcoin_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoin-gui.cpp +bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) $(LIBBITCOIN_IPC) $(LIBBITCOIN_UTIL) $(LIBMULTIPROCESS_LIBS) bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags) bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 8a5521eeb5..797e1f9a97 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -28,7 +28,7 @@ qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_ $(QT_INCLUDES) $(QT_TEST_INCLUDES) qt_test_test_bitcoin_qt_SOURCES = \ - init/bitcoind.cpp \ + init/bitcoin-qt.cpp \ qt/test/apptests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 765954c92e..8badc408ec 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -83,6 +84,13 @@ int main(int argc, char* argv[]) util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); #endif + + int exit_status; + std::unique_ptr init = interfaces::MakeWalletInit(argc, argv, exit_status); + if (!init) { + return exit_status; + } + SetupEnvironment(); RandomInit(); try { diff --git a/src/init/bitcoin-gui.cpp b/src/init/bitcoin-gui.cpp new file mode 100644 index 0000000000..c549ed3cc0 --- /dev/null +++ b/src/init/bitcoin-gui.cpp @@ -0,0 +1,47 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace init { +namespace { +const char* EXE_NAME = "bitcoin-gui"; + +class BitcoinGuiInit : public interfaces::Init +{ +public: + BitcoinGuiInit(const char* arg0) : m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) + { + m_node.args = &gArgs; + m_node.init = this; + } + std::unique_ptr makeNode() override { return interfaces::MakeNode(m_node); } + std::unique_ptr makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr makeWalletClient(interfaces::Chain& chain) override + { + return MakeWalletClient(chain, *Assert(m_node.args)); + } + std::unique_ptr makeEcho() override { return interfaces::MakeEcho(); } + interfaces::Ipc* ipc() override { return m_ipc.get(); } + NodeContext m_node; + std::unique_ptr m_ipc; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr MakeGuiInit(int argc, char* argv[]) +{ + return std::make_unique(argc > 0 ? argv[0] : ""); +} +} // namespace interfaces diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp new file mode 100644 index 0000000000..d71177e885 --- /dev/null +++ b/src/init/bitcoin-qt.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace init { +namespace { +class BitcoinQtInit : public interfaces::Init +{ +public: + BitcoinQtInit() + { + m_node.args = &gArgs; + m_node.init = this; + } + std::unique_ptr makeNode() override { return interfaces::MakeNode(m_node); } + std::unique_ptr makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr makeWalletClient(interfaces::Chain& chain) override + { + return MakeWalletClient(chain, *Assert(m_node.args)); + } + std::unique_ptr makeEcho() override { return interfaces::MakeEcho(); } + NodeContext m_node; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr MakeGuiInit(int argc, char* argv[]) +{ + return std::make_unique(); +} +} // namespace interfaces diff --git a/src/init/bitcoin-wallet.cpp b/src/init/bitcoin-wallet.cpp new file mode 100644 index 0000000000..e9dcde72fe --- /dev/null +++ b/src/init/bitcoin-wallet.cpp @@ -0,0 +1,12 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace interfaces { +std::unique_ptr MakeWalletInit(int argc, char* argv[], int& exit_status) +{ + return std::make_unique(); +} +} // namespace interfaces diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index d4895ea6ff..4715c7174b 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -462,9 +461,7 @@ int GuiMain(int argc, char* argv[]) std::tie(argc, argv) = winArgs.get(); #endif - NodeContext node_context; - int unused_exit_status; - std::unique_ptr init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); + std::unique_ptr init = interfaces::MakeGuiInit(argc, argv); SetupEnvironment(); util::ThreadSetInternalName("main"); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 884ed25637..e138704776 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -53,9 +53,7 @@ int main(int argc, char* argv[]) BasicTestingSetup dummy{CBaseChainParams::REGTEST}; } - NodeContext node_context; - int unused_exit_status; - std::unique_ptr init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); + std::unique_ptr init = interfaces::MakeGuiInit(argc, argv); gArgs.ForceSetArg("-listen", "0"); gArgs.ForceSetArg("-listenonion", "0"); gArgs.ForceSetArg("-discover", "0"); From f19ad404631010a5e2dac2c7cbecd057b005fe2a Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 16 Sep 2021 17:53:32 -0400 Subject: [PATCH 09/96] rpc, wallet: Descriptor wallets are no longer experimental --- src/wallet/rpcwallet.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e922f4ede9..4b2aca3078 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2771,7 +2771,6 @@ static RPCHelpMan createwallet() throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without sqlite support (required for descriptor wallets)"); #endif flags |= WALLET_FLAG_DESCRIPTORS; - warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } if (!request.params[7].isNull() && request.params[7].get_bool()) { #ifdef ENABLE_EXTERNAL_SIGNER From 9c1052a5218e191fd23c0d9fc06f2fca34b03411 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Thu, 16 Sep 2021 18:05:44 -0400 Subject: [PATCH 10/96] wallet: Default new wallets to descriptor wallets --- src/bitcoin-wallet.cpp | 1 + src/qt/forms/createwalletdialog.ui | 3 +++ src/wallet/rpcwallet.cpp | 4 ++-- src/wallet/wallettool.cpp | 18 +++++++++++++++++- test/functional/tool_wallet.py | 4 ++-- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 765954c92e..21d4df5b01 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -30,6 +30,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-dumpfile=", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS); argsman.AddArg("-debug=", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-descriptors", "Create descriptors wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); + argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); argsman.AddArg("-format=", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index b11fb026b0..56adbe17a5 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -107,6 +107,9 @@ Descriptor Wallet + + true + diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 4b2aca3078..211a33b29e 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2724,7 +2724,7 @@ static RPCHelpMan createwallet() {"blank", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{false}, "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, - {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, + {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, {"external_signer", RPCArg::Type::BOOL, RPCArg::Default{false}, "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, @@ -2766,7 +2766,7 @@ static RPCHelpMan createwallet() if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } - if (!request.params[5].isNull() && request.params[5].get_bool()) { + if (request.params[5].isNull() || request.params[5].get_bool()) { #ifndef USE_SQLITE throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without sqlite support (required for descriptor wallets)"); #endif diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 50b6c9d29f..4000c08ea9 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -116,6 +116,10 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (args.IsArgSet("-legacy") && command != "create") { + tfm::format(std::cerr, "The -legacy option can only be used with the 'create' command.\n"); + return false; + } if (command == "create" && !args.IsArgSet("-wallet")) { tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); return false; @@ -126,7 +130,19 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) if (command == "create") { DatabaseOptions options; options.require_create = true; - if (args.GetBoolArg("-descriptors", false)) { + // If -legacy is set, use it. Otherwise default to false. + bool make_legacy = args.GetBoolArg("-legacy", false); + // If neither -legacy nor -descriptors is set, default to true. If -descriptors is set, use its value. + bool make_descriptors = (!args.IsArgSet("-descriptors") && !args.IsArgSet("-legacy")) || (args.IsArgSet("-descriptors") && args.GetBoolArg("-descriptors", true)); + if (make_legacy && make_descriptors) { + tfm::format(std::cerr, "Only one of -legacy or -descriptors can be set to true, not both\n"); + return false; + } + if (!make_legacy && !make_descriptors) { + tfm::format(std::cerr, "One of -legacy or -descriptors must be set to true (or omitted)\n"); + return false; + } + if (make_descriptors) { options.create_flags |= WALLET_FLAG_DESCRIPTORS; options.require_format = DatabaseFormat::SQLITE; } diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 4bf3927879..3a0daf04ad 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -31,8 +31,8 @@ def skip_test_if_missing_module(self): def bitcoin_wallet_process(self, *args): binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] default_args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] - if self.options.descriptors and 'create' in args: - default_args.append('-descriptors') + if not self.options.descriptors and 'create' in args: + default_args.append('-legacy') return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) From fa2662c293ec0aaa93092b59b6632f74729c4283 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 20 Sep 2021 09:24:42 +0200 Subject: [PATCH 11/96] net: Avoid logging AlreadyHaveTx when disconnecting misbehaving peer Can be reviewed with --color-moved=dimmed-zebra --- src/net_processing.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 80655c61e7..8c95603e66 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2949,16 +2949,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, best_block = &inv.hash; } } else if (inv.IsGenTxMsg()) { + if (reject_tx_invs) { + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } const GenTxid gtxid = ToGenTxid(inv); const bool fAlreadyHave = AlreadyHaveTx(gtxid); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); pfrom.AddKnownTx(inv.hash); - if (reject_tx_invs) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); - pfrom.fDisconnect = true; - return; - } else if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { AddTxAnnouncement(pfrom, gtxid, current_time); } } else { From b1488c4dcecb9dda9d7c583c1c9e1bc0852c79b2 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 15 Aug 2021 17:23:02 +0200 Subject: [PATCH 12/96] test: fix reference to block processing test in p2p_segwit.py The block test was renamed from `p2p-fullblocks.py` to `feature_block.py` in commit ca6523d0c8 (PR #11774). --- test/functional/p2p_segwit.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index aa3b95fc4f..0ee6c11e6d 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -8,7 +8,12 @@ import struct import time -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER +from test_framework.blocktools import ( + WITNESS_COMMITMENT_HEADER, + add_witness_commitment, + create_block, + create_coinbase, +) from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, @@ -860,7 +865,7 @@ def test_block_malleability(self): @subtest # type: ignore def test_witness_block_size(self): # TODO: Test that non-witness carrying blocks can't exceed 1MB - # Skipping this test for now; this is covered in p2p-fullblocktest.py + # Skipping this test for now; this is covered in feature_block.py # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. block = self.build_next_block() From 4eb532ff8b5ea9338e6cb1b927abaa43f8e3c94e Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 15 Aug 2021 21:37:34 +0200 Subject: [PATCH 13/96] test: check for block reject reasons in p2p_segwit.py [1/2] This commit adds specific expected reject reasons for segwit blocks sent to the node, that are independent of whether multiple script threads are activated or not. --- test/functional/p2p_segwit.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 0ee6c11e6d..2fb0826c1b 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -790,7 +790,7 @@ def test_witness_commitments(self): block_3.rehash() block_3.solve() - test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False, reason='bad-witness-merkle-match') # Add a different commitment with different nonce, but in the # right location, and with some funds burned(!). @@ -856,7 +856,7 @@ def test_block_malleability(self): # Change the nonce -- should not cause the block to be permanently # failed block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(1)] - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, reason='bad-witness-merkle-match') # Changing the witness reserved value doesn't change the block hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] @@ -921,7 +921,7 @@ def test_witness_block_size(self): # limit assert len(block.serialize()) > 2 * 1024 * 1024 - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, reason='bad-blk-weight') # Now resize the second transaction to make the block fit. cur_length = len(block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0]) @@ -1180,7 +1180,7 @@ def serialize_with_witness(self): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, reason='bad-txnmrklroot') # Now try using a too short vtxinwit tx2.wit.vtxinwit.pop() @@ -1419,7 +1419,7 @@ def test_premature_coinbase_witness_spend(self): self.sync_blocks() block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) - test_witness_block(self.nodes[0], self.test_node, block2, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block2, accepted=False, reason='bad-txns-premature-spend-of-coinbase') # Advancing one more block should allow the spend. self.generate(self.nodes[0], 1) @@ -1922,7 +1922,7 @@ def test_witness_sigops(self): block_2 = self.build_next_block() self.update_witness_block_with_transactions(block_2, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block_2, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block_2, accepted=False, reason='bad-blk-sigops') # Try dropping the last input in tx2, and add an output that has # too many sigops (contributing to legacy sigop count). @@ -1935,7 +1935,7 @@ def test_witness_sigops(self): tx2.rehash() block_3 = self.build_next_block() self.update_witness_block_with_transactions(block_3, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False, reason='bad-blk-sigops') # If we drop the last checksig in this output, the tx should succeed. block_4 = self.build_next_block() From 45827fd718d51acffdbeb3bb12d1eb96e3fa8bc0 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Sun, 15 Aug 2021 22:26:57 +0200 Subject: [PATCH 14/96] test: check for block reject reasons in p2p_segwit.py [2/2] This commit adds specific expected reject reasons for segwit blocks sent to the node, that are only showing up if one script threads is used. For this reason, the node is started with the parameter `-par=1`. --- test/functional/p2p_segwit.py | 37 +++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 2fb0826c1b..a564984663 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -198,7 +198,7 @@ def set_test_params(self): self.num_nodes = 2 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. self.extra_args = [ - ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-whitelist=noban@127.0.0.1"], + ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-whitelist=noban@127.0.0.1", "-par=1"], ["-acceptnonstdtxn=0", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}"], ] self.supports_cli = False @@ -509,8 +509,8 @@ def test_v0_outputs_arent_spendable(self): # 'block-validation-failed' (if script check threads > 1) or # 'non-mandatory-script-verify-flag (Witness program was passed an # empty witness)' (otherwise). - # TODO: support multiple acceptable reject reasons. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False, + reason='non-mandatory-script-verify-flag (Witness program was passed an empty witness)') self.utxo.pop(0) self.utxo.append(UTXO(txid, 2, value)) @@ -993,7 +993,8 @@ def test_extra_witness_data(self): self.update_witness_block_with_transactions(block, [tx]) # Extra witness data should not be allowed. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Witness provided for non-witness script)') # Try extra signature data. Ok if we're not spending a witness output. block.vtx[1].wit.vtxinwit = [] @@ -1018,7 +1019,8 @@ def test_extra_witness_data(self): self.update_witness_block_with_transactions(block, [tx2]) # This has extra witness data, so it should fail. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Stack size must be exactly one after execution)') # Now get rid of the extra witness, but add extra scriptSig data tx2.vin[0].scriptSig = CScript([OP_TRUE]) @@ -1030,7 +1032,8 @@ def test_extra_witness_data(self): block.solve() # This has extra signature data for a witness input, so it should fail. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Witness requires empty scriptSig)') # Now get rid of the extra scriptsig on the witness input, and verify # success (even with extra scriptsig data in the non-witness input) @@ -1068,7 +1071,8 @@ def test_max_witness_push_length(self): tx2.rehash() self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Push value size limit exceeded)') # Now reduce the length of the stack element tx2.wit.vtxinwit[0].scriptWitness.stack[0] = b'a' * (MAX_SCRIPT_ELEMENT_SIZE) @@ -1108,7 +1112,8 @@ def test_max_witness_script_length(self): self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Script is too big)') # Try again with one less byte in the witness script witness_script = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) @@ -1188,6 +1193,8 @@ def serialize_with_witness(self): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) + # This block doesn't result in a specific reject reason, but an iostream exception: + # "Exception 'CDataStream::read(): end of data: unspecified iostream_category error' (...) caught" test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Now make one of the intermediate witnesses be incorrect @@ -1197,7 +1204,8 @@ def serialize_with_witness(self): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Operation not valid with the current stack size)') # Fix the broken witness and the block should be accepted. tx2.wit.vtxinwit[5].scriptWitness.stack = [b'a', witness_script] @@ -1568,13 +1576,17 @@ def test_signature_version_1(self): # Too-large input value sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue + 1, key) self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Script evaluated without error ' + 'but finished with a false/empty top stack element') # Too-small input value sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue - 1, key) block.vtx.pop() # remove last tx self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Script evaluated without error ' + 'but finished with a false/empty top stack element') # Now try correct value sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue, key) @@ -1676,7 +1688,8 @@ def test_signature_version_1(self): tx2.vin[0].scriptSig = CScript([signature, pubkey]) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Witness requires empty scriptSig)') # Move the signature to the witness. block.vtx.pop() From d95913fc432f0fde9dec743884b14c5df83727af Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Wed, 29 Sep 2021 17:50:26 +0200 Subject: [PATCH 15/96] rpc: fix "trusted" description in TransactionDescriptionString The helps for RPCs gettransaction, listtransactions, and listsinceblock returned by TransactionDescriptionString() state that the "trusted" boolean field is only present if the transaction is trusted and safe to spend from. The "trusted" boolean field is in fact returned by WalletTxToJSON() when the transaction has 0 confirmations, or negative confirmations, if conflicted, and it can be true or false. This commit updates TransactionDescriptionString() to a more accurate description for "trusted" and updates the existing line of test coverage to fail more helpfully. --- src/wallet/rpcwallet.cpp | 5 +++-- test/functional/wallet_import_rescan.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index c430b1db5c..e90bd207d7 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1389,7 +1389,8 @@ static const std::vector TransactionDescriptionString() return{{RPCResult::Type::NUM, "confirmations", "The number of confirmations for the transaction. Negative confirmations means the\n" "transaction conflicted that many blocks ago."}, {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if transaction only input is a coinbase one."}, - {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Only present if we consider transaction to be trusted and so safe to spend from."}, + {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Whether we consider the transaction to be trusted and safe to spend from.\n" + "Only present when the transaction has 0 confirmations (or negative confirmations, if conflicted)."}, {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash containing the transaction."}, {RPCResult::Type::NUM, "blockheight", /* optional */ true, "The block height containing the transaction."}, {RPCResult::Type::NUM, "blockindex", /* optional */ true, "The index of the transaction in the block that includes it."}, @@ -1407,7 +1408,7 @@ static const std::vector TransactionDescriptionString() {RPCResult::Type::NUM_TIME, "timereceived", "The time received expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::STR, "comment", /* optional */ true, "If a comment is associated with the transaction, only present if not empty."}, {RPCResult::Type::STR, "bip125-replaceable", "(\"yes|no|unknown\") Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" - "may be unknown for unconfirmed transactions not in the mempool"}}; + "may be unknown for unconfirmed transactions not in the mempool."}}; } static RPCHelpMan listtransactions() diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index cbe3e9bfdd..27a2a42dac 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -99,7 +99,7 @@ def check(self, txid=None, amount=None, confirmation_height=None): assert_equal(tx["label"], self.label) assert_equal(tx["txid"], txid) assert_equal(tx["confirmations"], 1 + current_height - confirmation_height) - assert_equal("trusted" not in tx, True) + assert "trusted" not in tx address, = [ad for ad in addresses if txid in ad["txids"]] assert_equal(address["address"], self.address["address"]) From 296cfa312fd9ce19f1f820aeafa37d87764ad21d Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Wed, 29 Sep 2021 18:10:12 +0200 Subject: [PATCH 16/96] test: add listtransactions/listsinceblock "trusted" coverage --- test/functional/wallet_listsinceblock.py | 11 +++++++++++ test/functional/wallet_listtransactions.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index bd3b29c81c..f4a00a8ec8 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -44,6 +44,14 @@ def run_test(self): def test_no_blockhash(self): self.log.info("Test no blockhash") txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) + self.sync_all() + assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, { + "category": "receive", + "amount": 1, + "confirmations": 0, + "trusted": False, + }) + blockhash, = self.generate(self.nodes[2], 1) blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() @@ -56,6 +64,9 @@ def test_no_blockhash(self): "blockheight": blockheight, "confirmations": 1, }) + assert_equal(len(txs), 1) + assert "trusted" not in txs[0] + assert_equal( self.nodes[0].listsinceblock(), {"lastblock": blockhash, diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index a14bfe345c..9e2f08960f 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -31,10 +31,10 @@ def run_test(self): self.sync_all() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, - {"category": "send", "amount": Decimal("-0.1"), "confirmations": 0}) + {"category": "send", "amount": Decimal("-0.1"), "confirmations": 0, "trusted": True}) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid}, - {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0}) + {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0, "trusted": False}) self.log.info("Test confirmations change after mining a block") blockhash = self.generate(self.nodes[0], 1)[0] blockheight = self.nodes[0].getblockheader(blockhash)['height'] From 66f6efc70a72cc1613906fd3c10281f9af0ba0db Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Wed, 29 Sep 2021 18:12:21 +0200 Subject: [PATCH 17/96] rpc: improve TransactionDescriptionString() "generated" help --- src/wallet/rpcwallet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e90bd207d7..f45cf94070 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1388,7 +1388,7 @@ static const std::vector TransactionDescriptionString() { return{{RPCResult::Type::NUM, "confirmations", "The number of confirmations for the transaction. Negative confirmations means the\n" "transaction conflicted that many blocks ago."}, - {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if transaction only input is a coinbase one."}, + {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if the transaction's only input is a coinbase one."}, {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Whether we consider the transaction to be trusted and safe to spend from.\n" "Only present when the transaction has 0 confirmations (or negative confirmations, if conflicted)."}, {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash containing the transaction."}, From cb1407196fba648aa75504e3ab3d46aa0181563a Mon Sep 17 00:00:00 2001 From: glozow Date: Thu, 30 Sep 2021 09:25:11 +0100 Subject: [PATCH 18/96] [refactor/bench] make mempool_stress bench reusable and parameterizable --- src/bench/mempool_stress.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index f28768efc8..21d7407fca 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -26,14 +26,8 @@ struct Available { Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){} }; -static void ComplexMemPool(benchmark::Bench& bench) +static std::vector CreateOrderedCoins(FastRandomContext& det_rand, int childTxs, int min_ancestors) { - int childTxs = 800; - if (bench.complexityN() > 1) { - childTxs = static_cast(bench.complexityN()); - } - - FastRandomContext det_rand{true}; std::vector available_coins; std::vector ordered_coins; // Create some base transactions @@ -58,8 +52,10 @@ static void ComplexMemPool(benchmark::Bench& bench) size_t idx = det_rand.randrange(available_coins.size()); Available coin = available_coins[idx]; uint256 hash = coin.ref->GetHash(); - // biased towards taking just one ancestor, but maybe more - size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left); + // biased towards taking min_ancestors parents, but maybe more + size_t n_to_take = det_rand.randrange(2) == 0 ? + min_ancestors : + min_ancestors + det_rand.randrange(coin.ref->vout.size() - coin.vin_left); for (size_t i = 0; i < n_to_take; ++i) { tx.vin.emplace_back(); tx.vin.back().prevout = COutPoint(hash, coin.vin_left++); @@ -79,6 +75,17 @@ static void ComplexMemPool(benchmark::Bench& bench) ordered_coins.emplace_back(MakeTransactionRef(tx)); available_coins.emplace_back(ordered_coins.back(), tx_counter++); } + return ordered_coins; +} + +static void ComplexMemPool(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + int childTxs = 800; + if (bench.complexityN() > 1) { + childTxs = static_cast(bench.complexityN()); + } + std::vector ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 1); const auto testing_setup = MakeNoLogFileContext(CBaseChainParams::MAIN); CTxMemPool pool; LOCK2(cs_main, pool.cs); From 30e240f65e69c6dffcd033afc63895345bd51f53 Mon Sep 17 00:00:00 2001 From: glozow Date: Thu, 30 Sep 2021 09:31:18 +0100 Subject: [PATCH 19/96] [bench] Benchmark CTxMemPool::check() --- src/bench/mempool_stress.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 21d7407fca..16c57881d4 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -98,4 +99,20 @@ static void ComplexMemPool(benchmark::Bench& bench) }); } +static void MempoolCheck(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + const int childTxs = bench.complexityN() > 1 ? static_cast(bench.complexityN()) : 2000; + const std::vector ordered_coins = CreateOrderedCoins(det_rand, childTxs, /* min_ancestors */ 5); + const auto testing_setup = MakeNoLogFileContext(CBaseChainParams::MAIN, {"-checkmempool=1"}); + CTxMemPool pool; + LOCK2(cs_main, pool.cs); + for (auto& tx : ordered_coins) AddTx(tx, pool); + + bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { + pool.check(testing_setup.get()->m_node.chainman->ActiveChainstate()); + }); +} + BENCHMARK(ComplexMemPool); +BENCHMARK(MempoolCheck); From 54c6f3c1da01090aee9691a2c2bee0984a054ce8 Mon Sep 17 00:00:00 2001 From: glozow Date: Thu, 30 Sep 2021 09:08:40 +0100 Subject: [PATCH 20/96] [mempool] speed up check() by using coins cache and iterating in topo order No behavior changes. Before, we're always adding transactions to the "check later" queue if they have any parents in the mempool. But there's no reason to do this if all of its inputs are already available from mempoolDuplicate. Instead, check for inputs, and only mark fDependsWait=true if the parents haven't been processed yet. Reduce the amount of "check later" transactions by looking at ancestors before descendants. Do this by iterating through them in ascending order by ancestor count. This works because a child will always have more in-mempool ancestors than its parent. We should never have any entries in the "check later" queue after this commit. --- src/txmempool.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 5a93f30c8a..10f7f66263 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -693,13 +693,14 @@ void CTxMemPool::check(CChainState& active_chainstate) const uint64_t checkTotal = 0; CAmount check_total_fee{0}; uint64_t innerUsage = 0; + uint64_t prev_ancestor_count{0}; CCoinsViewCache& active_coins_tip = active_chainstate.CoinsTip(); CCoinsViewCache mempoolDuplicate(const_cast(&active_coins_tip)); const int64_t spendheight = active_chainstate.m_chain.Height() + 1; std::list waitingOnDependants; - for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { + for (const auto& it : GetSortedDepthAndScore()) { unsigned int i = 0; checkTotal += it->GetTxSize(); check_total_fee += it->GetFee(); @@ -714,7 +715,7 @@ void CTxMemPool::check(CChainState& active_chainstate) const if (it2 != mapTx.end()) { const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); - fDependsWait = true; + if (!mempoolDuplicate.HaveCoin(txin.prevout)) fDependsWait = true; setParentCheck.insert(*it2); } else { assert(active_coins_tip.HaveCoin(txin.prevout)); @@ -751,6 +752,9 @@ void CTxMemPool::check(CChainState& active_chainstate) const assert(it->GetSizeWithAncestors() == nSizeCheck); assert(it->GetSigOpCostWithAncestors() == nSigOpCheck); assert(it->GetModFeesWithAncestors() == nFeesCheck); + // Sanity check: we are walking in ascending ancestor count order. + assert(prev_ancestor_count <= it->GetCountWithAncestors()); + prev_ancestor_count = it->GetCountWithAncestors(); // Check children against mapNextTx CTxMemPoolEntry::Children setChildrenCheck; From e8639ec26aaf4de3fae280963434bf1cf2017b6f Mon Sep 17 00:00:00 2001 From: glozow Date: Wed, 29 Sep 2021 18:38:24 +0100 Subject: [PATCH 21/96] [mempool] remove now-unnecessary code Remove variables used for keeping track of mempool transactions for which we haven't processed the parents yet. Since we're iterating in topological order now, they're always unused. --- src/txmempool.cpp | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 10f7f66263..2decc153f2 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -699,15 +699,12 @@ void CTxMemPool::check(CChainState& active_chainstate) const CCoinsViewCache mempoolDuplicate(const_cast(&active_coins_tip)); const int64_t spendheight = active_chainstate.m_chain.Height() + 1; - std::list waitingOnDependants; for (const auto& it : GetSortedDepthAndScore()) { - unsigned int i = 0; checkTotal += it->GetTxSize(); check_total_fee += it->GetFee(); innerUsage += it->DynamicMemoryUsage(); const CTransaction& tx = it->GetTx(); innerUsage += memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); - bool fDependsWait = false; CTxMemPoolEntry::Parents setParentCheck; for (const CTxIn &txin : tx.vin) { // Check that every mempool transaction's inputs refer to available coins, or other mempool tx's. @@ -715,7 +712,10 @@ void CTxMemPool::check(CChainState& active_chainstate) const if (it2 != mapTx.end()) { const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); - if (!mempoolDuplicate.HaveCoin(txin.prevout)) fDependsWait = true; + // We are iterating through the mempool entries sorted in order by ancestor count. + // All parents must have been checked before their children and their coins added to + // the mempoolDuplicate coins cache. + assert(mempoolDuplicate.HaveCoin(txin.prevout)); setParentCheck.insert(*it2); } else { assert(active_coins_tip.HaveCoin(txin.prevout)); @@ -725,7 +725,6 @@ void CTxMemPool::check(CChainState& active_chainstate) const assert(it3 != mapNextTx.end()); assert(it3->first == &txin.prevout); assert(it3->second == &tx); - i++; } auto comp = [](const CTxMemPoolEntry& a, const CTxMemPoolEntry& b) -> bool { return a.GetTx().GetHash() == b.GetTx().GetHash(); @@ -773,24 +772,7 @@ void CTxMemPool::check(CChainState& active_chainstate) const // just a sanity check, not definitive that this calc is correct... assert(it->GetSizeWithDescendants() >= child_sizes + it->GetTxSize()); - if (fDependsWait) - waitingOnDependants.push_back(&(*it)); - else { - CheckInputsAndUpdateCoins(tx, mempoolDuplicate, spendheight); - } - } - unsigned int stepsSinceLastRemove = 0; - while (!waitingOnDependants.empty()) { - const CTxMemPoolEntry* entry = waitingOnDependants.front(); - waitingOnDependants.pop_front(); - if (!mempoolDuplicate.HaveInputs(entry->GetTx())) { - waitingOnDependants.push_back(entry); - stepsSinceLastRemove++; - assert(stepsSinceLastRemove < waitingOnDependants.size()); - } else { - CheckInputsAndUpdateCoins(entry->GetTx(), mempoolDuplicate, spendheight); - stepsSinceLastRemove = 0; - } + CheckInputsAndUpdateCoins(tx, mempoolDuplicate, spendheight); } for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { uint256 hash = it->second->GetHash(); From 09d18916afb0ecae90700d4befd9d5dc52767970 Mon Sep 17 00:00:00 2001 From: glozow Date: Wed, 29 Sep 2021 19:10:44 +0100 Subject: [PATCH 22/96] MOVEONLY: remove single-use helper func CheckInputsAndUpdateCoins --- src/txmempool.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 2decc153f2..40e142dc47 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -671,15 +671,6 @@ void CTxMemPool::clear() _clear(); } -static void CheckInputsAndUpdateCoins(const CTransaction& tx, CCoinsViewCache& mempoolDuplicate, const int64_t spendheight) -{ - TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass - CAmount txfee = 0; - bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); - assert(fCheckResult); - UpdateCoins(tx, mempoolDuplicate, std::numeric_limits::max()); -} - void CTxMemPool::check(CChainState& active_chainstate) const { if (m_check_ratio == 0) return; @@ -772,7 +763,11 @@ void CTxMemPool::check(CChainState& active_chainstate) const // just a sanity check, not definitive that this calc is correct... assert(it->GetSizeWithDescendants() >= child_sizes + it->GetTxSize()); - CheckInputsAndUpdateCoins(tx, mempoolDuplicate, spendheight); + TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass + CAmount txfee = 0; + bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); + assert(fCheckResult); + UpdateCoins(tx, mempoolDuplicate, std::numeric_limits::max()); } for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { uint256 hash = it->second->GetHash(); From 9e8d7ad5d9cc4b013826daead9cee09aad539401 Mon Sep 17 00:00:00 2001 From: glozow Date: Mon, 4 Oct 2021 13:01:38 +0100 Subject: [PATCH 23/96] [validation/mempool] use Spend/AddCoin instead of UpdateCoins UpdateCoins is an unnecessary dependency on validation. All we need to do is add and remove coins to check inputs. We don't need the extra logic for checking coinbases and handling TxUndos. Also remove the wrapper function in validation.h which constructs a throwaway TxUndo object before calling UpdateCoins because it is now unused. --- src/txmempool.cpp | 4 +++- src/validation.cpp | 6 ------ src/validation.h | 3 --- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 40e142dc47..c1abe24af7 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -767,7 +768,8 @@ void CTxMemPool::check(CChainState& active_chainstate) const CAmount txfee = 0; bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); assert(fCheckResult); - UpdateCoins(tx, mempoolDuplicate, std::numeric_limits::max()); + for (const auto& input: tx.vin) mempoolDuplicate.SpendCoin(input.prevout); + AddCoins(mempoolDuplicate, tx, std::numeric_limits::max()); } for (auto it = mapNextTx.cbegin(); it != mapNextTx.cend(); it++) { uint256 hash = it->second->GetHash(); diff --git a/src/validation.cpp b/src/validation.cpp index 4504d2ca0a..863502e0d7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1240,12 +1240,6 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txund AddCoins(inputs, tx, nHeight); } -void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) -{ - CTxUndo txundo; - UpdateCoins(tx, inputs, txundo, nHeight); -} - bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; diff --git a/src/validation.h b/src/validation.h index b2282828ce..caa0832dd3 100644 --- a/src/validation.h +++ b/src/validation.h @@ -229,9 +229,6 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx const Package& txns, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Apply the effects of this transaction on the UTXO set represented by view */ -void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight); - /** Transaction validation functions */ /** From ed6115f1eae0eb4669601106a9aaff078a2f3a74 Mon Sep 17 00:00:00 2001 From: glozow Date: Mon, 4 Oct 2021 12:58:50 +0100 Subject: [PATCH 24/96] [mempool] simplify some check() logic No transaction in the mempool should ever be a coinbase. Since mempoolDuplicate's backend is the chainstate coins view, it should always contain the coins available. --- src/txmempool.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c1abe24af7..f65fb24f19 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -704,14 +704,12 @@ void CTxMemPool::check(CChainState& active_chainstate) const if (it2 != mapTx.end()) { const CTransaction& tx2 = it2->GetTx(); assert(tx2.vout.size() > txin.prevout.n && !tx2.vout[txin.prevout.n].IsNull()); - // We are iterating through the mempool entries sorted in order by ancestor count. - // All parents must have been checked before their children and their coins added to - // the mempoolDuplicate coins cache. - assert(mempoolDuplicate.HaveCoin(txin.prevout)); setParentCheck.insert(*it2); - } else { - assert(active_coins_tip.HaveCoin(txin.prevout)); } + // We are iterating through the mempool entries sorted in order by ancestor count. + // All parents must have been checked before their children and their coins added to + // the mempoolDuplicate coins cache. + assert(mempoolDuplicate.HaveCoin(txin.prevout)); // Check whether its inputs are marked in mapNextTx. auto it3 = mapNextTx.find(txin.prevout); assert(it3 != mapNextTx.end()); @@ -766,8 +764,8 @@ void CTxMemPool::check(CChainState& active_chainstate) const TxValidationState dummy_state; // Not used. CheckTxInputs() should always pass CAmount txfee = 0; - bool fCheckResult = tx.IsCoinBase() || Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee); - assert(fCheckResult); + assert(!tx.IsCoinBase()); + assert(Consensus::CheckTxInputs(tx, dummy_state, mempoolDuplicate, spendheight, txfee)); for (const auto& input: tx.vin) mempoolDuplicate.SpendCoin(input.prevout); AddCoins(mempoolDuplicate, tx, std::numeric_limits::max()); } From 082c5bf099c64e3d27abe9b68a71ce500b693e7e Mon Sep 17 00:00:00 2001 From: glozow Date: Wed, 29 Sep 2021 19:36:01 +0100 Subject: [PATCH 25/96] [refactor] pass coinsview and height to check() Removes check's dependency on validation.h --- src/bench/mempool_stress.cpp | 3 ++- src/net_processing.cpp | 6 ++++-- src/test/fuzz/tx_pool.cpp | 4 ++-- src/txmempool.cpp | 4 +--- src/txmempool.h | 2 +- src/validation.cpp | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 16c57881d4..a0a82ea359 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -107,10 +107,11 @@ static void MempoolCheck(benchmark::Bench& bench) const auto testing_setup = MakeNoLogFileContext(CBaseChainParams::MAIN, {"-checkmempool=1"}); CTxMemPool pool; LOCK2(cs_main, pool.cs); + const CCoinsViewCache& coins_tip = testing_setup.get()->m_node.chainman->ActiveChainstate().CoinsTip(); for (auto& tx : ordered_coins) AddTx(tx, pool); bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { - pool.check(testing_setup.get()->m_node.chainman->ActiveChainstate()); + pool.check(coins_tip, /* spendheight */ 2); }); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 008b4d679c..12c4eece56 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2298,7 +2298,8 @@ void PeerManagerImpl::ProcessOrphanTx(std::set& orphan_work_set) break; } } - m_mempool.check(m_chainman.ActiveChainstate()); + CChainState& active_chainstate = m_chainman.ActiveChainstate(); + m_mempool.check(active_chainstate.CoinsTip(), active_chainstate.m_chain.Height() + 1); } bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, @@ -3260,7 +3261,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - m_mempool.check(m_chainman.ActiveChainstate()); + CChainState& active_chainstate = m_chainman.ActiveChainstate(); + m_mempool.check(active_chainstate.CoinsTip(), active_chainstate.m_chain.Height() + 1); // As this version of the transaction was acceptable, we can forget about any // requests for it. m_txrequest.ForgetTxHash(tx.GetHash()); diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 6201cc813c..17b5ef88b9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -81,7 +81,7 @@ void SetMempoolConstraints(ArgsManager& args, FuzzedDataProvider& fuzzed_data_pr void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, CChainState& chainstate) { - WITH_LOCK(::cs_main, tx_pool.check(chainstate)); + WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); { BlockAssembler::Options options; options.nBlockMaxWeight = fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BLOCK_WEIGHT); @@ -97,7 +97,7 @@ void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, CCh std::vector all_txids; tx_pool.queryHashes(all_txids); assert(all_txids.size() < info_all.size()); - WITH_LOCK(::cs_main, tx_pool.check(chainstate)); + WITH_LOCK(::cs_main, tx_pool.check(chainstate.CoinsTip(), chainstate.m_chain.Height() + 1)); } SyncWithValidationInterfaceQueue(); } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index f65fb24f19..a0d9e2a6bf 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -672,7 +672,7 @@ void CTxMemPool::clear() _clear(); } -void CTxMemPool::check(CChainState& active_chainstate) const +void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const { if (m_check_ratio == 0) return; @@ -687,9 +687,7 @@ void CTxMemPool::check(CChainState& active_chainstate) const uint64_t innerUsage = 0; uint64_t prev_ancestor_count{0}; - CCoinsViewCache& active_coins_tip = active_chainstate.CoinsTip(); CCoinsViewCache mempoolDuplicate(const_cast(&active_coins_tip)); - const int64_t spendheight = active_chainstate.m_chain.Height() + 1; for (const auto& it : GetSortedDepthAndScore()) { checkTotal += it->GetTxSize(); diff --git a/src/txmempool.h b/src/txmempool.h index 27ee0628a7..a3a11eb72b 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -622,7 +622,7 @@ class CTxMemPool * all inputs are in the mapNextTx array). If sanity-checking is turned off, * check does nothing. */ - void check(CChainState& active_chainstate) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void check(const CCoinsViewCache& active_coins_tip, int64_t spendheight) const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); // addUnchecked must updated state for all ancestors of a given transaction, // to track size/count of descendant transactions. First version of diff --git a/src/validation.cpp b/src/validation.cpp index 863502e0d7..8f0ddd9064 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2486,7 +2486,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex // any disconnected transactions back to the mempool. MaybeUpdateMempoolForReorg(disconnectpool, true); } - if (m_mempool) m_mempool->check(*this); + if (m_mempool) m_mempool->check(this->CoinsTip(), this->m_chain.Height() + 1); CheckForkWarningConditions(); From 3cc95345ca49b87e8caca9a0e6418c63ae1e463a Mon Sep 17 00:00:00 2001 From: fyquah Date: Mon, 1 Mar 2021 20:03:35 +0000 Subject: [PATCH 26/96] rpc: Replace boolean argument for tx details with enum class. Co-authored-by: Luke Dashjr Co-authored-by: 0xB10C <19157360+0xB10C@users.noreply.github.com> --- src/bench/rpc_blockchain.cpp | 4 +-- src/core_io.h | 8 ++++++ src/rest.cpp | 8 +++--- src/rpc/blockchain.cpp | 49 +++++++++++++++++++++++------------- src/rpc/blockchain.h | 2 +- 5 files changed, 46 insertions(+), 25 deletions(-) diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index c8886a4c23..a294676133 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -40,7 +40,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -50,7 +50,7 @@ BENCHMARK(BlockToJsonVerbose); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/core_io.h b/src/core_io.h index be93a17efe..b545d0b59e 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -20,6 +20,14 @@ class uint256; class UniValue; class CTxUndo; +/** + * Verbose level for block's transaction + */ +enum class TxVerbosity { + SHOW_TXID, //!< Only TXID for each block's transaction + SHOW_DETAILS //!< Include TXID, inputs, outputs, and other common block's transaction information +}; + // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); diff --git a/src/rest.cpp b/src/rest.cpp index e50ab33e54..d442a5e9fb 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -260,7 +260,7 @@ static bool rest_headers(const std::any& context, static bool rest_block(const std::any& context, HTTPRequest* req, const std::string& strURIPart, - bool showTxDetails) + TxVerbosity tx_verbosity) { if (!CheckWarmup(req)) return false; @@ -312,7 +312,7 @@ static bool rest_block(const std::any& context, } case RetFormat::JSON: { - UniValue objBlock = blockToJSON(block, tip, pblockindex, showTxDetails); + UniValue objBlock = blockToJSON(block, tip, pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -327,12 +327,12 @@ static bool rest_block(const std::any& context, static bool rest_block_extended(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, true); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_DETAILS); } static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, false); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_TXID); } // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ac746de32f..e4a9fa47fd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -200,7 +200,7 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) { UniValue result = blockheaderToJSON(tip, blockindex); @@ -208,22 +208,28 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("weight", (int)::GetBlockWeight(block)); UniValue txs(UniValue::VARR); - if (txDetails) { - CBlockUndo blockUndo; - const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); - for (size_t i = 0; i < block.vtx.size(); ++i) { - const CTransactionRef& tx = block.vtx.at(i); - // coinbase transaction (i == 0) doesn't have undo data - const CTxUndo* txundo = (have_undo && i) ? &blockUndo.vtxundo.at(i - 1) : nullptr; - UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo); - txs.push_back(objTx); - } - } else { - for (const CTransactionRef& tx : block.vtx) { - txs.push_back(tx->GetHash().GetHex()); - } + + switch (verbosity) { + case TxVerbosity::SHOW_TXID: + for (const CTransactionRef& tx : block.vtx) { + txs.push_back(tx->GetHash().GetHex()); + } + break; + + case TxVerbosity::SHOW_DETAILS: + CBlockUndo blockUndo; + const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); + + for (size_t i = 0; i < block.vtx.size(); ++i) { + const CTransactionRef& tx = block.vtx.at(i); + // coinbase transaction (i.e. i == 0) doesn't have undo data + const CTxUndo* txundo = (have_undo && i) ? &blockUndo.vtxundo.at(i - 1) : nullptr; + UniValue objTx(UniValue::VOBJ); + TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo); + txs.push_back(objTx); + } } + result.pushKV("tx", txs); return result; @@ -931,7 +937,7 @@ static RPCHelpMan getblock() return RPCHelpMan{"getblock", "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block .\n" - "If verbosity is 2, returns an Object with information about block and information about each transaction. \n", + "If verbosity is 2, returns an Object with information about block and information about each transaction.\n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{1}, "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, @@ -1018,7 +1024,14 @@ static RPCHelpMan getblock() return strHex; } - return blockToJSON(block, tip, pblockindex, verbosity >= 2); + TxVerbosity tx_verbosity; + if (verbosity == 1) { + tx_verbosity = TxVerbosity::SHOW_TXID; + } else { + tx_verbosity = TxVerbosity::SHOW_DETAILS; + } + + return blockToJSON(block, tip, pblockindex, tx_verbosity); }, }; } diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 09e471afdd..5143de0196 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -38,7 +38,7 @@ double GetDifficulty(const CBlockIndex* blockindex); void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false) LOCKS_EXCLUDED(cs_main); +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity = TxVerbosity::SHOW_TXID) LOCKS_EXCLUDED(cs_main); /** Mempool information to JSON */ UniValue MempoolInfoToJSON(const CTxMemPool& pool); From 51dbc167e98daab317baa80cf80bfda337672dab Mon Sep 17 00:00:00 2001 From: fyquah Date: Sat, 27 Feb 2021 17:39:09 +0000 Subject: [PATCH 27/96] rpc: Add level 3 verbosity to getblock RPC call. Display the prevout in transaction inputs when calling getblock level 3 verbosity. Co-authored-by: Luke Dashjr Co-authored-by: 0xB10C <19157360+0xB10C@users.noreply.github.com> --- src/bench/rpc_blockchain.cpp | 4 ++-- src/core_io.h | 5 +++-- src/core_write.cpp | 23 +++++++++++++++++++++-- src/rest.cpp | 2 +- src/rpc/blockchain.cpp | 12 ++++++++---- src/rpc/blockchain.h | 3 ++- 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index a294676133..3bef64f720 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -40,7 +40,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -50,7 +50,7 @@ BENCHMARK(BlockToJsonVerbose); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/core_io.h b/src/core_io.h index b545d0b59e..4d7199ab12 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -25,7 +25,8 @@ class CTxUndo; */ enum class TxVerbosity { SHOW_TXID, //!< Only TXID for each block's transaction - SHOW_DETAILS //!< Include TXID, inputs, outputs, and other common block's transaction information + SHOW_DETAILS, //!< Include TXID, inputs, outputs, and other common block's transaction information + SHOW_DETAILS_AND_PREVOUT //!< The same as previous option with information about prevouts if available }; // core_read.cpp @@ -54,6 +55,6 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address = true); void ScriptToUniv(const CScript& script, UniValue& out); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_write.cpp b/src/core_write.cpp index 6b13e4c586..d14a3f306b 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -163,7 +163,7 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -204,8 +204,27 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, in.pushKV("txinwitness", txinwitness); } if (calculate_fee) { - const CTxOut& prev_txout = txundo->vprevout[i].out; + const Coin& prev_coin = txundo->vprevout[i]; + const CTxOut& prev_txout = prev_coin.out; + amt_total_in += prev_txout.nValue; + switch (verbosity) { + case TxVerbosity::SHOW_TXID: + case TxVerbosity::SHOW_DETAILS: + break; + + case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: + UniValue o_script_pub_key(UniValue::VOBJ); + ScriptPubKeyToUniv(prev_txout.scriptPubKey, o_script_pub_key, /* includeHex */ true); + + UniValue p(UniValue::VOBJ); + p.pushKV("generated", bool(prev_coin.fCoinBase)); + p.pushKV("height", uint64_t(prev_coin.nHeight)); + p.pushKV("value", ValueFromAmount(prev_txout.nValue)); + p.pushKV("scriptPubKey", o_script_pub_key); + in.pushKV("prevout", p); + break; + } } in.pushKV("sequence", (int64_t)txin.nSequence); vin.push_back(in); diff --git a/src/rest.cpp b/src/rest.cpp index d442a5e9fb..e21fd8dad5 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -327,7 +327,7 @@ static bool rest_block(const std::any& context, static bool rest_block_extended(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, TxVerbosity::SHOW_DETAILS); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); } static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, const std::string& strURIPart) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e4a9fa47fd..92e608a030 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -217,15 +217,16 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn break; case TxVerbosity::SHOW_DETAILS: + case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); // coinbase transaction (i.e. i == 0) doesn't have undo data - const CTxUndo* txundo = (have_undo && i) ? &blockUndo.vtxundo.at(i - 1) : nullptr; + const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo); + TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo, verbosity); txs.push_back(objTx); } } @@ -937,7 +938,8 @@ static RPCHelpMan getblock() return RPCHelpMan{"getblock", "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block .\n" - "If verbosity is 2, returns an Object with information about block and information about each transaction.\n", + "If verbosity is 2, returns an Object with information about block and information about each transaction.\n" + "If verbosity is 3, returns an Object with information about block and information about each transaction, including prevout information for inputs (only for unpruned blocks in the current best chain).\n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{1}, "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, @@ -1027,8 +1029,10 @@ static RPCHelpMan getblock() TxVerbosity tx_verbosity; if (verbosity == 1) { tx_verbosity = TxVerbosity::SHOW_TXID; - } else { + } else if (verbosity == 2) { tx_verbosity = TxVerbosity::SHOW_DETAILS; + } else { + tx_verbosity = TxVerbosity::SHOW_DETAILS_AND_PREVOUT; } return blockToJSON(block, tip, pblockindex, tx_verbosity); diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 5143de0196..d9c6761f47 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -6,6 +6,7 @@ #define BITCOIN_RPC_BLOCKCHAIN_H #include +#include #include #include @@ -38,7 +39,7 @@ double GetDifficulty(const CBlockIndex* blockindex); void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity = TxVerbosity::SHOW_TXID) LOCKS_EXCLUDED(cs_main); +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); /** Mempool information to JSON */ UniValue MempoolInfoToJSON(const CTxMemPool& pool); From 4330af6f72172848f5971a052a8f325ed50eb576 Mon Sep 17 00:00:00 2001 From: fyquah Date: Sat, 27 Feb 2021 14:19:32 +0000 Subject: [PATCH 28/96] rpc: Add test for level 3 verbosity getblock rpc call. --- test/functional/rpc_blockchain.py | 66 +++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index c3c6ade684..eea9ee26cb 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -434,17 +434,55 @@ def _test_getblock(self): miniwallet.send_self_transfer(fee_rate=fee_per_kb, from_node=node) blockhash = self.generate(node, 1)[0] - self.log.info("Test getblock with verbosity 1 doesn't include fee") - block = node.getblock(blockhash, 1) - assert 'fee' not in block['tx'][1] - - self.log.info('Test getblock with verbosity 2 includes expected fee') - block = node.getblock(blockhash, 2) - tx = block['tx'][1] - assert 'fee' in tx - assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) - - self.log.info("Test getblock with verbosity 2 still works with pruned Undo data") + def assert_fee_not_in_block(verbosity): + block = node.getblock(blockhash, verbosity) + assert 'fee' not in block['tx'][1] + + def assert_fee_in_block(verbosity): + block = node.getblock(blockhash, verbosity) + tx = block['tx'][1] + assert 'fee' in tx + assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) + + def assert_vin_contains_prevout(verbosity): + block = node.getblock(blockhash, verbosity) + tx = block["tx"][1] + total_vin = Decimal("0.00000000") + total_vout = Decimal("0.00000000") + for vin in tx["vin"]: + assert "prevout" in vin + assert_equal(set(vin["prevout"].keys()), set(("value", "height", "generated", "scriptPubKey"))) + assert_equal(vin["prevout"]["generated"], True) + total_vin += vin["prevout"]["value"] + for vout in tx["vout"]: + total_vout += vout["value"] + assert_equal(total_vin, total_vout + tx["fee"]) + + def assert_vin_does_not_contain_prevout(verbosity): + block = node.getblock(blockhash, verbosity) + tx = block["tx"][1] + if isinstance(tx, str): + # In verbosity level 1, only the transaction hashes are written + pass + else: + for vin in tx["vin"]: + assert "prevout" not in vin + + self.log.info("Test that getblock with verbosity 1 doesn't include fee") + assert_fee_not_in_block(1) + + self.log.info('Test that getblock with verbosity 2 and 3 includes expected fee') + assert_fee_in_block(2) + assert_fee_in_block(3) + + self.log.info("Test that getblock with verbosity 1 and 2 does not include prevout") + assert_vin_does_not_contain_prevout(1) + assert_vin_does_not_contain_prevout(2) + + self.log.info("Test that getblock with verbosity 3 includes prevout") + assert_vin_contains_prevout(3) + + self.log.info("Test that getblock with verbosity 2 and 3 still works with pruned Undo data") datadir = get_datadir_path(self.options.tmpdir, 0) self.log.info("Test getblock with invalid verbosity type returns proper error message") @@ -458,8 +496,10 @@ def move_block_file(old, new): # Move instead of deleting so we can restore chain state afterwards move_block_file('rev00000.dat', 'rev_wrong') - block = node.getblock(blockhash, 2) - assert 'fee' not in block['tx'][1] + assert_fee_not_in_block(2) + assert_fee_not_in_block(3) + assert_vin_does_not_contain_prevout(2) + assert_vin_does_not_contain_prevout(3) # Restore chain state move_block_file('rev_wrong', 'rev00000.dat') From 459104b2aae6eeaadfa5a7e47944f1a34780dacd Mon Sep 17 00:00:00 2001 From: fyquah Date: Sat, 27 Feb 2021 18:18:41 +0000 Subject: [PATCH 29/96] rest: Add test for prevout fields in getblock --- test/functional/interface_rest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index e0716fc54a..531c42ba2c 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -311,6 +311,15 @@ def run_test(self): if 'coinbase' not in tx['vin'][0]} assert_equal(non_coinbase_txs, set(txs)) + # Verify that the non-coinbase tx has "prevout" key set + for tx_obj in json_obj["tx"]: + for vin in tx_obj["vin"]: + if "coinbase" not in vin: + assert "prevout" in vin + assert_equal(vin["prevout"]["generated"], False) + else: + assert "prevout" not in vin + # Check the same but without tx details json_obj = self.test_rest_request(f"/block/notxdetails/{newblockhash[0]}") for tx in txs: From 8edf6204a87057a451160d1e61e79d8be112e81f Mon Sep 17 00:00:00 2001 From: fyquah Date: Fri, 16 Jul 2021 16:57:24 +0100 Subject: [PATCH 30/96] release-notes: Add release note about getblock verbosity level 3. --- doc/release-notes.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/release-notes.md b/doc/release-notes.md index 81e79dd3a9..3bf3e47169 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -76,6 +76,14 @@ Updated RPCs `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, `/rest/block` no longer return the `addresses` and `reqSigs` fields, which were previously deprecated in 22.0. (#22650) +- The `getblock` RPC command now supports verbose level 3 containing transaction inputs + `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain + this information too. Every `vin` field will contain an additional `prevout` subfield + describing the spent output. `prevout` contains the following keys: + - `generated` - true if the spent coins was a coinbase. + - `height` + - `value` + - `scriptPubKey` - `listunspent` now includes `ancestorcount`, `ancestorsize`, and `ancestorfees` for each transaction output that is still in the mempool. From 5c34507ecbbdc29c086276d1c62835b461823507 Mon Sep 17 00:00:00 2001 From: fyquah Date: Tue, 3 Aug 2021 23:00:31 +0100 Subject: [PATCH 31/96] core_write: Rename calculate_fee to have_undo for clarity --- src/core_write.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core_write.cpp b/src/core_write.cpp index d14a3f306b..468694b011 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -179,7 +179,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, // If available, use Undo data to calculate the fee. Note that txundo == nullptr // for coinbase transactions and for transactions where undo data is unavailable. - const bool calculate_fee = txundo != nullptr; + const bool have_undo = txundo != nullptr; CAmount amt_total_in = 0; CAmount amt_total_out = 0; @@ -203,7 +203,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, } in.pushKV("txinwitness", txinwitness); } - if (calculate_fee) { + if (have_undo) { const Coin& prev_coin = txundo->vprevout[i]; const CTxOut& prev_txout = prev_coin.out; @@ -245,13 +245,13 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, out.pushKV("scriptPubKey", o); vout.push_back(out); - if (calculate_fee) { + if (have_undo) { amt_total_out += txout.nValue; } } entry.pushKV("vout", vout); - if (calculate_fee) { + if (have_undo) { const CAmount fee = amt_total_in - amt_total_out; CHECK_NONFATAL(MoneyRange(fee)); entry.pushKV("fee", ValueFromAmount(fee)); From fac303c504ab19b863fddc7a0093068fee9d4ef3 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 29 Sep 2021 17:05:43 +0200 Subject: [PATCH 32/96] refactor: Remove unused MakeUCharSpan --- src/bloom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bloom.cpp b/src/bloom.cpp index 15e06389de..53f95bea3b 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -82,7 +82,7 @@ bool CBloomFilter::contains(const COutPoint& outpoint) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - return contains(MakeUCharSpan(stream)); + return contains(stream); } bool CBloomFilter::IsWithinSizeConstraints() const From fa1e5de2db2c7c95b96773a4ac231ab4249317e9 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 29 Sep 2021 17:22:44 +0200 Subject: [PATCH 33/96] scripted-diff: Move bloom to src/common -BEGIN VERIFY SCRIPT- # Move to directory mkdir src/common git mv src/bloom.cpp src/common/ git mv src/bloom.h src/common/ # Replace occurrences sed -i 's|\|common/bloom.cpp|g' $(git grep -l 'bloom.cpp') sed -i 's|\|common/bloom.h|g' $(git grep -l 'bloom.h') sed -i 's|BITCOIN_BLOOM_H|BITCOIN_COMMON_BLOOM_H|g' $(git grep -l 'BLOOM_H') -END VERIFY SCRIPT- --- src/Makefile.am | 4 ++-- src/banman.h | 2 +- src/bench/rollingbloom.cpp | 2 +- src/{ => common}/bloom.cpp | 2 +- src/{ => common}/bloom.h | 6 +++--- src/merkleblock.h | 2 +- src/net.h | 2 +- src/test/bloom_tests.cpp | 2 +- src/test/fuzz/bloom_filter.cpp | 2 +- src/test/fuzz/rolling_bloom_filter.cpp | 2 +- test/sanitizer_suppressions/ubsan | 4 ++-- 11 files changed, 15 insertions(+), 15 deletions(-) rename src/{ => common}/bloom.cpp (99%) rename src/{ => common}/bloom.h (98%) diff --git a/src/Makefile.am b/src/Makefile.am index b366252ba3..df17197198 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -123,7 +123,7 @@ BITCOIN_CORE_H = \ bech32.h \ blockencodings.h \ blockfilter.h \ - bloom.h \ + common/bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -538,7 +538,7 @@ libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bech32.cpp \ - bloom.cpp \ + common/bloom.cpp \ chainparams.cpp \ coins.cpp \ compressor.cpp \ diff --git a/src/banman.h b/src/banman.h index 8a03a9e3fc..f495dab49d 100644 --- a/src/banman.h +++ b/src/banman.h @@ -6,7 +6,7 @@ #define BITCOIN_BANMAN_H #include -#include +#include #include #include // For banmap_t #include diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 28167767db..30bc1d5fdf 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -4,7 +4,7 @@ #include -#include +#include static void RollingBloom(benchmark::Bench& bench) { diff --git a/src/bloom.cpp b/src/common/bloom.cpp similarity index 99% rename from src/bloom.cpp rename to src/common/bloom.cpp index 53f95bea3b..26b70b4d14 100644 --- a/src/bloom.cpp +++ b/src/common/bloom.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include diff --git a/src/bloom.h b/src/common/bloom.h similarity index 98% rename from src/bloom.h rename to src/common/bloom.h index 422646d8b9..25c16fbfe2 100644 --- a/src/bloom.h +++ b/src/common/bloom.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_BLOOM_H -#define BITCOIN_BLOOM_H +#ifndef BITCOIN_COMMON_BLOOM_H +#define BITCOIN_COMMON_BLOOM_H #include #include @@ -124,4 +124,4 @@ class CRollingBloomFilter int nHashFuncs; }; -#endif // BITCOIN_BLOOM_H +#endif // BITCOIN_COMMON_BLOOM_H diff --git a/src/merkleblock.h b/src/merkleblock.h index 0e4ed72130..ef6cc3cee9 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include diff --git a/src/net.h b/src/net.h index a97ed9946d..a4ee03d928 100644 --- a/src/net.h +++ b/src/net.h @@ -7,7 +7,7 @@ #define BITCOIN_NET_H #include -#include +#include #include #include #include diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 23ef2062ef..fe5ed0a3c8 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index c5bb8744a4..746591a176 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include #include diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index b9ed497e68..9c18ad49cb 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include #include diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index b52e105a33..1d608b9ec1 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -22,7 +22,7 @@ unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h unsigned-integer-overflow:bitcoin-tx.cpp -unsigned-integer-overflow:bloom.cpp +unsigned-integer-overflow:common/bloom.cpp unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h unsigned-integer-overflow:coded_stream.h @@ -48,7 +48,7 @@ implicit-integer-sign-change:*/new_allocator.h implicit-integer-sign-change:addrman.h implicit-integer-sign-change:arith_uint256.cpp implicit-integer-sign-change:bech32.cpp -implicit-integer-sign-change:bloom.cpp +implicit-integer-sign-change:common/bloom.cpp implicit-integer-sign-change:chain.cpp implicit-integer-sign-change:chain.h implicit-integer-sign-change:coins.h From fa2d611bedc2a755dcf84a82699c70b57b903cf6 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 29 Sep 2021 17:09:55 +0200 Subject: [PATCH 34/96] style: Sort --- src/Makefile.am | 4 ++-- src/merkleblock.h | 4 ++-- src/net.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index df17197198..8eff84a82d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -123,7 +123,6 @@ BITCOIN_CORE_H = \ bech32.h \ blockencodings.h \ blockfilter.h \ - common/bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -131,6 +130,7 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ + common/bloom.h \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ @@ -538,9 +538,9 @@ libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bech32.cpp \ - common/bloom.cpp \ chainparams.cpp \ coins.cpp \ + common/bloom.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ diff --git a/src/merkleblock.h b/src/merkleblock.h index ef6cc3cee9..70749b6378 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_MERKLEBLOCK_H #define BITCOIN_MERKLEBLOCK_H +#include +#include #include #include -#include -#include #include diff --git a/src/net.h b/src/net.h index a4ee03d928..658fddf699 100644 --- a/src/net.h +++ b/src/net.h @@ -7,8 +7,8 @@ #define BITCOIN_NET_H #include -#include #include +#include #include #include #include From 632aad9e6d8369750f4327a886ca5b3d3fed89bd Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 29 Sep 2021 16:22:44 -0400 Subject: [PATCH 35/96] Make CAddrman::Select_ select buckets, not positions, first The original CAddrMan behaviour (before commit e6b343d880f50d52390c5af8623afa15fcbc65a2) was to pick a uniformly random non-empty bucket, and then pick a random element from that bucket. That commit, which introduced deterministic placement of entries in buckets, changed this to picking a uniformly random non-empty bucket position instead. This commit reverts the original high-level behavior, but in the deterministic placement model. --- src/addrman.cpp | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/addrman.cpp b/src/addrman.cpp index c364a7710b..08aae6f2a4 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -709,38 +709,54 @@ std::pair AddrManImpl::Select_(bool newOnly) const // use a tried node double fChanceFactor = 1.0; while (1) { + // Pick a tried bucket, and an initial position in that bucket. int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvTried[nKBucket][nKBucketPos]; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE]; const auto it_found{mapInfo.find(nId)}; assert(it_found != mapInfo.end()); const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { return {info, info.nLastTry}; } + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } else { // use a new node double fChanceFactor = 1.0; while (1) { + // Pick a new bucket, and an initial position in that bucket. int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvNew[nUBucket][nUBucketPos]; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE]; const auto it_found{mapInfo.find(nId)}; assert(it_found != mapInfo.end()); const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { return {info, info.nLastTry}; } + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } From 35e814c1cfedf17e7751b730ae1e36099801be4c Mon Sep 17 00:00:00 2001 From: = Date: Tue, 5 Oct 2021 21:52:22 +0530 Subject: [PATCH 36/96] qt: never disable HD status icon Make the watch-only icon in the bottom bar enabled by default for a better user interface. Currently, it's disabled by default with a 50% opacity which makes it hard to see the icon in dark mode. --- src/qt/bitcoingui.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 610637360b..b68ce39b53 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -1304,8 +1304,6 @@ void BitcoinGUI::setHDStatus(bool privkeyDisabled, int hdEnabled) labelWalletHDStatusIcon->setThemedPixmap(privkeyDisabled ? QStringLiteral(":/icons/eye") : hdEnabled ? QStringLiteral(":/icons/hd_enabled") : QStringLiteral(":/icons/hd_disabled"), STATUSBAR_ICONSIZE, STATUSBAR_ICONSIZE); labelWalletHDStatusIcon->setToolTip(privkeyDisabled ? tr("Private key disabled") : hdEnabled ? tr("HD key generation is enabled") : tr("HD key generation is disabled")); labelWalletHDStatusIcon->show(); - // eventually disable the QLabel to set its opacity to 50% - labelWalletHDStatusIcon->setEnabled(hdEnabled); } void BitcoinGUI::setEncryptionStatus(int status) From faac1cda6e2ca1d86b1551fc90453132f249d511 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 6 Oct 2021 13:41:31 +0200 Subject: [PATCH 37/96] test: Use generate* from TestFramework, not TestNode --- test/functional/mempool_package_limits.py | 2 +- test/functional/wallet_transactiontime_rescan.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 2217628858..89a5c83826 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -244,7 +244,7 @@ def test_desc_count_limits_2(self): assert_equal(txres["package-error"], "package-mempool-limits") # Clear mempool and check that the package passes now - node.generate(1) + self.generate(node, 1) assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) def test_anc_count_limits(self): diff --git a/test/functional/wallet_transactiontime_rescan.py b/test/functional/wallet_transactiontime_rescan.py index 78859e6131..afa5139da7 100755 --- a/test/functional/wallet_transactiontime_rescan.py +++ b/test/functional/wallet_transactiontime_rescan.py @@ -63,7 +63,7 @@ def run_test(self): # generate some btc to create transactions and check blockcount initial_mine = COINBASE_MATURITY + 1 - minernode.generatetoaddress(initial_mine, m1) + self.generatetoaddress(minernode, initial_mine, m1) assert_equal(minernode.getblockcount(), initial_mine + 200) # synchronize nodes and time @@ -76,7 +76,7 @@ def run_test(self): miner_wallet.sendtoaddress(wo1, 10) # generate blocks and check blockcount - minernode.generatetoaddress(COINBASE_MATURITY, m1) + self.generatetoaddress(minernode, COINBASE_MATURITY, m1) assert_equal(minernode.getblockcount(), initial_mine + 300) # synchronize nodes and time @@ -89,7 +89,7 @@ def run_test(self): miner_wallet.sendtoaddress(wo2, 5) # generate blocks and check blockcount - minernode.generatetoaddress(COINBASE_MATURITY, m1) + self.generatetoaddress(minernode, COINBASE_MATURITY, m1) assert_equal(minernode.getblockcount(), initial_mine + 400) # synchronize nodes and time @@ -102,7 +102,7 @@ def run_test(self): miner_wallet.sendtoaddress(wo3, 1) # generate more blocks and check blockcount - minernode.generatetoaddress(COINBASE_MATURITY, m1) + self.generatetoaddress(minernode, COINBASE_MATURITY, m1) assert_equal(minernode.getblockcount(), initial_mine + 500) self.log.info('Check user\'s final balance and transaction count') From fac7f6102feb1eb1c47ea8cb1c75c4f4dbf2f6b0 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 6 Oct 2021 13:40:24 +0200 Subject: [PATCH 38/96] test: Use generate* node RPC, not wallet RPC --- test/functional/wallet_descriptor.py | 2 +- test/functional/wallet_importdescriptors.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 17a4c79da3..4ec44a8a6c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -84,7 +84,7 @@ def run_test(self): send_wrpc = self.nodes[0].get_wallet_rpc("desc1") # Generate some coins - self.generatetoaddress(send_wrpc, COINBASE_MATURITY + 1, send_wrpc.getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, send_wrpc.getnewaddress()) # Make transactions self.log.info("Test sending and receiving") diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index d86c3737fe..c8f9664885 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -74,7 +74,7 @@ def run_test(self): assert_equal(wpriv.getwalletinfo()['keypoolsize'], 0) self.log.info('Mining coins') - self.generatetoaddress(w0, COINBASE_MATURITY + 1, w0.getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, w0.getnewaddress()) # RPC importdescriptors ----------------------------------------------- @@ -405,7 +405,7 @@ def run_test(self): solvable=True, ismine=True) txid = w0.sendtoaddress(address, 49.99995540) - self.generatetoaddress(w0, 6, w0.getnewaddress()) + self.generatetoaddress(self.nodes[0], 6, w0.getnewaddress()) self.sync_blocks() tx = wpriv.createrawtransaction([{"txid": txid, "vout": 0}], {w0.getnewaddress(): 49.999}) signed_tx = wpriv.signrawtransactionwithwallet(tx) From fac62e6ff594f03832f5c0057f9b67c9118c21f4 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 27 Jul 2021 13:59:55 +0200 Subject: [PATCH 39/96] test: Delete generate* calls from TestNode --- test/functional/test_framework/test_framework.py | 8 ++++---- test/functional/test_framework/test_node.py | 16 ++++++++++++++-- test/functional/test_framework/wallet.py | 4 ++-- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index d87d0cacfd..727ac6aed9 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -628,19 +628,19 @@ def join_network(self): self.sync_all() def generate(self, generator, *args, **kwargs): - blocks = generator.generate(*args, **kwargs) + blocks = generator.generate(*args, invalid_call=False, **kwargs) return blocks def generateblock(self, generator, *args, **kwargs): - blocks = generator.generateblock(*args, **kwargs) + blocks = generator.generateblock(*args, invalid_call=False, **kwargs) return blocks def generatetoaddress(self, generator, *args, **kwargs): - blocks = generator.generatetoaddress(*args, **kwargs) + blocks = generator.generatetoaddress(*args, invalid_call=False, **kwargs) return blocks def generatetodescriptor(self, generator, *args, **kwargs): - blocks = generator.generatetodescriptor(*args, **kwargs) + blocks = generator.generatetodescriptor(*args, invalid_call=False, **kwargs) return blocks def sync_blocks(self, nodes=None, wait=1, timeout=60): diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index f9e2cfa2f5..e8ff41a46d 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -297,9 +297,21 @@ def wait_for_cookie_credentials(self): time.sleep(1.0 / poll_per_s) self._raise_assertion_error("Unable to retrieve cookie credentials after {}s".format(self.rpc_timeout)) - def generate(self, nblocks, maxtries=1000000): + def generate(self, nblocks, maxtries=1000000, **kwargs): self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`") - return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries) + return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries, **kwargs) + + def generateblock(self, *args, invalid_call, **kwargs): + assert not invalid_call + return self.__getattr__('generateblock')(*args, **kwargs) + + def generatetoaddress(self, *args, invalid_call, **kwargs): + assert not invalid_call + return self.__getattr__('generatetoaddress')(*args, **kwargs) + + def generatetodescriptor(self, *args, invalid_call, **kwargs): + assert not invalid_call + return self.__getattr__('generatetodescriptor')(*args, **kwargs) def get_wallet_rpc(self, wallet_name): if self.use_cli: diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index ef27cb3221..a420ea1cf8 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -109,9 +109,9 @@ def sign_tx(self, tx, fixed_length=True): break tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) - def generate(self, num_blocks): + def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" - blocks = self._test_node.generatetodescriptor(num_blocks, self.get_descriptor()) + blocks = self._test_node.generatetodescriptor(num_blocks, self.get_descriptor(), **kwargs) for b in blocks: cb_tx = self._test_node.getblock(blockhash=b, verbosity=2)['tx'][0] self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value']}) From fadf1186c899f45787a91c28120b0608bdc6c246 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 7 Oct 2021 13:22:51 +0200 Subject: [PATCH 40/96] p2p: Use mocktime for ping timeout --- src/net.cpp | 3 +-- src/net.h | 2 +- src/net_processing.cpp | 5 ++++- src/test/denialofservice_tests.cpp | 2 ++ src/test/util/net.h | 6 ++++++ test/functional/p2p_ping.py | 14 +++++++++++--- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 7271ff22b2..ad558dd598 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1295,9 +1295,8 @@ void CConnman::NotifyNumConnectionsChanged() } } -bool CConnman::ShouldRunInactivityChecks(const CNode& node, std::optional now_in) const +bool CConnman::ShouldRunInactivityChecks(const CNode& node, int64_t now) const { - const int64_t now = now_in ? now_in.value() : GetTimeSeconds(); return node.nTimeConnected + m_peer_connect_timeout < now; } diff --git a/src/net.h b/src/net.h index e2071414b4..86b58f37de 100644 --- a/src/net.h +++ b/src/net.h @@ -942,7 +942,7 @@ class CConnman std::chrono::microseconds PoissonNextSendInbound(std::chrono::microseconds now, std::chrono::seconds average_interval); /** Return true if we should disconnect the peer for failing an inactivity check. */ - bool ShouldRunInactivityChecks(const CNode& node, std::optional now=std::nullopt) const; + bool ShouldRunInactivityChecks(const CNode& node, int64_t secs_now) const; private: struct ListenSocket { diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 66b99aa2bb..22586ba7da 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -4312,8 +4312,11 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::microseconds now) { - if (m_connman.ShouldRunInactivityChecks(node_to) && peer.m_ping_nonce_sent && + if (m_connman.ShouldRunInactivityChecks(node_to, std::chrono::duration_cast(now).count()) && + peer.m_ping_nonce_sent && now > peer.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL}) { + // The ping timeout is using mocktime. To disable the check during + // testing, increase -peertimeout. LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), peer.m_id); node_to.fDisconnect = true; return; diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 0bfe6eecd9..668ff150ee 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -52,6 +52,8 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); auto connman = std::make_unique(0x1337, 0x1337, *m_node.addrman); + // Disable inactivity checks for this test to avoid interference + static_cast(connman.get())->SetPeerConnectTimeout(99999); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); diff --git a/src/test/util/net.h b/src/test/util/net.h index 939ec322ed..d89fc34b75 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -17,6 +17,12 @@ struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; + + void SetPeerConnectTimeout(int64_t timeout) + { + m_peer_connect_timeout = timeout; + } + void AddTestNode(CNode& node) { LOCK(cs_vNodes); diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py index 888e986fba..d67e97acf7 100755 --- a/test/functional/p2p_ping.py +++ b/test/functional/p2p_ping.py @@ -30,11 +30,16 @@ def on_ping(self, message): pass +TIMEOUT_INTERVAL = 20 * 60 + + class PingPongTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [['-peertimeout=3']] + # Set the peer connection timeout low. It does not matter for this + # test, as long as it is less than TIMEOUT_INTERVAL. + self.extra_args = [['-peertimeout=1']] def check_peer_info(self, *, pingtime, minping, pingwait): stats = self.nodes[0].getpeerinfo()[0] @@ -110,8 +115,11 @@ def run_test(self): self.nodes[0].ping() no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) with self.nodes[0].assert_debug_log(['ping timeout: 1201.000000s']): - self.mock_forward(20 * 60 + 1) - time.sleep(4) # peertimeout + 1 + self.mock_forward(TIMEOUT_INTERVAL // 2) + # Check that sending a ping does not prevent the disconnect + no_pong_node.sync_with_ping() + self.mock_forward(TIMEOUT_INTERVAL // 2 + 1) + no_pong_node.wait_for_disconnect() if __name__ == '__main__': From 9b49ed656fb2b687fbbe8a3236d18285957eee16 Mon Sep 17 00:00:00 2001 From: fanquake Date: Mon, 11 Oct 2021 20:45:56 +0800 Subject: [PATCH 41/96] Squashed 'src/univalue/' changes from 98fadc0909..a44caf65fe a44caf65fe Merge bitcoin-core/univalue-subtree#28: Import fixes for sanitizer reported issues 135254331e Import fixes for sanitizer reported issues d5fb86940e refactor: use c++11 range based for loop in checkObject ff9c379304 refactor: Use nullptr (c++11) instead of NULL 08a99754d5 build: use ax_cxx_compile_stdcxx.m4 to check for C++11 support 66d3713ce7 Merge bitcoin-core/univalue#29: ci: travis -> cirrus 808d487292 ci: travis -> cirrus c390ac375f Merge bitcoin-core/univalue#19: Split sources for easier buildsystem integration 4a5b0a1c65 build: Move source entries out to sources.mk 6c7d94b33c build: cleanup wonky gen usage a222637c6d Merge #23: Merge changes from jgarzik/univalue@1ae6a23 f77d0f718d Merge commit '1ae6a231a0169938eb3972c1d48dd17cba5947e1' into HEAD 1ae6a231a0 Merge pull request #57 from MarcoFalke/test_fix 92bdd11f0b univalue_write: remove unneeded sstream.h include ffb621c130 Merge pull request #56 from drodil/remove_sstream_header f33acf9fe8 Merge commit '7890db9~' into HEAD 66e0adec4d Remove unnecessary sstream header from univalue.h 88967f6586 Version 1.0.4 1dc113dbef Merge pull request #50 from luke-jr/pushKV_bool 72392fb227 [tests] test pushKV for boolean values c23132bcf4 Pushing boolean value to univalue correctly 81faab26a1 Merge pull request #48 from fwolfst/47-UPDATE_MIT_LINK_TO_HTTPS b17634ef24 Update URLs to MIT license. 88ab64f6b5 Merge pull request #46 from jasonbcox/master 35ed96da31 Merge pull request #44 from MarcoFalke/Mf1709-univalue-cherrypick-explicit 420c226290 Merge pull request #45 from MarcoFalke/Mf1710-univalue-revert-test git-subtree-dir: src/univalue git-subtree-split: a44caf65fe55b9dd8ddb08f04c0f70409efd53b3 --- .cirrus.yml | 44 ++ .travis.yml | 51 -- Makefile.am | 90 +-- build-aux/m4/ax_cxx_compile_stdcxx.m4 | 962 ++++++++++++++++++++++++++ configure.ac | 9 +- gen/gen.cpp | 4 +- include/univalue.h | 4 +- lib/univalue.cpp | 16 +- lib/univalue_escapes.h | 442 ++++++------ lib/univalue_get.cpp | 7 +- lib/univalue_read.cpp | 6 +- lib/univalue_utffilter.h | 2 +- lib/univalue_write.cpp | 7 +- sources.mk | 95 +++ test/object.cpp | 2 +- test/unitester.cpp | 4 +- 16 files changed, 1369 insertions(+), 376 deletions(-) create mode 100644 .cirrus.yml delete mode 100644 .travis.yml create mode 100644 build-aux/m4/ax_cxx_compile_stdcxx.m4 create mode 100644 sources.mk diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..f140fee12b --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,44 @@ +env: + MAKEJOBS: "-j4" + RUN_TESTS: "true" + BASE_OUTDIR: "$CIRRUS_WORKING_DIR/out_dir_base" + DEBIAN_FRONTEND: "noninteractive" + +task: + container: + image: ubuntu:focal + cpu: 1 + memory: 1G + greedy: true # https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4 + + matrix: + - name: "gcc" + env: + CC: "gcc" + CXX: "g++" + APT_PKGS: "gcc" + - name: "clang" + env: + CC: "clang" + CXX: "clang++" + APT_PKGS: "clang" + - name: "mingw" + env: + CC: "" + CXX: "" + UNIVALUE_CONFIG: "--host=x86_64-w64-mingw32" + APT_PKGS: "g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64" + RUN_TESTS: "false" + + install_script: + - apt update + - apt install -y pkg-config build-essential libtool autotools-dev automake bsdmainutils + - apt install -y $APT_PKGS + autogen_script: + - ./autogen.sh + configure_script: + - ./configure --cache-file=config.cache --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib $UNIVALUE_CONFIG + make_script: + - make $MAKEJOBS V=1 + test_script: + - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 43a1ed362e..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -language: cpp - -compiler: - - clang - - gcc - -os: - - linux - - osx - -sudo: false - -env: - global: - - MAKEJOBS=-j3 - - RUN_TESTS=true - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - -cache: - apt: true - -addons: - apt: - packages: - - pkg-config - -before_script: - - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - -script: - - if [ -n "$UNIVALUE_CONFIG" ]; then unset CC; unset CXX; fi - - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - - UNIVALUE_CONFIG_ALL="--prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - - ./configure --cache-file=config.cache $UNIVALUE_CONFIG_ALL $UNIVALUE_CONFIG || ( cat config.log && false) - - make -s $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL ; false ) - - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi - -matrix: - fast_finish: true - include: - - os: linux - compiler: gcc - env: UNIVALUE_CONFIG=--host=x86_64-w64-mingw32 RUN_TESTS=false - addons: - apt: - packages: - - g++-mingw-w64-x86-64 - - gcc-mingw-w64-x86-64 - - binutils-mingw-w64-x86-64 diff --git a/Makefile.am b/Makefile.am index 0f5ba59954..476f14b922 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,20 +1,17 @@ +include sources.mk ACLOCAL_AMFLAGS = -I build-aux/m4 -.PHONY: gen +.PHONY: gen FORCE .INTERMEDIATE: $(GENBIN) -include_HEADERS = include/univalue.h -noinst_HEADERS = lib/univalue_escapes.h lib/univalue_utffilter.h +include_HEADERS = $(UNIVALUE_DIST_HEADERS_INT) +noinst_HEADERS = $(UNIVALUE_LIB_HEADERS_INT) lib_LTLIBRARIES = libunivalue.la pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = pc/libunivalue.pc -libunivalue_la_SOURCES = \ - lib/univalue.cpp \ - lib/univalue_get.cpp \ - lib/univalue_read.cpp \ - lib/univalue_write.cpp +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) libunivalue_la_LDFLAGS = \ -version-info $(LIBUNIVALUE_CURRENT):$(LIBUNIVALUE_REVISION):$(LIBUNIVALUE_AGE) \ @@ -30,89 +27,32 @@ $(GENBIN): $(GEN_SRCS) @echo Building $@ $(AM_V_at)c++ -I$(top_srcdir)/include -o $@ $< -gen: lib/univalue_escapes.h $(GENBIN) - @echo Updating $< +gen: $(GENBIN) FORCE + @echo Updating lib/univalue_escapes.h $(AM_V_at)$(GENBIN) > lib/univalue_escapes.h noinst_PROGRAMS = $(TESTS) test/test_json -TEST_DATA_DIR=test - -test_unitester_SOURCES = test/unitester.cpp +test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) test_unitester_LDADD = libunivalue.la -test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(TEST_DATA_DIR)\" +test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_test_json_SOURCES = test/test_json.cpp +test_test_json_SOURCES = $(UNIVALUE_TEST_JSON_INT) test_test_json_LDADD = libunivalue.la test_test_json_CXXFLAGS = -I$(top_srcdir)/include test_test_json_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_no_nul_SOURCES = test/no_nul.cpp +test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) test_no_nul_LDADD = libunivalue.la test_no_nul_CXXFLAGS = -I$(top_srcdir)/include test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_object_SOURCES = test/object.cpp +test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) test_object_LDADD = libunivalue.la test_object_CXXFLAGS = -I$(top_srcdir)/include test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -TEST_FILES = \ - $(TEST_DATA_DIR)/fail10.json \ - $(TEST_DATA_DIR)/fail11.json \ - $(TEST_DATA_DIR)/fail12.json \ - $(TEST_DATA_DIR)/fail13.json \ - $(TEST_DATA_DIR)/fail14.json \ - $(TEST_DATA_DIR)/fail15.json \ - $(TEST_DATA_DIR)/fail16.json \ - $(TEST_DATA_DIR)/fail17.json \ - $(TEST_DATA_DIR)/fail18.json \ - $(TEST_DATA_DIR)/fail19.json \ - $(TEST_DATA_DIR)/fail1.json \ - $(TEST_DATA_DIR)/fail20.json \ - $(TEST_DATA_DIR)/fail21.json \ - $(TEST_DATA_DIR)/fail22.json \ - $(TEST_DATA_DIR)/fail23.json \ - $(TEST_DATA_DIR)/fail24.json \ - $(TEST_DATA_DIR)/fail25.json \ - $(TEST_DATA_DIR)/fail26.json \ - $(TEST_DATA_DIR)/fail27.json \ - $(TEST_DATA_DIR)/fail28.json \ - $(TEST_DATA_DIR)/fail29.json \ - $(TEST_DATA_DIR)/fail2.json \ - $(TEST_DATA_DIR)/fail30.json \ - $(TEST_DATA_DIR)/fail31.json \ - $(TEST_DATA_DIR)/fail32.json \ - $(TEST_DATA_DIR)/fail33.json \ - $(TEST_DATA_DIR)/fail34.json \ - $(TEST_DATA_DIR)/fail35.json \ - $(TEST_DATA_DIR)/fail36.json \ - $(TEST_DATA_DIR)/fail37.json \ - $(TEST_DATA_DIR)/fail38.json \ - $(TEST_DATA_DIR)/fail39.json \ - $(TEST_DATA_DIR)/fail40.json \ - $(TEST_DATA_DIR)/fail41.json \ - $(TEST_DATA_DIR)/fail42.json \ - $(TEST_DATA_DIR)/fail44.json \ - $(TEST_DATA_DIR)/fail45.json \ - $(TEST_DATA_DIR)/fail3.json \ - $(TEST_DATA_DIR)/fail4.json \ - $(TEST_DATA_DIR)/fail5.json \ - $(TEST_DATA_DIR)/fail6.json \ - $(TEST_DATA_DIR)/fail7.json \ - $(TEST_DATA_DIR)/fail8.json \ - $(TEST_DATA_DIR)/fail9.json \ - $(TEST_DATA_DIR)/pass1.json \ - $(TEST_DATA_DIR)/pass2.json \ - $(TEST_DATA_DIR)/pass3.json \ - $(TEST_DATA_DIR)/pass4.json \ - $(TEST_DATA_DIR)/round1.json \ - $(TEST_DATA_DIR)/round2.json \ - $(TEST_DATA_DIR)/round3.json \ - $(TEST_DATA_DIR)/round4.json \ - $(TEST_DATA_DIR)/round5.json \ - $(TEST_DATA_DIR)/round6.json \ - $(TEST_DATA_DIR)/round7.json - -EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS) +TEST_FILES = $(UNIVALUE_TEST_FILES_INT) + +EXTRA_DIST=$(UNIVALUE_TEST_FILES_INT) $(GEN_SRCS) diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000000..f7e5137003 --- /dev/null +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,962 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for no added switch, and then for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016, 2018 Krzesimir Nowak +# Copyright (c) 2019 Enji Cooper +# Copyright (c) 2020 Jason Merrill +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 12 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/configure.ac b/configure.ac index 8298332ac1..495b25a53d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [3]) -m4_define([libunivalue_interface_age], [3]) +m4_define([libunivalue_micro_version], [4]) +m4_define([libunivalue_interface_age], [4]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.3], +AC_INIT([univalue], [1.0.4], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used @@ -45,6 +45,9 @@ AC_SUBST(LIBUNIVALUE_AGE) LT_INIT LT_LANG([C++]) +dnl Require C++11 compiler (no GNU extensions) +AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) + case $host in *mingw*) LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static" diff --git a/gen/gen.cpp b/gen/gen.cpp index 85fe20924a..b8a6c73f4e 100644 --- a/gen/gen.cpp +++ b/gen/gen.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. // // To re-create univalue_escapes.h: @@ -45,7 +45,7 @@ static void outputEscape() for (unsigned int i = 0; i < 256; i++) { if (escapes[i].empty()) { - printf("\tNULL,\n"); + printf("\tnullptr,\n"); } else { printf("\t\""); diff --git a/include/univalue.h b/include/univalue.h index 048e162f7d..fc5cf402be 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef __UNIVALUE_H__ #define __UNIVALUE_H__ @@ -14,8 +14,6 @@ #include #include -#include // .get_int64() - class UniValue { public: enum VType { VNULL, VOBJ, VARR, VSTR, VNUM, VBOOL, }; diff --git a/lib/univalue.cpp b/lib/univalue.cpp index 4c9c15d63e..c4e59fae74 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -178,17 +178,19 @@ bool UniValue::findKey(const std::string& key, size_t& retIdx) const bool UniValue::checkObject(const std::map& t) const { - if (typ != VOBJ) + if (typ != VOBJ) { return false; + } - for (std::map::const_iterator it = t.begin(); - it != t.end(); ++it) { + for (const auto& object: t) { size_t idx = 0; - if (!findKey(it->first, idx)) + if (!findKey(object.first, idx)) { return false; + } - if (values.at(idx).getType() != it->second) + if (values.at(idx).getType() != object.second) { return false; + } } return true; @@ -228,7 +230,7 @@ const char *uvTypeName(UniValue::VType t) } // not reached - return NULL; + return nullptr; } const UniValue& find_value(const UniValue& obj, const std::string& name) diff --git a/lib/univalue_escapes.h b/lib/univalue_escapes.h index 74596aab6d..3f714f8e5b 100644 --- a/lib/univalue_escapes.h +++ b/lib/univalue_escapes.h @@ -34,229 +34,229 @@ static const char *escapes[256] = { "\\u001d", "\\u001e", "\\u001f", - NULL, - NULL, + nullptr, + nullptr, "\\\"", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, "\\\\", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, "\\u007f", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, }; #endif // BITCOIN_UNIVALUE_UNIVALUE_ESCAPES_H diff --git a/lib/univalue_get.cpp b/lib/univalue_get.cpp index 0ad6146545..5af89a3561 100644 --- a/lib/univalue_get.cpp +++ b/lib/univalue_get.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -11,6 +11,7 @@ #include #include #include +#include #include "univalue.h" @@ -31,7 +32,7 @@ bool ParseInt32(const std::string& str, int32_t *out) { if (!ParsePrechecks(str)) return false; - char *endp = NULL; + char *endp = nullptr; errno = 0; // strtol will not set errno if valid long int n = strtol(str.c_str(), &endp, 10); if(out) *out = (int32_t)n; @@ -47,7 +48,7 @@ bool ParseInt64(const std::string& str, int64_t *out) { if (!ParsePrechecks(str)) return false; - char *endp = NULL; + char *endp = nullptr; errno = 0; // strtoll will not set errno if valid long long int n = strtoll(str.c_str(), &endp, 10); if(out) *out = (int64_t)n; diff --git a/lib/univalue_read.cpp b/lib/univalue_read.cpp index 5c6a1acf75..be39bfe57a 100644 --- a/lib/univalue_read.cpp +++ b/lib/univalue_read.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -227,7 +227,7 @@ enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, } else { - writer.push_back(*raw); + writer.push_back(static_cast(*raw)); raw++; } } @@ -244,7 +244,7 @@ enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, } } -enum expect_bits { +enum expect_bits : unsigned { EXP_OBJ_NAME = (1U << 0), EXP_COLON = (1U << 1), EXP_ARR_VALUE = (1U << 2), diff --git a/lib/univalue_utffilter.h b/lib/univalue_utffilter.h index 20d4043009..c24ac58eaf 100644 --- a/lib/univalue_utffilter.h +++ b/lib/univalue_utffilter.h @@ -1,6 +1,6 @@ // Copyright 2016 Wladimir J. van der Laan // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef UNIVALUE_UTFFILTER_H #define UNIVALUE_UTFFILTER_H diff --git a/lib/univalue_write.cpp b/lib/univalue_write.cpp index 827eb9b271..3a2c580c7f 100644 --- a/lib/univalue_write.cpp +++ b/lib/univalue_write.cpp @@ -1,9 +1,8 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include -#include #include #include "univalue.h" #include "univalue_escapes.h" @@ -14,13 +13,13 @@ static std::string json_escape(const std::string& inS) outS.reserve(inS.size() * 2); for (unsigned int i = 0; i < inS.size(); i++) { - unsigned char ch = inS[i]; + unsigned char ch = static_cast(inS[i]); const char *escStr = escapes[ch]; if (escStr) outS += escStr; else - outS += ch; + outS += static_cast(ch); } return outS; diff --git a/sources.mk b/sources.mk new file mode 100644 index 0000000000..efab6d277f --- /dev/null +++ b/sources.mk @@ -0,0 +1,95 @@ +# - All variables are namespaced with UNIVALUE_ to avoid colliding with +# downstream makefiles. +# - All Variables ending in _HEADERS or _SOURCES confuse automake, so the +# _INT postfix is applied. +# - Convenience variables, for example a UNIVALUE_TEST_DIR should not be used +# as they interfere with automatic dependency generation +# - The %reldir% is the relative path from the Makefile.am. This allows +# downstreams to use these variables without having to manually account for +# the path change. + +UNIVALUE_INCLUDE_DIR_INT = %reldir%/include + +UNIVALUE_DIST_HEADERS_INT = +UNIVALUE_DIST_HEADERS_INT += %reldir%/include/univalue.h + +UNIVALUE_LIB_HEADERS_INT = +UNIVALUE_LIB_HEADERS_INT += %reldir%/lib/univalue_utffilter.h +UNIVALUE_LIB_HEADERS_INT += %reldir%/lib/univalue_escapes.h + +UNIVALUE_LIB_SOURCES_INT = +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_get.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_read.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_write.cpp + +UNIVALUE_TEST_DATA_DIR_INT = %reldir%/test + +UNIVALUE_TEST_UNITESTER_INT = +UNIVALUE_TEST_UNITESTER_INT += %reldir%/test/unitester.cpp + +UNIVALUE_TEST_JSON_INT = +UNIVALUE_TEST_JSON_INT += %reldir%/test/test_json.cpp + +UNIVALUE_TEST_NO_NUL_INT = +UNIVALUE_TEST_NO_NUL_INT += %reldir%/test/no_nul.cpp + +UNIVALUE_TEST_OBJECT_INT = +UNIVALUE_TEST_OBJECT_INT += %reldir%/test/object.cpp + +UNIVALUE_TEST_FILES_INT = +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail5.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail6.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail7.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail8.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail9.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail10.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail11.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail12.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail13.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail14.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail15.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail16.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail17.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail18.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail19.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail20.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail21.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail22.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail23.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail24.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail25.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail26.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail27.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail28.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail29.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail30.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail31.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail32.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail33.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail34.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail35.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail36.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail37.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail38.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail39.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail40.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail41.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail42.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail44.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail45.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round5.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round6.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round7.json diff --git a/test/object.cpp b/test/object.cpp index ccc1344836..c2f52f83ac 100644 --- a/test/object.cpp +++ b/test/object.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2014 BitPay Inc. // Copyright (c) 2014-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include diff --git a/test/unitester.cpp b/test/unitester.cpp index 2308afbcdf..02e1a83c6d 100644 --- a/test/unitester.cpp +++ b/test/unitester.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include #include @@ -58,7 +58,7 @@ static void runtest_file(const char *filename_) std::string basename(filename_); std::string filename = srcdir + "/" + basename; FILE *f = fopen(filename.c_str(), "r"); - assert(f != NULL); + assert(f != nullptr); std::string jdata; From 0f95247246344510c9a51810c14c633abb382e95 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 15 Jul 2021 18:42:17 +0000 Subject: [PATCH 42/96] Integrate univalue into our buildsystem This addresses issues like the one in #12467, where some of our compiler flags end up being dropped during the subconfigure of Univalue. Specifically, we're still using the compiler-default c++ version rather than forcing c++17. We can drop the need subconfigure completely in favor of a tighter build integration, where the sources are listed separately from the build recipes, so that they may be included directly by upstream projects. This is similar to the way leveldb build integration works in Core. Core benefits of this approach include: - Better caching (for ex. ccache and autoconf) - No need for a slow subconfigure - Faster autoconf - No more missing compile flags - Compile only the objects needed There are no benefits to Univalue itself that I can think of. These changes should be a no-op there, and to downstreams as well until they take advantage of the new sources.mk. This also removes the option to use an external univalue to avoid similar ABI issues with mystery binaries. Co-authored-by: fanquake --- ci/test/06_script_b.sh | 2 -- configure.ac | 38 ----------------------------------- contrib/guix/libexec/build.sh | 2 +- src/Makefile.am | 19 +++++------------- src/Makefile.test.include | 22 ++++++++++++++++++-- src/Makefile.univalue.include | 6 ++++++ 6 files changed, 32 insertions(+), 57 deletions(-) create mode 100644 src/Makefile.univalue.include diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 194b14beab..311a43755a 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -9,14 +9,12 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi diff --git a/configure.ac b/configure.ac index 5a6f21fe42..08ae53c921 100644 --- a/configure.ac +++ b/configure.ac @@ -262,12 +262,6 @@ if test "x$use_asm" = xyes; then AC_DEFINE(USE_ASM, 1, [Define this symbol to build in assembly routines]) fi -AC_ARG_WITH([system-univalue], - [AS_HELP_STRING([--with-system-univalue], - [Build with system UniValue (default is no)])], - [system_univalue=$withval], - [system_univalue=no] -) AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -1546,34 +1540,6 @@ if test "x$use_zmq" = xyes; then esac fi -dnl univalue check - -need_bundled_univalue=yes -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnononononononono; then - need_bundled_univalue=no -else - if test x$system_univalue != xno; then - PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) - if test x$found_univalue = xyes; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no - fi - fi - - if test x$need_bundled_univalue = xyes; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' - fi -fi - -AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) -AC_SUBST(UNIVALUE_CFLAGS) -AC_SUBST(UNIVALUE_LIBS) - dnl libmultiprocess library check libmultiprocess_found=no @@ -1936,10 +1902,6 @@ PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" -if test x$need_bundled_univalue = xyes; then - AC_CONFIG_SUBDIRS([src/univalue]) -fi - ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental" AC_CONFIG_SUBDIRS([src/secp256k1]) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 93526f8c45..bffe524dbc 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -297,7 +297,7 @@ mkdir -p "$DISTSRC" ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} - sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool + sed -i.old 's/-lstdc++ //g' config.status libtool # Build Bitcoin Core make --jobs="$JOBS" ${V:+V=1} diff --git a/src/Makefile.am b/src/Makefile.am index 12fdc9ad75..0deae6c387 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,7 +6,7 @@ print-%: FORCE @echo '$*'='$($*)' -DIST_SUBDIRS = secp256k1 univalue +DIST_SUBDIRS = secp256k1 AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) @@ -15,18 +15,7 @@ AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -if EMBEDDED_UNIVALUE -LIBUNIVALUE = univalue/libunivalue.la - -$(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -else -LIBUNIVALUE = $(UNIVALUE_LIBS) -endif - -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) - -BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -80,6 +69,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_ZMQ) lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) +noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = @@ -797,7 +787,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) clean-local: -$(MAKE) -C secp256k1 clean - -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h -rm -rf test/__pycache__ @@ -887,3 +876,5 @@ endif if ENABLE_QT_TESTS include Makefile.qttest.include endif + +include Makefile.univalue.include diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d70793ffa9..27f9382631 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -350,8 +350,26 @@ if ENABLE_BENCH endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check -if EMBEDDED_UNIVALUE - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check + +if !ENABLE_FUZZ +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +noinst_PROGRAMS += $(UNIVALUE_TESTS) +TESTS += $(UNIVALUE_TESTS) + +univalue_test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) +univalue_test_unitester_LDADD = $(LIBUNIVALUE) +univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" +univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) +univalue_test_no_nul_LDADD = $(LIBUNIVALUE) +univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) +univalue_test_object_LDADD = $(LIBUNIVALUE) +univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp diff --git a/src/Makefile.univalue.include b/src/Makefile.univalue.include new file mode 100644 index 0000000000..3644e36368 --- /dev/null +++ b/src/Makefile.univalue.include @@ -0,0 +1,6 @@ +include univalue/sources.mk + +LIBUNIVALUE = libunivalue.la +noinst_LTLIBRARIES += $(LIBUNIVALUE) +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) $(UNIVALUE_DIST_HEADERS_INT) $(UNIVALUE_LIB_HEADERS_INT) $(UNIVALUE_TEST_FILES_INT) +libunivalue_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) From a0efe529e4fd053b890450413b9ca5e1bcd8f2c2 Mon Sep 17 00:00:00 2001 From: Samuel Dobson Date: Tue, 12 Oct 2021 14:36:51 +1300 Subject: [PATCH 43/96] Fix outdated comments referring to ::ChainActive() --- src/index/base.cpp | 2 +- src/policy/fees.cpp | 2 +- src/txmempool.h | 2 +- src/validation.cpp | 2 +- src/validation.h | 2 +- src/wallet/wallet.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/index/base.cpp b/src/index/base.cpp index 3ca86a310e..fc6dd77a72 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -321,7 +321,7 @@ bool BaseIndex::BlockUntilSyncedToCurrentChain() const { // Skip the queue-draining stuff if we know we're caught up with - // ::ChainActive().Tip(). + // m_chain.Tip(). LOCK(cs_main); const CBlockIndex* chain_tip = m_chainstate->m_chain.Tip(); const CBlockIndex* best_block_index = m_best_block_index.load(); diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 2ae5798ebe..2e2061d0a1 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -549,7 +549,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo if (txHeight != nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random they don't // affect the estimate. We'll potentially double count transactions in 1-block reorgs. - // Ignore txs if BlockPolicyEstimator is not in sync with ::ChainActive().Tip(). + // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip(). // It will be synced next time a block is processed. return; } diff --git a/src/txmempool.h b/src/txmempool.h index 0be51db181..460e9d0ceb 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -542,7 +542,7 @@ class CTxMemPool * By design, it is guaranteed that: * * 1. Locking both `cs_main` and `mempool.cs` will give a view of mempool - * that is consistent with current chain tip (`::ChainActive()` and + * that is consistent with current chain tip (`ActiveChain()` and * `CoinsTip()`) and is fully populated. Fully populated means that if the * current active chain is missing transactions that were present in a * previously active chain, all the missing transactions will have been diff --git a/src/validation.cpp b/src/validation.cpp index 14dcd2c24b..c21a87e6d3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -219,7 +219,7 @@ bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) // If there are relative lock times then the maxInputBlock will be set // If there are no relative lock times, the LockPoints don't depend on the chain if (lp->maxInputBlock) { - // Check whether ::ChainActive() is an extension of the block at which the LockPoints + // Check whether active_chain is an extension of the block at which the LockPoints // calculation was valid. If not LockPoints are no longer valid if (!active_chain.Contains(lp->maxInputBlock)) { return false; diff --git a/src/validation.h b/src/validation.h index b10050f931..1c23b3ad5c 100644 --- a/src/validation.h +++ b/src/validation.h @@ -75,7 +75,7 @@ static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; static const unsigned int DEFAULT_CHECKLEVEL = 3; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index abfe9d7dba..180d9d652a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1294,7 +1294,7 @@ void CWallet::updatedBlockTip() void CWallet::BlockUntilSyncedToCurrentChain() const { AssertLockNotHeld(cs_wallet); // Skip the queue-draining stuff if we know we're caught up with - // ::ChainActive().Tip(), otherwise put a callback in the validation interface queue and wait + // chain().Tip(), otherwise put a callback in the validation interface queue and wait // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); From b7884dd1b68814c59ff4fb5f7a199e306b015e85 Mon Sep 17 00:00:00 2001 From: brunoerg Date: Mon, 11 Oct 2021 17:50:02 -0300 Subject: [PATCH 44/96] test: bip125-replaceable in listsinceblock --- test/functional/wallet_listtransactions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index a14bfe345c..bfcfdf7c2e 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -204,6 +204,15 @@ def get_unconfirmed_utxo_entry(node, txid_to_match): assert_equal(n.gettransaction(txid_3b)["bip125-replaceable"], "yes") assert_equal(n.gettransaction(txid_4)["bip125-replaceable"], "unknown") + self.log.info("Test bip125-replaceable status with listsinceblock") + for n in self.nodes[0:2]: + txs = {tx['txid']: tx['bip125-replaceable'] for tx in n.listsinceblock()['transactions']} + assert_equal(txs[txid_1], "no") + assert_equal(txs[txid_2], "no") + assert_equal(txs[txid_3], "yes") + assert_equal(txs[txid_3b], "yes") + assert_equal(txs[txid_4], "unknown") + self.log.info("Test mined transactions are no longer bip125-replaceable") self.generate(self.nodes[0], 1) assert txid_3b not in self.nodes[0].getrawmempool() From 58765a450c40152db8160bca8a6b0f5b754c5858 Mon Sep 17 00:00:00 2001 From: "W. J. van der Laan" Date: Wed, 13 Oct 2021 18:03:15 +0200 Subject: [PATCH 45/96] qt: Use only Qt translation primitives in GUI code Use `QObject::tr`, `QT_TR_NOOP`, and `QCoreApplication::translate` as appropriate instead of using `_()` which doesn't get picked up. --- src/qt/bitcoin.cpp | 14 ++++++++------ src/qt/splashscreen.cpp | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 7de56a648a..5b586b9d89 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -154,10 +154,11 @@ static bool InitSettings() std::vector errors; if (!gArgs.ReadSettingsFile(&errors)) { - bilingual_str error = _("Settings file could not be read"); - InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + std::string error = QT_TRANSLATE_NOOP("bitcoin-core", "Settings file could not be read"); + std::string error_translated = QCoreApplication::translate("bitcoin-core", error.c_str()).toStdString(); + InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors)))); - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort); + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Reset | QMessageBox::Abort); /*: Explanatory text shown on startup when the settings file cannot be read. Prompts user to make a choice between resetting or aborting. */ messagebox.setInformativeText(QObject::tr("Do you want to reset settings to default values, or to abort without making changes?")); @@ -176,10 +177,11 @@ static bool InitSettings() errors.clear(); if (!gArgs.WriteSettingsFile(&errors)) { - bilingual_str error = _("Settings file could not be written"); - InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + std::string error = QT_TRANSLATE_NOOP("bitcoin-core", "Settings file could not be written"); + std::string error_translated = QCoreApplication::translate("bitcoin-core", error.c_str()).toStdString(); + InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors)))); - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok); + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Ok); /*: Explanatory text shown on startup when the settings file could not be written. Prompts user to check that we have the ability to write to the file. Explains that the user has the option of running without a settings file.*/ diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 2292c01d6a..61b52fd08a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -184,8 +184,8 @@ static void InitMessage(SplashScreen *splash, const std::string &message) static void ShowProgress(SplashScreen *splash, const std::string &title, int nProgress, bool resume_possible) { InitMessage(splash, title + std::string("\n") + - (resume_possible ? _("(press q to shutdown and continue later)").translated - : _("press q to shutdown").translated) + + (resume_possible ? SplashScreen::tr("(press q to shutdown and continue later)").toStdString() + : SplashScreen::tr("press q to shutdown").toStdString()) + strprintf("\n%d", nProgress) + "%"); } From 4ac8c89ad96de9ad61cad756b10c9dee2d9e1405 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Fri, 15 Oct 2021 02:18:51 +0200 Subject: [PATCH 46/96] test: check that bumpfee RPC fails for txs with descendants in mempool This commit adds missing test coverage for the bumpfee RPC error "Transaction has descendants in the mempool", which is thrown if the bumped tx has descendants in the mempool and is _not_ connected to the bitcoin wallet. To achieve that, the test framework's MiniWallet is used. --- test/functional/wallet_bumpfee.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index a1676fffa5..46a5df4a8e 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -32,6 +32,8 @@ assert_greater_than, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 @@ -265,6 +267,14 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad tx = rbf_node.signrawtransactionwithwallet(tx) rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + + # create tx with descendant in the mempool by using MiniWallet + miniwallet = MiniWallet(rbf_node) + parent_id = spend_one_input(rbf_node, miniwallet.get_address()) + tx = rbf_node.gettransaction(txid=parent_id, verbose=True)['decoded'] + miniwallet.scan_tx(tx) + miniwallet.send_self_transfer(from_node=rbf_node) + assert_raises_rpc_error(-8, "Transaction has descendants in the mempool", rbf_node.bumpfee, parent_id) self.clear_mempool() From 17ae2601c786e6863cee1bd62297d79521219295 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 14 Oct 2021 19:33:06 +0000 Subject: [PATCH 47/96] build: remove build stubs for external leveldb Presumably these stubs indicate to packagers that external leveldb is meant to be supported in some way. It is not. Remove the stubs to avoid sending any mixed messages. --- configure.ac | 8 -------- src/Makefile.am | 3 --- src/Makefile.leveldb.include | 5 +++-- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index 9e9284015b..445ae22089 100644 --- a/configure.ac +++ b/configure.ac @@ -1225,14 +1225,6 @@ if test "x$have_any_system" != "xno"; then AC_DEFINE(HAVE_SYSTEM, 1, Define to 1 if std::system or ::wsystem is available.) fi -LEVELDB_CPPFLAGS= -LIBLEVELDB= -LIBMEMENV= -AM_CONDITIONAL([EMBEDDED_LEVELDB],[true]) -AC_SUBST(LEVELDB_CPPFLAGS) -AC_SUBST(LIBLEVELDB) -AC_SUBST(LIBMEMENV) - dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS]) dnl Replace -I with -isystem in $SOME_CPPFLAGS to suppress warnings from dnl headers from its include directories and return the result. diff --git a/src/Makefile.am b/src/Makefile.am index 9d15120b72..dd35dcfc70 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -852,11 +852,8 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif -if EMBEDDED_LEVELDB include Makefile.crc32c.include include Makefile.leveldb.include -endif - include Makefile.test_util.include include Makefile.test_fuzz.include diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index ce1f93f11f..3bec92482a 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -8,9 +8,10 @@ LIBMEMENV_INT = leveldb/libmemenv.a EXTRA_LIBRARIES += $(LIBLEVELDB_INT) EXTRA_LIBRARIES += $(LIBMEMENV_INT) -LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C) -LIBMEMENV += $(LIBMEMENV_INT) +LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) +LIBMEMENV = $(LIBMEMENV_INT) +LEVELDB_CPPFLAGS = LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv From a5595b1320d0ebd2c60833286799ee42108a7c01 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 18:50:47 -0400 Subject: [PATCH 48/96] tests: Remove global vCoins and testWallet from coinselector_tests To avoid issues with test data leaking across tests cases, the global vCoins and testWallet are removed from coinselector_tests and all of the relevant functions reworked to not need them. --- src/wallet/test/coinselector_tests.cpp | 397 ++++++++++++------------- 1 file changed, 196 insertions(+), 201 deletions(-) diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index f80c4637b8..6b4c2a2725 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -28,19 +28,9 @@ BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) typedef std::set CoinSet; -static std::vector vCoins; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -static CAmount balance = 0; - -CoinEligibilityFilter filter_standard(1, 6, 0); -CoinEligibilityFilter filter_confirmed(1, 1, 0); -CoinEligibilityFilter filter_standard_extra(6, 6, 0); -CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); +static const CoinEligibilityFilter filter_standard(1, 6, 0); +static const CoinEligibilityFilter filter_confirmed(1, 1, 0); +static const CoinEligibilityFilter filter_standard_extra(6, 6, 0); static void add_coin(const CAmount& nValue, int nInput, std::vector& set) { @@ -62,9 +52,8 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fe set.insert(coin); } -static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) +static void add_coin(std::vector& coins, CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) { - balance += nValue; static int nextLockTime = 0; CMutableTransaction tx; tx.nLockTime = nextLockTime++; // so all transactions get different hashes @@ -89,17 +78,7 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo wtx->m_is_cache_empty = false; } COutput output(wallet, *wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); - vCoins.push_back(output); -} -static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) -{ - add_coin(testWallet, nValue, nAge, fIsFromMe, nInput, spendable); -} - -static void empty_wallet(void) -{ - vCoins.clear(); - balance = 0; + coins.push_back(output); } static bool equal_sets(CoinSet a, CoinSet b) @@ -142,20 +121,20 @@ inline std::vector& GroupCoins(const std::vector& coins) return static_groups; } -inline std::vector& KnapsackGroupOutputs(const CoinEligibilityFilter& filter) +inline std::vector& KnapsackGroupOutputs(const std::vector& coins, CWallet& wallet, const CoinEligibilityFilter& filter) { + CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, + /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), + /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); static std::vector static_groups; - static_groups = GroupOutputs(testWallet, vCoins, coin_selection_params, filter, /* positive_only */false); + static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /* positive_only */false); return static_groups; } // Branch and bound coin selection tests BOOST_AUTO_TEST_CASE(bnb_search_test) { - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); - // Setup std::vector utxo_pool; CoinSet selection; @@ -288,196 +267,210 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(3000), /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); - CoinSet setCoinsRet; - CAmount nValueRet; - empty_wallet(); - add_coin(1); - vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); - - // Test fees subtracted from output: - empty_wallet(); - add_coin(1 * CENT); - vCoins.at(0).nInputBytes = 40; - coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); - BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); - - // Make sure that can use BnB when there are preset inputs - empty_wallet(); { std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); - add_coin(*wallet, 5 * CENT, 6 * 24, false, 0, true); - add_coin(*wallet, 3 * CENT, 6 * 24, false, 0, true); - add_coin(*wallet, 2 * CENT, 6 * 24, false, 0, true); + + std::vector coins; + CoinSet setCoinsRet; + CAmount nValueRet; + + add_coin(coins, *wallet, 1); + coins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail + BOOST_CHECK(!SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); + + // Test fees subtracted from output: + coins.clear(); + add_coin(coins, *wallet, 1 * CENT); + coins.at(0).nInputBytes = 40; + coin_selection_params_bnb.m_subtract_fee_outputs = true; + BOOST_CHECK(SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); + BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); + } + + { + std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + wallet->SetupLegacyScriptPubKeyMan(); + LOCK(wallet->cs_wallet); + + std::vector coins; + CoinSet setCoinsRet; + CAmount nValueRet; + + add_coin(coins, *wallet, 5 * CENT, 6 * 24, false, 0, true); + add_coin(coins, *wallet, 3 * CENT, 6 * 24, false, 0, true); + add_coin(coins, *wallet, 2 * CENT, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.fAllowOtherInputs = true; - coin_control.Select(COutPoint(vCoins.at(0).tx->GetHash(), vCoins.at(0).i)); + coin_control.Select(COutPoint(coins.at(0).tx->GetHash(), coins.at(0).i)); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); - BOOST_CHECK(SelectCoins(*wallet, vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); + BOOST_CHECK(SelectCoins(*wallet, coins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); } } BOOST_AUTO_TEST_CASE(knapsack_solver_test) { + std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + wallet->SetupLegacyScriptPubKeyMan(); + LOCK(wallet->cs_wallet); + CoinSet setCoinsRet, setCoinsRet2; CAmount nValueRet; - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); + std::vector coins; // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) { - empty_wallet(); + coins.clear(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); - add_coin(1*CENT, 4); // add a new 1 cent coin + add_coin(coins, *wallet, 1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // but we can find a new 1 cent - BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); - add_coin(2*CENT); // add a mature 2 cent coin + add_coin(coins, *wallet, 2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // we can make 3 cents of new coins - BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); - add_coin(5*CENT); // add a mature 5 cent coin, - add_coin(10*CENT, 3, true); // a new 10 cent coin sent from one of our own addresses - add_coin(20*CENT); // and a mature 20 cent coin + add_coin(coins, *wallet, 5*CENT); // add a mature 5 cent coin, + add_coin(coins, *wallet, 10*CENT, 3, true); // a new 10 cent coin sent from one of our own addresses + add_coin(coins, *wallet, 20*CENT); // and a mature 20 cent coin // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard_extra), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), setCoinsRet, nValueRet)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // now clear out the wallet and start again to test choosing between subsets of smaller coins and the next biggest coin - empty_wallet(); + coins.clear(); - add_coin( 6*CENT); - add_coin( 7*CENT); - add_coin( 8*CENT); - add_coin(20*CENT); - add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total + add_coin(coins, *wallet, 6*CENT); + add_coin(coins, *wallet, 7*CENT); + add_coin(coins, *wallet, 8*CENT); + add_coin(coins, *wallet, 20*CENT); + add_coin(coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); - BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total + add_coin(coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); - add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 + add_coin(coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // check that the smallest bigger coin is used - add_coin( 1*COIN); - add_coin( 2*COIN); - add_coin( 3*COIN); - add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + add_coin(coins, *wallet, 1*COIN); + add_coin(coins, *wallet, 2*COIN); + add_coin(coins, *wallet, 3*COIN); + add_coin(coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents + BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // empty the wallet and start again, now with fractions of a cent, to test small change avoidance - empty_wallet(); - add_coin(MIN_CHANGE * 1 / 10); - add_coin(MIN_CHANGE * 2 / 10); - add_coin(MIN_CHANGE * 3 / 10); - add_coin(MIN_CHANGE * 4 / 10); - add_coin(MIN_CHANGE * 5 / 10); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 1 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 2 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 3 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided - add_coin(1111*MIN_CHANGE); + add_coin(coins, *wallet, 1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change - empty_wallet(); + coins.clear(); for (int j = 0; j < 20; j++) - add_coin(50000 * COIN); + add_coin(coins, *wallet, 50000 * COIN); - BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -485,79 +478,79 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // we need to try finding an exact subset anyway // sometimes it will fail, and so we use the next biggest coin: - empty_wallet(); - add_coin(MIN_CHANGE * 5 / 10); - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 7 / 10); - add_coin(1111 * MIN_CHANGE); - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, 1111 * MIN_CHANGE); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // but sometimes it's possible, and we use an exact subset (0.4 + 0.6 = 1.0) - empty_wallet(); - add_coin(MIN_CHANGE * 4 / 10); - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 8 / 10); - add_coin(1111 * MIN_CHANGE); - BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 8 / 10); + add_coin(coins, *wallet, 1111 * MIN_CHANGE); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 // test avoiding small change - empty_wallet(); - add_coin(MIN_CHANGE * 5 / 100); - add_coin(MIN_CHANGE * 1); - add_coin(MIN_CHANGE * 100); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 100); + add_coin(coins, *wallet, MIN_CHANGE * 1); + add_coin(coins, *wallet, MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); - } - - // test with many inputs - for (CAmount amt=1500; amt < COIN; amt*=10) { - empty_wallet(); - // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) - for (uint16_t j = 0; j < 676; j++) - add_coin(amt); - - // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. - for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); - - if (amt - 2000 < MIN_CHANGE) { - // needs more than one input: - uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); - CAmount returnValue = amt * returnSize; - BOOST_CHECK_EQUAL(nValueRet, returnValue); - BOOST_CHECK_EQUAL(setCoinsRet.size(), returnSize); - } else { - // one input is sufficient: - BOOST_CHECK_EQUAL(nValueRet, amt); - BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - } - } - } - - // test randomness - { - empty_wallet(); - for (int i2 = 0; i2 < 100; i2++) - add_coin(COIN); - - // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. - for (int i = 0; i < RUN_TESTS; i++) { + } + + // test with many inputs + for (CAmount amt=1500; amt < COIN; amt*=10) { + coins.clear(); + // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) + for (uint16_t j = 0; j < 676; j++) + add_coin(coins, *wallet, amt); + + // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. + for (int i = 0; i < RUN_TESTS; i++) { + BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); + + if (amt - 2000 < MIN_CHANGE) { + // needs more than one input: + uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); + CAmount returnValue = amt * returnSize; + BOOST_CHECK_EQUAL(nValueRet, returnValue); + BOOST_CHECK_EQUAL(setCoinsRet.size(), returnSize); + } else { + // one input is sufficient: + BOOST_CHECK_EQUAL(nValueRet, amt); + BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); + } + } + } + + // test randomness + { + coins.clear(); + for (int i2 = 0; i2 < 100; i2++) + add_coin(coins, *wallet, COIN); + + // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. + for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(coins), setCoinsRet2, nValueRet)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -567,66 +560,65 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // When choosing 1 from 100 identical coins, 1% of the time, this test will choose the same coin twice // which will cause it to fail. // To avoid that issue, run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(coins), setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } BOOST_CHECK_NE(fails, RANDOM_REPEATS); - } - - // add 75 cents in small change. not enough to make 90 cents, - // then try making 90 cents. there are multiple competing "smallest bigger" coins, - // one of which should be picked at random - add_coin(5 * CENT); - add_coin(10 * CENT); - add_coin(15 * CENT); - add_coin(20 * CENT); - add_coin(25 * CENT); - - for (int i = 0; i < RUN_TESTS; i++) { + } + + // add 75 cents in small change. not enough to make 90 cents, + // then try making 90 cents. there are multiple competing "smallest bigger" coins, + // one of which should be picked at random + add_coin(coins, *wallet, 5 * CENT); + add_coin(coins, *wallet, 10 * CENT); + add_coin(coins, *wallet, 15 * CENT); + add_coin(coins, *wallet, 20 * CENT); + add_coin(coins, *wallet, 25 * CENT); + + for (int i = 0; i < RUN_TESTS; i++) { int fails = 0; for (int j = 0; j < RANDOM_REPEATS; j++) { - BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(coins), setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } BOOST_CHECK_NE(fails, RANDOM_REPEATS); - } - } - - empty_wallet(); + } + } } BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { + std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + wallet->SetupLegacyScriptPubKeyMan(); + LOCK(wallet->cs_wallet); + CoinSet setCoinsRet; CAmount nValueRet; - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); - - empty_wallet(); + std::vector coins; // Test vValue sort order for (int i = 0; i < 1000; i++) - add_coin(1000 * COIN); - add_coin(3 * COIN); + add_coin(coins, *wallet, 1000 * COIN); + add_coin(coins, *wallet, 3 * COIN); - BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); - - empty_wallet(); } // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); + std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + wallet->SetupLegacyScriptPubKeyMan(); + LOCK(wallet->cs_wallet); // Random generator stuff std::default_random_engine generator; @@ -636,12 +628,15 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) // Run this test 100 times for (int i = 0; i < 100; ++i) { - empty_wallet(); + std::vector coins; + CAmount balance{0}; // Make a wallet with 1000 exponentially distributed random inputs for (int j = 0; j < 1000; ++j) { - add_coin((CAmount)(distribution(generator)*10000000)); + CAmount val = distribution(generator)*10000000; + add_coin(coins, *wallet, val); + balance += val; } // Generate a random fee rate in the range of 100 - 400 @@ -658,7 +653,7 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CoinSet out_set; CAmount out_value = 0; CCoinControl cc; - BOOST_CHECK(SelectCoins(testWallet, vCoins, target, out_set, out_value, cc, cs_params)); + BOOST_CHECK(SelectCoins(*wallet, coins, target, out_set, out_value, cc, cs_params)); BOOST_CHECK_GE(out_value, target); } } From 5e54aa9b90c5d4d472be47a7fca969c5e7b92e88 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Wed, 13 Oct 2021 14:09:27 -0400 Subject: [PATCH 49/96] bench: remove global testWallet from CoinSelection benchmark --- src/bench/coin_selection.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 934b574f8b..4286b9031e 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -65,10 +65,6 @@ static void CoinSelection(benchmark::Bench& bench) } typedef std::set CoinSet; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -std::vector> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector& set) @@ -76,10 +72,9 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - std::unique_ptr wtx = std::make_unique(MakeTransactionRef(std::move(tx))); + CInputCoin coin(MakeTransactionRef(tx), nInput); set.emplace_back(); - set.back().Insert(COutput(testWallet, *wtx, nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); - wtxn.emplace_back(std::move(wtx)); + set.back().Insert(coin, 0, true, 0, 0, false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector& utxo_pool) @@ -97,7 +92,6 @@ static CAmount make_hard_case(int utxos, std::vector& utxo_pool) static void BnBExhaustion(benchmark::Bench& bench) { // Setup - testWallet.SetupLegacyScriptPubKeyMan(); std::vector utxo_pool; CoinSet selection; CAmount value_ret = 0; From 9bf02438727e1052c69d906252fc2a451c923409 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 14:25:38 -0400 Subject: [PATCH 50/96] bench: Use DescriptorScriptPubKeyMan for wallet things For wallet related benchmarks that need a ScriptPubKeyMan for operation, use a DescriptorScriptPubKeyMan --- src/bench/coin_selection.cpp | 1 - src/bench/wallet_balance.cpp | 16 ++++++++-------- src/test/util/wallet.cpp | 12 ------------ 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 4286b9031e..fd5145950b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -33,7 +33,6 @@ static void CoinSelection(benchmark::Bench& bench) NodeContext node; auto chain = interfaces::MakeChain(node); CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); - wallet.SetupLegacyScriptPubKeyMan(); std::vector> wtxs; LOCK(wallet.cs_wallet); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index a205d8b6e7..166ed16042 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -14,7 +14,7 @@ #include -static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_watchonly, const bool add_mine) +static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext(); @@ -22,13 +22,14 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; { - wallet.SetupLegacyScriptPubKeyMan(); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); const std::optional address_mine{add_mine ? std::optional{getnewaddress(wallet)} : std::nullopt}; - if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { generatetoaddress(test_setup->m_node, address_mine.value_or(ADDRESS_WATCHONLY)); @@ -42,14 +43,13 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b if (set_dirty) wallet.MarkDirty(); bal = GetBalance(wallet); if (add_mine) assert(bal.m_mine_trusted > 0); - if (add_watchonly) assert(bal.m_watchonly_trusted > 0); }); } -static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ false, /* add_mine */ true); } -static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ false); } +static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_mine */ true); } +static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ false); } BENCHMARK(WalletBalanceDirty); BENCHMARK(WalletBalanceClean); diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 061659818f..76c1bf93a5 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -25,16 +25,4 @@ std::string getnewaddress(CWallet& w) return EncodeDestination(dest); } -void importaddress(CWallet& wallet, const std::string& address) -{ - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); - const auto dest = DecodeDestination(address); - assert(IsValidDestination(dest)); - const auto script = GetScriptForDestination(dest); - wallet.MarkDirty(); - assert(!spk_man->HaveWatchOnly(script)); - if (!spk_man->AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); - wallet.SetAddressBook(dest, /* label */ "", "receive"); -} #endif // ENABLE_WALLET From 811319fea4295bfff05c23c0dcab1e24c85e8544 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 16:12:59 -0400 Subject: [PATCH 51/96] tests, gui: Use DescriptorScriptPubKeyMan in GUI tests --- src/qt/test/addressbooktests.cpp | 6 +++++- src/qt/test/wallettests.cpp | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 0de781661a..729957699a 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -64,8 +64,12 @@ void TestAddAddressesToSendBook(interfaces::Node& node) test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); std::shared_ptr wallet = std::make_shared(node.context()->chain.get(), "", CreateMockWalletDatabase()); - wallet->SetupLegacyScriptPubKeyMan(); wallet->LoadWallet(); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + { + LOCK(wallet->cs_wallet); + wallet->SetupDescriptorScriptPubKeyMans(); + } auto build_address = [&wallet]() { CKey key; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 62b135d3f1..c74c8f25b3 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -143,11 +143,20 @@ void TestGUI(interfaces::Node& node) node.setContext(&test.m_node); std::shared_ptr wallet = std::make_shared(node.context()->chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); - spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + LOCK(wallet->cs_wallet); + wallet->SetupDescriptorScriptPubKeyMans(); + + // Add the coinbase key + FlatSigningProvider provider; + std::string error; + std::unique_ptr desc = Parse("combo(" + EncodeSecret(test.coinbaseKey) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet->AddWalletDescriptor(w_desc, provider, "", false)) assert(false); + CTxDestination dest = GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type); + wallet->SetAddressBook(dest, "", "receive"); wallet->SetLastBlockProcessed(105, node.context()->chainman->ActiveChain().Tip()->GetBlockHash()); } { From 4b1588c6bd96743b333cc291e19a9fc76dc8cdf1 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 20:43:12 -0400 Subject: [PATCH 52/96] tests: Use DescriptorScriptPubKeyMan in coinselector_tests --- src/wallet/test/coinselector_tests.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 6b4c2a2725..e880e13845 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -270,8 +270,9 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) { std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); std::vector coins; CoinSet setCoinsRet; @@ -293,8 +294,9 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) { std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); std::vector coins; CoinSet setCoinsRet; @@ -315,8 +317,9 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) { std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); CoinSet setCoinsRet, setCoinsRet2; CAmount nValueRet; @@ -595,8 +598,9 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); CoinSet setCoinsRet; CAmount nValueRet; @@ -617,8 +621,9 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) { std::unique_ptr wallet = std::make_unique(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); // Random generator stuff std::default_random_engine generator; From dcd6eeb64adb2b532f5003cbb86ba65b3c08a87b Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 20:43:34 -0400 Subject: [PATCH 53/96] tests: Use descriptors in psbt_wallet_tests --- src/wallet/test/psbt_wallet_tests.cpp | 43 ++++++++++++--------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 8a97f7779d..120a20749e 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -13,10 +13,21 @@ BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) +static void import_descriptor(CWallet& wallet, const std::string& descriptor) +{ + LOCK(wallet.cs_wallet); + FlatSigningProvider provider; + std::string error; + std::unique_ptr desc = Parse(descriptor, provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 10, 0); + wallet.AddWalletDescriptor(w_desc, provider, "", false); +} + BOOST_AUTO_TEST_CASE(psbt_updater_test) { - auto spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(m_wallet.cs_wallet, spk_man->cs_KeyStore); + LOCK(m_wallet.cs_wallet); + m_wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); @@ -29,27 +40,10 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) s_prev_tx2 >> prev_tx2; m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2)); - // Add scripts - CScript rs1; - CDataStream s_rs1(ParseHex("475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae"), SER_NETWORK, PROTOCOL_VERSION); - s_rs1 >> rs1; - spk_man->AddCScript(rs1); - - CScript rs2; - CDataStream s_rs2(ParseHex("2200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903"), SER_NETWORK, PROTOCOL_VERSION); - s_rs2 >> rs2; - spk_man->AddCScript(rs2); - - CScript ws1; - CDataStream s_ws1(ParseHex("47522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae"), SER_NETWORK, PROTOCOL_VERSION); - s_ws1 >> ws1; - spk_man->AddCScript(ws1); - - // Add hd seed - CKey key = DecodeSecret("5KSSJQ7UNfFGwVgpCZDSHm5rVNhMFcFtvWM3zQ8mW4qNDEN7LFd"); // Mainnet and uncompressed form of cUkG8i1RFfWGWy5ziR11zJ5V4U4W3viSFCfyJmZnvQaUsd1xuF3T - CPubKey master_pub_key = spk_man->DeriveNewSeed(key); - spk_man->SetHDSeed(master_pub_key); - spk_man->NewKeyPool(); + // Import descriptors for keys and scripts + import_descriptor(m_wallet, "sh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/0h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/1h))"); + import_descriptor(m_wallet, "sh(wsh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/2h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/3h)))"); + import_descriptor(m_wallet, "wpkh(xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/*h)"); // Call FillPSBT PartiallySignedTransaction psbtx; @@ -71,7 +65,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; - BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); + BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); + //BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) From 99516285b7cf2664563712d95d95f54e1985c0c2 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 22:07:18 -0400 Subject: [PATCH 54/96] tests: Use legacy change type in subtract fee from outputs test The subtract fee from outputs assumes that the leftover input amount will be dropped to fees. However this only happens if that amount is less than the cost of change. In the event that it is higher than the cost of change, the leftover amount will actually become a change output. To avoid this scenario, force a change type which has a high cost of change. --- src/wallet/test/spend_tests.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index becef70729..d88d8eabdb 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -33,6 +33,8 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) CCoinControl coin_control; coin_control.m_feerate.emplace(10000); coin_control.fOverrideFeeRate = true; + // We need to use a change type with high cost of change so that the leftover amount will be dropped to fee instead of added as a change output + coin_control.m_change_type = OutputType::LEGACY; FeeCalculation fee_calc; BOOST_CHECK(CreateTransaction(*wallet, {recipient}, tx, fee, change_pos, error, coin_control, fee_calc)); BOOST_CHECK_EQUAL(tx->vout.size(), 1); From 2d2edc1248a2e49636409b07448676e5bfe44956 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 12 Oct 2021 20:54:33 -0400 Subject: [PATCH 55/96] tests: Use Descriptor wallets for generic wallet tests For the generic wallet tests, make DescriptorScriptPubKeyMans. There are still some wallet tests that test legacy wallet things. Those remain unchanged. --- src/wallet/test/util.cpp | 14 +++++++-- src/wallet/test/wallet_tests.cpp | 52 +++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index c3061b93c0..2990fc8f8d 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,9 +24,16 @@ std::unique_ptr CreateSyncedWallet(interfaces::Chain& chain, CChain& cc } wallet->LoadWallet(); { - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - spk_man->AddKeyPubKey(key, key.GetPubKey()); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + FlatSigningProvider provider; + std::string error; + std::unique_ptr desc = Parse("combo(" + EncodeSecret(key) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet->AddWalletDescriptor(w_desc, provider, "", false)) assert(false); } WalletRescanReserver reserver(*wallet); reserver.reserve(); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 94b5abfba7..0965128ade 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,7 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static std::shared_ptr TestLoadWallet(WalletContext& context) { DatabaseOptions options; + options.create_flags = WALLET_FLAG_DESCRIPTORS; DatabaseStatus status; bilingual_str error; std::vector warnings; @@ -77,9 +79,13 @@ static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t in static void AddKey(CWallet& wallet, const CKey& key) { - auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); - spk_man->AddKeyPubKey(key, key.GetPubKey()); + LOCK(wallet.cs_wallet); + FlatSigningProvider provider; + std::string error; + std::unique_ptr desc = Parse("combo(" + EncodeSecret(key) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet.AddWalletDescriptor(w_desc, provider, "", false)) assert(false); } BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) @@ -95,6 +101,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -114,6 +121,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -140,6 +148,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -165,6 +174,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -320,10 +330,12 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); - auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(m_coinbase_txns.back()); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); + wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash(), 0); @@ -336,7 +348,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); - BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); + AddKey(wallet, coinbaseKey); BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); } @@ -593,14 +605,26 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { - std::shared_ptr wallet = std::make_shared(m_node.chain.get(), "", CreateDummyWalletDatabase()); - wallet->SetupLegacyScriptPubKeyMan(); - wallet->SetMinVersion(FEATURE_LATEST); - wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - BOOST_CHECK(!wallet->TopUpKeyPool(1000)); - CTxDestination dest; - bilingual_str error; - BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + { + std::shared_ptr wallet = std::make_shared(m_node.chain.get(), "", CreateDummyWalletDatabase()); + wallet->SetupLegacyScriptPubKeyMan(); + wallet->SetMinVersion(FEATURE_LATEST); + wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + BOOST_CHECK(!wallet->TopUpKeyPool(1000)); + CTxDestination dest; + bilingual_str error; + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + } + { + std::shared_ptr wallet = std::make_shared(m_node.chain.get(), "", CreateDummyWalletDatabase()); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetMinVersion(FEATURE_LATEST); + wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + CTxDestination dest; + bilingual_str error; + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + } } // Explicit calculation which is used to test the wallet constant From 1946af2c45372e3de39000a45a5954bb5870bc1b Mon Sep 17 00:00:00 2001 From: Kennan Mell Date: Sat, 16 Oct 2021 13:35:57 -0700 Subject: [PATCH 56/96] Add comment to COIN constant. The COIN constant is critical in understanding Bitcoin's supply, but what it represents isn't clear from the name of the constant. Adding a comment clarifies the meaning of the constant for future readers. --- src/consensus/amount.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/consensus/amount.h b/src/consensus/amount.h index 8b41a2277d..96566ea13f 100644 --- a/src/consensus/amount.h +++ b/src/consensus/amount.h @@ -11,6 +11,7 @@ /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; +/** The amount of satoshis in one BTC. */ static constexpr CAmount COIN = 100000000; /** No amount larger than this (in satoshi) is valid. From 130ee481082d2612d452d7d69131ade935b225b5 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Fri, 15 Oct 2021 16:11:30 +0200 Subject: [PATCH 57/96] test: get and decode tx with a single `gettransaction` RPC call Rather than subsequently calling `gettransaction` and `decoderawtransaction` to get the decoded information for a specific tx-id, we can simply use the verbose version of `gettransaction`, which returns this in a 'decoded' key. I.e. node.decoderawtransaction(node.gettransaction(txid)['hex']) can be replaced by: node.gettransaction(txid=txid, verbose=True)['decoded'] --- test/functional/mempool_packages.py | 3 +-- test/functional/wallet_address_types.py | 3 +-- test/functional/wallet_basic.py | 2 +- test/functional/wallet_create_tx.py | 4 ++-- test/functional/wallet_hd.py | 2 +- test/functional/wallet_importdescriptors.py | 12 ++++-------- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 3943bba489..ff5e45519f 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -65,8 +65,7 @@ def run_test(self): value = sent_value chain.append(txid) # We need the wtxids to check P2P announcements - fulltx = self.nodes[0].getrawtransaction(txid) - witnesstx = self.nodes[0].decoderawtransaction(fulltx, True) + witnesstx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] witness_chain.append(witnesstx['hash']) # Check that listunspent ancestor{count, size, fees} yield the correct results diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index bdee22e62b..7a448e8590 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -204,8 +204,7 @@ def test_desc(self, node, address, multisig, typ, utxo): def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) - raw_tx = self.nodes[node_sender].getrawtransaction(txid) - tx = self.nodes[node_sender].decoderawtransaction(raw_tx) + tx = self.nodes[node_sender].gettransaction(txid=txid, verbose=True)['decoded'] # Make sure the transaction has change: assert_equal(len(tx["vout"]), len(destinations) + 1) diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 599e506f98..92da54d97c 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -666,7 +666,7 @@ def run_test(self): self.generate(self.nodes[0], 1) destination = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(destination, 0.123) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] output_addresses = [vout['scriptPubKey']['address'] for vout in tx["vout"]] assert len(output_addresses) > 1 for address in output_addresses: diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index c8b92ef1bf..00ee08002e 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -34,13 +34,13 @@ def test_anti_fee_sniping(self): self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] assert_equal(tx['locktime'], 0) self.log.info('Check that anti-fee-sniping is enabled when we mine a recent block') self.generate(self.nodes[0], 1) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] assert 0 < tx['locktime'] <= 201 def test_tx_size_too_large(self): diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 974ce7f381..f54ae89c04 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -129,7 +129,7 @@ def run_test(self): # send a tx and make sure its using the internal chain for the changeoutput txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'] + outs = self.nodes[1].gettransaction(txid=txid, verbose=True)['decoded']['vout'] keypath = "" for out in outs: if out['value'] != 1: diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index d86c3737fe..cec978cb8a 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -454,7 +454,7 @@ def run_test(self): self.generate(self.nodes[0], 6) self.sync_all() send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) - decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) + decoded = wmulti_priv.gettransaction(txid=send_txid, verbose=True)['decoded'] assert_equal(len(decoded['vin'][0]['txinwitness']), 4) self.generate(self.nodes[0], 6) self.sync_all() @@ -586,7 +586,7 @@ def run_test(self): self.sync_all() # It is standard and would relay. txid = wmulti_priv_big.sendtoaddress(w0.getnewaddress(), 9.999) - decoded = wmulti_priv_big.decoderawtransaction(wmulti_priv_big.gettransaction(txid)['hex']) + decoded = wmulti_priv_big.gettransaction(txid=txid, verbose=True)['decoded'] # 20 sigs + dummy + witness script assert_equal(len(decoded['vin'][0]['txinwitness']), 22) @@ -620,12 +620,8 @@ def run_test(self): self.generate(self.nodes[0], 6) self.sync_all() # It is standard and would relay. - txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", - True) - decoded = multi_priv_big.decoderawtransaction( - multi_priv_big.gettransaction(txid)['hex'] - ) - + txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", True) + decoded = multi_priv_big.gettransaction(txid=txid, verbose=True)['decoded'] self.log.info("Amending multisig with new private keys") self.nodes[1].createwallet(wallet_name="wmulti_priv3", descriptors=True) From acb9400ab602065d0996f3901de418b710a18159 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:13:14 +0300 Subject: [PATCH 58/96] build: Drop non-existent share/pkgconfig directory --- depends/packages/qt.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 9004b064d6..12e0494ad4 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -248,7 +248,6 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef From 4a37c268dbeed3a361286dcd090aea779527d996 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:16:09 +0300 Subject: [PATCH 59/96] build: Remove unneeded share/doc directory from expat package --- depends/packages/expat.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 902fe43be2..41c1114be0 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -23,5 +23,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/*.la endef From 9067c6c451262222a11785ce9622dd6627644cf1 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:18:33 +0300 Subject: [PATCH 60/96] build: Remove empty var/cache/fontconfig directory from fontconfig --- depends/packages/fontconfig.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 0d5f94f380..22b5022f06 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -29,5 +29,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf var lib/*.la endef From 6c25c83050a8401a76502a1f0ace0ca1428e2916 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:20:05 +0300 Subject: [PATCH 61/96] build: Remove unneeded share/man directory from freetype package --- depends/packages/freetype.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index a1584608e1..aebc8a5f3b 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -23,5 +23,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share/man lib/*.la endef From 539ca409c939a31bc51f41f14ebb8bf8f48e0073 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 23 Aug 2021 20:21:49 +0300 Subject: [PATCH 62/96] build: Remove unneeded share/man directory from libXau package --- depends/packages/libXau.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 4c55c2df04..24e0e9d325 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -30,5 +30,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/*.la endef From 53c9fa9e6253ea89ba1057b35e018ad1a25fb97e Mon Sep 17 00:00:00 2001 From: 0xb10c <0xb10c@gmail.com> Date: Mon, 18 Oct 2021 13:19:13 +0200 Subject: [PATCH 63/96] tracing: drop block_connected hash.toString() arg The tracepoint `validation:block_connected` was introduced in #22006. The first argument was the hash of the connected block as a pointer to a C-like String. The last argument passed the hash of the connected block as a pointer to 32 bytes. The hash was only passed as string to allow `bpftrace` scripts to print the hash. It was (incorrectly) assumed that `bpftrace` cannot hex-format and print the block hash given only the hash as bytes. The block hash can be printed in `bpftrace` by calling `printf("%02x")` for each byte of the hash in an `unroll () {...}`. By starting from the last byte of the hash, it can be printed in big-endian (the block-explorer format). ```C $p = $hash + 31; unroll(32) { $b = *(uint8*)$p; printf("%02x", $b); $p -= 1; } ``` See also: https://github.com/bitcoin/bitcoin/pull/22902#discussion_r705176691 This is a breaking change to the block_connected tracepoint API, however this tracepoint has not yet been included in a release. --- contrib/tracing/README.md | 7 +------ contrib/tracing/connectblock_benchmark.bt | 22 ++++++++++++++-------- doc/tracing.md | 9 +-------- src/validation.cpp | 7 +++---- 4 files changed, 19 insertions(+), 26 deletions(-) diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index 047354cda1..1f93474fa0 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -176,17 +176,12 @@ third acts as a duration threshold in milliseconds. When the `ConnectBlock()` function takes longer than the threshold, information about the block, is printed. For more details, see the header comment in the script. -By default, `bpftrace` limits strings to 64 bytes due to the limited stack size -in the kernel VM. Block hashes as zero-terminated hex strings are 65 bytes which -exceed the string limit. The string size limit can be set to 65 bytes with the -environment variable `BPFTRACE_STRLEN`. - The following command can be used to benchmark, for example, `ConnectBlock()` between height 20000 and 38000 on SigNet while logging all blocks that take longer than 25ms to connect. ``` -$ BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 +$ bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 ``` In a different terminal, starting Bitcoin Core in SigNet mode and with diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt index d268eff7f8..6e7a98ef07 100755 --- a/contrib/tracing/connectblock_benchmark.bt +++ b/contrib/tracing/connectblock_benchmark.bt @@ -4,11 +4,8 @@ USAGE: - BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt + bpftrace contrib/tracing/connectblock_benchmark.bt - - The environment variable BPFTRACE_STRLEN needs to be set to 65 chars as - strings are limited to 64 chars by default. Hex strings with Bitcoin block - hashes are 64 hex chars + 1 null-termination char. - sets the height at which the benchmark should start. Setting the start height to 0 starts the benchmark immediately, even before the first block is connected. @@ -23,7 +20,7 @@ EXAMPLES: - BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 + bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 When run together 'bitcoind -reindex', this benchmarks the time it takes to connect the blocks between height 300.000 and 680.000 (inclusive) and prints @@ -31,7 +28,7 @@ histogram with block connection times when the benchmark is finished. - BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 + bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 When running together 'bitcoind', all newly connected blocks that take longer than 500ms to connect are logged. A histogram with block @@ -107,14 +104,23 @@ usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 */ usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / { - $hash_str = str(arg0); + $hash = arg0; $height = (int32) arg1; $transactions = (uint64) arg2; $inputs = (int32) arg3; $sigops = (int64) arg4; $duration = (int64) arg5; - printf("Block %d (%s) %4d tx %5d ins %5d sigops took %4d ms\n", $height, $hash_str, $transactions, $inputs, $sigops, (uint64) $duration / 1000); + + printf("Block %d (", $height); + /* Prints each byte of the block hash as hex in big-endian (the block-explorer format) */ + $p = $hash + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p -= 1; + } + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); } diff --git a/doc/tracing.md b/doc/tracing.md index 87fc9603fe..57104c43a0 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -101,19 +101,12 @@ Is called *after* a block is connected to the chain. Can, for example, be used to benchmark block connections together with `-reindex`. Arguments passed: -1. Block Header Hash as `pointer to C-style String` (64 characters) +1. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) 2. Block Height as `int32` 3. Transactions in the Block as `uint64` 4. Inputs spend in the Block as `int32` 5. SigOps in the Block (excluding coinbase SigOps) `uint64` 6. Time it took to connect the Block in microseconds (µs) as `uint64` -7. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) - -Note: The 7th argument can't be accessed by bpftrace and is purposefully chosen -to be the block header hash as bytes. See [bpftrace argument limit] for more -details. - -[bpftrace argument limit]: #bpftrace-argument-limit ## Adding tracepoints to Bitcoin Core diff --git a/src/validation.cpp b/src/validation.cpp index 4b9a61320c..78559a8ee6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1877,14 +1877,13 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); - TRACE7(validation, block_connected, - block.GetHash().ToString().c_str(), + TRACE6(validation, block_connected, + block.GetHash().data(), pindex->nHeight, block.vtx.size(), nInputs, nSigOpsCost, - GetTimeMicros() - nTimeStart, // in microseconds (µs) - block.GetHash().data() + GetTimeMicros() - nTimeStart // in microseconds (µs) ); return true; From ffdd94d753ccb8de86eacfb50ffe733c43c1c7c1 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 18 Oct 2021 23:14:56 +0300 Subject: [PATCH 64/96] test: Fix wallet_multisig_descriptor_psbt.py --- test/functional/wallet_multisig_descriptor_psbt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index 68c206b038..ed855d2525 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -104,13 +104,13 @@ def run_test(self): self.log.info("Get a mature utxo to send to the multisig...") coordinator_wallet = participants["signers"][0] - coordinator_wallet.generatetoaddress(101, coordinator_wallet.getnewaddress()) + self.generatetoaddress(self.nodes[0], 101, coordinator_wallet.getnewaddress()) deposit_amount = 6.15 multisig_receiving_address = participants["multisigs"][0].getnewaddress() self.log.info("Send funds to the resulting multisig receiving address...") coordinator_wallet.sendtoaddress(multisig_receiving_address, deposit_amount) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() for participant in participants["multisigs"]: assert_approx(participant.getbalance(), deposit_amount, vspan=0.001) @@ -136,7 +136,7 @@ def run_test(self): coordinator_wallet.sendrawtransaction(finalized["hex"]) self.log.info("Check that balances are correct after the transaction has been included in a block.") - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - value, vspan=0.001) assert_equal(participants["signers"][self.N - 1].getbalance(), value) @@ -153,7 +153,7 @@ def run_test(self): coordinator_wallet.sendrawtransaction(finalized["hex"]) self.log.info("Check that balances are correct after the transaction has been included in a block.") - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - (value * 2), vspan=0.001) assert_equal(participants["signers"][self.N - 1].getbalance(), value * 2) From b65a25a84666d41a0af4ad98ffadfa4ac802d1bb Mon Sep 17 00:00:00 2001 From: Martin Zumsande Date: Mon, 30 Aug 2021 00:18:13 +0200 Subject: [PATCH 65/96] log: improve addrman logging --- src/addrman.cpp | 32 ++++++++++++++----------- test/functional/p2p_invalid_messages.py | 2 +- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/addrman.cpp b/src/addrman.cpp index c364a7710b..832c3b3cb9 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -45,10 +45,7 @@ int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector& asmap { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket); - return tried_bucket; + return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector& asmap) const @@ -56,10 +53,7 @@ int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std:: std::vector vchSourceGroupKey = src.GetGroup(asmap); uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket); - return new_bucket; + return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const @@ -481,6 +475,7 @@ void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } @@ -532,6 +527,8 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", + infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); @@ -582,17 +579,20 @@ void AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // Will moving this address into tried evict another entry? if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - // Output the entry we'd be colliding with, for debugging purposes - auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { m_tried_collisions.insert(nId); } + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", + colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", + addr.ToString(), + m_tried_collisions.size()); } else { - LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); - // move nId to the tried tables MakeTried(info, nId); + LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); } } @@ -662,6 +662,8 @@ bool AddrManImpl::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTi ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; + LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -720,6 +722,7 @@ std::pair AddrManImpl::Select_(bool newOnly) const assert(it_found != mapInfo.end()); const AddrInfo& info{it_found->second}; if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); return {info, info.nLastTry}; } fChanceFactor *= 1.2; @@ -739,6 +742,7 @@ std::pair AddrManImpl::Select_(bool newOnly) const assert(it_found != mapInfo.end()); const AddrInfo& info{it_found->second}; if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); return {info, info.nLastTry}; } fChanceFactor *= 1.2; @@ -780,7 +784,7 @@ std::vector AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct addresses.push_back(ai); } - + LogPrint(BCLog::ADDRMAN, "GetAddr returned %d random addresses\n", addresses.size()); return addresses; } diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index f3b80abb59..82c7e94c59 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -209,7 +209,7 @@ def test_addrv2_unrecognized_network(self): self.test_addrv2('unrecognized network', [ 'received: addrv2 (25 bytes)', - 'IP 9.9.9.9 mapped', + '9.9.9.9:8333 mapped', 'Added 1 addresses', ], bytes.fromhex( From fa44b071fd9dd3c64d551e881ae084fcc650a592 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 19 Oct 2021 11:13:40 +0200 Subject: [PATCH 66/96] test: Remove unused node from mining_prioritisetransaction --- test/functional/mining_prioritisetransaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index da85ee54be..35274d3500 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -13,7 +13,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 2 + self.num_nodes = 1 self.extra_args = [[ "-printpriority=1", "-acceptnonstdtxn=1", From faf13e272cad44917c4e5516172617fe8d68c00a Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 18 Oct 2021 11:51:08 +0200 Subject: [PATCH 67/96] Add missing gettimeofday to syscall sandbox Also, sort entries. Can be reviewed with: --color-moved=dimmed-zebra --- src/util/syscall_sandbox.cpp | 37 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp index b361b09568..bc69df44f4 100644 --- a/src/util/syscall_sandbox.cpp +++ b/src/util/syscall_sandbox.cpp @@ -169,6 +169,10 @@ const std::map LINUX_SYSCALLS{ {__NR_ftruncate, "ftruncate"}, {__NR_futex, "futex"}, {__NR_futimesat, "futimesat"}, + {__NR_get_kernel_syms, "get_kernel_syms"}, + {__NR_get_mempolicy, "get_mempolicy"}, + {__NR_get_robust_list, "get_robust_list"}, + {__NR_get_thread_area, "get_thread_area"}, {__NR_getcpu, "getcpu"}, {__NR_getcwd, "getcwd"}, {__NR_getdents, "getdents"}, @@ -178,8 +182,6 @@ const std::map LINUX_SYSCALLS{ {__NR_getgid, "getgid"}, {__NR_getgroups, "getgroups"}, {__NR_getitimer, "getitimer"}, - {__NR_get_kernel_syms, "get_kernel_syms"}, - {__NR_get_mempolicy, "get_mempolicy"}, {__NR_getpeername, "getpeername"}, {__NR_getpgid, "getpgid"}, {__NR_getpgrp, "getpgrp"}, @@ -191,12 +193,10 @@ const std::map LINUX_SYSCALLS{ {__NR_getresgid, "getresgid"}, {__NR_getresuid, "getresuid"}, {__NR_getrlimit, "getrlimit"}, - {__NR_get_robust_list, "get_robust_list"}, {__NR_getrusage, "getrusage"}, {__NR_getsid, "getsid"}, {__NR_getsockname, "getsockname"}, {__NR_getsockopt, "getsockopt"}, - {__NR_get_thread_area, "get_thread_area"}, {__NR_gettid, "gettid"}, {__NR_gettimeofday, "gettimeofday"}, {__NR_getuid, "getuid"}, @@ -207,15 +207,15 @@ const std::map LINUX_SYSCALLS{ {__NR_inotify_init1, "inotify_init1"}, {__NR_inotify_rm_watch, "inotify_rm_watch"}, {__NR_io_cancel, "io_cancel"}, - {__NR_ioctl, "ioctl"}, {__NR_io_destroy, "io_destroy"}, {__NR_io_getevents, "io_getevents"}, + {__NR_io_setup, "io_setup"}, + {__NR_io_submit, "io_submit"}, + {__NR_ioctl, "ioctl"}, {__NR_ioperm, "ioperm"}, {__NR_iopl, "iopl"}, {__NR_ioprio_get, "ioprio_get"}, {__NR_ioprio_set, "ioprio_set"}, - {__NR_io_setup, "io_setup"}, - {__NR_io_submit, "io_submit"}, {__NR_kcmp, "kcmp"}, {__NR_kexec_file_load, "kexec_file_load"}, {__NR_kexec_load, "kexec_load"}, @@ -271,8 +271,8 @@ const std::map LINUX_SYSCALLS{ {__NR_newfstatat, "newfstatat"}, {__NR_nfsservctl, "nfsservctl"}, {__NR_open, "open"}, - {__NR_openat, "openat"}, {__NR_open_by_handle_at, "open_by_handle_at"}, + {__NR_openat, "openat"}, {__NR_pause, "pause"}, {__NR_perf_event_open, "perf_event_open"}, {__NR_personality, "personality"}, @@ -307,6 +307,7 @@ const std::map LINUX_SYSCALLS{ #ifdef __NR_pwritev2 {__NR_pwritev2, "pwritev2"}, #endif + {__NR__sysctl, "_sysctl"}, {__NR_query_module, "query_module"}, {__NR_quotactl, "quotactl"}, {__NR_read, "read"}, @@ -334,11 +335,11 @@ const std::map LINUX_SYSCALLS{ {__NR_rt_sigsuspend, "rt_sigsuspend"}, {__NR_rt_sigtimedwait, "rt_sigtimedwait"}, {__NR_rt_tgsigqueueinfo, "rt_tgsigqueueinfo"}, + {__NR_sched_get_priority_max, "sched_get_priority_max"}, + {__NR_sched_get_priority_min, "sched_get_priority_min"}, {__NR_sched_getaffinity, "sched_getaffinity"}, {__NR_sched_getattr, "sched_getattr"}, {__NR_sched_getparam, "sched_getparam"}, - {__NR_sched_get_priority_max, "sched_get_priority_max"}, - {__NR_sched_get_priority_min, "sched_get_priority_min"}, {__NR_sched_getscheduler, "sched_getscheduler"}, {__NR_sched_rr_get_interval, "sched_rr_get_interval"}, {__NR_sched_setaffinity, "sched_setaffinity"}, @@ -357,6 +358,10 @@ const std::map LINUX_SYSCALLS{ {__NR_sendmmsg, "sendmmsg"}, {__NR_sendmsg, "sendmsg"}, {__NR_sendto, "sendto"}, + {__NR_set_mempolicy, "set_mempolicy"}, + {__NR_set_robust_list, "set_robust_list"}, + {__NR_set_thread_area, "set_thread_area"}, + {__NR_set_tid_address, "set_tid_address"}, {__NR_setdomainname, "setdomainname"}, {__NR_setfsgid, "setfsgid"}, {__NR_setfsuid, "setfsuid"}, @@ -364,7 +369,6 @@ const std::map LINUX_SYSCALLS{ {__NR_setgroups, "setgroups"}, {__NR_sethostname, "sethostname"}, {__NR_setitimer, "setitimer"}, - {__NR_set_mempolicy, "set_mempolicy"}, {__NR_setns, "setns"}, {__NR_setpgid, "setpgid"}, {__NR_setpriority, "setpriority"}, @@ -373,11 +377,8 @@ const std::map LINUX_SYSCALLS{ {__NR_setresuid, "setresuid"}, {__NR_setreuid, "setreuid"}, {__NR_setrlimit, "setrlimit"}, - {__NR_set_robust_list, "set_robust_list"}, {__NR_setsid, "setsid"}, {__NR_setsockopt, "setsockopt"}, - {__NR_set_thread_area, "set_thread_area"}, - {__NR_set_tid_address, "set_tid_address"}, {__NR_settimeofday, "settimeofday"}, {__NR_setuid, "setuid"}, {__NR_setxattr, "setxattr"}, @@ -402,7 +403,6 @@ const std::map LINUX_SYSCALLS{ {__NR_sync, "sync"}, {__NR_sync_file_range, "sync_file_range"}, {__NR_syncfs, "syncfs"}, - {__NR__sysctl, "_sysctl"}, {__NR_sysfs, "sysfs"}, {__NR_sysinfo, "sysinfo"}, {__NR_syslog, "syslog"}, @@ -411,12 +411,12 @@ const std::map LINUX_SYSCALLS{ {__NR_time, "time"}, {__NR_timer_create, "timer_create"}, {__NR_timer_delete, "timer_delete"}, - {__NR_timerfd_create, "timerfd_create"}, - {__NR_timerfd_gettime, "timerfd_gettime"}, - {__NR_timerfd_settime, "timerfd_settime"}, {__NR_timer_getoverrun, "timer_getoverrun"}, {__NR_timer_gettime, "timer_gettime"}, {__NR_timer_settime, "timer_settime"}, + {__NR_timerfd_create, "timerfd_create"}, + {__NR_timerfd_gettime, "timerfd_gettime"}, + {__NR_timerfd_settime, "timerfd_settime"}, {__NR_times, "times"}, {__NR_tkill, "tkill"}, {__NR_truncate, "truncate"}, @@ -650,6 +650,7 @@ class SeccompPolicyBuilder { allowed_syscalls.insert(__NR_clock_getres); // find the resolution (precision) of the specified clock allowed_syscalls.insert(__NR_clock_gettime); // retrieve the time of the specified clock + allowed_syscalls.insert(__NR_gettimeofday); // get timeval } void AllowGlobalProcessEnvironment() From 13ae56864e0960fa996ceb82384f21c9656f7b0b Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Tue, 19 Oct 2021 21:33:02 +0300 Subject: [PATCH 68/96] ci: Bump vcpkg release tag In the new release the "Binary Caching" and "Manifest Mode" features are available by default. --- .cirrus.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 05b264fb73..44aaf005f0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -85,8 +85,7 @@ task: env: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f' - VCPKG_FEATURE_FLAGS: 'manifests' + VCPKG_TAG: '2021.05.12' QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.12.11' From 1d13c44a4cb02b0acef6c6a108410cfc7eb20f71 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 19 Oct 2021 18:41:05 -0400 Subject: [PATCH 69/96] tests: Use descriptors for feature_segwit multisig setup When setting up the multisig addresses in feature_segwit.py, use descriptors rather than addmultisigaddress. --- test/functional/feature_segwit.py | 38 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 2f9ab34e99..362120e42e 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -17,6 +17,7 @@ send_to_witness, witness_script, ) +from test_framework.descriptors import descsum_create from test_framework.messages import ( COIN, COutPoint, @@ -49,6 +50,9 @@ assert_raises_rpc_error, try_rpc, ) +from test_framework.wallet_util import ( + get_generate_key, +) NODE_0 = 0 NODE_2 = 2 @@ -142,13 +146,39 @@ def run_test(self): p2sh_ids = [] # p2sh_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE embedded in p2sh wit_ids = [] # wit_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE via bare witness for i in range(3): - newaddress = self.nodes[i].getnewaddress() - self.pubkey.append(self.nodes[i].getaddressinfo(newaddress)["pubkey"]) + key = get_generate_key() + self.pubkey.append(key.pubkey) + multiscript = CScript([OP_1, bytes.fromhex(self.pubkey[-1]), OP_1, OP_CHECKMULTISIG]) - p2sh_ms_addr = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]], '', 'p2sh-segwit')['address'] - bip173_ms_addr = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]], '', 'bech32')['address'] + p2sh_ms_addr = self.nodes[i].createmultisig(1, [self.pubkey[-1]], 'p2sh-segwit')['address'] + bip173_ms_addr = self.nodes[i].createmultisig(1, [self.pubkey[-1]], 'bech32')['address'] assert_equal(p2sh_ms_addr, script_to_p2sh_p2wsh(multiscript)) assert_equal(bip173_ms_addr, script_to_p2wsh(multiscript)) + + p2sh_ms_desc = descsum_create(f"sh(wsh(multi(1,{key.privkey})))") + bip173_ms_desc = descsum_create(f"wsh(multi(1,{key.privkey}))") + assert_equal(self.nodes[i].deriveaddresses(p2sh_ms_desc)[0], p2sh_ms_addr) + assert_equal(self.nodes[i].deriveaddresses(bip173_ms_desc)[0], bip173_ms_addr) + + sh_wpkh_desc = descsum_create(f"sh(wpkh({key.privkey}))") + wpkh_desc = descsum_create(f"wpkh({key.privkey})") + assert_equal(self.nodes[i].deriveaddresses(sh_wpkh_desc)[0], key.p2sh_p2wpkh_addr) + assert_equal(self.nodes[i].deriveaddresses(wpkh_desc)[0], key.p2wpkh_addr) + + if self.options.descriptors: + res = self.nodes[i].importdescriptors([ + {"desc": p2sh_ms_desc, "timestamp": "now"}, + {"desc": bip173_ms_desc, "timestamp": "now"}, + {"desc": sh_wpkh_desc, "timestamp": "now"}, + {"desc": wpkh_desc, "timestamp": "now"}, + ]) + else: + # The nature of the legacy wallet is that this import results in also adding all of the necessary scripts + res = self.nodes[i].importmulti([ + {"desc": p2sh_ms_desc, "timestamp": "now"}, + ]) + assert all([r["success"] for r in res]) + p2sh_ids.append([]) wit_ids.append([]) for _ in range(2): From ae6cbcc90926b65099ed8747e7a13a4aefba787a Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 19 Oct 2021 18:41:46 -0400 Subject: [PATCH 70/96] tests: restrict feature_segwit legacy wallet import tests A portion of feature_segwit deals with the legacy wallet IsMine and import behavior. This is now hidden behind --legacy-wallet --- test/functional/feature_segwit.py | 557 +++++++++++++++--------------- 1 file changed, 279 insertions(+), 278 deletions(-) diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 362120e42e..5abe989e55 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -341,284 +341,285 @@ def run_test(self): # Mine a block to clear the gbt cache again. self.generate(self.nodes[0], 1) - self.log.info("Verify behaviour of importaddress and listunspent") - - # Some public keys to be used later - pubkeys = [ - "0363D44AABD0F1699138239DF2F042C3282C0671CC7A76826A55C8203D90E39242", # cPiM8Ub4heR9NBYmgVzJQiUH1if44GSBGiqaeJySuL2BKxubvgwb - "02D3E626B3E616FC8662B489C123349FECBFC611E778E5BE739B257EAE4721E5BF", # cPpAdHaD6VoYbW78kveN2bsvb45Q7G5PhaPApVUGwvF8VQ9brD97 - "04A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538A62F5BD8EC85C2477F39650BD391EA6250207065B2A81DA8B009FC891E898F0E", # 91zqCU5B9sdWxzMt1ca3VzbtVm2YM6Hi5Rxn4UDtxEaN9C9nzXV - "02A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538", # cPQFjcVRpAUBG8BA9hzr2yEzHwKoMgLkJZBBtK9vJnvGJgMjzTbd - "036722F784214129FEB9E8129D626324F3F6716555B603FFE8300BBCB882151228", # cQGtcm34xiLjB1v7bkRa4V3aAc9tS2UTuBZ1UnZGeSeNy627fN66 - "0266A8396EE936BF6D99D17920DB21C6C7B1AB14C639D5CD72B300297E416FD2EC", # cTW5mR5M45vHxXkeChZdtSPozrFwFgmEvTNnanCW6wrqwaCZ1X7K - "0450A38BD7F0AC212FEBA77354A9B036A32E0F7C81FC4E0C5ADCA7C549C4505D2522458C2D9AE3CEFD684E039194B72C8A10F9CB9D4764AB26FCC2718D421D3B84", # 92h2XPssjBpsJN5CqSP7v9a7cf2kgDunBC6PDFwJHMACM1rrVBJ - ] - - # Import a compressed key and an uncompressed key, generate some multisig addresses - self.nodes[0].importprivkey("92e6XLo5jVAVwrQKPNTs93oQco8f8sDNBcpv73Dsrs397fQtFQn") - uncompressed_spendable_address = ["mvozP4UwyGD2mGZU4D2eMvMLPB9WkMmMQu"] - self.nodes[0].importprivkey("cNC8eQ5dg3mFAVePDX4ddmPYpPbw41r9bm2jd1nLJT77e6RrzTRR") - compressed_spendable_address = ["mmWQubrDomqpgSYekvsU7HWEVjLFHAakLe"] - assert not self.nodes[0].getaddressinfo(uncompressed_spendable_address[0])['iscompressed'] - assert self.nodes[0].getaddressinfo(compressed_spendable_address[0])['iscompressed'] - - self.nodes[0].importpubkey(pubkeys[0]) - compressed_solvable_address = [key_to_p2pkh(pubkeys[0])] - self.nodes[0].importpubkey(pubkeys[1]) - compressed_solvable_address.append(key_to_p2pkh(pubkeys[1])) - self.nodes[0].importpubkey(pubkeys[2]) - uncompressed_solvable_address = [key_to_p2pkh(pubkeys[2])] - - spendable_anytime = [] # These outputs should be seen anytime after importprivkey and addmultisigaddress - spendable_after_importaddress = [] # These outputs should be seen after importaddress - solvable_after_importaddress = [] # These outputs should be seen after importaddress but not spendable - unsolvable_after_importaddress = [] # These outputs should be unsolvable after importaddress - solvable_anytime = [] # These outputs should be solvable after importpubkey - unseen_anytime = [] # These outputs should never be seen - - uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], compressed_spendable_address[0]])['address']) - uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], uncompressed_spendable_address[0]])['address']) - compressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_spendable_address[0]])['address']) - uncompressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], uncompressed_solvable_address[0]])['address']) - compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_solvable_address[0]])['address']) - compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_solvable_address[0], compressed_solvable_address[1]])['address']) - - # Test multisig_without_privkey - # We have 2 public keys without private keys, use addmultisigaddress to add to wallet. - # Money sent to P2SH of multisig of this should only be seen after importaddress with the BASE58 P2SH address. - - multisig_without_privkey_address = self.nodes[0].addmultisigaddress(2, [pubkeys[3], pubkeys[4]])['address'] - script = CScript([OP_2, bytes.fromhex(pubkeys[3]), bytes.fromhex(pubkeys[4]), OP_2, OP_CHECKMULTISIG]) - solvable_after_importaddress.append(script_to_p2sh_script(script)) - - for i in compressed_spendable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - # p2sh multisig with compressed keys should always be spendable - spendable_anytime.extend([p2sh]) - # bare multisig can be watched and signed, but is not treated as ours - solvable_after_importaddress.extend([bare]) - # P2WSH and P2SH(P2WSH) multisig with compressed keys are spendable after direct importaddress - spendable_after_importaddress.extend([p2wsh, p2sh_p2wsh]) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # normal P2PKH and P2PK with compressed keys should always be spendable - spendable_anytime.extend([p2pkh, p2pk]) - # P2SH_P2PK, P2SH_P2PKH with compressed keys are spendable after direct importaddress - spendable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) - # P2WPKH and P2SH_P2WPKH with compressed keys should always be spendable - spendable_anytime.extend([p2wpkh, p2sh_p2wpkh]) - - for i in uncompressed_spendable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - # p2sh multisig with uncompressed keys should always be spendable - spendable_anytime.extend([p2sh]) - # bare multisig can be watched and signed, but is not treated as ours - solvable_after_importaddress.extend([bare]) - # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen - unseen_anytime.extend([p2wsh, p2sh_p2wsh]) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # normal P2PKH and P2PK with uncompressed keys should always be spendable - spendable_anytime.extend([p2pkh, p2pk]) - # P2SH_P2PK and P2SH_P2PKH are spendable after direct importaddress - spendable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh]) - # Witness output types with uncompressed keys are never seen - unseen_anytime.extend([p2wpkh, p2sh_p2wpkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) - - for i in compressed_solvable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - # Multisig without private is not seen after addmultisigaddress, but seen after importaddress - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - solvable_after_importaddress.extend([bare, p2sh, p2wsh, p2sh_p2wsh]) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # normal P2PKH, P2PK, P2WPKH and P2SH_P2WPKH with compressed keys should always be seen - solvable_anytime.extend([p2pkh, p2pk, p2wpkh, p2sh_p2wpkh]) - # P2SH_P2PK, P2SH_P2PKH with compressed keys are seen after direct importaddress - solvable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) - - for i in uncompressed_solvable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - # Base uncompressed multisig without private is not seen after addmultisigaddress, but seen after importaddress - solvable_after_importaddress.extend([bare, p2sh]) - # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen - unseen_anytime.extend([p2wsh, p2sh_p2wsh]) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # normal P2PKH and P2PK with uncompressed keys should always be seen - solvable_anytime.extend([p2pkh, p2pk]) - # P2SH_P2PK, P2SH_P2PKH with uncompressed keys are seen after direct importaddress - solvable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh]) - # Witness output types with uncompressed keys are never seen - unseen_anytime.extend([p2wpkh, p2sh_p2wpkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) - - op1 = CScript([OP_1]) - op0 = CScript([OP_0]) - # 2N7MGY19ti4KDMSzRfPAssP6Pxyuxoi6jLe is the P2SH(P2PKH) version of mjoE3sSrb8ByYEvgnC3Aox86u1CHnfJA4V - unsolvable_address_key = bytes.fromhex("02341AEC7587A51CDE5279E0630A531AEA2615A9F80B17E8D9376327BAEAA59E3D") - unsolvablep2pkh = key_to_p2pkh_script(unsolvable_address_key) - unsolvablep2wshp2pkh = script_to_p2wsh_script(unsolvablep2pkh) - p2shop0 = script_to_p2sh_script(op0) - p2wshop1 = script_to_p2wsh_script(op1) - unsolvable_after_importaddress.append(unsolvablep2pkh) - unsolvable_after_importaddress.append(unsolvablep2wshp2pkh) - unsolvable_after_importaddress.append(op1) # OP_1 will be imported as script - unsolvable_after_importaddress.append(p2wshop1) - unseen_anytime.append(op0) # OP_0 will be imported as P2SH address with no script provided - unsolvable_after_importaddress.append(p2shop0) - - spendable_txid = [] - solvable_txid = [] - spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime, 2)) - solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime, 1)) - self.mine_and_test_listunspent(spendable_after_importaddress + solvable_after_importaddress + unseen_anytime + unsolvable_after_importaddress, 0) - - importlist = [] - for i in compressed_spendable_address + uncompressed_spendable_address + compressed_solvable_address + uncompressed_solvable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - bare = bytes.fromhex(v['hex']) - importlist.append(bare.hex()) - importlist.append(script_to_p2wsh_script(bare).hex()) - else: - pubkey = bytes.fromhex(v['pubkey']) - p2pk = key_to_p2pk_script(pubkey) - p2pkh = key_to_p2pkh_script(pubkey) - importlist.append(p2pk.hex()) - importlist.append(p2pkh.hex()) - importlist.append(key_to_p2wpkh_script(pubkey).hex()) - importlist.append(script_to_p2wsh_script(p2pk).hex()) - importlist.append(script_to_p2wsh_script(p2pkh).hex()) - - importlist.append(unsolvablep2pkh.hex()) - importlist.append(unsolvablep2wshp2pkh.hex()) - importlist.append(op1.hex()) - importlist.append(p2wshop1.hex()) - - for i in importlist: - # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC - # exceptions and continue. - try_rpc(-4, "The wallet already contains the private key for this address or script", self.nodes[0].importaddress, i, "", False, True) - - self.nodes[0].importaddress(script_to_p2sh(op0)) # import OP_0 as address only - self.nodes[0].importaddress(multisig_without_privkey_address) # Test multisig_without_privkey - - spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime + spendable_after_importaddress, 2)) - solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime + solvable_after_importaddress, 1)) - self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) - self.mine_and_test_listunspent(unseen_anytime, 0) - - spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime + spendable_after_importaddress, 2)) - solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime + solvable_after_importaddress, 1)) - self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) - self.mine_and_test_listunspent(unseen_anytime, 0) - - # Repeat some tests. This time we don't add witness scripts with importaddress - # Import a compressed key and an uncompressed key, generate some multisig addresses - self.nodes[0].importprivkey("927pw6RW8ZekycnXqBQ2JS5nPyo1yRfGNN8oq74HeddWSpafDJH") - uncompressed_spendable_address = ["mguN2vNSCEUh6rJaXoAVwY3YZwZvEmf5xi"] - self.nodes[0].importprivkey("cMcrXaaUC48ZKpcyydfFo8PxHAjpsYLhdsp6nmtB3E2ER9UUHWnw") - compressed_spendable_address = ["n1UNmpmbVUJ9ytXYXiurmGPQ3TRrXqPWKL"] - - self.nodes[0].importpubkey(pubkeys[5]) - compressed_solvable_address = [key_to_p2pkh(pubkeys[5])] - self.nodes[0].importpubkey(pubkeys[6]) - uncompressed_solvable_address = [key_to_p2pkh(pubkeys[6])] - - unseen_anytime = [] # These outputs should never be seen - solvable_anytime = [] # These outputs should be solvable after importpubkey - unseen_anytime = [] # These outputs should never be seen - - uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], compressed_spendable_address[0]])['address']) - uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], uncompressed_spendable_address[0]])['address']) - compressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_spendable_address[0]])['address']) - uncompressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_solvable_address[0], uncompressed_solvable_address[0]])['address']) - compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_solvable_address[0]])['address']) - - premature_witaddress = [] - - for i in compressed_spendable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - premature_witaddress.append(script_to_p2sh(p2wsh)) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # P2WPKH, P2SH_P2WPKH are always spendable - spendable_anytime.extend([p2wpkh, p2sh_p2wpkh]) - - for i in uncompressed_spendable_address + uncompressed_solvable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen - unseen_anytime.extend([p2wsh, p2sh_p2wsh]) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # P2WPKH, P2SH_P2WPKH with uncompressed keys are never seen - unseen_anytime.extend([p2wpkh, p2sh_p2wpkh]) - - for i in compressed_solvable_address: - v = self.nodes[0].getaddressinfo(i) - if v['isscript']: - [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) - premature_witaddress.append(script_to_p2sh(p2wsh)) - else: - [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) - # P2SH_P2PK, P2SH_P2PKH with compressed keys are always solvable - solvable_anytime.extend([p2wpkh, p2sh_p2wpkh]) - - self.mine_and_test_listunspent(spendable_anytime, 2) - self.mine_and_test_listunspent(solvable_anytime, 1) - self.mine_and_test_listunspent(unseen_anytime, 0) - - # Check that createrawtransaction/decoderawtransaction with non-v0 Bech32 works - v1_addr = program_to_witness(1, [3, 5]) - v1_tx = self.nodes[0].createrawtransaction([getutxo(spendable_txid[0])], {v1_addr: 1}) - v1_decoded = self.nodes[1].decoderawtransaction(v1_tx) - assert_equal(v1_decoded['vout'][0]['scriptPubKey']['address'], v1_addr) - assert_equal(v1_decoded['vout'][0]['scriptPubKey']['hex'], "51020305") - - # Check that spendable outputs are really spendable - self.create_and_mine_tx_from_txids(spendable_txid) - - # import all the private keys so solvable addresses become spendable - self.nodes[0].importprivkey("cPiM8Ub4heR9NBYmgVzJQiUH1if44GSBGiqaeJySuL2BKxubvgwb") - self.nodes[0].importprivkey("cPpAdHaD6VoYbW78kveN2bsvb45Q7G5PhaPApVUGwvF8VQ9brD97") - self.nodes[0].importprivkey("91zqCU5B9sdWxzMt1ca3VzbtVm2YM6Hi5Rxn4UDtxEaN9C9nzXV") - self.nodes[0].importprivkey("cPQFjcVRpAUBG8BA9hzr2yEzHwKoMgLkJZBBtK9vJnvGJgMjzTbd") - self.nodes[0].importprivkey("cQGtcm34xiLjB1v7bkRa4V3aAc9tS2UTuBZ1UnZGeSeNy627fN66") - self.nodes[0].importprivkey("cTW5mR5M45vHxXkeChZdtSPozrFwFgmEvTNnanCW6wrqwaCZ1X7K") - self.create_and_mine_tx_from_txids(solvable_txid) - - # Test that importing native P2WPKH/P2WSH scripts works - for use_p2wsh in [False, True]: - if use_p2wsh: - scriptPubKey = "00203a59f3f56b713fdcf5d1a57357f02c44342cbf306ffe0c4741046837bf90561a" - transaction = "01000000000100e1f505000000002200203a59f3f56b713fdcf5d1a57357f02c44342cbf306ffe0c4741046837bf90561a00000000" - else: - scriptPubKey = "a9142f8c469c2f0084c48e11f998ffbe7efa7549f26d87" - transaction = "01000000000100e1f5050000000017a9142f8c469c2f0084c48e11f998ffbe7efa7549f26d8700000000" - - self.nodes[1].importaddress(scriptPubKey, "", False) - rawtxfund = self.nodes[1].fundrawtransaction(transaction)['hex'] - rawtxfund = self.nodes[1].signrawtransactionwithwallet(rawtxfund)["hex"] - txid = self.nodes[1].sendrawtransaction(rawtxfund) - - assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) - assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) - - # Assert it is properly saved - self.restart_node(1) - assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) - assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) + if not self.options.descriptors: + self.log.info("Verify behaviour of importaddress and listunspent") + + # Some public keys to be used later + pubkeys = [ + "0363D44AABD0F1699138239DF2F042C3282C0671CC7A76826A55C8203D90E39242", # cPiM8Ub4heR9NBYmgVzJQiUH1if44GSBGiqaeJySuL2BKxubvgwb + "02D3E626B3E616FC8662B489C123349FECBFC611E778E5BE739B257EAE4721E5BF", # cPpAdHaD6VoYbW78kveN2bsvb45Q7G5PhaPApVUGwvF8VQ9brD97 + "04A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538A62F5BD8EC85C2477F39650BD391EA6250207065B2A81DA8B009FC891E898F0E", # 91zqCU5B9sdWxzMt1ca3VzbtVm2YM6Hi5Rxn4UDtxEaN9C9nzXV + "02A47F2CBCEFFA7B9BCDA184E7D5668D3DA6F9079AD41E422FA5FD7B2D458F2538", # cPQFjcVRpAUBG8BA9hzr2yEzHwKoMgLkJZBBtK9vJnvGJgMjzTbd + "036722F784214129FEB9E8129D626324F3F6716555B603FFE8300BBCB882151228", # cQGtcm34xiLjB1v7bkRa4V3aAc9tS2UTuBZ1UnZGeSeNy627fN66 + "0266A8396EE936BF6D99D17920DB21C6C7B1AB14C639D5CD72B300297E416FD2EC", # cTW5mR5M45vHxXkeChZdtSPozrFwFgmEvTNnanCW6wrqwaCZ1X7K + "0450A38BD7F0AC212FEBA77354A9B036A32E0F7C81FC4E0C5ADCA7C549C4505D2522458C2D9AE3CEFD684E039194B72C8A10F9CB9D4764AB26FCC2718D421D3B84", # 92h2XPssjBpsJN5CqSP7v9a7cf2kgDunBC6PDFwJHMACM1rrVBJ + ] + + # Import a compressed key and an uncompressed key, generate some multisig addresses + self.nodes[0].importprivkey("92e6XLo5jVAVwrQKPNTs93oQco8f8sDNBcpv73Dsrs397fQtFQn") + uncompressed_spendable_address = ["mvozP4UwyGD2mGZU4D2eMvMLPB9WkMmMQu"] + self.nodes[0].importprivkey("cNC8eQ5dg3mFAVePDX4ddmPYpPbw41r9bm2jd1nLJT77e6RrzTRR") + compressed_spendable_address = ["mmWQubrDomqpgSYekvsU7HWEVjLFHAakLe"] + assert not self.nodes[0].getaddressinfo(uncompressed_spendable_address[0])['iscompressed'] + assert self.nodes[0].getaddressinfo(compressed_spendable_address[0])['iscompressed'] + + self.nodes[0].importpubkey(pubkeys[0]) + compressed_solvable_address = [key_to_p2pkh(pubkeys[0])] + self.nodes[0].importpubkey(pubkeys[1]) + compressed_solvable_address.append(key_to_p2pkh(pubkeys[1])) + self.nodes[0].importpubkey(pubkeys[2]) + uncompressed_solvable_address = [key_to_p2pkh(pubkeys[2])] + + spendable_anytime = [] # These outputs should be seen anytime after importprivkey and addmultisigaddress + spendable_after_importaddress = [] # These outputs should be seen after importaddress + solvable_after_importaddress = [] # These outputs should be seen after importaddress but not spendable + unsolvable_after_importaddress = [] # These outputs should be unsolvable after importaddress + solvable_anytime = [] # These outputs should be solvable after importpubkey + unseen_anytime = [] # These outputs should never be seen + + uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], compressed_spendable_address[0]])['address']) + uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], uncompressed_spendable_address[0]])['address']) + compressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_spendable_address[0]])['address']) + uncompressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], uncompressed_solvable_address[0]])['address']) + compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_solvable_address[0]])['address']) + compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_solvable_address[0], compressed_solvable_address[1]])['address']) + + # Test multisig_without_privkey + # We have 2 public keys without private keys, use addmultisigaddress to add to wallet. + # Money sent to P2SH of multisig of this should only be seen after importaddress with the BASE58 P2SH address. + + multisig_without_privkey_address = self.nodes[0].addmultisigaddress(2, [pubkeys[3], pubkeys[4]])['address'] + script = CScript([OP_2, bytes.fromhex(pubkeys[3]), bytes.fromhex(pubkeys[4]), OP_2, OP_CHECKMULTISIG]) + solvable_after_importaddress.append(script_to_p2sh_script(script)) + + for i in compressed_spendable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + # p2sh multisig with compressed keys should always be spendable + spendable_anytime.extend([p2sh]) + # bare multisig can be watched and signed, but is not treated as ours + solvable_after_importaddress.extend([bare]) + # P2WSH and P2SH(P2WSH) multisig with compressed keys are spendable after direct importaddress + spendable_after_importaddress.extend([p2wsh, p2sh_p2wsh]) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # normal P2PKH and P2PK with compressed keys should always be spendable + spendable_anytime.extend([p2pkh, p2pk]) + # P2SH_P2PK, P2SH_P2PKH with compressed keys are spendable after direct importaddress + spendable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + # P2WPKH and P2SH_P2WPKH with compressed keys should always be spendable + spendable_anytime.extend([p2wpkh, p2sh_p2wpkh]) + + for i in uncompressed_spendable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + # p2sh multisig with uncompressed keys should always be spendable + spendable_anytime.extend([p2sh]) + # bare multisig can be watched and signed, but is not treated as ours + solvable_after_importaddress.extend([bare]) + # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen + unseen_anytime.extend([p2wsh, p2sh_p2wsh]) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # normal P2PKH and P2PK with uncompressed keys should always be spendable + spendable_anytime.extend([p2pkh, p2pk]) + # P2SH_P2PK and P2SH_P2PKH are spendable after direct importaddress + spendable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh]) + # Witness output types with uncompressed keys are never seen + unseen_anytime.extend([p2wpkh, p2sh_p2wpkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + + for i in compressed_solvable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + # Multisig without private is not seen after addmultisigaddress, but seen after importaddress + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + solvable_after_importaddress.extend([bare, p2sh, p2wsh, p2sh_p2wsh]) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # normal P2PKH, P2PK, P2WPKH and P2SH_P2WPKH with compressed keys should always be seen + solvable_anytime.extend([p2pkh, p2pk, p2wpkh, p2sh_p2wpkh]) + # P2SH_P2PK, P2SH_P2PKH with compressed keys are seen after direct importaddress + solvable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + + for i in uncompressed_solvable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + # Base uncompressed multisig without private is not seen after addmultisigaddress, but seen after importaddress + solvable_after_importaddress.extend([bare, p2sh]) + # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen + unseen_anytime.extend([p2wsh, p2sh_p2wsh]) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # normal P2PKH and P2PK with uncompressed keys should always be seen + solvable_anytime.extend([p2pkh, p2pk]) + # P2SH_P2PK, P2SH_P2PKH with uncompressed keys are seen after direct importaddress + solvable_after_importaddress.extend([p2sh_p2pk, p2sh_p2pkh]) + # Witness output types with uncompressed keys are never seen + unseen_anytime.extend([p2wpkh, p2sh_p2wpkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh]) + + op1 = CScript([OP_1]) + op0 = CScript([OP_0]) + # 2N7MGY19ti4KDMSzRfPAssP6Pxyuxoi6jLe is the P2SH(P2PKH) version of mjoE3sSrb8ByYEvgnC3Aox86u1CHnfJA4V + unsolvable_address_key = bytes.fromhex("02341AEC7587A51CDE5279E0630A531AEA2615A9F80B17E8D9376327BAEAA59E3D") + unsolvablep2pkh = key_to_p2pkh_script(unsolvable_address_key) + unsolvablep2wshp2pkh = script_to_p2wsh_script(unsolvablep2pkh) + p2shop0 = script_to_p2sh_script(op0) + p2wshop1 = script_to_p2wsh_script(op1) + unsolvable_after_importaddress.append(unsolvablep2pkh) + unsolvable_after_importaddress.append(unsolvablep2wshp2pkh) + unsolvable_after_importaddress.append(op1) # OP_1 will be imported as script + unsolvable_after_importaddress.append(p2wshop1) + unseen_anytime.append(op0) # OP_0 will be imported as P2SH address with no script provided + unsolvable_after_importaddress.append(p2shop0) + + spendable_txid = [] + solvable_txid = [] + spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime, 2)) + solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime, 1)) + self.mine_and_test_listunspent(spendable_after_importaddress + solvable_after_importaddress + unseen_anytime + unsolvable_after_importaddress, 0) + + importlist = [] + for i in compressed_spendable_address + uncompressed_spendable_address + compressed_solvable_address + uncompressed_solvable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + bare = bytes.fromhex(v['hex']) + importlist.append(bare.hex()) + importlist.append(script_to_p2wsh_script(bare).hex()) + else: + pubkey = bytes.fromhex(v['pubkey']) + p2pk = key_to_p2pk_script(pubkey) + p2pkh = key_to_p2pkh_script(pubkey) + importlist.append(p2pk.hex()) + importlist.append(p2pkh.hex()) + importlist.append(key_to_p2wpkh_script(pubkey).hex()) + importlist.append(script_to_p2wsh_script(p2pk).hex()) + importlist.append(script_to_p2wsh_script(p2pkh).hex()) + + importlist.append(unsolvablep2pkh.hex()) + importlist.append(unsolvablep2wshp2pkh.hex()) + importlist.append(op1.hex()) + importlist.append(p2wshop1.hex()) + + for i in importlist: + # import all generated addresses. The wallet already has the private keys for some of these, so catch JSON RPC + # exceptions and continue. + try_rpc(-4, "The wallet already contains the private key for this address or script", self.nodes[0].importaddress, i, "", False, True) + + self.nodes[0].importaddress(script_to_p2sh(op0)) # import OP_0 as address only + self.nodes[0].importaddress(multisig_without_privkey_address) # Test multisig_without_privkey + + spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime + spendable_after_importaddress, 2)) + solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime + solvable_after_importaddress, 1)) + self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) + self.mine_and_test_listunspent(unseen_anytime, 0) + + spendable_txid.append(self.mine_and_test_listunspent(spendable_anytime + spendable_after_importaddress, 2)) + solvable_txid.append(self.mine_and_test_listunspent(solvable_anytime + solvable_after_importaddress, 1)) + self.mine_and_test_listunspent(unsolvable_after_importaddress, 1) + self.mine_and_test_listunspent(unseen_anytime, 0) + + # Repeat some tests. This time we don't add witness scripts with importaddress + # Import a compressed key and an uncompressed key, generate some multisig addresses + self.nodes[0].importprivkey("927pw6RW8ZekycnXqBQ2JS5nPyo1yRfGNN8oq74HeddWSpafDJH") + uncompressed_spendable_address = ["mguN2vNSCEUh6rJaXoAVwY3YZwZvEmf5xi"] + self.nodes[0].importprivkey("cMcrXaaUC48ZKpcyydfFo8PxHAjpsYLhdsp6nmtB3E2ER9UUHWnw") + compressed_spendable_address = ["n1UNmpmbVUJ9ytXYXiurmGPQ3TRrXqPWKL"] + + self.nodes[0].importpubkey(pubkeys[5]) + compressed_solvable_address = [key_to_p2pkh(pubkeys[5])] + self.nodes[0].importpubkey(pubkeys[6]) + uncompressed_solvable_address = [key_to_p2pkh(pubkeys[6])] + + unseen_anytime = [] # These outputs should never be seen + solvable_anytime = [] # These outputs should be solvable after importpubkey + unseen_anytime = [] # These outputs should never be seen + + uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], compressed_spendable_address[0]])['address']) + uncompressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [uncompressed_spendable_address[0], uncompressed_spendable_address[0]])['address']) + compressed_spendable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_spendable_address[0]])['address']) + uncompressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_solvable_address[0], uncompressed_solvable_address[0]])['address']) + compressed_solvable_address.append(self.nodes[0].addmultisigaddress(2, [compressed_spendable_address[0], compressed_solvable_address[0]])['address']) + + premature_witaddress = [] + + for i in compressed_spendable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + premature_witaddress.append(script_to_p2sh(p2wsh)) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # P2WPKH, P2SH_P2WPKH are always spendable + spendable_anytime.extend([p2wpkh, p2sh_p2wpkh]) + + for i in uncompressed_spendable_address + uncompressed_solvable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen + unseen_anytime.extend([p2wsh, p2sh_p2wsh]) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # P2WPKH, P2SH_P2WPKH with uncompressed keys are never seen + unseen_anytime.extend([p2wpkh, p2sh_p2wpkh]) + + for i in compressed_solvable_address: + v = self.nodes[0].getaddressinfo(i) + if v['isscript']: + [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) + premature_witaddress.append(script_to_p2sh(p2wsh)) + else: + [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] = self.p2pkh_address_to_script(v) + # P2SH_P2PK, P2SH_P2PKH with compressed keys are always solvable + solvable_anytime.extend([p2wpkh, p2sh_p2wpkh]) + + self.mine_and_test_listunspent(spendable_anytime, 2) + self.mine_and_test_listunspent(solvable_anytime, 1) + self.mine_and_test_listunspent(unseen_anytime, 0) + + # Check that createrawtransaction/decoderawtransaction with non-v0 Bech32 works + v1_addr = program_to_witness(1, [3, 5]) + v1_tx = self.nodes[0].createrawtransaction([getutxo(spendable_txid[0])], {v1_addr: 1}) + v1_decoded = self.nodes[1].decoderawtransaction(v1_tx) + assert_equal(v1_decoded['vout'][0]['scriptPubKey']['address'], v1_addr) + assert_equal(v1_decoded['vout'][0]['scriptPubKey']['hex'], "51020305") + + # Check that spendable outputs are really spendable + self.create_and_mine_tx_from_txids(spendable_txid) + + # import all the private keys so solvable addresses become spendable + self.nodes[0].importprivkey("cPiM8Ub4heR9NBYmgVzJQiUH1if44GSBGiqaeJySuL2BKxubvgwb") + self.nodes[0].importprivkey("cPpAdHaD6VoYbW78kveN2bsvb45Q7G5PhaPApVUGwvF8VQ9brD97") + self.nodes[0].importprivkey("91zqCU5B9sdWxzMt1ca3VzbtVm2YM6Hi5Rxn4UDtxEaN9C9nzXV") + self.nodes[0].importprivkey("cPQFjcVRpAUBG8BA9hzr2yEzHwKoMgLkJZBBtK9vJnvGJgMjzTbd") + self.nodes[0].importprivkey("cQGtcm34xiLjB1v7bkRa4V3aAc9tS2UTuBZ1UnZGeSeNy627fN66") + self.nodes[0].importprivkey("cTW5mR5M45vHxXkeChZdtSPozrFwFgmEvTNnanCW6wrqwaCZ1X7K") + self.create_and_mine_tx_from_txids(solvable_txid) + + # Test that importing native P2WPKH/P2WSH scripts works + for use_p2wsh in [False, True]: + if use_p2wsh: + scriptPubKey = "00203a59f3f56b713fdcf5d1a57357f02c44342cbf306ffe0c4741046837bf90561a" + transaction = "01000000000100e1f505000000002200203a59f3f56b713fdcf5d1a57357f02c44342cbf306ffe0c4741046837bf90561a00000000" + else: + scriptPubKey = "a9142f8c469c2f0084c48e11f998ffbe7efa7549f26d87" + transaction = "01000000000100e1f5050000000017a9142f8c469c2f0084c48e11f998ffbe7efa7549f26d8700000000" + + self.nodes[1].importaddress(scriptPubKey, "", False) + rawtxfund = self.nodes[1].fundrawtransaction(transaction)['hex'] + rawtxfund = self.nodes[1].signrawtransactionwithwallet(rawtxfund)["hex"] + txid = self.nodes[1].sendrawtransaction(rawtxfund) + + assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) + assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) + + # Assert it is properly saved + self.restart_node(1) + assert_equal(self.nodes[1].gettransaction(txid, True)["txid"], txid) + assert_equal(self.nodes[1].listtransactions("*", 1, 0, True)[0]["txid"], txid) def mine_and_test_listunspent(self, script_list, ismine): utxo = find_spendable_utxo(self.nodes[0], 50) From e9ade032f3283971025943d750b1305bc8da56fc Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Tue, 19 Oct 2021 18:43:18 -0400 Subject: [PATCH 71/96] tests: Add feature_segwit.py --descriptors to test_runner.py --- test/functional/test_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index b91b294108..916cd94b79 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -100,6 +100,7 @@ 'p2p_compactblocks.py', 'p2p_compactblocks_blocksonly.py', 'feature_segwit.py --legacy-wallet', + 'feature_segwit.py --descriptors', # vv Tests less than 2m vv 'wallet_basic.py --legacy-wallet', 'wallet_basic.py --descriptors', From 7b3c9e4ee8feb552dc0fc4347db2d06e60894a9f Mon Sep 17 00:00:00 2001 From: lsilva01 Date: Wed, 20 Oct 2021 00:30:28 -0300 Subject: [PATCH 72/96] Make explicit the node param in init_wallet() --- test/functional/feature_rbf.py | 2 +- test/functional/rpc_invalid_address_message.py | 2 +- test/functional/test_framework/test_framework.py | 8 ++++---- test/functional/wallet_backup.py | 6 +++--- test/functional/wallet_listdescriptors.py | 2 +- test/functional/wallet_taproot.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 4eaaf46454..420147542e 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -539,7 +539,7 @@ def test_rpc(self): assert_equal(json1["vin"][0]["sequence"], 4294967295) if self.is_wallet_compiled(): - self.init_wallet(0) + self.init_wallet(node=0) rawtx2 = self.nodes[0].createrawtransaction([], outs) frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": True}) frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": False}) diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index 7ab5a5e90d..085f6582b5 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -90,7 +90,7 @@ def run_test(self): self.test_validateaddress() if self.is_wallet_compiled(): - self.init_wallet(0) + self.init_wallet(node=0) self.test_getaddressinfo() diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 727ac6aed9..ec3561b1f2 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -423,12 +423,12 @@ def setup_nodes(self): def import_deterministic_coinbase_privkeys(self): for i in range(self.num_nodes): - self.init_wallet(i) + self.init_wallet(node=i) - def init_wallet(self, i): - wallet_name = self.default_wallet_name if self.wallet_names is None else self.wallet_names[i] if i < len(self.wallet_names) else False + def init_wallet(self, *, node): + wallet_name = self.default_wallet_name if self.wallet_names is None else self.wallet_names[node] if node < len(self.wallet_names) else False if wallet_name is not False: - n = self.nodes[i] + n = self.nodes[node] if wallet_name is not None: n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors, load_on_startup=True) n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase') diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index bc6d6206e5..a07c28c8a4 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -124,9 +124,9 @@ def restore_wallet_existent_name(self): assert_raises_rpc_error(-8, "Wallet name already exists.", node.restorewallet, wallet_name, wallet_file) def init_three(self): - self.init_wallet(0) - self.init_wallet(1) - self.init_wallet(2) + self.init_wallet(node=0) + self.init_wallet(node=1) + self.init_wallet(node=2) def run_test(self): self.log.info("Generating initial blockchain") diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index 221f5262d9..436bbdcfcc 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -23,7 +23,7 @@ def skip_test_if_missing_module(self): self.skip_if_no_sqlite() # do not create any wallet by default - def init_wallet(self, i): + def init_wallet(self, *, node): return def run_test(self): diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index 4f84dbd125..80c125df97 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -185,7 +185,7 @@ def skip_test_if_missing_module(self): def setup_network(self): self.setup_nodes() - def init_wallet(self, i): + def init_wallet(self, *, node): pass @staticmethod From a78137ec3393fab8b2253995f91b22800886f82b Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 20 Oct 2021 12:28:04 +0800 Subject: [PATCH 73/96] build: fix python detection post #23182 23182 was broken. Fix up the changes, and add python3.11 as suggested. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 06e0f3f25a..5d21c50744 100644 --- a/configure.ac +++ b/configure.ac @@ -107,7 +107,7 @@ AC_PATH_TOOL(GCOV, gcov) AC_PATH_TOOL(LLVM_COV, llvm-cov) AC_PATH_PROG(LCOV, lcov) dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9, python3.10, python3 python]) +AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) From be7f4130f996b2564041719177f0a907e5c2011b Mon Sep 17 00:00:00 2001 From: = Date: Wed, 13 Oct 2021 21:33:07 +0530 Subject: [PATCH 74/96] Fix K1/K2 use in the comments in ChaCha20-Poly1305 AEAD This is done for the ChaCha20-Poly1305 AEAD test vector and for the K1/K2 ChaCha20 cipher instances in chacha_poly_aead.h --- src/crypto/chacha_poly_aead.h | 4 ++-- src/test/crypto_tests.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crypto/chacha_poly_aead.h b/src/crypto/chacha_poly_aead.h index 0afe8fcc14..6a7998335d 100644 --- a/src/crypto/chacha_poly_aead.h +++ b/src/crypto/chacha_poly_aead.h @@ -117,8 +117,8 @@ static constexpr int AAD_PACKAGES_PER_ROUND = 21; /* 64 / 3 round down*/ class ChaCha20Poly1305AEAD { private: - ChaCha20 m_chacha_main; // payload and poly1305 key-derivation cipher instance - ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) + ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) and poly1305 key-derivation cipher instance + ChaCha20 m_chacha_main; // payload unsigned char m_aad_keystream_buffer[CHACHA20_ROUND_OUTPUT]; // aad keystream cache uint64_t m_cached_aad_seqnr; // aad keystream cache hint diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 5b3b39fdb8..1483bd3cb3 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -694,8 +694,8 @@ BOOST_AUTO_TEST_CASE(chacha20_poly1305_aead_testvector) TestChaCha20Poly1305AEAD(true, 0, /* m */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k1 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k2 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", + /* k1 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", + /* k2 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", /* AAD keystream */ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", /* encrypted message & MAC */ "76b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32d2fc11829c1b6c1df1f551cd6131ff08", /* encrypted message & MAC at sequence 999 */ "b0a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3aaa7aa16ec62c5e24f040c08bb20c3598"); From d7524546abf1fa5be8e6317ee50585e966ae6b4c Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 20 Oct 2021 16:13:23 +0800 Subject: [PATCH 75/96] build: explicitly disable libsecp256k1 openssl based tests These tests are failing when run against OpenSSL 3, and have been removed upstream, https://github.com/bitcoin-core/secp256k1/pull/983, so disabled them for now to avoid `make check` failures. Note that this will also remove warning output from our build, due to the use of deprecated OpenSSL API functions. See #23048. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 5d21c50744..ae8ee783eb 100644 --- a/configure.ac +++ b/configure.ac @@ -1878,7 +1878,7 @@ PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental --disable-openssl-tests" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT From 96f469f91bc02a19703344cc439eab064b72081a Mon Sep 17 00:00:00 2001 From: Jon Atack Date: Wed, 20 Oct 2021 14:26:06 +0200 Subject: [PATCH 76/96] netinfo: print peer counts for all reachable networks instead of only for networks we have peer connections to. Users reported the previous behavior caused confusion, as no column was printed when a network was reachable but no peers were connected. Users expected a column to be printed with 0 peers. This commit aligns behavior with that expectation. --- src/bitcoin-cli.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 43e986a765..b6344ec413 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -551,15 +551,26 @@ class NetinfoRequestHandler : public BaseRequestHandler } // Report peer connection totals by type. - result += " ipv4 ipv6 onion"; - const bool any_i2p_peers = m_counts.at(2).at(3); // false if total i2p peers count is 0, otherwise true - if (any_i2p_peers) result += " i2p"; + result += " "; + std::vector reachable_networks; + for (const UniValue& network : networkinfo["networks"].getValues()) { + if (network["reachable"].get_bool()) { + const std::string& network_name{network["name"].get_str()}; + const int8_t network_id{NetworkStringToId(network_name)}; + if (network_id == UNKNOWN_NETWORK) continue; + result += strprintf("%8s", network_name); // column header + reachable_networks.push_back(network_id); + } + }; result += " total block"; if (m_manual_peers_count) result += " manual"; + const std::array rows{"in", "out", "total"}; - for (uint8_t i = 0; i < 3; ++i) { - result += strprintf("\n%-5s %5i %5i %5i", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2)); // ipv4/ipv6/onion peers counts - if (any_i2p_peers) result += strprintf(" %5i", m_counts.at(i).at(3)); // i2p peers count + for (size_t i = 0; i < rows.size(); ++i) { + result += strprintf("\n%-5s", rows[i]); // row header + for (int8_t n : reachable_networks) { + result += strprintf("%8i", m_counts.at(i).at(n)); // network peers count + } result += strprintf(" %5i", m_counts.at(i).at(m_networks.size())); // total peers count if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts result += strprintf(" %5i", m_block_relay_peers_count); From fa38d98aa98bcf34b5b59ff894bbb5da67355b29 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 20 Oct 2021 13:26:51 +0200 Subject: [PATCH 77/96] doc: Add note on deleting past-EOL release branches --- doc/release-process.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/release-process.md b/doc/release-process.md index 6a5202d0f9..14567d4f15 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -60,7 +60,7 @@ Release Process To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: - ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 23.0) This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. @@ -253,6 +253,10 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - bitcoincore.org maintained versions update: [table](https://github.com/bitcoin-core/bitcoincore.org/commits/master/_includes/posts/maintenance-table.md) + - Delete post-EOL [release branches](https://github.com/bitcoin/bitcoin/branches/all) and create a tag `v${branch_name}-final`. + + - Delete ["Needs backport" labels](https://github.com/bitcoin/bitcoin/labels?q=backport) for non-existing branches. + - bitcoincore.org RPC documentation update - Install [golang](https://golang.org/doc/install) From da791c7f66243080177f92ce5f38c49305da63dc Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Tue, 19 Oct 2021 23:53:23 +0300 Subject: [PATCH 78/96] wallet: Use PACKAGE_NAME to mention our software --- src/wallet/bdb.cpp | 2 +- src/wallet/sqlite.cpp | 2 +- test/functional/feature_filelock.py | 2 +- test/functional/tool_wallet.py | 2 +- test/functional/wallet_multiwallet.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 2290e119fd..74fc10ab25 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -132,7 +132,7 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) fs::path pathIn = fs::PathFromString(strPath); TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { - LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); + LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance may be using it.\n", strPath); err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory()))); return false; } diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 650e083e8e..c493b96248 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -228,7 +228,7 @@ void SQLiteDatabase::Open() // Now begin a transaction to acquire the exclusive lock. This lock won't be released until we close because of the exclusive locking mode. int ret = sqlite3_exec(m_db, "BEGIN EXCLUSIVE TRANSACTION", nullptr, nullptr, nullptr); if (ret != SQLITE_OK) { - throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?\n"); + throw std::runtime_error("SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of " PACKAGE_NAME "?\n"); } ret = sqlite3_exec(m_db, "COMMIT", nullptr, nullptr, nullptr); if (ret != SQLITE_OK) { diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index e09107802b..0fc654e10a 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -35,7 +35,7 @@ def check_wallet_filelock(descriptors): wallet_dir = os.path.join(datadir, 'wallets') self.log.info("Check that we can't start a second bitcoind instance using the same wallet") if descriptors: - expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" + expected_msg = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" else: expected_msg = "Error: Error initializing wallet database environment" self.nodes[1].assert_start_raises_init_error(extra_args=[f'-walletdir={wallet_dir}', f'-wallet={wallet_name}', '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 4bf3927879..a276d20b43 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -193,7 +193,7 @@ def test_invalid_tool_commands_and_args(self): locked_dir = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets") error = 'Error initializing wallet database environment "{}"!'.format(locked_dir) if self.options.descriptors: - error = "SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" + error = f"SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" self.assert_raises_tool_error( error, '-wallet=' + self.default_wallet_name, diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index d4768f5043..68ca005649 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -202,7 +202,7 @@ def wallet_file(name): self.restart_node(0, ['-nowallet', '-walletdir=' + competing_wallet_dir]) self.nodes[0].createwallet(self.default_wallet_name) if self.options.descriptors: - exp_stderr = r"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" + exp_stderr = f"Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?" else: exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\S*\"!" self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX) @@ -303,7 +303,7 @@ def wallet_file(name): # Fail to load duplicate wallets path = os.path.join(self.options.tmpdir, "node0", "regtest", "wallets", "w1", "wallet.dat") if self.options.descriptors: - assert_raises_rpc_error(-4, "Wallet file verification failed. SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?", self.nodes[0].loadwallet, wallet_names[0]) + assert_raises_rpc_error(-4, f"Wallet file verification failed. SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another instance of {self.config['environment']['PACKAGE_NAME']}?", self.nodes[0].loadwallet, wallet_names[0]) else: assert_raises_rpc_error(-35, "Wallet file verification failed. Refusing to load database. Data file '{}' is already loaded.".format(path), self.nodes[0].loadwallet, wallet_names[0]) From fa44406ffd34670af929f14484042e1de29ffcdd Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 20 Oct 2021 21:08:19 +0200 Subject: [PATCH 79/96] ci: Disable syscall sandbox in valgrind functional tests --- ci/test/00_setup_env_native_valgrind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 78af869e70..0058a042f5 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -11,6 +11,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--nosandbox --exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI From b00646bc770e0b70400fdc4545756405fb307dba Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Wed, 20 Oct 2021 21:58:43 +0300 Subject: [PATCH 80/96] ci, refactor: Rename VCPKG_TAG variable and vcpkg_cache script The VCPKG_TAG variable renamed to CI_VCPKG_TAG to prevent any possible name clash with vcpkg-specific variables. The vcpkg_cache script renamed into more meaningful one. --- .cirrus.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 44aaf005f0..4dbc2f5f3a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -85,7 +85,7 @@ task: env: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - VCPKG_TAG: '2021.05.12' + CI_VCPKG_TAG: '2021.05.12' QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.12.11' @@ -120,7 +120,7 @@ task: - ..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix %QTBASEDIR% - jom - jom install - vcpkg_cache: + vcpkg_binary_cache: folder: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' install_python_script: - choco install --yes --no-progress python3 --version=3.9.6 @@ -130,7 +130,7 @@ task: - cd .. - git clone --quiet https://github.com/microsoft/vcpkg.git - cd vcpkg - - git -c advice.detachedHead=false checkout %VCPKG_TAG% + - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - .\bootstrap-vcpkg -disableMetrics - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - .\vcpkg integrate install From e8692cf2c151d2abc206ca699e04ae05d4c31dcd Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Wed, 20 Oct 2021 22:04:50 +0300 Subject: [PATCH 81/96] ci: Improve vcpkg binary cache settings This change comes with two improvements: 1) using the VCPKG_DEFAULT_BINARY_CACHE variable drops dependency on vcpkg default cached archives location, and improves readability 2) two obvious cases when binary cache is invalidated are defined, that guaranties it won't grow boundlessly when, for example, vcpkg has being updated. --- .cirrus.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 4dbc2f5f3a..1f32dd5231 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -86,6 +86,7 @@ task: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 CI_VCPKG_TAG: '2021.05.12' + VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.12.11' @@ -121,7 +122,13 @@ task: - jom - jom install vcpkg_binary_cache: - folder: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' + folder: '%VCPKG_DEFAULT_BINARY_CACHE%' + reupload_on_changes: true + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version + populate_script: + - mkdir %VCPKG_DEFAULT_BINARY_CACHE% install_python_script: - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq From ea4b61a1570178ebe5851b5fb4065222e3926f7e Mon Sep 17 00:00:00 2001 From: Pasta Date: Mon, 4 Oct 2021 22:49:21 -0400 Subject: [PATCH 82/96] refactor: remove references to deprecated values under std::allocator Includes allocator::pointer, allocator::const_pointer, allocator::reference and allocator::const_reference which are deprecated in c++17 and removed in c++20. See https://en.cppreference.com/w/cpp/memory/allocator Also prefer `using` over `typedef` see: https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rt-using --- src/support/allocators/secure.h | 17 ++++++++--------- src/support/allocators/zeroafterfree.h | 16 +++++++--------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 0e31ad3ce3..c4923dc56f 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -9,6 +9,7 @@ #include #include +#include #include // @@ -17,15 +18,13 @@ // template struct secure_allocator : public std::allocator { - // MSVC8 default copy constructor is broken - typedef std::allocator base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; + using base = std::allocator; + using traits = std::allocator_traits; + using size_type = typename traits::size_type; + using difference_type = typename traits::difference_type; + using pointer = typename traits::pointer; + using const_pointer = typename traits::const_pointer; + using value_type = typename traits::value_type; secure_allocator() noexcept {} secure_allocator(const secure_allocator& a) noexcept : base(a) {} template diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 418f0ee656..77de4b1e69 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -13,15 +13,13 @@ template struct zero_after_free_allocator : public std::allocator { - // MSVC8 default copy constructor is broken - typedef std::allocator base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; + using base = std::allocator; + using traits = std::allocator_traits; + using size_type = typename traits::size_type; + using difference_type = typename traits::difference_type; + using pointer = typename traits::pointer; + using const_pointer = typename traits::const_pointer; + using value_type = typename traits::value_type; zero_after_free_allocator() noexcept {} zero_after_free_allocator(const zero_after_free_allocator& a) noexcept : base(a) {} template From fa6c62f34b50818102ad58f2ffd152189f54d973 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 21 Oct 2021 11:17:27 +0200 Subject: [PATCH 83/96] test: Replace log with assert_equal in wallet_abandonconflict This will make the test fail as soon as the bug is fixed, forcing it to be updated. Otherwise, the bug might be fixed (or made worse) accidentally, leaving the test in an inconsistent state. --- test/functional/wallet_abandonconflict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index d6766097f6..8f54e50598 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -213,7 +213,7 @@ def run_test(self): #assert_equal(newbalance, balance - Decimal("10")) self.log.info("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") - self.log.info(str(balance) + " -> " + str(newbalance) + " ?") + assert_equal(balance, newbalance) if __name__ == '__main__': From 6911ab95f19d2b1f60f2d0b2f3961fa6639d4f31 Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Thu, 21 Oct 2021 16:03:03 +0200 Subject: [PATCH 84/96] wallet: fix segfault by avoiding invalid default-ctored `external_spk_managers` entry In the method `CWallet::LoadActiveScriptPubKeyMan`, the map `external_spk_managers` (or `internal_spk_managers`, if parameter `internal` is false) is accessed via std::map::operator[], which means that a default-ctored entry is created with a null-pointer as value, if the key doesn't exist. As soon as this value is dereferenced, a segmentation fault occurs, e.g. in `CWallet::KeypoolCountExternalKeys`. The bevaviour can be reproduced by the following steps (starting with empty regtest datadir): $ ./src/bitcoind -regtest -daemon $ ./src/bitcoin-cli -regtest -named createwallet_name=wallet descriptors=true blank=true $ cat regtest-descriptors.txt [ { "desc": "tr([e4445899/49'/1'/0']tprv8ZgxMBicQKsPd8jCeBWsYLEoWxbVgzJDatJ7XkwQ6G3uF4FsHuaziHQ5JZAW4K515nj6kVVwPaNWZSMEcR7aFCwL4tQqTcaoprMKTTtm6Zg/1/*)#mr3llm7f", "timestamp": 1634652324, "active": true, "internal": true, "range": [ 0, 999 ], "next": 0 } ] $ ./src/bitcoin-cli -regtest importdescriptors "$(cat regtest-descriptors.txt)" [ { "success": true } ] $ ./src/bitcoin-cli -regtest getwalletinfo error: timeout on transient error: Could not connect to the server 127.0.0.1:18443 (error code 1 - "EOF reached") Bug reported by Josef Vondrlik (josef-v). --- src/wallet/wallet.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index a749ab8897..803e88cda2 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3235,7 +3235,8 @@ void CWallet::LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool intern auto spk_man = m_spk_managers.at(id).get(); spk_mans[type] = spk_man; - if (spk_mans_other[type] == spk_man) { + const auto it = spk_mans_other.find(type); + if (it != spk_mans_other.end() && it->second == spk_man) { spk_mans_other.erase(type); } From d50fbd4c5b4bc72415854d582cedf94541a46983 Mon Sep 17 00:00:00 2001 From: glozow Date: Thu, 21 Oct 2021 16:15:01 +0100 Subject: [PATCH 85/96] create explicit GenTxid::{Txid, Wtxid} ctors --- src/primitives/transaction.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 46db39f8db..d7a85015ef 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -393,6 +393,8 @@ class GenTxid uint256 m_hash; public: GenTxid(bool is_wtxid, const uint256& hash) : m_is_wtxid(is_wtxid), m_hash(hash) {} + static GenTxid Txid(const uint256& hash) { return GenTxid{false, hash}; } + static GenTxid Wtxid(const uint256& hash) { return GenTxid{true, hash}; } bool IsWtxid() const { return m_is_wtxid; } const uint256& GetHash() const { return m_hash; } friend bool operator==(const GenTxid& a, const GenTxid& b) { return a.m_is_wtxid == b.m_is_wtxid && a.m_hash == b.m_hash; } From 4307849256761fe2440d82bbec892d0e8e6b4dd4 Mon Sep 17 00:00:00 2001 From: glozow Date: Wed, 20 Oct 2021 16:41:45 +0100 Subject: [PATCH 86/96] [mempool] delete exists(uint256) function Allowing callers to pass in a uint256 (which could be txid or wtxid) but then always assuming that it's a txid is a footgunny interface. --- src/net_processing.cpp | 2 +- src/node/interfaces.cpp | 2 +- src/policy/rbf.cpp | 4 ++-- src/rpc/blockchain.cpp | 2 +- src/test/mempool_tests.cpp | 36 ++++++++++++++++++------------------ src/txmempool.cpp | 4 ++-- src/txmempool.h | 3 +-- src/validation.cpp | 6 +++--- 8 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 66b99aa2bb..bb731c14a4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3247,7 +3247,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing // the node to function as a gateway for nodes hidden behind it. - if (!m_mempool.exists(tx.GetHash())) { + if (!m_mempool.exists(GenTxid::Txid(tx.GetHash()))) { LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 73f4036057..192caf7994 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -555,7 +555,7 @@ class ChainImpl : public Chain { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - return m_node.mempool->exists(txid); + return m_node.mempool->exists(GenTxid::Txid(txid)); } bool hasDescendantsInMempool(const uint256& txid) override { diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 7ac2e22006..7e6b0cf245 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -22,7 +22,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If this transaction is not in our mempool, then we can't be sure // we will know about all its inputs. - if (!pool.exists(tx.GetHash())) { + if (!pool.exists(GenTxid::Txid(tx.GetHash()))) { return RBFTransactionState::UNKNOWN; } @@ -98,7 +98,7 @@ std::optional HasNoNewUnconfirmed(const CTransaction& tx, if (!parents_of_conflicts.count(tx.vin[j].prevout.hash)) { // Rather than check the UTXO set - potentially expensive - it's cheaper to just check // if the new input refers to a tx that's in the mempool. - if (pool.exists(tx.vin[j].prevout.hash)) { + if (pool.exists(GenTxid::Txid(tx.vin[j].prevout.hash))) { return strprintf("replacement %s adds unconfirmed input, idx %d", tx.GetHash().ToString(), j); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index dadd82e03f..aa7a55e7a9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -516,7 +516,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool std::set setDepends; for (const CTxIn& txin : tx.vin) { - if (pool.exists(txin.prevout.hash)) + if (pool.exists(GenTxid::Txid(txin.prevout.hash))) setDepends.insert(txin.prevout.hash.ToString()); } diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index bf36f8a6c9..b3497b8ef8 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -444,12 +444,12 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(entry.Fee(5000LL).FromTx(tx2)); pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing - BOOST_CHECK(pool.exists(tx1.GetHash())); - BOOST_CHECK(pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx2.GetHash()))); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // should remove the lower-feerate transaction - BOOST_CHECK(pool.exists(tx1.GetHash())); - BOOST_CHECK(!pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); pool.addUnchecked(entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); @@ -462,14 +462,14 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(entry.Fee(20000LL).FromTx(tx3)); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) - BOOST_CHECK(!pool.exists(tx1.GetHash())); - BOOST_CHECK(pool.exists(tx2.GetHash())); - BOOST_CHECK(pool.exists(tx3.GetHash())); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx2.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx3.GetHash()))); pool.TrimToSize(GetVirtualTransactionSize(CTransaction(tx1))); // mempool is limited to tx1's size in memory usage, so nothing fits - BOOST_CHECK(!pool.exists(tx1.GetHash())); - BOOST_CHECK(!pool.exists(tx2.GetHash())); - BOOST_CHECK(!pool.exists(tx3.GetHash())); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx3.GetHash()))); CFeeRate maxFeeRateRemoved(25000, GetVirtualTransactionSize(CTransaction(tx3)) + GetVirtualTransactionSize(CTransaction(tx2))); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); @@ -529,19 +529,19 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) // we only require this to remove, at max, 2 txn, because it's not clear what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); - BOOST_CHECK(pool.exists(tx4.GetHash())); - BOOST_CHECK(pool.exists(tx6.GetHash())); - BOOST_CHECK(!pool.exists(tx7.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); - if (!pool.exists(tx5.GetHash())) + if (!pool.exists(GenTxid::Txid(tx5.GetHash()))) pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 - BOOST_CHECK(pool.exists(tx4.GetHash())); - BOOST_CHECK(!pool.exists(tx5.GetHash())); - BOOST_CHECK(pool.exists(tx6.GetHash())); - BOOST_CHECK(!pool.exists(tx7.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx5.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 5a93f30c8a..b945659c0d 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -969,7 +969,7 @@ CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set& hashes) c bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (unsigned int i = 0; i < tx.vin.size(); i++) - if (exists(tx.vin[i].prevout.hash)) + if (exists(GenTxid::Txid(tx.vin[i].prevout.hash))) return false; return true; } @@ -1140,7 +1140,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector* pvNoSpends if (pvNoSpendsRemaining) { for (const CTransaction& tx : txn) { for (const CTxIn& txin : tx.vin) { - if (exists(txin.prevout.hash)) continue; + if (exists(GenTxid::Txid(txin.prevout.hash))) continue; pvNoSpendsRemaining->push_back(txin.prevout); } } diff --git a/src/txmempool.h b/src/txmempool.h index 460e9d0ceb..1fd0c70891 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -782,7 +782,6 @@ class CTxMemPool } return (mapTx.count(gtxid.GetHash()) != 0); } - bool exists(const uint256& txid) const { return exists(GenTxid{false, txid}); } CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) @@ -802,7 +801,7 @@ class CTxMemPool LOCK(cs); // Sanity check the transaction is in the mempool & insert into // unbroadcast set. - if (exists(txid)) m_unbroadcast_txids.insert(txid); + if (exists(GenTxid::Txid(txid))) m_unbroadcast_txids.insert(txid); }; /** Removes a transaction from the unbroadcast set */ diff --git a/src/validation.cpp b/src/validation.cpp index ff71020ebb..8938948d06 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -355,7 +355,7 @@ void CChainState::MaybeUpdateMempoolForReorg( // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); - } else if (m_mempool->exists((*it)->GetHash())) { + } else if (m_mempool->exists(GenTxid::Txid((*it)->GetHash()))) { vHashUpdate.push_back((*it)->GetHash()); } ++it; @@ -908,7 +908,7 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) // trim mempool and check if tx was trimmed if (!bypass_limits) { LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); - if (!m_pool.exists(hash)) + if (!m_pool.exists(GenTxid::Txid(hash))) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } return true; @@ -4500,7 +4500,7 @@ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mocka // wallet(s) having loaded it while we were processing // mempool transactions; consider these as valid, instead of // failed, but mark them as 'already there' - if (pool.exists(tx->GetHash())) { + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { ++already_there; } else { ++failed; From fa8fef6ef2287cd36ae14fbf10e852ddef7e62ac Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 21 Sep 2021 15:28:42 +0200 Subject: [PATCH 87/96] doc: Fix CWalletTx::Confirmation doc Follow-up to: * commit 700c42b85d20e624bef4228eef062c93084efab5, which replaced pIndex with block_hash in AddToWalletIfInvolvingMe. * commit 9700fcb47feca9d78e005b8d18b41148c8f6b25f, which replaced posInBlock with confirm.nIndex. --- src/wallet/transaction.h | 3 ++- src/wallet/wallet.cpp | 6 +++--- src/wallet/wallet.h | 8 +++----- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 6fc1bd1eed..1ccef31056 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -162,7 +162,8 @@ class CWalletTx int block_height; uint256 hashBlock; int nIndex; - Confirmation(Status s = UNCONFIRMED, int b = 0, uint256 h = uint256(), int i = 0) : status(s), block_height(b), hashBlock(h), nIndex(i) {} + Confirmation(Status status = UNCONFIRMED, int block_height = 0, uint256 block_hash = uint256(), int block_index = 0) + : status{status}, block_height{block_height}, hashBlock{block_hash}, nIndex{block_index} {} }; Confirmation m_confirm; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index a749ab8897..1b0d55bffa 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1212,7 +1212,7 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio void CWallet::transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) { LOCK(cs_wallet); - SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { @@ -1253,7 +1253,7 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // distinguishing between conflicted and unconfirmed transactions are // imperfect, and could be improved in general, see // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking - SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); } } @@ -1281,7 +1281,7 @@ void CWallet::blockDisconnected(const CBlock& block, int height) m_last_block_processed_height = height - 1; m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /* block height */ 0, /* block hash */ {}, /* index */ 0}); + SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 767b24adbb..c911eb461c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -260,9 +260,9 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati void AddToSpends(const uint256& wtxid, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** - * Add a transaction to the wallet, or update it. pIndex and posInBlock should + * Add a transaction to the wallet, or update it. confirm.block_* should * be set when the transaction was known to be included in a block. When - * pIndex == nullptr, then wallet state is not updated in AddToWallet, but + * block_hash.IsNull(), then wallet state is not updated in AddToWallet, but * notifications happen and cached balances are marked dirty. * * If fUpdate is true, existing transactions will be updated. @@ -270,7 +270,7 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati * assumption that any further notification of a transaction that was considered * abandoned is an indication that it is not safe to be considered abandoned. * Abandoned state should probably be more carefully tracked via different - * posInBlock signals or by checking mempool presence when necessary. + * chain notifications or by checking mempool presence when necessary. * * Should be called with rescanning_old_block set to true, if the transaction is * not discovered in real time, but during a rescan of old blocks. @@ -285,8 +285,6 @@ class CWallet final : public WalletStorage, public interfaces::Chain::Notificati void SyncMetaData(std::pair) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. - * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true, bool rescanning_old_block = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** WalletFlags set on this wallet. */ From 077a875d94b51e3c87381133657be98989c8643e Mon Sep 17 00:00:00 2001 From: Joan Karadimov Date: Fri, 22 Oct 2021 01:19:27 +0300 Subject: [PATCH 88/96] refactor: include a missing header in fs.cpp ... needed for std::numeric_limits::max on WIN32 --- src/fs.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fs.cpp b/src/fs.cpp index 8cae7f32c6..7a99444eef 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -16,6 +16,7 @@ #define NOMINMAX #endif #include +#include #include #endif From faeb9a575367119dbff60c35fa2c13547718e179 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 22 Oct 2021 12:15:15 +0200 Subject: [PATCH 89/96] remove unused CTxMemPool::info(const uint256& txid) --- src/txmempool.cpp | 2 -- src/txmempool.h | 1 - 2 files changed, 3 deletions(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index b945659c0d..70084ea1d1 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -895,8 +895,6 @@ TxMempoolInfo CTxMemPool::info(const GenTxid& gtxid) const return GetInfo(i); } -TxMempoolInfo CTxMemPool::info(const uint256& txid) const { return info(GenTxid{false, txid}); } - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { diff --git a/src/txmempool.h b/src/txmempool.h index 1fd0c70891..c63522225a 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -789,7 +789,6 @@ class CTxMemPool AssertLockHeld(cs); return mapTx.project<0>(mapTx.get().find(wtxid)); } - TxMempoolInfo info(const uint256& hash) const; TxMempoolInfo info(const GenTxid& gtxid) const; std::vector infoAll() const; From fa4ec1c0bdaef9f082a6661d7faf16149774e145 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 22 Oct 2021 12:28:14 +0200 Subject: [PATCH 90/96] Make GenTxid boolean constructor private --- src/net_processing.cpp | 4 ++-- src/primitives/transaction.h | 3 ++- src/protocol.cpp | 2 +- src/test/fuzz/txrequest.cpp | 4 ++-- src/test/txrequest_tests.cpp | 6 +++--- src/txrequest.cpp | 2 +- src/validation.cpp | 4 ++-- 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 9c8909f742..6d559d89f1 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3242,7 +3242,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (AlreadyHaveTx(GenTxid(/* is_wtxid=*/true, wtxid))) { + if (AlreadyHaveTx(GenTxid::Wtxid(wtxid))) { if (pfrom.HasPermission(NetPermissionFlags::ForceRelay)) { // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing @@ -3312,7 +3312,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // wtxidrelay peers. // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. - const GenTxid gtxid{/* is_wtxid=*/false, parent_txid}; + const auto gtxid{GenTxid::Txid(parent_txid)}; pfrom.AddKnownTx(parent_txid); if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); } diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index d7a85015ef..947c1c60bb 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -391,8 +391,9 @@ class GenTxid { bool m_is_wtxid; uint256 m_hash; -public: GenTxid(bool is_wtxid, const uint256& hash) : m_is_wtxid(is_wtxid), m_hash(hash) {} + +public: static GenTxid Txid(const uint256& hash) { return GenTxid{false, hash}; } static GenTxid Wtxid(const uint256& hash) { return GenTxid{true, hash}; } bool IsWtxid() const { return m_is_wtxid; } diff --git a/src/protocol.cpp b/src/protocol.cpp index 2e70b41e4c..7506c81815 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -223,5 +223,5 @@ std::vector serviceFlagsToStr(uint64_t flags) GenTxid ToGenTxid(const CInv& inv) { assert(inv.IsGenTxMsg()); - return {inv.IsMsgWtx(), inv.hash}; + return inv.IsMsgWtx() ? GenTxid::Wtxid(inv.hash) : GenTxid::Txid(inv.hash); } diff --git a/src/test/fuzz/txrequest.cpp b/src/test/fuzz/txrequest.cpp index 72438ff2d7..a73bbcfc25 100644 --- a/src/test/fuzz/txrequest.cpp +++ b/src/test/fuzz/txrequest.cpp @@ -204,7 +204,7 @@ class Tester } // Call TxRequestTracker's implementation. - m_tracker.ReceivedInv(peer, GenTxid{is_wtxid, TXHASHES[txhash]}, preferred, reqtime); + m_tracker.ReceivedInv(peer, is_wtxid ? GenTxid::Wtxid(TXHASHES[txhash]) : GenTxid::Txid(TXHASHES[txhash]), preferred, reqtime); } void RequestedTx(int peer, int txhash, std::chrono::microseconds exptime) @@ -252,7 +252,7 @@ class Tester for (int peer2 = 0; peer2 < MAX_PEERS; ++peer2) { Announcement& ann2 = m_announcements[txhash][peer2]; if (ann2.m_state == State::REQUESTED && ann2.m_time <= m_now) { - expected_expired.emplace_back(peer2, GenTxid{ann2.m_is_wtxid, TXHASHES[txhash]}); + expected_expired.emplace_back(peer2, ann2.m_is_wtxid ? GenTxid::Wtxid(TXHASHES[txhash]) : GenTxid::Txid(TXHASHES[txhash])); ann2.m_state = State::COMPLETED; break; } diff --git a/src/test/txrequest_tests.cpp b/src/test/txrequest_tests.cpp index 1d137b03b1..99d41882c9 100644 --- a/src/test/txrequest_tests.cpp +++ b/src/test/txrequest_tests.cpp @@ -221,7 +221,7 @@ class Scenario /** Generate a random GenTxid; the txhash follows NewTxHash; the is_wtxid flag is random. */ GenTxid NewGTxid(const std::vector>& orders = {}) { - return {InsecureRandBool(), NewTxHash(orders)}; + return InsecureRandBool() ? GenTxid::Wtxid(NewTxHash(orders)) : GenTxid::Txid(NewTxHash(orders)); } /** Generate a new random NodeId to use as peer. The same NodeId is never returned twice @@ -494,8 +494,8 @@ void BuildWtxidTest(Scenario& scenario, int config) auto peerT = scenario.NewPeer(); auto peerW = scenario.NewPeer(); auto txhash = scenario.NewTxHash(); - GenTxid txid{false, txhash}; - GenTxid wtxid{true, txhash}; + auto txid{GenTxid::Txid(txhash)}; + auto wtxid{GenTxid::Wtxid(txhash)}; auto reqtimeT = InsecureRandBool() ? MIN_TIME : scenario.Now() + RandomTime8s(); auto reqtimeW = InsecureRandBool() ? MIN_TIME : scenario.Now() + RandomTime8s(); diff --git a/src/txrequest.cpp b/src/txrequest.cpp index f8d7a1ece8..7d478a5b26 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -300,7 +300,7 @@ std::map ComputeTxHashInfo(const Index& index, const Priori GenTxid ToGenTxid(const Announcement& ann) { - return {ann.m_is_wtxid, ann.m_txhash}; + return ann.m_is_wtxid ? GenTxid::Wtxid(ann.m_txhash) : GenTxid::Txid(ann.m_txhash); } } // namespace diff --git a/src/validation.cpp b/src/validation.cpp index 8938948d06..8d64a9a460 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -585,10 +585,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); - if (m_pool.exists(GenTxid(true, tx.GetWitnessHash()))) { + if (m_pool.exists(GenTxid::Wtxid(tx.GetWitnessHash()))) { // Exact transaction already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); - } else if (m_pool.exists(GenTxid(false, tx.GetHash()))) { + } else if (m_pool.exists(GenTxid::Txid(tx.GetHash()))) { // Transaction with the same non-witness data but different witness (same txid, different // wtxid) already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-same-nonwitness-data-in-mempool"); From f778845d972578e7abdced64ec6129d809c816f6 Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Fri, 22 Oct 2021 12:02:20 +0300 Subject: [PATCH 91/96] ci: Add vcpkg tools cache This change avoids downloading of the cached vcpkg tools that could fail accidentally, and it makes CI task more robust. --- .cirrus.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 1f32dd5231..f7962e951c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -86,6 +86,7 @@ task: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 CI_VCPKG_TAG: '2021.05.12' + VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' @@ -121,6 +122,12 @@ task: - ..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix %QTBASEDIR% - jom - jom install + vcpkg_tools_cache: + folder: '%VCPKG_DOWNLOADS%\tools' + reupload_on_changes: false + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version vcpkg_binary_cache: folder: '%VCPKG_DEFAULT_BINARY_CACHE%' reupload_on_changes: true From 92617b7a758c0425330fba4b886296730567927c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 5 Oct 2021 16:13:41 -0400 Subject: [PATCH 92/96] Make AddrMan support multiple ports per IP --- src/addrman.cpp | 18 +----------------- src/addrman_impl.h | 6 +++--- src/netaddress.h | 36 +++++++++++++++++++----------------- src/test/addrman_tests.cpp | 22 +++++++++++----------- src/test/fuzz/addrman.cpp | 33 +++++++++++++++++++-------------- 5 files changed, 53 insertions(+), 62 deletions(-) diff --git a/src/addrman.cpp b/src/addrman.cpp index c364a7710b..92937abcb5 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -401,7 +401,7 @@ void AddrManImpl::Unserialize(Stream& s_) } } -AddrInfo* AddrManImpl::Find(const CNetAddr& addr, int* pnId) +AddrInfo* AddrManImpl::Find(const CService& addr, int* pnId) { AssertLockHeld(cs); @@ -556,10 +556,6 @@ void AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT AddrInfo& info = *pinfo; - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - // update info info.nLastSuccess = nTime; info.nLastTry = nTime; @@ -683,10 +679,6 @@ void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTi AddrInfo& info = *pinfo; - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - // update info info.nLastTry = nTime; if (fCountFailure && info.nLastCountAttempt < nLastGood) { @@ -796,10 +788,6 @@ void AddrManImpl::Connected_(const CService& addr, int64_t nTime) AddrInfo& info = *pinfo; - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - // update info int64_t nUpdateInterval = 20 * 60; if (nTime - info.nTime > nUpdateInterval) @@ -818,10 +806,6 @@ void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) AddrInfo& info = *pinfo; - // check whether we are talking about the exact same CService (including same port) - if (info != addr) - return; - // update info info.nServices = nServices; } diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 1dc7f25f9c..f8191d6b85 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -179,8 +179,8 @@ class AddrManImpl //! table with information about all nIds std::unordered_map mapInfo GUARDED_BY(cs); - //! find an nId based on its network address - std::unordered_map mapAddr GUARDED_BY(cs); + //! find an nId based on its network address and port. + std::unordered_map mapAddr GUARDED_BY(cs); //! randomly-ordered vector of all nIds //! This is mutable because it is unobservable outside the class, so any @@ -225,7 +225,7 @@ class AddrManImpl const std::vector m_asmap; //! Find an entry. - AddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/netaddress.h b/src/netaddress.h index 66c8c48f08..57eb8bc72f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -253,7 +253,6 @@ class CNetAddr } } - friend class CNetAddrHash; friend class CSubNet; private: @@ -467,22 +466,6 @@ class CNetAddr } }; -class CNetAddrHash -{ -public: - size_t operator()(const CNetAddr& a) const noexcept - { - CSipHasher hasher(m_salt_k0, m_salt_k1); - hasher.Write(a.m_net); - hasher.Write(a.m_addr.data(), a.m_addr.size()); - return static_cast(hasher.Finalize()); - } - -private: - const uint64_t m_salt_k0 = GetRand(std::numeric_limits::max()); - const uint64_t m_salt_k1 = GetRand(std::numeric_limits::max()); -}; - class CSubNet { protected: @@ -565,6 +548,25 @@ class CService : public CNetAddr READWRITEAS(CNetAddr, obj); READWRITE(Using>(obj.port)); } + + friend class CServiceHash; +}; + +class CServiceHash +{ +public: + size_t operator()(const CService& a) const noexcept + { + CSipHasher hasher(m_salt_k0, m_salt_k1); + hasher.Write(a.m_net); + hasher.Write(a.port); + hasher.Write(a.m_addr.data(), a.m_addr.size()); + return static_cast(hasher.Finalize()); + } + +private: + const uint64_t m_salt_k0 = GetRand(std::numeric_limits::max()); + const uint64_t m_salt_k1 = GetRand(std::numeric_limits::max()); }; #endif // BITCOIN_NETADDRESS_H diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index bd6f470219..991bfa5efc 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -89,7 +89,7 @@ class AddrManTest : public AddrMan deterministic = makeDeterministic; } - AddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) + AddrInfo* Find(const CService& addr, int* pnId = nullptr) { LOCK(m_impl->cs); return m_impl->Find(addr, pnId); @@ -222,15 +222,15 @@ BOOST_AUTO_TEST_CASE(addrman_ports) BOOST_CHECK_EQUAL(addrman.size(), 1U); CService addr1_port = ResolveService("250.1.1.1", 8334); - BOOST_CHECK(!addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK(addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman.size(), 2U); auto addr_ret2 = addrman.Select().first; - BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); + BOOST_CHECK(addr_ret2.ToString() == "250.1.1.1:8333" || addr_ret2.ToString() == "250.1.1.1:8334"); - // Test: Add same IP but diff port to tried table, it doesn't get added. - // Perhaps this is not ideal behavior but it is the current behavior. + // Test: Add same IP but diff port to tried table; this converts the entry with + // the specified port to tried, but not the other. addrman.Good(CAddress(addr1_port, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK_EQUAL(addrman.size(), 2U); bool newOnly = true; auto addr_ret3 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); @@ -369,18 +369,18 @@ BOOST_AUTO_TEST_CASE(addrman_find) CNetAddr source2 = ResolveIP("250.1.2.2"); BOOST_CHECK(addrman.Add({addr1}, source1)); - BOOST_CHECK(!addrman.Add({addr2}, source2)); + BOOST_CHECK(addrman.Add({addr2}, source2)); BOOST_CHECK(addrman.Add({addr3}, source1)); - // Test: ensure Find returns an IP matching what we searched on. + // Test: ensure Find returns an IP/port matching what we searched on. AddrInfo* info1 = addrman.Find(addr1); BOOST_REQUIRE(info1); BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); - // Test 18; Find does not discriminate by port number. + // Test; Find discriminates by port number. AddrInfo* info2 = addrman.Find(addr2); BOOST_REQUIRE(info2); - BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); + BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:9999"); // Test: Find returns another IP matching what we searched on. AddrInfo* info3 = addrman.Find(addr3); diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 8df3707fc9..c6df6a0e61 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -137,24 +137,29 @@ class AddrManDeterministic : public AddrMan // Check that all values in `mapInfo` are equal to all values in `other.mapInfo`. // Keys may be different. - using AddrInfoHasher = std::function; - using AddrInfoEq = std::function; - - CNetAddrHash netaddr_hasher; - - AddrInfoHasher addrinfo_hasher = [&netaddr_hasher](const AddrInfo& a) { - return netaddr_hasher(static_cast(a)) ^ netaddr_hasher(a.source) ^ - a.nLastSuccess ^ a.nAttempts ^ a.nRefCount ^ a.fInTried; + auto addrinfo_hasher = [](const AddrInfo& a) { + CSipHasher hasher(0, 0); + auto addr_key = a.GetKey(); + auto source_key = a.source.GetAddrBytes(); + hasher.Write(a.nLastSuccess); + hasher.Write(a.nAttempts); + hasher.Write(a.nRefCount); + hasher.Write(a.fInTried); + hasher.Write(a.GetNetwork()); + hasher.Write(a.source.GetNetwork()); + hasher.Write(addr_key.size()); + hasher.Write(source_key.size()); + hasher.Write(addr_key.data(), addr_key.size()); + hasher.Write(source_key.data(), source_key.size()); + return (size_t)hasher.Finalize(); }; - AddrInfoEq addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { - return static_cast(lhs) == static_cast(rhs) && - lhs.source == rhs.source && lhs.nLastSuccess == rhs.nLastSuccess && - lhs.nAttempts == rhs.nAttempts && lhs.nRefCount == rhs.nRefCount && - lhs.fInTried == rhs.fInTried; + auto addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { + return std::tie(static_cast(lhs), lhs.source, lhs.nLastSuccess, lhs.nAttempts, lhs.nRefCount, lhs.fInTried) == + std::tie(static_cast(rhs), rhs.source, rhs.nLastSuccess, rhs.nAttempts, rhs.nRefCount, rhs.fInTried); }; - using Addresses = std::unordered_set; + using Addresses = std::unordered_set; const size_t num_addresses{m_impl->mapInfo.size()}; From a78c2298080f173d0266e708267458a72eb2f600 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 22 Oct 2021 16:26:18 -0400 Subject: [PATCH 93/96] tests: Place into mapWallet in coinselector_tests Instead of using AddToWallet so that making a COutput will work, directly add the transaction into wallet.mapWallet. This bypasses many checks that AddToWallet will do which are pointless and just slow down this test. --- src/wallet/test/coinselector_tests.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index e880e13845..8606924bb3 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -71,13 +71,18 @@ static void add_coin(std::vector& coins, CWallet& wallet, const CAmount // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - CWalletTx* wtx = wallet.AddToWallet(MakeTransactionRef(std::move(tx)), /* confirm= */ {}); + uint256 txid = tx.GetHash(); + + LOCK(wallet.cs_wallet); + auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)))); + assert(ret.second); + CWalletTx& wtx = (*ret.first).second; if (fIsFromMe) { - wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); - wtx->m_is_cache_empty = false; + wtx.m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); + wtx.m_is_cache_empty = false; } - COutput output(wallet, *wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); + COutput output(wallet, wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); coins.push_back(output); } From a52f1d13409e4ef46277596ec13fa8b421fa1329 Mon Sep 17 00:00:00 2001 From: Andrew Chow Date: Fri, 22 Oct 2021 17:33:24 -0400 Subject: [PATCH 94/96] walletdb: Use SQLiteDatabase for mock wallet databases Default to SQLiteDatabase instead of BerkeleyDatabase for CreateDummyWalletDatabase. Most tests already use descriptor wallets and the mock db doesn't really matter for tests. The tests where it does matter will make the db directly. --- src/wallet/walletdb.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index a6839f1f78..c920d4af51 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1188,9 +1188,9 @@ std::unique_ptr CreateDummyWalletDatabase() /** Return object for accessing temporary in-memory database. */ std::unique_ptr CreateMockWalletDatabase() { -#ifdef USE_BDB - return std::make_unique(std::make_shared(), ""); -#elif USE_SQLITE +#ifdef USE_SQLITE return std::make_unique("", "", true); +#elif USE_BDB + return std::make_unique(std::make_shared(), ""); #endif } From 4718897ce3a7c728ff7aebbadabcc8ed7a0b8d6e Mon Sep 17 00:00:00 2001 From: Sebastian Falbesoner Date: Mon, 18 Oct 2021 22:25:04 +0200 Subject: [PATCH 95/96] test: add script_util helper for creating bare multisig scripts --- test/functional/feature_segwit.py | 7 +++---- test/functional/mempool_accept.py | 6 ++---- test/functional/test_framework/blocktools.py | 6 +++--- test/functional/test_framework/script_util.py | 13 +++++++++++++ test/functional/test_framework/wallet_util.py | 9 ++------- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 5abe989e55..acb7469c6a 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -30,8 +30,6 @@ CScript, OP_0, OP_1, - OP_2, - OP_CHECKMULTISIG, OP_DROP, OP_TRUE, ) @@ -39,6 +37,7 @@ key_to_p2pk_script, key_to_p2pkh_script, key_to_p2wpkh_script, + keys_to_multisig_script, script_to_p2sh_script, script_to_p2wsh_script, ) @@ -149,7 +148,7 @@ def run_test(self): key = get_generate_key() self.pubkey.append(key.pubkey) - multiscript = CScript([OP_1, bytes.fromhex(self.pubkey[-1]), OP_1, OP_CHECKMULTISIG]) + multiscript = keys_to_multisig_script([self.pubkey[-1]]) p2sh_ms_addr = self.nodes[i].createmultisig(1, [self.pubkey[-1]], 'p2sh-segwit')['address'] bip173_ms_addr = self.nodes[i].createmultisig(1, [self.pubkey[-1]], 'bech32')['address'] assert_equal(p2sh_ms_addr, script_to_p2sh_p2wsh(multiscript)) @@ -389,7 +388,7 @@ def run_test(self): # Money sent to P2SH of multisig of this should only be seen after importaddress with the BASE58 P2SH address. multisig_without_privkey_address = self.nodes[0].addmultisigaddress(2, [pubkeys[3], pubkeys[4]])['address'] - script = CScript([OP_2, bytes.fromhex(pubkeys[3]), bytes.fromhex(pubkeys[4]), OP_2, OP_CHECKMULTISIG]) + script = keys_to_multisig_script([pubkeys[3], pubkeys[4]]) solvable_after_importaddress.append(script_to_p2sh_script(script)) for i in compressed_spendable_address: diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 2ee440bcb7..71be2b4a82 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -22,13 +22,11 @@ from test_framework.script import ( CScript, OP_0, - OP_2, - OP_3, - OP_CHECKMULTISIG, OP_HASH160, OP_RETURN, ) from test_framework.script_util import ( + keys_to_multisig_script, script_to_p2sh_script, ) from test_framework.util import ( @@ -283,7 +281,7 @@ def run_test(self): key = ECKey() key.generate() pubkey = key.get_pubkey().get_bytes() - tx.vout[0].scriptPubKey = CScript([OP_2, pubkey, pubkey, pubkey, OP_3, OP_CHECKMULTISIG]) # Some bare multisig script (2-of-3) + tx.vout[0].scriptPubKey = keys_to_multisig_script([pubkey] * 3, k=2) # Some bare multisig script (2-of-3) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bare-multisig'}], rawtxs=[tx.serialize().hex()], diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 85e3c2a383..5d0113465f 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -32,13 +32,13 @@ CScriptNum, CScriptOp, OP_1, - OP_CHECKMULTISIG, OP_RETURN, OP_TRUE, ) from .script_util import ( key_to_p2pk_script, key_to_p2wpkh_script, + keys_to_multisig_script, script_to_p2wsh_script, ) from .util import assert_equal @@ -209,7 +209,7 @@ def witness_script(use_p2wsh, pubkey): pkscript = key_to_p2wpkh_script(pubkey) else: # 1-of-1 multisig - witness_script = CScript([OP_1, bytes.fromhex(pubkey), OP_1, OP_CHECKMULTISIG]) + witness_script = keys_to_multisig_script([pubkey]) pkscript = script_to_p2wsh_script(witness_script) return pkscript.hex() @@ -218,7 +218,7 @@ def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): Optionally wrap the segwit output using P2SH.""" if use_p2wsh: - program = CScript([OP_1, bytes.fromhex(pubkey), OP_1, OP_CHECKMULTISIG]) + program = keys_to_multisig_script([pubkey]) addr = script_to_p2sh_p2wsh(program) if encode_p2sh else script_to_p2wsh(program) else: addr = key_to_p2sh_p2wpkh(pubkey) if encode_p2sh else key_to_p2wpkh(pubkey) diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 82a9067dd2..cbc4a560db 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -5,7 +5,9 @@ """Useful Script constants and utils.""" from test_framework.script import ( CScript, + CScriptOp, OP_0, + OP_CHECKMULTISIG, OP_CHECKSIG, OP_DUP, OP_EQUAL, @@ -41,6 +43,17 @@ def key_to_p2pk_script(key): return CScript([key, OP_CHECKSIG]) +def keys_to_multisig_script(keys, *, k=None): + n = len(keys) + if k is None: # n-of-n multisig by default + k = n + assert k <= n + op_k = CScriptOp.encode_op_n(k) + op_n = CScriptOp.encode_op_n(n) + checked_keys = [check_key(key) for key in keys] + return CScript([op_k] + checked_keys + [op_n, OP_CHECKMULTISIG]) + + def keyhash_to_p2pkh_script(hash): assert len(hash) == 20 return CScript([OP_DUP, OP_HASH160, hash, OP_EQUALVERIFY, OP_CHECKSIG]) diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index 1ee55aa3b7..c307ded542 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -15,15 +15,10 @@ script_to_p2wsh, ) from test_framework.key import ECKey -from test_framework.script import ( - CScript, - OP_2, - OP_3, - OP_CHECKMULTISIG, -) from test_framework.script_util import ( key_to_p2pkh_script, key_to_p2wpkh_script, + keys_to_multisig_script, script_to_p2sh_script, script_to_p2wsh_script, ) @@ -92,7 +87,7 @@ def get_multisig(node): addr = node.getaddressinfo(node.getnewaddress()) addrs.append(addr['address']) pubkeys.append(addr['pubkey']) - script_code = CScript([OP_2] + [bytes.fromhex(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG]) + script_code = keys_to_multisig_script(pubkeys, k=2) witness_script = script_to_p2wsh_script(script_code) return Multisig(privkeys=[node.dumpprivkey(addr) for addr in addrs], pubkeys=pubkeys, From 749470f8922b4ee47ed9e4b183648e5d22921c50 Mon Sep 17 00:00:00 2001 From: Jarol Rodriguez Date: Wed, 27 Oct 2021 14:49:32 -0400 Subject: [PATCH 96/96] multiprocess: add new bitcoin-qt init implementation to qml-gui This incorporates the new bitcoin-qt init implementation into the qml-gui. --- src/qml/bitcoin.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/qml/bitcoin.cpp b/src/qml/bitcoin.cpp index 425b8fb2cf..4b02e2ea86 100644 --- a/src/qml/bitcoin.cpp +++ b/src/qml/bitcoin.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -102,15 +101,12 @@ int QmlGuiMain(int argc, char* argv[]) auto handler_message_box = ::uiInterface.ThreadSafeMessageBox_connect(InitErrorMessageBox); - NodeContext node_context; - int unused_exit_status; - std::unique_ptr init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); + std::unique_ptr init = interfaces::MakeGuiInit(argc, argv); SetupEnvironment(); util::ThreadSetInternalName("main"); /// Parse command-line options. We do this after qt in order to show an error if there are problems parsing these. - node_context.args = &gArgs; SetupServerArgs(gArgs); SetupUIArgs(gArgs); std::string error;