Re-claim forwarded HTLCs on startup

Now that we let `commitment_signed` `ChannelMonitorUpdate`s from a
downstream channel complete prior to the preimage
`ChannelMonitorUpdate` on the upstream channel, we may not get a
`update_fulfill_htlc` replay on startup. Thus, we have to ensure
any payment preimages contained in that downstream update are
re-claimed on startup.

Here we do this during the existing walk of the `ChannelMonitor`
preimages for closed channels.

Author: TheBlueMatt

lightning/src/chain/channelmonitor.rs

@@ -69,7 +69,7 @@ use crate::sync::{Mutex, LockTestExt};
 /// much smaller than a full [`ChannelMonitor`]. However, for large single commitment transaction
 /// updates (e.g. ones during which there are hundreds of HTLCs pending on the commitment
 /// transaction), a single update may reach upwards of 1 MiB in serialized size.
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 #[must_use]
 pub struct ChannelMonitorUpdate {
 	pub(crate) updates: Vec<ChannelMonitorUpdateStep>,
@@ -490,7 +490,7 @@ impl_writeable_tlv_based_enum_upgradable!(OnchainEvent,
 
 );
 
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) enum ChannelMonitorUpdateStep {
 	LatestHolderCommitmentTXInfo {
 		commitment_tx: HolderCommitmentTransaction,

lightning/src/ln/chan_utils.rs

@@ -437,7 +437,7 @@ pub fn derive_public_revocation_key<T: secp256k1::Verification>(secp_ctx: &Secp2
 /// channel basepoints via the new function, or they were obtained via
 /// CommitmentTransaction.trust().keys() because we trusted the source of the
 /// pre-calculated keys.
-#[derive(PartialEq, Eq, Clone)]
+#[derive(PartialEq, Eq, Clone, Debug)]
 pub struct TxCreationKeys {
 	/// The broadcaster's per-commitment public key which was used to derive the other keys.
 	pub per_commitment_point: PublicKey,
@@ -949,7 +949,7 @@ impl<'a> DirectedChannelTransactionParameters<'a> {
 /// Information needed to build and sign a holder's commitment transaction.
 ///
 /// The transaction is only signed once we are ready to broadcast.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct HolderCommitmentTransaction {
 	inner: CommitmentTransaction,
 	/// Our counterparty's signature for the transaction
@@ -1052,7 +1052,7 @@ impl HolderCommitmentTransaction {
 }
 
 /// A pre-built Bitcoin commitment transaction and its txid.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct BuiltCommitmentTransaction {
 	/// The commitment transaction
 	pub transaction: Transaction,
@@ -1215,7 +1215,7 @@ impl<'a> TrustedClosingTransaction<'a> {
 ///
 /// This class can be used inside a signer implementation to generate a signature given the relevant
 /// secret key.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct CommitmentTransaction {
 	commitment_number: u64,
 	to_broadcaster_value_sat: u64,

lightning/src/ln/chanmon_update_fail_tests.rs

@@ -3021,7 +3021,7 @@ fn test_blocked_chan_preimage_release() {
 	expect_payment_sent(&nodes[2], payment_preimage_2, None, true, true);
 }
 
-fn do_test_inverted_mon_completion_order(complete_bc_commitment_dance: bool) {
+fn do_test_inverted_mon_completion_order(with_latest_manager: bool, complete_bc_commitment_dance: bool) {
 	// When we forward a payment and receive an `update_fulfill_htlc` message from the downstream
 	// channel, we immediately claim the HTLC on the upstream channel, before even doing a
 	// `commitment_signed` dance on the downstream channel. This implies that our
@@ -3049,6 +3049,10 @@ fn do_test_inverted_mon_completion_order(complete_bc_commitment_dance: bool) {
 	let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 100_000);
 
 	let mon_ab = get_monitor!(nodes[1], chan_id_ab).encode();
+	let mut manager_b = Vec::new();
+	if !with_latest_manager {
+		manager_b = nodes[1].node.encode();
+	}
 
 	nodes[2].node.claim_funds(payment_preimage);
 	check_added_monitors(&nodes[2], 1);
@@ -3085,58 +3089,227 @@ fn do_test_inverted_mon_completion_order(complete_bc_commitment_dance: bool) {
 	}
 
 	// Now reload node B
-	let manager_b = nodes[1].node.encode();
+	if with_latest_manager {
+		manager_b = nodes[1].node.encode();
+	}
 
 	let mon_bc = get_monitor!(nodes[1], chan_id_bc).encode();
 	reload_node!(nodes[1], &manager_b, &[&mon_ab, &mon_bc], persister, new_chain_monitor, nodes_1_deserialized);
 
 	nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 	nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 
-	// If we used the latest ChannelManager to reload from, we should have both channels still
-	// live. The B <-> C channel's final RAA ChannelMonitorUpdate must still be blocked as
-	// before - the ChannelMonitorUpdate for the A <-> B channel hasn't completed.
-	// When we call `timer_tick_occurred` we will get that monitor update back, which we'll
-	// complete after reconnecting to our peers.
-	persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
-	nodes[1].node.timer_tick_occurred();
-	check_added_monitors(&nodes[1], 1);
-	assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+	if with_latest_manager {
+		// If we used the latest ChannelManager to reload from, we should have both channels still
+		// live. The B <-> C channel's final RAA ChannelMonitorUpdate must still be blocked as
+		// before - the ChannelMonitorUpdate for the A <-> B channel hasn't completed.
+		// When we call `timer_tick_occurred` we will get that monitor update back, which we'll
+		// complete after reconnecting to our peers.
+		persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+		nodes[1].node.timer_tick_occurred();
+		check_added_monitors(&nodes[1], 1);
+		assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
-	// Now reconnect B to both A and C. If the B <-> C commitment signed dance wasn't run to
-	// the end go ahead and do that, though the -2 in `reconnect_nodes` indicates that we
-	// expect to *not* receive the final RAA ChannelMonitorUpdate.
-	if complete_bc_commitment_dance {
-		reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+		// Now reconnect B to both A and C. If the B <-> C commitment signed dance wasn't run to
+		// the end go ahead and do that, though the -2 in `reconnect_nodes` indicates that we
+		// expect to *not* receive the final RAA ChannelMonitorUpdate.
+		if complete_bc_commitment_dance {
+			reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+		} else {
+			reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, -2), (0, 0), (0, 0), (0, 0), (0, 0), (false, true));
+		}
+
+		reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+		// (Finally) complete the A <-> B ChannelMonitorUpdate, ensuring the preimage is durably on
+		// disk in the proper ChannelMonitor, unblocking the B <-> C ChannelMonitor updating
+		// process.
+		let (outpoint, _, ab_update_id) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_id_ab).unwrap().clone();
+		nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(outpoint, ab_update_id).unwrap();
+
+		// When we fetch B's HTLC update messages here (now that the ChannelMonitorUpdate has
+		// completed), it will also release the final RAA ChannelMonitorUpdate on the B <-> C
+		// channel.
+		let bs_updates = get_htlc_update_msgs(&nodes[1], &nodes[0].node.get_our_node_id());
+		check_added_monitors(&nodes[1], 1);
+
+		nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
+		do_commitment_signed_dance(&nodes[0], &nodes[1], &bs_updates.commitment_signed, false, false);
+
+		expect_payment_forwarded!(nodes[1], &nodes[0], &nodes[2], Some(1_000), false, false);
 	} else {
-		reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, -2), (0, 0), (0, 0), (0, 0), (0, 0), (false, true));
-	}
+		// If the ChannelManager used in the reload was stale, check that the B <-> C channel was
+		// closed.
+		//
+		// Note that this will also process the ChannelMonitorUpdates which were queued up when we
+		// reloaded the ChannelManager. This will re-emit the A<->B preimage as well as the B<->C
+		// force-closure ChannelMonitorUpdate. Once the A<->B preimage update completes, the claim
+		// commitment update will be allowed to go out.
+		check_added_monitors(&nodes[1], 0);
+		persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+		persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+		check_closed_event(&nodes[1], 1, ClosureReason::OutdatedChannelManager, false);
+		check_added_monitors(&nodes[1], 2);
 
-	reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+		nodes[1].node.timer_tick_occurred();
+		check_added_monitors(&nodes[1], 0);
 
-	// (Finally) complete the A <-> B ChannelMonitorUpdate, ensuring the preimage is durably on
-	// disk in the proper ChannelMonitor, unblocking the B <-> C ChannelMonitor updating
-	// process.
-	let (outpoint, _, ab_update_id) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_id_ab).unwrap().clone();
-	nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(outpoint, ab_update_id).unwrap();
+		// Don't bother to reconnect B to C - that channel has been closed. We don't need to
+		// exchange any messages here even though there's a pending commitment update because the
+		// ChannelMonitorUpdate hasn't yet completed.
+		reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 
-	// When we fetch B's HTLC update messages here (now that the ChannelMonitorUpdate has
-	// completed), it will also release the final RAA ChannelMonitorUpdate on the B <-> C
-	// channel.
-	let bs_updates = get_htlc_update_msgs(&nodes[1], &nodes[0].node.get_our_node_id());
-	check_added_monitors(&nodes[1], 1);
+		let (outpoint, _, ab_update_id) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_id_ab).unwrap().clone();
+		nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(outpoint, ab_update_id).unwrap();
 
-	nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
-	do_commitment_signed_dance(&nodes[0], &nodes[1], &bs_updates.commitment_signed, false, false);
+		// The ChannelMonitorUpdate which was completed prior to the reconnect only contained the
+		// preimage (as it was a replay of the original ChannelMonitorUpdate from before we
+		// restarted). When we go to fetch the commitment transaction updates we'll poll the
+		// ChannelMonitorUpdate completion, then generate (and complete) a new ChannelMonitorUpdate
+		// with the actual commitment transaction, which will allow us to fulfill the HTLC with
+		// node A.
+		let bs_updates = get_htlc_update_msgs(&nodes[1], &nodes[0].node.get_our_node_id());
+		check_added_monitors(&nodes[1], 1);
 
-	expect_payment_forwarded!(nodes[1], &nodes[0], &nodes[2], Some(1_000), false, false);
+		nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
+		do_commitment_signed_dance(&nodes[0], &nodes[1], &bs_updates.commitment_signed, false, false);
+	}
 
 	// Finally, check that the payment was, ultimately, seen as sent by node A.
 	expect_payment_sent(&nodes[0], payment_preimage, None, true, true);
 }
 
 #[test]
 fn test_inverted_mon_completion_order() {
-	do_test_inverted_mon_completion_order(true);
-	do_test_inverted_mon_completion_order(false);
+	do_test_inverted_mon_completion_order(true, true);
+	do_test_inverted_mon_completion_order(true, false);
+	do_test_inverted_mon_completion_order(false, true);
+	do_test_inverted_mon_completion_order(false, false);
+}
+
+fn do_test_durable_preimages_on_closed_channel(close_chans_before_reload: bool, close_only_a: bool) {
+	// Test that we can apply a `ChannelMonitorUpdate` with a payment preimage even if the channel
+	// is force-closed between when we generate the update on reload and when we go to handle the
+	// update or prior to generating the update at all.
+
+	if !close_chans_before_reload && close_only_a {
+		// If we're not closing, it makes no sense to "only close A"
+		panic!();
+	}
+
+	let chanmon_cfgs = create_chanmon_cfgs(3);
+	let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+
+	let persister;
+	let new_chain_monitor;
+	let nodes_1_deserialized;
+
+	let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+	let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+
+	let chan_id_ab = create_announced_chan_between_nodes(&nodes, 0, 1).2;
+	let chan_id_bc = create_announced_chan_between_nodes(&nodes, 1, 2).2;
+
+	// Route a payment from A, through B, to C, then claim it on C. Once we pass B the
+	// `update_fulfill_htlc` we have a monitor update for both of B's channels. We complete the one
+	// on the B<->C channel but leave the A<->B monitor update pending, then reload B.
+	let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
+
+	let mon_ab = get_monitor!(nodes[1], chan_id_ab).encode();
+
+	nodes[2].node.claim_funds(payment_preimage);
+	check_added_monitors(&nodes[2], 1);
+	expect_payment_claimed!(nodes[2], payment_hash, 1_000_000);
+
+	chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+	let cs_updates = get_htlc_update_msgs(&nodes[2], &nodes[1].node.get_our_node_id());
+	nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
+
+	// B generates a new monitor update for the A <-> B channel, but doesn't send the new messages
+	// for it since the monitor update is marked in-progress.
+	check_added_monitors(&nodes[1], 1);
+	assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+
+	// Now step the Commitment Signed Dance between B and C forward a bit, ensuring we won't get
+	// the preimage when the nodes reconnect, at which point we have to ensure we get it from the
+	// ChannelMonitor.
+	nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &cs_updates.commitment_signed);
+	check_added_monitors(&nodes[1], 1);
+	let _ = get_revoke_commit_msgs!(nodes[1], nodes[2].node.get_our_node_id());
+
+	let mon_bc = get_monitor!(nodes[1], chan_id_bc).encode();
+
+	if close_chans_before_reload {
+		if !close_only_a {
+			chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+			nodes[1].node.force_close_broadcasting_latest_txn(&chan_id_bc, &nodes[2].node.get_our_node_id()).unwrap();
+			check_closed_broadcast(&nodes[1], 1, true);
+			check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
+		}
+
+		chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
+		nodes[1].node.force_close_broadcasting_latest_txn(&chan_id_ab, &nodes[0].node.get_our_node_id()).unwrap();
+		check_closed_broadcast(&nodes[1], 1, true);
+		check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
+	}
+
+	// Now reload node B
+	let manager_b = nodes[1].node.encode();
+	reload_node!(nodes[1], &manager_b, &[&mon_ab, &mon_bc], persister, new_chain_monitor, nodes_1_deserialized);
+
+	nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
+	nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
+
+	if close_chans_before_reload {
+		// If the channels were already closed, B will rebroadcast its closing transactions here.
+		let bs_close_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+		if close_only_a {
+			assert_eq!(bs_close_txn.len(), 2);
+		} else {
+			assert_eq!(bs_close_txn.len(), 3);
+		}
+	}
+
+	nodes[0].node.force_close_broadcasting_latest_txn(&chan_id_ab, &nodes[1].node.get_our_node_id()).unwrap();
+	check_closed_event(&nodes[0], 1, ClosureReason::HolderForceClosed, false);
+	let as_closing_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+	assert_eq!(as_closing_tx.len(), 1);
+
+	// In order to give B A's closing transaction without processing background events first, use
+	// the _without_checks utility method. This is similar to connecting blocks during startup
+	// prior to the node being full initialized.
+	mine_transaction_without_checks(&nodes[1], &as_closing_tx[0]);
+
+	// After a timer tick a payment preimage ChannelMonitorUpdate is applied to the A<->B
+	// ChannelMonitor, even though the channel has since been closed.
+	check_added_monitors(&nodes[1], 0);
+	nodes[1].node.timer_tick_occurred();
+	check_added_monitors(&nodes[1], if close_chans_before_reload && !close_only_a { 3 } else { 2 });
+
+	// Finally, check that B created a payment preimage transaction and close out the payment.
+	let bs_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+	let bs_preimage_tx = if close_chans_before_reload && !close_only_a {
+		assert_eq!(bs_txn.len(), 2);
+		&bs_txn[1]
+	} else {
+		assert_eq!(bs_txn.len(), 1);
+		&bs_txn[0]
+	};
+	check_spends!(bs_preimage_tx, as_closing_tx[0]);
+
+	if !close_chans_before_reload {
+		check_closed_broadcast(&nodes[1], 1, true);
+		check_closed_event(&nodes[1], 1, ClosureReason::CommitmentTxConfirmed, false);
+	}
+
+	mine_transactions(&nodes[0], &[&as_closing_tx[0], bs_preimage_tx]);
+	check_closed_broadcast(&nodes[0], 1, true);
+	expect_payment_sent(&nodes[0], payment_preimage, None, true, true);
+}
+
+#[test]
+fn test_durable_preimages_on_closed_channel() {
+	do_test_durable_preimages_on_closed_channel(true, true);
+	do_test_durable_preimages_on_closed_channel(true, false);
+	do_test_durable_preimages_on_closed_channel(false, false);
 }