From cb81d27f42209204451b6fbdf67e640c604cb09d Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 9 Feb 2023 13:21:52 -0600 Subject: [PATCH 1/8] Pass payment hash into pay_internal Useful for generating Payment(Path)Failed events in this method --- lightning/src/ln/outbound_payment.rs | 32 +++++++++++++++------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index f252d88ac36..9fbca12ec11 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -408,7 +408,7 @@ impl OutboundPayments { SP: Fn(&Vec, &Option, &PaymentHash, &Option, u64, u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError>, { - self.pay_internal(payment_id, Some((payment_hash, payment_secret, None, retry_strategy)), + self.pay_internal(payment_id, payment_hash, Some((payment_secret, None, retry_strategy)), route_params, router, first_hops, &compute_inflight_htlcs, entropy_source, node_signer, best_block_height, logger, &send_payment_along_path) .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) @@ -449,7 +449,7 @@ impl OutboundPayments { let preimage = payment_preimage .unwrap_or_else(|| PaymentPreimage(entropy_source.get_secure_random_bytes())); let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner()); - self.pay_internal(payment_id, Some((payment_hash, &None, Some(preimage), retry_strategy)), + self.pay_internal(payment_id, payment_hash, Some((&None, Some(preimage), retry_strategy)), route_params, router, first_hops, &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, &send_payment_along_path) .map(|()| payment_hash) @@ -501,9 +501,9 @@ impl OutboundPayments { let mut retry_id_route_params = None; for (pmt_id, pmt) in outbounds.iter_mut() { if pmt.is_auto_retryable_now() { - if let PendingOutboundPayment::Retryable { pending_amt_msat, total_msat, payment_params: Some(params), .. } = pmt { + if let PendingOutboundPayment::Retryable { payment_hash, pending_amt_msat, total_msat, payment_params: Some(params), .. } = pmt { if pending_amt_msat < total_msat { - retry_id_route_params = Some((*pmt_id, RouteParameters { + retry_id_route_params = Some((*pmt_id, *payment_hash, RouteParameters { final_value_msat: *total_msat - *pending_amt_msat, final_cltv_expiry_delta: if let Some(delta) = params.final_cltv_expiry_delta { delta } @@ -519,8 +519,8 @@ impl OutboundPayments { } } core::mem::drop(outbounds); - if let Some((payment_id, route_params)) = retry_id_route_params { - if let Err(e) = self.pay_internal(payment_id, None, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, &send_payment_along_path) { + if let Some((payment_id, payment_hash, route_params)) = retry_id_route_params { + if let Err(e) = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, &send_payment_along_path) { log_info!(logger, "Errored retrying payment: {:?}", e); // If we error on retry, there is no chance of the payment succeeding and no HTLCs have // been irrevocably committed to, so we can safely abandon. @@ -553,8 +553,8 @@ impl OutboundPayments { /// Will return `Ok(())` iff at least one HTLC is sent for the payment. fn pay_internal( - &self, payment_id: PaymentId, - initial_send_info: Option<(PaymentHash, &Option, Option, Retry)>, + &self, payment_id: PaymentId, payment_hash: PaymentHash, + initial_send_info: Option<(&Option, Option, Retry)>, route_params: RouteParameters, router: &R, first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, send_payment_along_path: &SP, @@ -583,7 +583,7 @@ impl OutboundPayments { err: format!("Failed to find a route for payment {}: {:?}", log_bytes!(payment_id.0), e), // TODO: add APIError::RouteNotFound }))?; - let res = if let Some((payment_hash, payment_secret, keysend_preimage, retry_strategy)) = initial_send_info { + let res = if let Some((payment_secret, keysend_preimage, retry_strategy)) = initial_send_info { let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, keysend_preimage, &route, Some(retry_strategy), Some(route_params.payment_params.clone()), entropy_source, best_block_height)?; self.pay_route_internal(&route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs, node_signer, best_block_height, send_payment_along_path) } else { @@ -591,7 +591,7 @@ impl OutboundPayments { }; match res { Err(PaymentSendFailure::AllFailedResendSafe(_)) => { - let retry_res = self.pay_internal(payment_id, None, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, send_payment_along_path); + let retry_res = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, send_payment_along_path); log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), retry_res); if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = &retry_res { if err.starts_with("Retries exhausted ") { return res; } @@ -602,7 +602,7 @@ impl OutboundPayments { // Some paths were sent, even if we failed to send the full MPP value our recipient may // misbehave and claim the funds, at which point we have to consider the payment sent, so // return `Ok()` here, ignoring any retry errors. - let retry_res = self.pay_internal(payment_id, None, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, send_payment_along_path); + let retry_res = self.pay_internal(payment_id, payment_hash, None, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, send_payment_along_path); log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), retry_res); Ok(()) }, @@ -1279,8 +1279,9 @@ mod tests { }; let err = if on_retry { outbound_payments.pay_internal( - PaymentId([0; 32]), None, expired_route_params, &&router, vec![], &|| InFlightHtlcs::new(), - &&keys_manager, &&keys_manager, 0, &&logger, &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + PaymentId([0; 32]), PaymentHash([0; 32]), None, expired_route_params, &&router, vec![], + &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() } else { outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), expired_route_params, @@ -1322,8 +1323,9 @@ mod tests { &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); outbound_payments.pay_internal( - PaymentId([0; 32]), None, route_params, &&router, vec![], &|| InFlightHtlcs::new(), - &&keys_manager, &&keys_manager, 0, &&logger, &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + PaymentId([0; 32]), PaymentHash([0; 32]), None, route_params, &&router, vec![], + &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() } else { outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params, From 9f41bd7f6492ca146a852c779f072d5b87581fed Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 9 Feb 2023 13:34:37 -0600 Subject: [PATCH 2/8] Pass pending_events into pay_internal Useful for generating Payment(Path)Failed events in this method --- lightning/src/ln/channelmanager.rs | 3 ++- lightning/src/ln/outbound_payment.rs | 28 ++++++++++++++++------------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 77a335fed99..ad942d25dd3 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -2537,6 +2537,7 @@ where .send_payment(payment_hash, payment_secret, payment_id, retry_strategy, route_params, &self.router, self.list_usable_channels(), || self.compute_inflight_htlcs(), &self.entropy_source, &self.node_signer, best_block_height, &self.logger, + &self.pending_events, |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv| self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv)) } @@ -2613,7 +2614,7 @@ where self.pending_outbound_payments.send_spontaneous_payment(payment_preimage, payment_id, retry_strategy, route_params, &self.router, self.list_usable_channels(), || self.compute_inflight_htlcs(), &self.entropy_source, &self.node_signer, best_block_height, - &self.logger, + &self.logger, &self.pending_events, |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv| self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv)) } diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 9fbca12ec11..2ddd3e2e006 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -397,7 +397,8 @@ impl OutboundPayments { &self, payment_hash: PaymentHash, payment_secret: &Option, payment_id: PaymentId, retry_strategy: Retry, route_params: RouteParameters, router: &R, first_hops: Vec, compute_inflight_htlcs: IH, entropy_source: &ES, - node_signer: &NS, best_block_height: u32, logger: &L, send_payment_along_path: SP, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: SP, ) -> Result<(), PaymentSendFailure> where R::Target: Router, @@ -410,7 +411,7 @@ impl OutboundPayments { { self.pay_internal(payment_id, payment_hash, Some((payment_secret, None, retry_strategy)), route_params, router, first_hops, &compute_inflight_htlcs, entropy_source, node_signer, - best_block_height, logger, &send_payment_along_path) + best_block_height, logger, pending_events, &send_payment_along_path) .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) } @@ -435,7 +436,8 @@ impl OutboundPayments { &self, payment_preimage: Option, payment_id: PaymentId, retry_strategy: Retry, route_params: RouteParameters, router: &R, first_hops: Vec, inflight_htlcs: IH, entropy_source: &ES, - node_signer: &NS, best_block_height: u32, logger: &L, send_payment_along_path: SP + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: SP ) -> Result where R::Target: Router, @@ -451,7 +453,7 @@ impl OutboundPayments { let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner()); self.pay_internal(payment_id, payment_hash, Some((&None, Some(preimage), retry_strategy)), route_params, router, first_hops, &inflight_htlcs, entropy_source, node_signer, - best_block_height, logger, &send_payment_along_path) + best_block_height, logger, pending_events, &send_payment_along_path) .map(|()| payment_hash) .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) } @@ -520,7 +522,7 @@ impl OutboundPayments { } core::mem::drop(outbounds); if let Some((payment_id, payment_hash, route_params)) = retry_id_route_params { - if let Err(e) = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, &send_payment_along_path) { + if let Err(e) = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) { log_info!(logger, "Errored retrying payment: {:?}", e); // If we error on retry, there is no chance of the payment succeeding and no HTLCs have // been irrevocably committed to, so we can safely abandon. @@ -557,7 +559,7 @@ impl OutboundPayments { initial_send_info: Option<(&Option, Option, Retry)>, route_params: RouteParameters, router: &R, first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, - logger: &L, send_payment_along_path: &SP, + logger: &L, pending_events: &Mutex>, send_payment_along_path: &SP, ) -> Result<(), PaymentSendFailure> where R::Target: Router, @@ -591,7 +593,7 @@ impl OutboundPayments { }; match res { Err(PaymentSendFailure::AllFailedResendSafe(_)) => { - let retry_res = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, send_payment_along_path); + let retry_res = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), retry_res); if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = &retry_res { if err.starts_with("Retries exhausted ") { return res; } @@ -602,7 +604,7 @@ impl OutboundPayments { // Some paths were sent, even if we failed to send the full MPP value our recipient may // misbehave and claim the funds, at which point we have to consider the payment sent, so // return `Ok()` here, ignoring any retry errors. - let retry_res = self.pay_internal(payment_id, payment_hash, None, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, send_payment_along_path); + let retry_res = self.pay_internal(payment_id, payment_hash, None, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), retry_res); Ok(()) }, @@ -1277,16 +1279,17 @@ mod tests { final_value_msat: 0, final_cltv_expiry_delta: 0, }; + let pending_events = Mutex::new(Vec::new()); let err = if on_retry { outbound_payments.pay_internal( PaymentId([0; 32]), PaymentHash([0; 32]), None, expired_route_params, &&router, vec![], - &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() } else { outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), expired_route_params, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, - |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + &pending_events, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() }; if let PaymentSendFailure::ParameterError(APIError::APIMisuseError { err }) = err { assert!(err.contains("Invoice expired")); @@ -1318,19 +1321,20 @@ mod tests { router.expect_find_route(route_params.clone(), Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })); + let pending_events = Mutex::new(Vec::new()); let err = if on_retry { outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None, &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); outbound_payments.pay_internal( PaymentId([0; 32]), PaymentHash([0; 32]), None, route_params, &&router, vec![], - &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() } else { outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, - |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + &pending_events, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() }; if let PaymentSendFailure::ParameterError(APIError::APIMisuseError { err }) = err { assert!(err.contains("Failed to find a route")); From 5e4f0bcff0cea5e4c0652c2ef3c86ddc19585e21 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Wed, 22 Feb 2023 14:09:58 -0500 Subject: [PATCH 3/8] Fix InvalidRoute error to be ChannelUnavailable InvalidRoute is reserved for malformed routes, not routes where a channel or its peer is unavailable --- lightning/src/ln/channelmanager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index ad942d25dd3..39cb9a7b5fe 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -2386,7 +2386,7 @@ where let per_peer_state = self.per_peer_state.read().unwrap(); let peer_state_mutex = per_peer_state.get(&counterparty_node_id) - .ok_or_else(|| APIError::InvalidRoute{err: "No peer matching the path's first hop found!" })?; + .ok_or_else(|| APIError::ChannelUnavailable{err: "No peer matching the path's first hop found!".to_owned() })?; let mut peer_state_lock = peer_state_mutex.lock().unwrap(); let peer_state = &mut *peer_state_lock; if let hash_map::Entry::Occupied(mut chan) = peer_state.channel_by_id.entry(id) { From d471d9746c234a31ff3431d17e89be9085994d40 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Fri, 10 Feb 2023 15:09:01 -0600 Subject: [PATCH 4/8] Rework auto retry send errors Prior to this, we returned PaymentSendFailure from auto retry send payment methods. This implied that we might return a PartialFailure from them, which has never been the case. So it makes sense to rework the errors to be a better fit for the methods. We're taking error handling in a totally different direction now to make it more asynchronous, see send_payment_internal for more information. --- lightning-invoice/src/payment.rs | 4 +- lightning/src/ln/channelmanager.rs | 6 +- lightning/src/ln/outbound_payment.rs | 278 +++++++++++++++++++-------- lightning/src/ln/payment_tests.rs | 40 +++- 4 files changed, 241 insertions(+), 87 deletions(-) diff --git a/lightning-invoice/src/payment.rs b/lightning-invoice/src/payment.rs index 0ffbae5fad5..db47f995480 100644 --- a/lightning-invoice/src/payment.rs +++ b/lightning-invoice/src/payment.rs @@ -17,7 +17,7 @@ use lightning::chain; use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator}; use lightning::chain::keysinterface::{NodeSigner, SignerProvider, EntropySource}; use lightning::ln::{PaymentHash, PaymentSecret}; -use lightning::ln::channelmanager::{ChannelManager, PaymentId, PaymentSendFailure, Retry}; +use lightning::ln::channelmanager::{ChannelManager, PaymentId, Retry, RetryableSendFailure}; use lightning::routing::router::{PaymentParameters, RouteParameters, Router}; use lightning::util::logger::Logger; @@ -172,7 +172,7 @@ pub enum PaymentError { /// An error resulting from the provided [`Invoice`] or payment hash. Invoice(&'static str), /// An error occurring when sending a payment. - Sending(PaymentSendFailure), + Sending(RetryableSendFailure), } /// A trait defining behavior of an [`Invoice`] payer. diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 39cb9a7b5fe..3fd472925cf 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -76,7 +76,7 @@ use core::time::Duration; use core::ops::Deref; // Re-export this for use in the public API. -pub use crate::ln::outbound_payment::{PaymentSendFailure, Retry}; +pub use crate::ln::outbound_payment::{PaymentSendFailure, Retry, RetryableSendFailure}; // We hold various information about HTLC relay in the HTLC objects in Channel itself: // @@ -2531,7 +2531,7 @@ where /// Similar to [`ChannelManager::send_payment`], but will automatically find a route based on /// `route_params` and retry failed payment paths based on `retry_strategy`. - pub fn send_payment_with_retry(&self, payment_hash: PaymentHash, payment_secret: &Option, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result<(), PaymentSendFailure> { + pub fn send_payment_with_retry(&self, payment_hash: PaymentHash, payment_secret: &Option, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result<(), RetryableSendFailure> { let best_block_height = self.best_block.read().unwrap().height(); self.pending_outbound_payments .send_payment(payment_hash, payment_secret, payment_id, retry_strategy, route_params, @@ -2609,7 +2609,7 @@ where /// payments. /// /// [`PaymentParameters::for_keysend`]: crate::routing::router::PaymentParameters::for_keysend - pub fn send_spontaneous_payment_with_retry(&self, payment_preimage: Option, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result { + pub fn send_spontaneous_payment_with_retry(&self, payment_preimage: Option, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result { let best_block_height = self.best_block.read().unwrap().height(); self.pending_outbound_payments.send_spontaneous_payment(payment_preimage, payment_id, retry_strategy, route_params, &self.router, self.list_usable_channels(), diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 2ddd3e2e006..2d79033ac2e 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -312,9 +312,35 @@ impl Display for PaymentAttemptsUsingTime { } } -/// If a payment fails to send, it can be in one of several states. This enum is returned as the -/// Err() type describing which state the payment is in, see the description of individual enum -/// states for more. +/// Indicates an immediate error on [`ChannelManager::send_payment_with_retry`]. Further errors +/// may be surfaced later via [`Event::PaymentPathFailed`] and [`Event::PaymentFailed`]. +/// +/// [`ChannelManager::send_payment_with_retry`]: crate::ln::channelmanager::ChannelManager::send_payment_with_retry +/// [`Event::PaymentPathFailed`]: crate::util::events::Event::PaymentPathFailed +/// [`Event::PaymentFailed`]: crate::util::events::Event::PaymentFailed +#[derive(Clone, Debug)] +pub enum RetryableSendFailure { + /// The provided [`PaymentParameters::expiry_time`] indicated that the payment has expired. Note + /// that this error is *not* caused by [`Retry::Timeout`]. + /// + /// [`PaymentParameters::expiry_time`]: crate::routing::router::PaymentParameters::expiry_time + PaymentExpired, + /// We were unable to find a route to the destination. + RouteNotFound, + /// Indicates that a payment for the provided [`PaymentId`] is already in-flight and has not + /// yet completed (i.e. generated an [`Event::PaymentSent`] or [`Event::PaymentFailed`]). + /// + /// [`PaymentId`]: crate::ln::channelmanager::PaymentId + /// [`Event::PaymentSent`]: crate::util::events::Event::PaymentSent + /// [`Event::PaymentFailed`]: crate::util::events::Event::PaymentFailed + DuplicatePayment, +} + +/// If a payment fails to send with [`ChannelManager::send_payment`], it can be in one of several +/// states. This enum is returned as the Err() type describing which state the payment is in, see +/// the description of individual enum states for more. +/// +/// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment #[derive(Clone, Debug)] pub enum PaymentSendFailure { /// A parameter which was passed to send_payment was invalid, preventing us from attempting to @@ -399,7 +425,7 @@ impl OutboundPayments { first_hops: Vec, compute_inflight_htlcs: IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, pending_events: &Mutex>, send_payment_along_path: SP, - ) -> Result<(), PaymentSendFailure> + ) -> Result<(), RetryableSendFailure> where R::Target: Router, ES::Target: EntropySource, @@ -409,10 +435,9 @@ impl OutboundPayments { SP: Fn(&Vec, &Option, &PaymentHash, &Option, u64, u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError>, { - self.pay_internal(payment_id, payment_hash, Some((payment_secret, None, retry_strategy)), + self.send_payment_internal(payment_id, payment_hash, payment_secret, None, retry_strategy, route_params, router, first_hops, &compute_inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) - .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) } pub(super) fn send_payment_with_route( @@ -438,7 +463,7 @@ impl OutboundPayments { first_hops: Vec, inflight_htlcs: IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, pending_events: &Mutex>, send_payment_along_path: SP - ) -> Result + ) -> Result where R::Target: Router, ES::Target: EntropySource, @@ -451,11 +476,10 @@ impl OutboundPayments { let preimage = payment_preimage .unwrap_or_else(|| PaymentPreimage(entropy_source.get_secure_random_bytes())); let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner()); - self.pay_internal(payment_id, payment_hash, Some((&None, Some(preimage), retry_strategy)), - route_params, router, first_hops, &inflight_htlcs, entropy_source, node_signer, - best_block_height, logger, pending_events, &send_payment_along_path) + self.send_payment_internal(payment_id, payment_hash, &None, Some(preimage), retry_strategy, + route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, + best_block_height, logger, pending_events, send_payment_along_path) .map(|()| payment_hash) - .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) } pub(super) fn send_spontaneous_payment_with_route( @@ -522,12 +546,7 @@ impl OutboundPayments { } core::mem::drop(outbounds); if let Some((payment_id, payment_hash, route_params)) = retry_id_route_params { - if let Err(e) = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) { - log_info!(logger, "Errored retrying payment: {:?}", e); - // If we error on retry, there is no chance of the payment succeeding and no HTLCs have - // been irrevocably committed to, so we can safely abandon. - self.abandon_payment(payment_id, pending_events); - } + self.retry_payment_internal(payment_id, payment_hash, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) } else { break } } @@ -553,14 +572,18 @@ impl OutboundPayments { !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 && !pmt.is_fulfilled()) } - /// Will return `Ok(())` iff at least one HTLC is sent for the payment. - fn pay_internal( - &self, payment_id: PaymentId, payment_hash: PaymentHash, - initial_send_info: Option<(&Option, Option, Retry)>, - route_params: RouteParameters, router: &R, first_hops: Vec, - inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, - logger: &L, pending_events: &Mutex>, send_payment_along_path: &SP, - ) -> Result<(), PaymentSendFailure> + /// Errors immediately on [`RetryableSendFailure`] error conditions. Otherwise, further errors may + /// be surfaced asynchronously via [`Event::PaymentPathFailed`] and [`Event::PaymentFailed`]. + /// + /// [`Event::PaymentPathFailed`]: crate::util::events::Event::PaymentPathFailed + /// [`Event::PaymentFailed`]: crate::util::events::Event::PaymentFailed + fn send_payment_internal( + &self, payment_id: PaymentId, payment_hash: PaymentHash, payment_secret: &Option, + keysend_preimage: Option, retry_strategy: Retry, route_params: RouteParameters, + router: &R, first_hops: Vec, inflight_htlcs: IH, entropy_source: &ES, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: SP, + ) -> Result<(), RetryableSendFailure> where R::Target: Router, ES::Target: EntropySource, @@ -568,53 +591,148 @@ impl OutboundPayments { L::Target: Logger, IH: Fn() -> InFlightHtlcs, SP: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> { #[cfg(feature = "std")] { if has_expired(&route_params) { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Invoice expired for payment id {}", log_bytes!(payment_id.0)), - })) + return Err(RetryableSendFailure::PaymentExpired) } } let route = router.find_route( &node_signer.get_node_id(Recipient::Node).unwrap(), &route_params, - Some(&first_hops.iter().collect::>()), &inflight_htlcs(), - ).map_err(|e| PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Failed to find a route for payment {}: {:?}", log_bytes!(payment_id.0), e), // TODO: add APIError::RouteNotFound - }))?; - - let res = if let Some((payment_secret, keysend_preimage, retry_strategy)) = initial_send_info { - let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, keysend_preimage, &route, Some(retry_strategy), Some(route_params.payment_params.clone()), entropy_source, best_block_height)?; - self.pay_route_internal(&route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs, node_signer, best_block_height, send_payment_along_path) - } else { - self.retry_payment_with_route(&route, payment_id, entropy_source, node_signer, best_block_height, send_payment_along_path) + Some(&first_hops.iter().collect::>()), &inflight_htlcs() + ).map_err(|_| RetryableSendFailure::RouteNotFound)?; + + let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, + payment_id, keysend_preimage, &route, Some(retry_strategy), + Some(route_params.payment_params.clone()), entropy_source, best_block_height) + .map_err(|_| RetryableSendFailure::DuplicatePayment)?; + + let res = self.pay_route_internal(&route, payment_hash, payment_secret, None, payment_id, None, + onion_session_privs, node_signer, best_block_height, &send_payment_along_path); + log_info!(logger, "Result sending payment with id {}: {:?}", log_bytes!(payment_id.0), res); + if let Err(e) = res { + self.handle_pay_route_err(e, payment_id, payment_hash, route, route_params, router, first_hops, &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path); + } + Ok(()) + } + + fn retry_payment_internal( + &self, payment_id: PaymentId, payment_hash: PaymentHash, route_params: RouteParameters, + router: &R, first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: &SP, + ) + where + R::Target: Router, + ES::Target: EntropySource, + NS::Target: NodeSigner, + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &Option, &PaymentHash, &Option, u64, + u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + { + #[cfg(feature = "std")] { + if has_expired(&route_params) { + log_error!(logger, "Payment params expired on retry, abandoning payment {}", log_bytes!(payment_id.0)); + self.abandon_payment(payment_id, pending_events); + return + } + } + + let route = match router.find_route( + &node_signer.get_node_id(Recipient::Node).unwrap(), &route_params, + Some(&first_hops.iter().collect::>()), &inflight_htlcs() + ) { + Ok(route) => route, + Err(e) => { + log_error!(logger, "Failed to find a route on retry, abandoning payment {}: {:#?}", log_bytes!(payment_id.0), e); + self.abandon_payment(payment_id, pending_events); + return + } }; - match res { - Err(PaymentSendFailure::AllFailedResendSafe(_)) => { - let retry_res = self.pay_internal(payment_id, payment_hash, None, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); - log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), retry_res); - if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = &retry_res { - if err.starts_with("Retries exhausted ") { return res; } - } - retry_res + + let res = self.retry_payment_with_route(&route, payment_id, entropy_source, node_signer, + best_block_height, send_payment_along_path); + log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), res); + if let Err(e) = res { + self.handle_pay_route_err(e, payment_id, payment_hash, route, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + } + } + + fn handle_pay_route_err( + &self, err: PaymentSendFailure, payment_id: PaymentId, payment_hash: PaymentHash, route: Route, + route_params: RouteParameters, router: &R, first_hops: Vec, inflight_htlcs: &IH, + entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: &SP, + ) + where + R::Target: Router, + ES::Target: EntropySource, + NS::Target: NodeSigner, + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &Option, &PaymentHash, &Option, u64, + u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + { + match err { + PaymentSendFailure::AllFailedResendSafe(errs) => { + Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, errs.into_iter().map(|e| Err(e)), pending_events); + self.retry_payment_internal(payment_id, payment_hash, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, - Err(PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), .. }) => { + PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), results, .. } => { + Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, results.into_iter(), pending_events); // Some paths were sent, even if we failed to send the full MPP value our recipient may // misbehave and claim the funds, at which point we have to consider the payment sent, so // return `Ok()` here, ignoring any retry errors. - let retry_res = self.pay_internal(payment_id, payment_hash, None, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); - log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), retry_res); - Ok(()) + self.retry_payment_internal(payment_id, payment_hash, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, - Err(PaymentSendFailure::PartialFailure { failed_paths_retry: None, .. }) => { + PaymentSendFailure::PartialFailure { failed_paths_retry: None, .. } => { // This may happen if we send a payment and some paths fail, but only due to a temporary // monitor failure or the like, implying they're really in-flight, but we haven't sent the // initial HTLC-Add messages yet. - Ok(()) }, - res => res, + PaymentSendFailure::PathParameterError(results) => { + Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, results.into_iter(), pending_events); + self.abandon_payment(payment_id, pending_events); + }, + PaymentSendFailure::ParameterError(e) => { + log_error!(logger, "Failed to send to route due to parameter error: {:?}. Your router is buggy", e); + self.abandon_payment(payment_id, pending_events); + }, + PaymentSendFailure::DuplicatePayment => debug_assert!(false), // unreachable + } + } + + fn push_payment_path_failed_evs>>( + payment_id: PaymentId, payment_hash: PaymentHash, paths: Vec>, path_results: I, + pending_events: &Mutex> + ) { + let mut events = pending_events.lock().unwrap(); + debug_assert_eq!(paths.len(), path_results.len()); + for (path, path_res) in paths.into_iter().zip(path_results) { + if let Err(e) = path_res { + let failed_scid = if let APIError::InvalidRoute { .. } = e { + None + } else { + Some(path[0].short_channel_id) + }; + events.push(events::Event::PaymentPathFailed { + payment_id: Some(payment_id), + payment_hash, + payment_failed_permanently: false, + network_update: None, + all_paths_failed: false, + path, + short_channel_id: failed_scid, + retry: None, + #[cfg(test)] + error_code: None, + #[cfg(test)] + error_data: None, + }); + } } } @@ -1243,13 +1361,13 @@ mod tests { use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; use crate::ln::PaymentHash; - use crate::ln::channelmanager::{PaymentId, PaymentSendFailure}; + use crate::ln::channelmanager::PaymentId; use crate::ln::msgs::{ErrorAction, LightningError}; - use crate::ln::outbound_payment::{OutboundPayments, Retry}; + use crate::ln::outbound_payment::{OutboundPayments, Retry, RetryableSendFailure}; use crate::routing::gossip::NetworkGraph; use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteParameters}; use crate::sync::{Arc, Mutex}; - use crate::util::errors::APIError; + use crate::util::events::Event; use crate::util::test_utils; #[test] @@ -1280,20 +1398,24 @@ mod tests { final_cltv_expiry_delta: 0, }; let pending_events = Mutex::new(Vec::new()); - let err = if on_retry { - outbound_payments.pay_internal( - PaymentId([0; 32]), PaymentHash([0; 32]), None, expired_route_params, &&router, vec![], + if on_retry { + outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None, + &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), + Some(expired_route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); + outbound_payments.retry_payment_internal( + PaymentId([0; 32]), PaymentHash([0; 32]), expired_route_params, &&router, vec![], &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, - &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + &|_, _, _, _, _, _, _, _, _| Ok(())); + let events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 1); + if let Event::PaymentFailed { .. } = events[0] { } else { panic!("Unexpected event"); } } else { - outbound_payments.send_payment( + let err = outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), expired_route_params, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, - &pending_events, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() - }; - if let PaymentSendFailure::ParameterError(APIError::APIMisuseError { err }) = err { - assert!(err.contains("Invoice expired")); - } else { panic!("Unexpected error"); } + &pending_events, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err(); + if let RetryableSendFailure::PaymentExpired = err { } else { panic!("Unexpected error"); } + } } #[test] @@ -1322,22 +1444,24 @@ mod tests { Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })); let pending_events = Mutex::new(Vec::new()); - let err = if on_retry { + if on_retry { outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None, &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); - outbound_payments.pay_internal( - PaymentId([0; 32]), PaymentHash([0; 32]), None, route_params, &&router, vec![], + outbound_payments.retry_payment_internal( + PaymentId([0; 32]), PaymentHash([0; 32]), route_params, &&router, vec![], &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, - &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + &|_, _, _, _, _, _, _, _, _| Ok(())); + let events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 1); + if let Event::PaymentFailed { .. } = events[0] { } else { panic!("Unexpected event"); } } else { - outbound_payments.send_payment( + let err = outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params, &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, - &pending_events, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() - }; - if let PaymentSendFailure::ParameterError(APIError::APIMisuseError { err }) = err { - assert!(err.contains("Failed to find a route")); - } else { panic!("Unexpected error"); } + &pending_events, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err(); + if let RetryableSendFailure::RouteNotFound = err { + } else { panic!("Unexpected error"); } + } } } diff --git a/lightning/src/ln/payment_tests.rs b/lightning/src/ln/payment_tests.rs index 3285cf4d7a5..7c90f560104 100644 --- a/lightning/src/ln/payment_tests.rs +++ b/lightning/src/ln/payment_tests.rs @@ -1869,15 +1869,23 @@ fn auto_retry_partial_failure() { // Send a payment that will partially fail on send, then partially fail on retry, then succeed. nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap(); let closed_chan_events = nodes[0].node.get_and_clear_pending_events(); - assert_eq!(closed_chan_events.len(), 2); + assert_eq!(closed_chan_events.len(), 4); match closed_chan_events[0] { Event::ChannelClosed { .. } => {}, _ => panic!("Unexpected event"), } match closed_chan_events[1] { + Event::PaymentPathFailed { .. } => {}, + _ => panic!("Unexpected event"), + } + match closed_chan_events[2] { Event::ChannelClosed { .. } => {}, _ => panic!("Unexpected event"), } + match closed_chan_events[3] { + Event::PaymentPathFailed { .. } => {}, + _ => panic!("Unexpected event"), + } // Pass the first part of the payment along the path. check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed @@ -1993,11 +2001,13 @@ fn auto_retry_zero_attempts_send_error() { }; chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure); - let err = nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap_err(); - if let PaymentSendFailure::AllFailedResendSafe(_) = err { - } else { panic!("Unexpected error"); } + nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap(); assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages - assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 1); // channel close event + let events = nodes[0].node.get_and_clear_pending_events(); + assert_eq!(events.len(), 3); + if let Event::ChannelClosed { .. } = events[0] { } else { panic!(); } + if let Event::PaymentPathFailed { .. } = events[1] { } else { panic!(); } + if let Event::PaymentFailed { .. } = events[2] { } else { panic!(); } check_added_monitors!(nodes[0], 2); } @@ -2121,6 +2131,16 @@ fn retry_multi_path_single_failed_payment() { } nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); + let events = nodes[0].node.get_and_clear_pending_events(); + assert_eq!(events.len(), 1); + match events[0] { + Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false, + network_update: None, all_paths_failed: false, short_channel_id: Some(expected_scid), .. } => { + assert_eq!(payment_hash, ev_payment_hash); + assert_eq!(expected_scid, route.paths[1][0].short_channel_id); + }, + _ => panic!("Unexpected event"), + } let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events(); assert_eq!(htlc_msgs.len(), 2); check_added_monitors!(nodes[0], 2); @@ -2182,6 +2202,16 @@ fn immediate_retry_on_failure() { }, Ok(route.clone())); nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); + let events = nodes[0].node.get_and_clear_pending_events(); + assert_eq!(events.len(), 1); + match events[0] { + Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false, + network_update: None, all_paths_failed: false, short_channel_id: Some(expected_scid), .. } => { + assert_eq!(payment_hash, ev_payment_hash); + assert_eq!(expected_scid, route.paths[1][0].short_channel_id); + }, + _ => panic!("Unexpected event"), + } let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events(); assert_eq!(htlc_msgs.len(), 2); check_added_monitors!(nodes[0], 2); From b826d1735d4fdfb162bc46d87deee495f4fb3363 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sun, 19 Feb 2023 16:12:24 -0500 Subject: [PATCH 5/8] In-line retry_with_route method Since it's only used one place now --- lightning/src/ln/outbound_payment.rs | 180 +++++++++++++-------------- 1 file changed, 88 insertions(+), 92 deletions(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 2d79033ac2e..2e2dc6b4008 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -527,9 +527,9 @@ impl OutboundPayments { let mut retry_id_route_params = None; for (pmt_id, pmt) in outbounds.iter_mut() { if pmt.is_auto_retryable_now() { - if let PendingOutboundPayment::Retryable { payment_hash, pending_amt_msat, total_msat, payment_params: Some(params), .. } = pmt { + if let PendingOutboundPayment::Retryable { pending_amt_msat, total_msat, payment_params: Some(params), .. } = pmt { if pending_amt_msat < total_msat { - retry_id_route_params = Some((*pmt_id, *payment_hash, RouteParameters { + retry_id_route_params = Some((*pmt_id, RouteParameters { final_value_msat: *total_msat - *pending_amt_msat, final_cltv_expiry_delta: if let Some(delta) = params.final_cltv_expiry_delta { delta } @@ -545,8 +545,8 @@ impl OutboundPayments { } } core::mem::drop(outbounds); - if let Some((payment_id, payment_hash, route_params)) = retry_id_route_params { - self.retry_payment_internal(payment_id, payment_hash, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) + if let Some((payment_id, route_params)) = retry_id_route_params { + self.retry_payment_internal(payment_id, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) } else { break } } @@ -619,10 +619,10 @@ impl OutboundPayments { } fn retry_payment_internal( - &self, payment_id: PaymentId, payment_hash: PaymentHash, route_params: RouteParameters, - router: &R, first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, - node_signer: &NS, best_block_height: u32, logger: &L, - pending_events: &Mutex>, send_payment_along_path: &SP, + &self, payment_id: PaymentId, route_params: RouteParameters, router: &R, + first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, + best_block_height: u32, logger: &L, pending_events: &Mutex>, + send_payment_along_path: &SP, ) where R::Target: Router, @@ -652,9 +652,81 @@ impl OutboundPayments { return } }; + for path in route.paths.iter() { + if path.len() == 0 { + log_error!(logger, "length-0 path in route"); + self.abandon_payment(payment_id, pending_events); + return + } + } - let res = self.retry_payment_with_route(&route, payment_id, entropy_source, node_signer, - best_block_height, send_payment_along_path); + const RETRY_OVERFLOW_PERCENTAGE: u64 = 10; + let mut onion_session_privs = Vec::with_capacity(route.paths.len()); + for _ in 0..route.paths.len() { + onion_session_privs.push(entropy_source.get_secure_random_bytes()); + } + + macro_rules! abandon_with_entry { + ($payment_id: expr, $payment_hash: expr, $payment: expr, $pending_events: expr) => { + if $payment.get_mut().mark_abandoned().is_ok() && $payment.get().remaining_parts() == 0 { + $pending_events.lock().unwrap().push(events::Event::PaymentFailed { + payment_id: $payment_id, + payment_hash: $payment_hash, + }); + $payment.remove(); + } + } + } + let (total_msat, payment_hash, payment_secret, keysend_preimage) = { + let mut outbounds = self.pending_outbound_payments.lock().unwrap(); + match outbounds.entry(payment_id) { + hash_map::Entry::Occupied(mut payment) => { + let res = match payment.get() { + PendingOutboundPayment::Retryable { + total_msat, payment_hash, keysend_preimage, payment_secret, pending_amt_msat, .. + } => { + let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum(); + if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 { + log_error!(logger, "retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat); + let payment_hash = *payment_hash; + abandon_with_entry!(payment_id, payment_hash, payment, pending_events); + return + } + (*total_msat, *payment_hash, *payment_secret, *keysend_preimage) + }, + PendingOutboundPayment::Legacy { .. } => { + log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102"); + return + }, + PendingOutboundPayment::Fulfilled { .. } => { + log_error!(logger, "Payment already completed"); + return + }, + PendingOutboundPayment::Abandoned { .. } => { + log_error!(logger, "Payment already abandoned (with some HTLCs still pending)"); + return + }, + }; + if !payment.get().is_retryable_now() { + log_error!(logger, "Retries exhausted for payment id {}", log_bytes!(payment_id.0)); + abandon_with_entry!(payment_id, res.1, payment, pending_events); + return + } + payment.get_mut().increment_attempts(); + for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { + assert!(payment.get_mut().insert(*session_priv_bytes, path)); + } + res + }, + hash_map::Entry::Vacant(_) => { + log_error!(logger, "Payment with ID {} not found", log_bytes!(payment_id.0)); + return + } + } + }; + let res = self.pay_route_internal(&route, payment_hash, &payment_secret, keysend_preimage, + payment_id, Some(total_msat), onion_session_privs, node_signer, best_block_height, + &send_payment_along_path); log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), res); if let Err(e) = res { self.handle_pay_route_err(e, payment_id, payment_hash, route, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); @@ -679,14 +751,14 @@ impl OutboundPayments { match err { PaymentSendFailure::AllFailedResendSafe(errs) => { Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, errs.into_iter().map(|e| Err(e)), pending_events); - self.retry_payment_internal(payment_id, payment_hash, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + self.retry_payment_internal(payment_id, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), results, .. } => { Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, results.into_iter(), pending_events); // Some paths were sent, even if we failed to send the full MPP value our recipient may // misbehave and claim the funds, at which point we have to consider the payment sent, so // return `Ok()` here, ignoring any retry errors. - self.retry_payment_internal(payment_id, payment_hash, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + self.retry_payment_internal(payment_id, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, PaymentSendFailure::PartialFailure { failed_paths_retry: None, .. } => { // This may happen if we send a payment and some paths fail, but only due to a temporary @@ -736,82 +808,6 @@ impl OutboundPayments { } } - pub(super) fn retry_payment_with_route( - &self, route: &Route, payment_id: PaymentId, entropy_source: &ES, node_signer: &NS, best_block_height: u32, - send_payment_along_path: F - ) -> Result<(), PaymentSendFailure> - where - ES::Target: EntropySource, - NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> - { - const RETRY_OVERFLOW_PERCENTAGE: u64 = 10; - for path in route.paths.iter() { - if path.len() == 0 { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "length-0 path in route".to_string() - })) - } - } - - let mut onion_session_privs = Vec::with_capacity(route.paths.len()); - for _ in 0..route.paths.len() { - onion_session_privs.push(entropy_source.get_secure_random_bytes()); - } - - let (total_msat, payment_hash, payment_secret, keysend_preimage) = { - let mut outbounds = self.pending_outbound_payments.lock().unwrap(); - match outbounds.get_mut(&payment_id) { - Some(payment) => { - let res = match payment { - PendingOutboundPayment::Retryable { - total_msat, payment_hash, keysend_preimage, payment_secret, pending_amt_msat, .. - } => { - let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum(); - if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string() - })) - } - (*total_msat, *payment_hash, *payment_secret, *keysend_preimage) - }, - PendingOutboundPayment::Legacy { .. } => { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string() - })) - }, - PendingOutboundPayment::Fulfilled { .. } => { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "Payment already completed".to_owned() - })); - }, - PendingOutboundPayment::Abandoned { .. } => { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "Payment already abandoned (with some HTLCs still pending)".to_owned() - })); - }, - }; - if !payment.is_retryable_now() { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Retries exhausted for payment id {}", log_bytes!(payment_id.0)), - })) - } - payment.increment_attempts(); - for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { - assert!(payment.insert(*session_priv_bytes, path)); - } - res - }, - None => - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)), - })), - } - }; - self.pay_route_internal(route, payment_hash, &payment_secret, keysend_preimage, payment_id, Some(total_msat), onion_session_privs, node_signer, best_block_height, &send_payment_along_path) - } - pub(super) fn send_probe( &self, hops: Vec, probing_cookie_secret: [u8; 32], entropy_source: &ES, node_signer: &NS, best_block_height: u32, send_payment_along_path: F @@ -1403,8 +1399,8 @@ mod tests { &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), Some(expired_route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); outbound_payments.retry_payment_internal( - PaymentId([0; 32]), PaymentHash([0; 32]), expired_route_params, &&router, vec![], - &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, + PaymentId([0; 32]), expired_route_params, &&router, vec![], &|| InFlightHtlcs::new(), + &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, &|_, _, _, _, _, _, _, _, _| Ok(())); let events = pending_events.lock().unwrap(); assert_eq!(events.len(), 1); @@ -1449,8 +1445,8 @@ mod tests { &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); outbound_payments.retry_payment_internal( - PaymentId([0; 32]), PaymentHash([0; 32]), route_params, &&router, vec![], - &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, + PaymentId([0; 32]), route_params, &&router, vec![], &|| InFlightHtlcs::new(), + &&keys_manager, &&keys_manager, 0, &&logger, &pending_events, &|_, _, _, _, _, _, _, _, _| Ok(())); let events = pending_events.lock().unwrap(); assert_eq!(events.len(), 1); From 1224dac862ad5fa994efeb0830c0b8db88e22647 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 16 Feb 2023 16:40:51 -0500 Subject: [PATCH 6/8] On initial send retries, avoid previously failed scids Previously, we could have tried the same failed channels over and over until retries are exhausted. --- lightning/src/ln/outbound_payment.rs | 22 ++++++++++++---------- lightning/src/ln/payment_tests.rs | 20 +++++++++++++------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 2e2dc6b4008..a28169032d6 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -735,8 +735,8 @@ impl OutboundPayments { fn handle_pay_route_err( &self, err: PaymentSendFailure, payment_id: PaymentId, payment_hash: PaymentHash, route: Route, - route_params: RouteParameters, router: &R, first_hops: Vec, inflight_htlcs: &IH, - entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, + mut route_params: RouteParameters, router: &R, first_hops: Vec, + inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, pending_events: &Mutex>, send_payment_along_path: &SP, ) where @@ -750,11 +750,11 @@ impl OutboundPayments { { match err { PaymentSendFailure::AllFailedResendSafe(errs) => { - Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, errs.into_iter().map(|e| Err(e)), pending_events); + Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, errs.into_iter().map(|e| Err(e)), pending_events); self.retry_payment_internal(payment_id, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); }, - PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), results, .. } => { - Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, results.into_iter(), pending_events); + PaymentSendFailure::PartialFailure { failed_paths_retry: Some(mut retry), results, .. } => { + Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut retry, route.paths, results.into_iter(), pending_events); // Some paths were sent, even if we failed to send the full MPP value our recipient may // misbehave and claim the funds, at which point we have to consider the payment sent, so // return `Ok()` here, ignoring any retry errors. @@ -766,7 +766,7 @@ impl OutboundPayments { // initial HTLC-Add messages yet. }, PaymentSendFailure::PathParameterError(results) => { - Self::push_payment_path_failed_evs(payment_id, payment_hash, route.paths, results.into_iter(), pending_events); + Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, results.into_iter(), pending_events); self.abandon_payment(payment_id, pending_events); }, PaymentSendFailure::ParameterError(e) => { @@ -777,9 +777,9 @@ impl OutboundPayments { } } - fn push_payment_path_failed_evs>>( - payment_id: PaymentId, payment_hash: PaymentHash, paths: Vec>, path_results: I, - pending_events: &Mutex> + fn push_path_failed_evs_and_scids>>( + payment_id: PaymentId, payment_hash: PaymentHash, route_params: &mut RouteParameters, + paths: Vec>, path_results: I, pending_events: &Mutex> ) { let mut events = pending_events.lock().unwrap(); debug_assert_eq!(paths.len(), path_results.len()); @@ -788,7 +788,9 @@ impl OutboundPayments { let failed_scid = if let APIError::InvalidRoute { .. } = e { None } else { - Some(path[0].short_channel_id) + let scid = path[0].short_channel_id; + route_params.payment_params.previously_failed_channels.push(scid); + Some(scid) }; events.push(events::Event::PaymentPathFailed { payment_id: Some(payment_id), diff --git a/lightning/src/ln/payment_tests.rs b/lightning/src/ln/payment_tests.rs index 7c90f560104..1d6eba5ade6 100644 --- a/lightning/src/ln/payment_tests.rs +++ b/lightning/src/ln/payment_tests.rs @@ -1857,13 +1857,15 @@ fn auto_retry_partial_failure() { payment_params: Some(route_params.payment_params.clone()), }; nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route)); + let mut payment_params = route_params.payment_params.clone(); + payment_params.previously_failed_channels.push(chan_2_id); nodes[0].router.expect_find_route(RouteParameters { - payment_params: route_params.payment_params.clone(), - final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV + payment_params, final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV }, Ok(retry_1_route)); + let mut payment_params = route_params.payment_params.clone(); + payment_params.previously_failed_channels.push(chan_3_id); nodes[0].router.expect_find_route(RouteParameters { - payment_params: route_params.payment_params.clone(), - final_value_msat: amt_msat / 4, final_cltv_expiry_delta: TEST_FINAL_CLTV + payment_params, final_value_msat: amt_msat / 4, final_cltv_expiry_delta: TEST_FINAL_CLTV }, Ok(retry_2_route)); // Send a payment that will partially fail on send, then partially fail on retry, then succeed. @@ -2113,8 +2115,10 @@ fn retry_multi_path_single_failed_payment() { // On retry, split the payment across both channels. route.paths[0][0].fee_msat = 50_000_001; route.paths[1][0].fee_msat = 50_000_000; + let mut pay_params = route.payment_params.clone().unwrap(); + pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap()); nodes[0].router.expect_find_route(RouteParameters { - payment_params: route.payment_params.clone().unwrap(), + payment_params: pay_params, // Note that the second request here requests the amount we originally failed to send, // not the amount remaining on the full payment, which should be changed. final_value_msat: 100_000_001, final_cltv_expiry_delta: TEST_FINAL_CLTV @@ -2196,9 +2200,11 @@ fn immediate_retry_on_failure() { route.paths[0][0].short_channel_id = chans[1].short_channel_id.unwrap(); route.paths[0][0].fee_msat = 50_000_000; route.paths[1][0].fee_msat = 50_000_001; + let mut pay_params = route_params.payment_params.clone(); + pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap()); nodes[0].router.expect_find_route(RouteParameters { - payment_params: route_params.payment_params.clone(), - final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV + payment_params: pay_params, final_value_msat: amt_msat, + final_cltv_expiry_delta: TEST_FINAL_CLTV }, Ok(route.clone())); nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap(); From 12bcc9ae43a5f00d43551e42bac96eaff5933562 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Sat, 11 Feb 2023 19:35:48 -0500 Subject: [PATCH 7/8] Fix outdated PendingOutboundPayment::Abandoned docs --- lightning/src/ln/outbound_payment.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index a28169032d6..e1c5fa547f8 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -64,13 +64,8 @@ pub(crate) enum PendingOutboundPayment { payment_hash: Option, timer_ticks_without_htlcs: u8, }, - /// When a payer gives up trying to retry a payment, they inform us, letting us generate a - /// `PaymentFailed` event when all HTLCs have irrevocably failed. This avoids a number of race - /// conditions in MPP-aware payment retriers (1), where the possibility of multiple - /// `PaymentPathFailed` events with `all_paths_failed` can be pending at once, confusing a - /// downstream event handler as to when a payment has actually failed. - /// - /// (1) + /// When we've decided to give up retrying a payment, we mark it as abandoned so we can eventually + /// generate a `PaymentFailed` event when all HTLCs have irrevocably failed. Abandoned { session_privs: HashSet<[u8; 32]>, payment_hash: PaymentHash, From a6e9123d3fdfeaa23282ea87d318982ffe041b85 Mon Sep 17 00:00:00 2001 From: Valentine Wallace Date: Thu, 23 Feb 2023 15:49:44 -0500 Subject: [PATCH 8/8] Clarify Retry::Timeout vs PaymentParams::expiry_time in docs --- lightning/src/ln/outbound_payment.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index e1c5fa547f8..1d8d46a916a 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -235,7 +235,10 @@ pub enum Retry { /// were retried along a route from a single call to [`Router::find_route`]. Attempts(usize), #[cfg(not(feature = "no-std"))] - /// Time elapsed before abandoning retries for a payment. + /// Time elapsed before abandoning retries for a payment. At least one attempt at payment is made; + /// see [`PaymentParameters::expiry_time`] to avoid any attempt at payment after a specific time. + /// + /// [`PaymentParameters::expiry_time`]: crate::routing::router::PaymentParameters::expiry_time Timeout(core::time::Duration), }