Merge branch 'lightningdevkit:main' into main

Author: Sharmalm

.github/workflows/build.yml

@@ -168,7 +168,7 @@ jobs:
           done
           # check no-std compatibility across dependencies
           cd no-std-check
-          cargo check --verbose --color always
+          cargo check --verbose --color always --features lightning-transaction-sync
       - name: Build no-std-check on Rust ${{ matrix.toolchain }} for ARM Embedded
         if: "matrix.build-no-std && matrix.platform == 'ubuntu-latest'"
         run: |

CHANGELOG.md

@@ -1,3 +1,117 @@
+# 0.0.114 - Mar 3, 2023 - "Faster Async BOLT12 Retries"
+
+## API Updates
+ * `InvoicePayer` has been removed and its features moved directly into
+   `ChannelManager`. As such it now requires a simplified `Router` and supports
+   `send_payment_with_retry` (and friends). `ChannelManager::retry_payment` was
+   removed in favor of the automated retries. Invoice payment utilities in
+   `lightning-invoice` now call the new code (#1812, #1916, #1929, #2007, etc).
+ * `Sign`/`BaseSign` has been renamed `ChannelSigner`, with `EcdsaChannelSigner`
+   split out in anticipation of future schnorr/taproot support (#1967).
+ * The catch-all `KeysInterface` was split into `EntropySource`, `NodeSigner`,
+   and `SignerProvider`. `KeysManager` implements all three (#1910, #1930).
+ * `KeysInterface::get_node_secret` is now `KeysManager::get_node_secret_key`
+   and is no longer required for external signers (#1951, #2070).
+ * A `lightning-transaction-sync` crate has been added which implements keeping
+   LDK in sync with the chain via an esplora server (#1870). Note that it can
+   only be used on nodes that *never* ran a previous version of LDK.
+ * `Score` is updated in `BackgroundProcessor` instead of via `Router` (#1996).
+ * `ChainAccess::get_utxo` (now `UtxoAccess`) can now be resolved async (#1980).
+ * BOLT12 `Offer`, `InvoiceRequest`, `Invoice` and `Refund` structs as well as
+   associated builders have been added. Such invoices cannot yet be paid due to
+   missing support for blinded path payments (#1927, #1908, #1926).
+ * A `lightning-custom-message` crate has been added to make combining multiple
+   custom messages into one enum/handler easier (#1832).
+ * `Event::PaymentPathFailure` is now generated for failure to send an HTLC
+   over the first hop on our local channel (#2014, #2043).
+ * `lightning-net-tokio` no longer requires an `Arc` on `PeerManager` (#1968).
+ * `ChannelManager::list_recent_payments` was added (#1873).
+ * `lightning-background-processor` `std` is now optional in async mode (#1962).
+ * `create_phantom_invoice` can now be used in `no-std` (#1985).
+ * The required final CLTV delta on inbound payments is now configurable (#1878)
+ * bitcoind RPC error code and message are now surfaced in `block-sync` (#2057).
+ * Get `historical_estimated_channel_liquidity_probabilities` was added (#1961).
+ * `ChannelManager::fail_htlc_backwards_with_reason` was added (#1948).
+ * Macros which implement serialization using TLVs or straight writing of struct
+   fields are now public (#1823, #1976, #1977).
+
+## Backwards Compatibility
+ * Any inbound payments with a custom final CLTV delta will be rejected by LDK
+   if you downgrade prior to receipt (#1878).
+ * `Event::PaymentPathFailed::network_update` will always be `None` if an
+   0.0.114-generated event is read by a prior version of LDK (#2043).
+ * `Event::PaymentPathFailed::all_paths_removed` will always be false if an
+   0.0.114-generated event is read by a prior version of LDK. Users who rely on
+   it to determine payment retries should migrate to `Event::PaymentFailed`, in
+   a separate release prior to upgrading to LDK 0.0.114 if downgrading is
+   supported (#2043).
+
+## Performance Improvements
+ * Channel data is now stored per-peer and channel updates across multiple
+   peers can be operated on simultaneously (#1507).
+ * Routefinding is roughly 1.5x faster (#1799).
+ * Deserializing a `NetworkGraph` is roughly 6x faster (#2016).
+ * Memory usage for a `NetworkGraph` has been reduced substantially (#2040).
+ * `KeysInterface::get_secure_random_bytes` is roughly 200x faster (#1974).
+
+## Bug Fixes
+ * Fixed a bug where a delay in processing a `PaymentSent` event longer than the
+   time taken to persist a `ChannelMonitor` update, when occurring immediately
+   prior to a crash, may result in the `PaymentSent` event being lost (#2048).
+ * Fixed spurious rejections of rapid gossip sync data when the graph has been
+   updated by other means between gossip syncs (#2046).
+ * Fixed a panic in `KeysManager` when the high bit of `starting_time_nanos`
+   is set (#1935).
+ * Resolved an issue where the `ChannelManager::get_persistable_update_future`
+   future would fail to wake until a second notification occurs (#2064).
+ * Resolved a memory leak when using `ChannelManager::send_probe` (#2037).
+ * Fixed a deadlock on some platforms at least when using async `ChannelMonitor`
+   updating (#2006).
+ * Removed debug-only assertions which were reachable in threaded code (#1964).
+ * In some cases when payment sending fails on our local channel retries no
+   longer take the same path and thus never succeed (#2014).
+ * Retries for spontaneous payments have been fixed (#2002).
+ * Return an `Err` if `lightning-persister` fails to read the directory listing
+   rather than panicing (#1943).
+ * `peer_disconnected` will now never be called without `peer_connected` (#2035)
+
+## Security
+0.0.114 fixes several denial-of-service vulnerabilities which are reachable from
+untrusted input from channel counterparties or in deployments accepting inbound
+connections or channels. It also fixes a denial-of-service vulnerability in rare
+cases in the route finding logic.
+ * The number of pending un-funded channels as well as peers without funded
+   channels is now limited to avoid denial of service (#1988).
+ * A second `channel_ready` message received immediately after the first could
+   lead to a spurious panic (#2071). This issue was introduced with 0conf
+   support in LDK 0.0.107.
+ * A division-by-zero issue was fixed in the `ProbabilisticScorer` if the amount
+   being sent (including previous-hop fees) is equal to a channel's capacity
+   while walking the graph (#2072). The division-by-zero was introduced with
+   historical data tracking in LDK 0.0.112.
+
+In total, this release features 130 files changed, 21457 insertions, 10113
+deletions in 343 commits from 18 authors, in alphabetical order:
+ * Alec Chen
+ * Allan Douglas R. de Oliveira
+ * Andrei
+ * Arik Sosman
+ * Daniel Granhão
+ * Duncan Dean
+ * Elias Rohrer
+ * Jeffrey Czyz
+ * John Cantrell
+ * Kurtsley
+ * Matt Corallo
+ * Max Fang
+ * Omer Yacine
+ * Valentine Wallace
+ * Viktor Tigerström
+ * Wilmer Paulino
+ * benthecarman
+ * jurvis
+
+
 # 0.0.113 - Dec 16, 2022 - "Big Movement Intercepted"
 
 ## API Updates

Cargo.toml

@@ -16,14 +16,20 @@ exclude = [
     "no-std-check",
 ]
 
-# Our tests do actual crypto and lots of work, the tradeoff for -O1 is well worth it.
-# Ideally we would only do this in profile.test, but profile.test only applies to
-# the test binary, not dependencies, which means most of the critical code still
-# gets compiled as -O0. See
+# Our tests do actual crypto and lots of work, the tradeoff for -O2 is well
+# worth it. Note that we only apply optimizations to dependencies, not workspace
+# crates themselves.
 # https://doc.rust-lang.org/cargo/reference/profiles.html#profile-selection
+[profile.dev.package."*"]
+opt-level = 2
+
+# It appears some minimal optimizations are required to inline many std methods
+# and reduce the otherwise-substantial time spent in std self-checks. We do so
+# here but ensure we keep LTO disabled as otherwise we're re-optimizing all our
+# dependencies every time we make any local changes.
 [profile.dev]
 opt-level = 1
-panic = "abort"
+lto = "off"
 
 [profile.release]
 opt-level = 3

fuzz/src/full_stack.rs

@@ -42,7 +42,7 @@ use lightning::ln::msgs::{self, DecodeError};
 use lightning::ln::script::ShutdownScript;
 use lightning::routing::gossip::{P2PGossipSync, NetworkGraph};
 use lightning::routing::utxo::UtxoLookup;
-use lightning::routing::router::{find_route, InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters, Router};
+use lightning::routing::router::{find_route, InFlightHtlcs, PaymentParameters, Route, RouteParameters, Router};
 use lightning::routing::scoring::FixedPenaltyScorer;
 use lightning::util::config::UserConfig;
 use lightning::util::errors::APIError;

fuzz/src/router.rs

@@ -17,7 +17,7 @@ use lightning::ln::msgs;
 use lightning::routing::gossip::{NetworkGraph, RoutingFees};
 use lightning::routing::utxo::{UtxoFuture, UtxoLookup, UtxoLookupError, UtxoResult};
 use lightning::routing::router::{find_route, PaymentParameters, RouteHint, RouteHintHop, RouteParameters};
-use lightning::routing::scoring::FixedPenaltyScorer;
+use lightning::routing::scoring::ProbabilisticScorer;
 use lightning::util::config::UserConfig;
 use lightning::util::ser::Readable;
 
@@ -292,7 +292,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
 						}]));
 					}
 				}
-				let scorer = FixedPenaltyScorer::with_penalty(0);
+				let scorer = ProbabilisticScorer::new(Default::default(), &net_graph, &logger);
 				let random_seed_bytes: [u8; 32] = [get_slice!(1)[0]; 32];
 				for target in node_pks.iter() {
 					let final_value_msat = slice_to_be64(get_slice!(8));

lightning-background-processor/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-background-processor"
-version = "0.0.113"
+version = "0.0.114"
 authors = ["Valentine Wallace <[email protected]>"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -21,11 +21,11 @@ default = ["std"]
 
 [dependencies]
 bitcoin = { version = "0.29.0", default-features = false }
-lightning = { version = "0.0.113", path = "../lightning", default-features = false }
-lightning-rapid-gossip-sync = { version = "0.0.113", path = "../lightning-rapid-gossip-sync", default-features = false }
+lightning = { version = "0.0.114", path = "../lightning", default-features = false }
+lightning-rapid-gossip-sync = { version = "0.0.114", path = "../lightning-rapid-gossip-sync", default-features = false }
 futures-util = { version = "0.3", default-features = false, features = ["async-await-macro"], optional = true }
 
 [dev-dependencies]
-lightning = { version = "0.0.113", path = "../lightning", features = ["_test_utils"] }
-lightning-invoice = { version = "0.21.0", path = "../lightning-invoice" }
-lightning-persister = { version = "0.0.113", path = "../lightning-persister" }
+lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
+lightning-invoice = { version = "0.22.0", path = "../lightning-invoice" }
+lightning-persister = { version = "0.0.114", path = "../lightning-persister" }

lightning-background-processor/src/lib.rs

@@ -33,7 +33,9 @@ use lightning::routing::gossip::{NetworkGraph, P2PGossipSync};
 use lightning::routing::utxo::UtxoLookup;
 use lightning::routing::router::Router;
 use lightning::routing::scoring::{Score, WriteableScore};
-use lightning::util::events::{Event, EventHandler, EventsProvider, PathFailure};
+use lightning::util::events::{Event, PathFailure};
+#[cfg(feature = "std")]
+use lightning::util::events::{EventHandler, EventsProvider};
 use lightning::util::logger::Logger;
 use lightning::util::persist::Persister;
 use lightning_rapid_gossip_sync::RapidGossipSync;

lightning-block-sync/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-block-sync"
-version = "0.0.113"
+version = "0.0.114"
 authors = ["Jeffrey Czyz", "Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -19,13 +19,13 @@ rpc-client = [ "serde", "serde_json", "chunked_transfer" ]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.113", path = "../lightning" }
+lightning = { version = "0.0.114", path = "../lightning" }
 futures-util = { version = "0.3" }
 tokio = { version = "1.0", features = [ "io-util", "net", "time" ], optional = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
 chunked_transfer = { version = "1.4", optional = true }
 
 [dev-dependencies]
-lightning = { version = "0.0.113", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
 tokio = { version = "~1.14", features = [ "macros", "rt" ] }

lightning-block-sync/src/lib.rs

@@ -132,6 +132,9 @@ impl BlockSourceError {
 	}
 
 	/// Converts the error into the underlying error.
+	///
+	/// May contain an [`std::io::Error`] from the [`BlockSource`]. See implementations for further
+	/// details, if any.
 	pub fn into_inner(self) -> Box<dyn std::error::Error + Send + Sync> {
 		self.error
 	}

lightning-block-sync/src/rpc.rs

@@ -13,9 +13,31 @@ use serde_json;
 
 use std::convert::TryFrom;
 use std::convert::TryInto;
+use std::error::Error;
+use std::fmt;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
+/// An error returned by the RPC server.
+#[derive(Debug)]
+pub struct RpcError {
+	/// The error code.
+	pub code: i64,
+	/// The error message.
+	pub message: String,
+}
+
+impl fmt::Display for RpcError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "RPC error {}: {}", self.code, self.message)
+    }
+}
+
+impl Error for RpcError {}
+
 /// A simple RPC client for calling methods using HTTP `POST`.
+///
+/// Implements [`BlockSource`] and may return an `Err` containing [`RpcError`]. See
+/// [`RpcClient::call_method`] for details.
 pub struct RpcClient {
 	basic_auth: String,
 	endpoint: HttpEndpoint,
@@ -38,6 +60,9 @@ impl RpcClient {
 	}
 
 	/// Calls a method with the response encoded in JSON format and interpreted as type `T`.
+	///
+	/// When an `Err` is returned, [`std::io::Error::into_inner`] may contain an [`RpcError`] if
+	/// [`std::io::Error::kind`] is [`std::io::ErrorKind::Other`].
 	pub async fn call_method<T>(&self, method: &str, params: &[serde_json::Value]) -> std::io::Result<T>
 	where JsonResponse: TryFrom<Vec<u8>, Error = std::io::Error> + TryInto<T, Error = std::io::Error> {
 		let host = format!("{}:{}", self.endpoint.host(), self.endpoint.port());
@@ -69,8 +94,11 @@ impl RpcClient {
 		let error = &response["error"];
 		if !error.is_null() {
 			// TODO: Examine error code for a more precise std::io::ErrorKind.
-			let message = error["message"].as_str().unwrap_or("unknown error");
-			return Err(std::io::Error::new(std::io::ErrorKind::Other, message));
+			let rpc_error = RpcError { 
+				code: error["code"].as_i64().unwrap_or(-1), 
+				message: error["message"].as_str().unwrap_or("unknown error").to_string() 
+			};
+			return Err(std::io::Error::new(std::io::ErrorKind::Other, rpc_error));
 		}
 
 		let result = &mut response["result"];
@@ -163,7 +191,9 @@ mod tests {
 		match client.call_method::<u64>("getblock", &[invalid_block_hash]).await {
 			Err(e) => {
 				assert_eq!(e.kind(), std::io::ErrorKind::Other);
-				assert_eq!(e.get_ref().unwrap().to_string(), "invalid parameter");
+				let rpc_error: Box<RpcError> = e.into_inner().unwrap().downcast().unwrap();
+				assert_eq!(rpc_error.code, -8);
+				assert_eq!(rpc_error.message, "invalid parameter");
 			},
 			Ok(_) => panic!("Expected error"),
 		}

lightning-custom-message/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-custom-message"
-version = "0.0.113"
+version = "0.0.114"
 authors = ["Jeffrey Czyz"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -15,4 +15,4 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.113", path = "../lightning" }
+lightning = { version = "0.0.114", path = "../lightning" }

lightning-invoice/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lightning-invoice"
 description = "Data structures to parse and serialize BOLT11 lightning invoices"
-version = "0.21.0"
+version = "0.22.0"
 authors = ["Sebastian Geisler <[email protected]>"]
 documentation = "https://docs.rs/lightning-invoice/"
 license = "MIT OR Apache-2.0"
@@ -21,14 +21,14 @@ std = ["bitcoin_hashes/std", "num-traits/std", "lightning/std", "bech32/std"]
 
 [dependencies]
 bech32 = { version = "0.9.0", default-features = false }
-lightning = { version = "0.0.113", path = "../lightning", default-features = false }
+lightning = { version = "0.0.114", path = "../lightning", default-features = false }
 secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"] }
 num-traits = { version = "0.2.8", default-features = false }
 bitcoin_hashes = { version = "0.11", default-features = false }
 hashbrown = { version = "0.8", optional = true }
 serde = { version = "1.0.118", optional = true }
 
 [dev-dependencies]
-lightning = { version = "0.0.113", path = "../lightning", default-features = false, features = ["_test_utils"] }
+lightning = { version = "0.0.114", path = "../lightning", default-features = false, features = ["_test_utils"] }
 hex = "0.4"
 serde_json = { version = "1"}

lightning-net-tokio/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-net-tokio"
-version = "0.0.113"
+version = "0.0.114"
 authors = ["Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning/"
@@ -16,9 +16,9 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.113", path = "../lightning" }
+lightning = { version = "0.0.114", path = "../lightning" }
 tokio = { version = "1.0", features = [ "io-util", "macros", "rt", "sync", "net", "time" ] }
 
 [dev-dependencies]
 tokio = { version = "~1.14", features = [ "io-util", "macros", "rt", "rt-multi-thread", "sync", "net", "time" ] }
-lightning = { version = "0.0.113", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }

lightning-persister/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-persister"
-version = "0.0.113"
+version = "0.0.114"
 authors = ["Valentine Wallace", "Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning/"
@@ -18,11 +18,11 @@ _bench_unstable = ["lightning/_bench_unstable"]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.113", path = "../lightning" }
+lightning = { version = "0.0.114", path = "../lightning" }
 libc = "0.2"
 
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["winbase"] }
 
 [dev-dependencies]
-lightning = { version = "0.0.113", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }