Skip to content

Commit 082a19b

Browse files
TheBlueMattTheBlueMatt
authored
Merge pull request #2612 from TheBlueMatt/2023-09-namespace-split
Refer to top-level persistence namespaces as primary_namespace
2 parents 1e6707d + 47e1148 commit 082a19b

File tree

7 files changed

+202
-191
lines changed

7 files changed

+202
-191
lines changed

‎lightning-background-processor/src/lib.rs

+22-19
Original file line numberDiff line numberDiff line change
@@ -506,10 +506,10 @@ use core::task;
506506
/// # use lightning_background_processor::{process_events_async, GossipSync};
507507
/// # struct MyStore {}
508508
/// # impl lightning::util::persist::KVStore for MyStore {
509-
/// # fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> io::Result<Vec<u8>> { Ok(Vec::new()) }
510-
/// # fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) }
511-
/// # fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) }
512-
/// # fn list(&self, namespace: &str, sub_namespace: &str) -> io::Result<Vec<String>> { Ok(Vec::new()) }
509+
/// # fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> io::Result<Vec<u8>> { Ok(Vec::new()) }
510+
/// # fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> io::Result<()> { Ok(()) }
511+
/// # fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> io::Result<()> { Ok(()) }
512+
/// # fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result<Vec<String>> { Ok(Vec::new()) }
513513
/// # }
514514
/// # struct MyEventHandler {}
515515
/// # impl MyEventHandler {
@@ -868,7 +868,10 @@ mod tests {
868868
use lightning::util::config::UserConfig;
869869
use lightning::util::ser::Writeable;
870870
use lightning::util::test_utils;
871-
use lightning::util::persist::{KVStore, CHANNEL_MANAGER_PERSISTENCE_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY, NETWORK_GRAPH_PERSISTENCE_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY, SCORER_PERSISTENCE_NAMESPACE, SCORER_PERSISTENCE_SUB_NAMESPACE, SCORER_PERSISTENCE_KEY};
871+
use lightning::util::persist::{KVStore,
872+
CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE, CHANNEL_MANAGER_PERSISTENCE_KEY,
873+
NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE, NETWORK_GRAPH_PERSISTENCE_KEY,
874+
SCORER_PERSISTENCE_PRIMARY_NAMESPACE, SCORER_PERSISTENCE_SECONDARY_NAMESPACE, SCORER_PERSISTENCE_KEY};
872875
use lightning_persister::fs_store::FilesystemStore;
873876
use std::collections::VecDeque;
874877
use std::{fs, env};
@@ -983,22 +986,22 @@ mod tests {
983986
}
984987

985988
impl KVStore for Persister {
986-
fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> lightning::io::Result<Vec<u8>> {
987-
self.kv_store.read(namespace, sub_namespace, key)
989+
fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> lightning::io::Result<Vec<u8>> {
990+
self.kv_store.read(primary_namespace, secondary_namespace, key)
988991
}
989992

990-
fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> {
991-
if namespace == CHANNEL_MANAGER_PERSISTENCE_NAMESPACE &&
992-
sub_namespace == CHANNEL_MANAGER_PERSISTENCE_SUB_NAMESPACE &&
993+
fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> lightning::io::Result<()> {
994+
if primary_namespace == CHANNEL_MANAGER_PERSISTENCE_PRIMARY_NAMESPACE &&
995+
secondary_namespace == CHANNEL_MANAGER_PERSISTENCE_SECONDARY_NAMESPACE &&
993996
key == CHANNEL_MANAGER_PERSISTENCE_KEY
994997
{
995998
if let Some((error, message)) = self.manager_error {
996999
return Err(std::io::Error::new(error, message))
9971000
}
9981001
}
9991002

1000-
if namespace == NETWORK_GRAPH_PERSISTENCE_NAMESPACE &&
1001-
sub_namespace == NETWORK_GRAPH_PERSISTENCE_SUB_NAMESPACE &&
1003+
if primary_namespace == NETWORK_GRAPH_PERSISTENCE_PRIMARY_NAMESPACE &&
1004+
secondary_namespace == NETWORK_GRAPH_PERSISTENCE_SECONDARY_NAMESPACE &&
10021005
key == NETWORK_GRAPH_PERSISTENCE_KEY
10031006
{
10041007
if let Some(sender) = &self.graph_persistence_notifier {
@@ -1013,24 +1016,24 @@ mod tests {
10131016
}
10141017
}
10151018

1016-
if namespace == SCORER_PERSISTENCE_NAMESPACE &&
1017-
sub_namespace == SCORER_PERSISTENCE_SUB_NAMESPACE &&
1019+
if primary_namespace == SCORER_PERSISTENCE_PRIMARY_NAMESPACE &&
1020+
secondary_namespace == SCORER_PERSISTENCE_SECONDARY_NAMESPACE &&
10181021
key == SCORER_PERSISTENCE_KEY
10191022
{
10201023
if let Some((error, message)) = self.scorer_error {
10211024
return Err(std::io::Error::new(error, message))
10221025
}
10231026
}
10241027

1025-
self.kv_store.write(namespace, sub_namespace, key, buf)
1028+
self.kv_store.write(primary_namespace, secondary_namespace, key, buf)
10261029
}
10271030

1028-
fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> {
1029-
self.kv_store.remove(namespace, sub_namespace, key, lazy)
1031+
fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> lightning::io::Result<()> {
1032+
self.kv_store.remove(primary_namespace, secondary_namespace, key, lazy)
10301033
}
10311034

1032-
fn list(&self, namespace: &str, sub_namespace: &str) -> lightning::io::Result<Vec<String>> {
1033-
self.kv_store.list(namespace, sub_namespace)
1035+
fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> lightning::io::Result<Vec<String>> {
1036+
self.kv_store.list(primary_namespace, secondary_namespace)
10341037
}
10351038
}
10361039

‎lightning-persister/src/fs_store.rs

+23-23
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ impl FilesystemStore {
6767
}
6868
}
6969

70-
fn get_dest_dir_path(&self, namespace: &str, sub_namespace: &str) -> std::io::Result<PathBuf> {
70+
fn get_dest_dir_path(&self, primary_namespace: &str, secondary_namespace: &str) -> std::io::Result<PathBuf> {
7171
let mut dest_dir_path = {
7272
#[cfg(target_os = "windows")]
7373
{
@@ -81,20 +81,20 @@ impl FilesystemStore {
8181
}
8282
};
8383

84-
dest_dir_path.push(namespace);
85-
if !sub_namespace.is_empty() {
86-
dest_dir_path.push(sub_namespace);
84+
dest_dir_path.push(primary_namespace);
85+
if !secondary_namespace.is_empty() {
86+
dest_dir_path.push(secondary_namespace);
8787
}
8888

8989
Ok(dest_dir_path)
9090
}
9191
}
9292

9393
impl KVStore for FilesystemStore {
94-
fn read(&self, namespace: &str, sub_namespace: &str, key: &str) -> std::io::Result<Vec<u8>> {
95-
check_namespace_key_validity(namespace, sub_namespace, Some(key), "read")?;
94+
fn read(&self, primary_namespace: &str, secondary_namespace: &str, key: &str) -> std::io::Result<Vec<u8>> {
95+
check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "read")?;
9696

97-
let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
97+
let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
9898
dest_file_path.push(key);
9999

100100
let mut buf = Vec::new();
@@ -114,10 +114,10 @@ impl KVStore for FilesystemStore {
114114
Ok(buf)
115115
}
116116

117-
fn write(&self, namespace: &str, sub_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> {
118-
check_namespace_key_validity(namespace, sub_namespace, Some(key), "write")?;
117+
fn write(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: &[u8]) -> std::io::Result<()> {
118+
check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "write")?;
119119

120-
let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
120+
let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
121121
dest_file_path.push(key);
122122

123123
let parent_directory = dest_file_path
@@ -201,10 +201,10 @@ impl KVStore for FilesystemStore {
201201
res
202202
}
203203

204-
fn remove(&self, namespace: &str, sub_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> {
205-
check_namespace_key_validity(namespace, sub_namespace, Some(key), "remove")?;
204+
fn remove(&self, primary_namespace: &str, secondary_namespace: &str, key: &str, lazy: bool) -> std::io::Result<()> {
205+
check_namespace_key_validity(primary_namespace, secondary_namespace, Some(key), "remove")?;
206206

207-
let mut dest_file_path = self.get_dest_dir_path(namespace, sub_namespace)?;
207+
let mut dest_file_path = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
208208
dest_file_path.push(key);
209209

210210
if !dest_file_path.is_file() {
@@ -290,10 +290,10 @@ impl KVStore for FilesystemStore {
290290
Ok(())
291291
}
292292

293-
fn list(&self, namespace: &str, sub_namespace: &str) -> std::io::Result<Vec<String>> {
294-
check_namespace_key_validity(namespace, sub_namespace, None, "list")?;
293+
fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> std::io::Result<Vec<String>> {
294+
check_namespace_key_validity(primary_namespace, secondary_namespace, None, "list")?;
295295

296-
let prefixed_dest = self.get_dest_dir_path(namespace, sub_namespace)?;
296+
let prefixed_dest = self.get_dest_dir_path(primary_namespace, secondary_namespace)?;
297297
let mut keys = Vec::new();
298298

299299
if !Path::new(&prefixed_dest).exists() {
@@ -320,17 +320,17 @@ impl KVStore for FilesystemStore {
320320

321321
let metadata = p.metadata()?;
322322

323-
// We allow the presence of directories in the empty namespace and just skip them.
323+
// We allow the presence of directories in the empty primary namespace and just skip them.
324324
if metadata.is_dir() {
325325
continue;
326326
}
327327

328328
// If we otherwise don't find a file at the given path something went wrong.
329329
if !metadata.is_file() {
330330
debug_assert!(false, "Failed to list keys of {}/{}: file couldn't be accessed.",
331-
PrintableString(namespace), PrintableString(sub_namespace));
331+
PrintableString(primary_namespace), PrintableString(secondary_namespace));
332332
let msg = format!("Failed to list keys of {}/{}: file couldn't be accessed.",
333-
PrintableString(namespace), PrintableString(sub_namespace));
333+
PrintableString(primary_namespace), PrintableString(secondary_namespace));
334334
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
335335
}
336336

@@ -342,17 +342,17 @@ impl KVStore for FilesystemStore {
342342
}
343343
} else {
344344
debug_assert!(false, "Failed to list keys of {}/{}: file path is not valid UTF-8",
345-
PrintableString(namespace), PrintableString(sub_namespace));
345+
PrintableString(primary_namespace), PrintableString(secondary_namespace));
346346
let msg = format!("Failed to list keys of {}/{}: file path is not valid UTF-8",
347-
PrintableString(namespace), PrintableString(sub_namespace));
347+
PrintableString(primary_namespace), PrintableString(secondary_namespace));
348348
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
349349
}
350350
}
351351
Err(e) => {
352352
debug_assert!(false, "Failed to list keys of {}/{}: {}",
353-
PrintableString(namespace), PrintableString(sub_namespace), e);
353+
PrintableString(primary_namespace), PrintableString(secondary_namespace), e);
354354
let msg = format!("Failed to list keys of {}/{}: {}",
355-
PrintableString(namespace), PrintableString(sub_namespace), e);
355+
PrintableString(primary_namespace), PrintableString(secondary_namespace), e);
356356
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
357357
}
358358
}

‎lightning-persister/src/test_utils.rs

+13-12
Original file line numberDiff line numberDiff line change
@@ -12,34 +12,35 @@ use std::panic::RefUnwindSafe;
1212
pub(crate) fn do_read_write_remove_list_persist<K: KVStore + RefUnwindSafe>(kv_store: &K) {
1313
let data = [42u8; 32];
1414

15-
let namespace = "testspace";
16-
let sub_namespace = "testsubspace";
15+
let primary_namespace = "testspace";
16+
let secondary_namespace = "testsubspace";
1717
let key = "testkey";
1818

1919
// Test the basic KVStore operations.
20-
kv_store.write(namespace, sub_namespace, key, &data).unwrap();
20+
kv_store.write(primary_namespace, secondary_namespace, key, &data).unwrap();
2121

22-
// Test empty namespace/sub_namespace is allowed, but not empty namespace and non-empty
23-
// sub-namespace, and not empty key.
22+
// Test empty primary_namespace/secondary_namespace is allowed, but not empty primary_namespace
23+
// and non-empty secondary_namespace, and not empty key.
2424
kv_store.write("", "", key, &data).unwrap();
25-
let res = std::panic::catch_unwind(|| kv_store.write("", sub_namespace, key, &data));
25+
let res = std::panic::catch_unwind(|| kv_store.write("", secondary_namespace, key, &data));
2626
assert!(res.is_err());
27-
let res = std::panic::catch_unwind(|| kv_store.write(namespace, sub_namespace, "", &data));
27+
let res = std::panic::catch_unwind(|| kv_store.write(primary_namespace, secondary_namespace, "", &data));
2828
assert!(res.is_err());
2929

30-
let listed_keys = kv_store.list(namespace, sub_namespace).unwrap();
30+
let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap();
3131
assert_eq!(listed_keys.len(), 1);
3232
assert_eq!(listed_keys[0], key);
3333

34-
let read_data = kv_store.read(namespace, sub_namespace, key).unwrap();
34+
let read_data = kv_store.read(primary_namespace, secondary_namespace, key).unwrap();
3535
assert_eq!(data, &*read_data);
3636

37-
kv_store.remove(namespace, sub_namespace, key, false).unwrap();
37+
kv_store.remove(primary_namespace, secondary_namespace, key, false).unwrap();
3838

39-
let listed_keys = kv_store.list(namespace, sub_namespace).unwrap();
39+
let listed_keys = kv_store.list(primary_namespace, secondary_namespace).unwrap();
4040
assert_eq!(listed_keys.len(), 0);
4141

42-
// Ensure we have no issue operating with namespace/sub_namespace/key being KVSTORE_NAMESPACE_KEY_MAX_LEN
42+
// Ensure we have no issue operating with primary_namespace/secondary_namespace/key being
43+
// KVSTORE_NAMESPACE_KEY_MAX_LEN
4344
let max_chars: String = std::iter::repeat('A').take(KVSTORE_NAMESPACE_KEY_MAX_LEN).collect();
4445
kv_store.write(&max_chars, &max_chars, &max_chars, &data).unwrap();
4546

‎lightning-persister/src/utils.rs

+25-23
Original file line numberDiff line numberDiff line change
@@ -6,51 +6,53 @@ pub(crate) fn is_valid_kvstore_str(key: &str) -> bool {
66
key.len() <= KVSTORE_NAMESPACE_KEY_MAX_LEN && key.chars().all(|c| KVSTORE_NAMESPACE_KEY_ALPHABET.contains(c))
77
}
88

9-
pub(crate) fn check_namespace_key_validity(namespace: &str, sub_namespace: &str, key: Option<&str>, operation: &str) -> Result<(), std::io::Error> {
9+
pub(crate) fn check_namespace_key_validity(
10+
primary_namespace: &str, secondary_namespace: &str, key: Option<&str>, operation: &str)
11+
-> Result<(), std::io::Error> {
1012
if let Some(key) = key {
1113
if key.is_empty() {
1214
debug_assert!(false, "Failed to {} {}/{}/{}: key may not be empty.", operation,
13-
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
15+
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
1416
let msg = format!("Failed to {} {}/{}/{}: key may not be empty.", operation,
15-
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
17+
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
1618
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
1719
}
1820

19-
if namespace.is_empty() && !sub_namespace.is_empty() {
21+
if primary_namespace.is_empty() && !secondary_namespace.is_empty() {
2022
debug_assert!(false,
21-
"Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
23+
"Failed to {} {}/{}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
2224
operation,
23-
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
25+
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
2426
let msg = format!(
25-
"Failed to {} {}/{}/{}: namespace may not be empty if a non-empty sub-namespace is given.", operation,
26-
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
27+
"Failed to {} {}/{}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.", operation,
28+
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
2729
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
2830
}
2931

30-
if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) || !is_valid_kvstore_str(key) {
31-
debug_assert!(false, "Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.",
32+
if !is_valid_kvstore_str(primary_namespace) || !is_valid_kvstore_str(secondary_namespace) || !is_valid_kvstore_str(key) {
33+
debug_assert!(false, "Failed to {} {}/{}/{}: primary namespace, secondary namespace, and key must be valid.",
3234
operation,
33-
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
34-
let msg = format!("Failed to {} {}/{}/{}: namespace, sub-namespace, and key must be valid.",
35+
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
36+
let msg = format!("Failed to {} {}/{}/{}: primary namespace, secondary namespace, and key must be valid.",
3537
operation,
36-
PrintableString(namespace), PrintableString(sub_namespace), PrintableString(key));
38+
PrintableString(primary_namespace), PrintableString(secondary_namespace), PrintableString(key));
3739
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
3840
}
3941
} else {
40-
if namespace.is_empty() && !sub_namespace.is_empty() {
42+
if primary_namespace.is_empty() && !secondary_namespace.is_empty() {
4143
debug_assert!(false,
42-
"Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
43-
operation, PrintableString(namespace), PrintableString(sub_namespace));
44+
"Failed to {} {}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
45+
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
4446
let msg = format!(
45-
"Failed to {} {}/{}: namespace may not be empty if a non-empty sub-namespace is given.",
46-
operation, PrintableString(namespace), PrintableString(sub_namespace));
47+
"Failed to {} {}/{}: primary namespace may not be empty if a non-empty secondary namespace is given.",
48+
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
4749
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
4850
}
49-
if !is_valid_kvstore_str(namespace) || !is_valid_kvstore_str(sub_namespace) {
50-
debug_assert!(false, "Failed to {} {}/{}: namespace and sub-namespace must be valid.",
51-
operation, PrintableString(namespace), PrintableString(sub_namespace));
52-
let msg = format!("Failed to {} {}/{}: namespace and sub-namespace must be valid.",
53-
operation, PrintableString(namespace), PrintableString(sub_namespace));
51+
if !is_valid_kvstore_str(primary_namespace) || !is_valid_kvstore_str(secondary_namespace) {
52+
debug_assert!(false, "Failed to {} {}/{}: primary namespace and secondary namespace must be valid.",
53+
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
54+
let msg = format!("Failed to {} {}/{}: primary namespace and secondary namespace must be valid.",
55+
operation, PrintableString(primary_namespace), PrintableString(secondary_namespace));
5456
return Err(std::io::Error::new(std::io::ErrorKind::Other, msg));
5557
}
5658
}

0 commit comments

Comments
 (0)