Serialization macro for TLV streams

BOLT 12's offer message is encoded as a TLV stream (i.e., a sequence of
TLV records). impl_writeable_tlv_based can't be used because it writes
the overall length of the struct, whereas TLV streams only include the
length of each TLV record. Add a `tlv_stream` macro for defining structs
used in encoding.

TLV records containing a single variable-length type should not encode
the types length in the value since it is redundant. Add a wrapper type
that can be used within a TLV stream to support the correct behavior
during serialization and de-serialization.

Author: jkczyz

lightning/src/util/ser.rs

@@ -452,6 +452,15 @@ macro_rules! impl_writeable_primitive {
 				}
 			}
 		}
+		impl From<$val_type> for HighZeroBytesDroppedBigSize<$val_type> {
+			fn from(val: $val_type) -> Self { Self(val) }
+		}
+		impl From<&$val_type> for HighZeroBytesDroppedBigSize<$val_type> {
+			fn from(val: &$val_type) -> Self { Self(*val) }
+		}
+		impl From<HighZeroBytesDroppedBigSize<$val_type>> for $val_type {
+			fn from(val: HighZeroBytesDroppedBigSize<$val_type>) -> Self { val.0 }
+		}
 	}
 }
 
@@ -524,6 +533,62 @@ impl_array!(PUBLIC_KEY_SIZE); // for PublicKey
 impl_array!(COMPACT_SIGNATURE_SIZE); // for Signature
 impl_array!(1300); // for OnionPacket.hop_data
 
+/// For variable-length values within TLV record where the length is encoded as part of the record.
+/// Used to prevent encoding the length twice.
+#[derive(Clone)]
+pub(crate) struct WithoutLength<T>(pub T);
+
+impl Writeable for WithoutLength<String> {
+	#[inline]
+	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+		w.write_all(self.0.as_bytes())
+	}
+}
+impl Writeable for WithoutLength<&String> {
+	#[inline]
+	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+		w.write_all(self.0.as_bytes())
+	}
+}
+impl Readable for WithoutLength<String> {
+	#[inline]
+	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+		let v: VecReadWrapper<u8> = Readable::read(r)?;
+		Ok(Self(String::from_utf8(v.0).map_err(|_| DecodeError::InvalidValue)?))
+	}
+}
+impl<'a> From<&'a String> for WithoutLength<&'a String> {
+	fn from(s: &'a String) -> Self { Self(s) }
+}
+impl From<WithoutLength<String>> for String {
+	fn from(s: WithoutLength<String>) -> Self { s.0 }
+}
+
+impl<T: Writeable> Writeable for WithoutLength<Vec<T>> {
+	#[inline]
+	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+		VecWriteWrapper(&self.0).write(w)
+	}
+}
+impl<T: Writeable> Writeable for WithoutLength<&Vec<T>> {
+	#[inline]
+	fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+		VecWriteWrapper(self.0).write(w)
+	}
+}
+impl<T: Readable> Readable for WithoutLength<Vec<T>> {
+	#[inline]
+	fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+		Ok(Self(<VecReadWrapper<T> as Readable>::read(r)?.0))
+	}
+}
+impl<'a, T> From<&'a Vec<T>> for WithoutLength<&'a Vec<T>> {
+	fn from(v: &'a Vec<T>) -> Self { Self(v) }
+}
+impl<T> From<WithoutLength<Vec<T>>> for Vec<T> {
+	fn from(s: WithoutLength<Vec<T>>) -> Self { s.0 }
+}
+
 // HashMap
 impl<K, V> Writeable for HashMap<K, V>
 	where K: Writeable + Eq + Hash,

lightning/src/util/ser_macros.rs

@@ -427,6 +427,132 @@ macro_rules! impl_writeable_tlv_based {
 	}
 }
 
+/// Defines a struct for a TLV stream and a similar struct using references for non-primitive types,
+/// implementing [`Readable`] for the former and [`Writeable`] for the latter. Useful as an
+/// intermediary format when reading or writing a type encoded as a TLV stream.
+///
+/// [`Readable`]: crate::util::ser::Readable
+/// [`Writeable`]: crate::util::ser::Writeable
+macro_rules! tlv_stream {
+	(struct $name:ident {
+		$(($type:expr, $field:ident : $fieldty:ident$(<$gen:ident>)?)),* $(,)*
+	}) => {
+		#[derive(Debug)]
+		struct $name {
+			$(
+				$field: Option<tlv_record_type!($fieldty$(<$gen>)?)>,
+			)*
+		}
+
+		pub(crate) mod reference {
+			$(
+				tlv_record_import!($fieldty$(<$gen>)?);
+			)*
+
+			pub(crate) struct $name<'a> {
+				$(
+					pub(crate) $field: Option<tlv_record_ref_type!($fieldty$(<$gen>)?)>,
+				)*
+			}
+
+			impl<'a> ::util::ser::Writeable for $name<'a> {
+				fn write<W: ::util::ser::Writer>(&self, writer: &mut W) -> Result<(), $crate::io::Error> {
+					encode_tlv_stream!(writer, { $(($type, self.$field, option)),* });
+					Ok(())
+				}
+			}
+		}
+
+		impl ::util::ser::Readable for $name {
+			fn read<R: $crate::io::Read>(reader: &mut R) -> Result<Self, ::ln::msgs::DecodeError> {
+				$(
+					init_tlv_field_var!($field, option);
+				)*
+				decode_tlv_stream!(reader, {$(($type, $field, option)),*});
+
+				Ok(Self {
+					$(
+						$field: init_tlv_based_struct_field!($field, option)
+					),*
+				})
+			}
+		}
+	}
+}
+
+macro_rules! tlv_record_type {
+	(u8) => {
+		u8
+	};
+	(u16) => {
+		::util::ser::HighZeroBytesDroppedBigSize<u16>
+	};
+	(u32) => {
+		::util::ser::HighZeroBytesDroppedBigSize<u32>
+	};
+	(u64) => {
+		::util::ser::HighZeroBytesDroppedBigSize<u64>
+	};
+	(char) => {
+		char
+	};
+	(String) => {
+		::util::ser::WithoutLength<String>
+	};
+	(Vec<$type:ty>) => {
+		::util::ser::WithoutLength<Vec<$type>>
+	};
+	($type:ident$(<$gen:ident>)?) => {
+		$type$(<$gen>)?
+	};
+}
+
+macro_rules! tlv_record_ref_type {
+	(u8) => {
+		u8
+	};
+	(u16) => {
+		::util::ser::HighZeroBytesDroppedBigSize<u16>
+	};
+	(u32) => {
+		::util::ser::HighZeroBytesDroppedBigSize<u32>
+	};
+	(u64) => {
+		::util::ser::HighZeroBytesDroppedBigSize<u64>
+	};
+	(char) => {
+		char
+	};
+	(String) => {
+		::util::ser::WithoutLength<&'a String>
+	};
+	(Vec<$type:ty>) => {
+		::util::ser::WithoutLength<&'a Vec<$type>>
+	};
+	($type:ident$(<$gen:ident>)?) => {
+		&'a $type$(<$gen>)?
+	};
+}
+
+macro_rules! tlv_record_import {
+	(u8) => {};
+	(u16) => {};
+	(u32) => {};
+	(u64) => {};
+	(char) => {};
+	(String) => {};
+	(Vec<$type:ident>) => {
+		tlv_record_import!($type);
+	};
+	($type:ident<$gen:ident>) => {
+		tlv_record_import!($type);
+		tlv_record_import!($gen);
+	};
+	($type:ident) => {
+		use super::$type;
+	};
+}
+
 macro_rules! _impl_writeable_tlv_based_enum_common {
 	($st: ident, $(($variant_id: expr, $variant_name: ident) =>
 		{$(($type: expr, $field: ident, $fieldty: tt)),* $(,)*}