Skip to content

Commit a320240

Browse files
committed
explain the size
1 parent 55d57e0 commit a320240

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

crates/iceberg/src/writer/file_writer/rolling_writer.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ impl<B: FileWriterBuilder> RollingFileWriterBuilder<B> {
3939
///
4040
/// * `inner_builder` - The builder for the underlying file writer
4141
/// * `target_file_size` - The target size in bytes before rolling over to a new file
42+
///
43+
/// NOTE: The `target_file_size` does not exactly reflect the final size on physical storage.
44+
/// This is because the input size is based on the Arrow in-memory format, which differs from the on-disk file format.
4245
pub fn new(inner_builder: B, target_file_size: usize) -> Self {
4346
Self {
4447
inner_builder,
@@ -90,6 +93,8 @@ impl<B: FileWriterBuilder> RollingFileWriter<B> {
9093

9194
impl<B: FileWriterBuilder> FileWriter for RollingFileWriter<B> {
9295
async fn write(&mut self, input: &RecordBatch) -> Result<()> {
96+
// The input size is estimated using the Arrow in-memory format
97+
// and will differ from the final on-disk file size.
9398
let input_size = input.get_array_memory_size();
9499

95100
if self.inner.is_none() {

0 commit comments

Comments
 (0)