File tree Expand file tree Collapse file tree 1 file changed +5
-0
lines changed
crates/iceberg/src/writer/file_writer Expand file tree Collapse file tree 1 file changed +5
-0
lines changed Original file line number Diff line number Diff line change @@ -39,6 +39,9 @@ impl<B: FileWriterBuilder> RollingFileWriterBuilder<B> {
3939 ///
4040 /// * `inner_builder` - The builder for the underlying file writer
4141 /// * `target_file_size` - The target size in bytes before rolling over to a new file
42+ ///
43+ /// NOTE: The `target_file_size` does not exactly reflect the final size on physical storage.
44+ /// This is because the input size is based on the Arrow in-memory format, which differs from the on-disk file format.
4245 pub fn new ( inner_builder : B , target_file_size : usize ) -> Self {
4346 Self {
4447 inner_builder,
@@ -90,6 +93,8 @@ impl<B: FileWriterBuilder> RollingFileWriter<B> {
9093
9194impl < B : FileWriterBuilder > FileWriter for RollingFileWriter < B > {
9295 async fn write ( & mut self , input : & RecordBatch ) -> Result < ( ) > {
96+ // The input size is estimated using the Arrow in-memory format
97+ // and will differ from the final on-disk file size.
9398 let input_size = input. get_array_memory_size ( ) ;
9499
95100 if self . inner . is_none ( ) {
You can’t perform that action at this time.
0 commit comments