Skip to content

Commit 55cca03

Browse files
ZENOTMEZENOTME
authored
refine: refine interface of ManifestWriter (#738)
Co-authored-by: ZENOTME <[email protected]>
1 parent 425efc8 commit 55cca03

File tree

5 files changed

+716
-279
lines changed

5 files changed

+716
-279
lines changed

‎crates/iceberg/src/io/object_cache.rs‎

Lines changed: 26 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -192,9 +192,8 @@ mod tests {
192192
use super::*;
193193
use crate::io::{FileIO, OutputFile};
194194
use crate::spec::{
195-
DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, Literal, Manifest,
196-
ManifestContentType, ManifestEntry, ManifestListWriter, ManifestMetadata, ManifestStatus,
197-
ManifestWriter, Struct, TableMetadata,
195+
DataContentType, DataFileBuilder, DataFileFormat, Literal, ManifestEntry,
196+
ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Struct, TableMetadata,
198197
};
199198
use crate::table::Table;
200199
use crate::TableIdent;
@@ -264,37 +263,33 @@ mod tests {
264263
let current_partition_spec = self.table.metadata().default_partition_spec();
265264

266265
// Write data files
267-
let data_file_manifest = ManifestWriter::new(
266+
let mut writer = ManifestWriterBuilder::new(
268267
self.next_manifest_file(),
269-
current_snapshot.snapshot_id(),
268+
Some(current_snapshot.snapshot_id()),
270269
vec![],
270+
current_schema.clone(),
271+
current_partition_spec.as_ref().clone(),
271272
)
272-
.write(Manifest::new(
273-
ManifestMetadata::builder()
274-
.schema(current_schema.clone())
275-
.content(ManifestContentType::Data)
276-
.format_version(FormatVersion::V2)
277-
.partition_spec((**current_partition_spec).clone())
278-
.schema_id(current_schema.schema_id())
279-
.build(),
280-
vec![ManifestEntry::builder()
281-
.status(ManifestStatus::Added)
282-
.data_file(
283-
DataFileBuilder::default()
284-
.content(DataContentType::Data)
285-
.file_path(format!("{}/1.parquet", &self.table_location))
286-
.file_format(DataFileFormat::Parquet)
287-
.file_size_in_bytes(100)
288-
.record_count(1)
289-
.partition(Struct::from_iter([Some(Literal::long(100))]))
290-
.key_metadata(None)
291-
.build()
292-
.unwrap(),
293-
)
294-
.build()],
295-
))
296-
.await
297-
.unwrap();
273+
.build_v2_data();
274+
writer
275+
.add_entry(
276+
ManifestEntry::builder()
277+
.status(ManifestStatus::Added)
278+
.data_file(
279+
DataFileBuilder::default()
280+
.content(DataContentType::Data)
281+
.file_path(format!("{}/1.parquet", &self.table_location))
282+
.file_format(DataFileFormat::Parquet)
283+
.file_size_in_bytes(100)
284+
.record_count(1)
285+
.partition(Struct::from_iter([Some(Literal::long(100))]))
286+
.build()
287+
.unwrap(),
288+
)
289+
.build(),
290+
)
291+
.unwrap();
292+
let data_file_manifest = writer.write_manifest_file().await.unwrap();
298293

299294
// Write to manifest list
300295
let mut manifest_list_write = ManifestListWriter::v2(

‎crates/iceberg/src/scan.rs‎

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -983,9 +983,9 @@ pub mod tests {
983983
use crate::io::{FileIO, OutputFile};
984984
use crate::scan::FileScanTask;
985985
use crate::spec::{
986-
DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, Manifest,
987-
ManifestContentType, ManifestEntry, ManifestListWriter, ManifestMetadata, ManifestStatus,
988-
ManifestWriter, NestedField, PrimitiveType, Schema, Struct, TableMetadata, Type,
986+
DataContentType, DataFileBuilder, DataFileFormat, Datum, Literal, ManifestEntry,
987+
ManifestListWriter, ManifestStatus, ManifestWriterBuilder, NestedField, PrimitiveType,
988+
Schema, Struct, TableMetadata, Type,
989989
};
990990
use crate::table::Table;
991991
use crate::TableIdent;
@@ -1059,20 +1059,16 @@ pub mod tests {
10591059
let current_partition_spec = self.table.metadata().default_partition_spec();
10601060

10611061
// Write data files
1062-
let data_file_manifest = ManifestWriter::new(
1062+
let mut writer = ManifestWriterBuilder::new(
10631063
self.next_manifest_file(),
1064-
current_snapshot.snapshot_id(),
1064+
Some(current_snapshot.snapshot_id()),
10651065
vec![],
1066+
current_schema.clone(),
1067+
current_partition_spec.as_ref().clone(),
10661068
)
1067-
.write(Manifest::new(
1068-
ManifestMetadata::builder()
1069-
.schema(current_schema.clone())
1070-
.content(ManifestContentType::Data)
1071-
.format_version(FormatVersion::V2)
1072-
.partition_spec((**current_partition_spec).clone())
1073-
.schema_id(current_schema.schema_id())
1074-
.build(),
1075-
vec![
1069+
.build_v2_data();
1070+
writer
1071+
.add_entry(
10761072
ManifestEntry::builder()
10771073
.status(ManifestStatus::Added)
10781074
.data_file(
@@ -1088,6 +1084,10 @@ pub mod tests {
10881084
.unwrap(),
10891085
)
10901086
.build(),
1087+
)
1088+
.unwrap();
1089+
writer
1090+
.add_delete_entry(
10911091
ManifestEntry::builder()
10921092
.status(ManifestStatus::Deleted)
10931093
.snapshot_id(parent_snapshot.snapshot_id())
@@ -1105,6 +1105,10 @@ pub mod tests {
11051105
.unwrap(),
11061106
)
11071107
.build(),
1108+
)
1109+
.unwrap();
1110+
writer
1111+
.add_existing_entry(
11081112
ManifestEntry::builder()
11091113
.status(ManifestStatus::Existing)
11101114
.snapshot_id(parent_snapshot.snapshot_id())
@@ -1122,10 +1126,9 @@ pub mod tests {
11221126
.unwrap(),
11231127
)
11241128
.build(),
1125-
],
1126-
))
1127-
.await
1128-
.unwrap();
1129+
)
1130+
.unwrap();
1131+
let data_file_manifest = writer.write_manifest_file().await.unwrap();
11291132

11301133
// Write to manifest list
11311134
let mut manifest_list_write = ManifestListWriter::v2(

0 commit comments

Comments
 (0)