refine test

Author: ZENOTME

crates/e2e_test/testdata/docker-compose.yaml

@@ -24,7 +24,6 @@ services:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password
       - AWS_REGION=us-east-1
-      - CATALOG_CATOLOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog
       - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory
       - CATALOG_WAREHOUSE=s3://icebergdata/demo
       - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
@@ -37,7 +36,7 @@ services:
       - 8181
 
   minio:
-    image: minio/minio:RELEASE.2024-03-07T00-43-48Z
+    image: minio/minio:latest
     environment:
       - MINIO_ROOT_USER=admin
       - MINIO_ROOT_PASSWORD=password
@@ -50,7 +49,7 @@ services:
   mc:
     depends_on:
       - minio
-    image: minio/mc:RELEASE.2024-03-07T00-31-49Z
+    image: minio/mc:latest
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password

crates/e2e_test/tests/append_data_file_test.rs

@@ -32,6 +32,7 @@ use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation};
 use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
 use iceberg_test_utils::docker::DockerCompose;
 use iceberg_test_utils::{normalize_test_name, set_up};
+use parquet::arrow::ParquetRecordBatchStreamBuilder;
 use parquet::file::properties::WriterProperties;
 use port_scanner::scan_port_addr;
 use std::collections::HashMap;
@@ -169,6 +170,26 @@ async fn test_append_data_file() {
     data_file_writer.write(batch.clone()).await.unwrap();
     let data_file = data_file_writer.close().await.unwrap();
 
+    // check parquet file schema
+    let batch_stream_builder = ParquetRecordBatchStreamBuilder::new(
+        table
+            .file_io()
+            .new_input(data_file[0].file_path())
+            .unwrap()
+            .reader()
+            .await
+            .unwrap(),
+    )
+    .await
+    .unwrap();
+    let field_ids: Vec<i32> = batch_stream_builder
+        .parquet_schema()
+        .columns()
+        .iter()
+        .map(|col| col.self_type().get_basic_info().id())
+        .collect();
+    assert_eq!(field_ids, vec![1, 2, 3]);
+
     // commit result
     let tx = Transaction::new(&table);
     let mut append_action = tx.fast_append(None, vec![]).unwrap();

crates/iceberg/src/spec/values.rs

@@ -1147,6 +1147,11 @@ impl Struct {
             },
         )
     }
+
+    /// Return fields in the struct.
+    pub fn fields(&self) -> &[Literal] {
+        &self.fields
+    }
 }
 
 impl Index<usize> for Struct {

crates/iceberg/src/transaction.rs

@@ -358,6 +358,8 @@ impl<'a> FastAppendAction<'a> {
         writer.write(manifest).await
     }
 
+    // # TODO:
+    // Complete the summary.
     fn summary(&self) -> Summary {
         Summary {
             operation: crate::spec::Operation::Append,
@@ -750,7 +752,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_merge_snapshot_actio() {
+    async fn test_merge_snapshot_action() {
         let table = make_v2_minimal_table();
         let tx = Transaction::new(&table);