diff --git a/pom.xml b/pom.xml index f8c4e3023..470a58a4f 100644 --- a/pom.xml +++ b/pom.xml @@ -70,7 +70,7 @@ 1.18.36 1.18.20.0 3.4.1 - 1.1.0 + 1.2.0-SNAPSHOT 2.29.40 3.1.3 3.3.1 diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java index a56c2170a..1017569f7 100644 --- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java @@ -42,6 +42,7 @@ import org.apache.parquet.io.api.Binary; import org.apache.hudi.avro.model.HoodieMetadataColumnStats; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ParquetUtils; @@ -197,7 +198,8 @@ private HudiFileStats computeColumnStatsForFile( .filter( e -> HoodieTableMetadataUtil.isColumnTypeSupported( - schemaConverter.fromInternalSchema(e.getValue().getSchema()), + HoodieSchema.fromAvroSchema( + schemaConverter.fromInternalSchema(e.getValue().getSchema())), Option.empty(), indexVersion)) .map(Map.Entry::getKey) diff --git a/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java b/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java index 28e3f9857..f3bcb2574 100644 --- a/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java +++ b/xtable-core/src/test/java/org/apache/xtable/TestAbstractHudiTable.java @@ -443,9 +443,7 @@ protected HoodieWriteConfig generateWriteConfig(Schema schema, TypedProperties k HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder() .enable(true) - // TODO: Hudi 1.1 MDT col-stats generation fails for array and map types. - // https://github.com/apache/incubator-xtable/issues/773 - .withMetadataIndexColumnStats(false) + .withMetadataIndexColumnStats(true) .withColumnStatsIndexForColumns(getColumnsFromSchema(schema)) .build(); Properties lockProperties = new Properties(); diff --git a/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java b/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java index 2696a036e..c445ed491 100644 --- a/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java +++ b/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java @@ -53,7 +53,6 @@ import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.util.HadoopOutputFile; import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -134,8 +133,6 @@ public class TestHudiFileStatsExtractor { .build(); @Test - @Disabled( - "Hudi 1.1 MDT col-stats generation fails for array and map types - https://github.com/apache/incubator-xtable/issues/773") void columnStatsWithMetadataTable(@TempDir Path tempDir) throws Exception { String tableName = GenericTable.getTableName(); String basePath; @@ -148,6 +145,7 @@ void columnStatsWithMetadataTable(@TempDir Path tempDir) throws Exception { table.insertRecords(true, records); basePath = table.getBasePath(); metaClient = table.getMetaClient(); + metaClient.reloadTableConfig(); } HoodieTableMetadata tableMetadata = new HoodieBackedTableMetadata(