25.3 Antalya port of #761 - Iceberg + tests

src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h

@@ -280,6 +280,7 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
     std::string getEngineName() const override { return getImpl().getEngineName(); }
     std::string getNamespaceType() const override { return getImpl().getNamespaceType(); }
 
+    Path getFullPath() const override { return getImpl().getFullPath(); }
     Path getPath() const override { return getImpl().getPath(); }
     void setPath(const Path & path) override { getImpl().setPath(path); }
 
@@ -296,9 +297,14 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
         ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) override
         { getImpl().addStructureAndFormatToArgsIfNeeded(args, structure_, format_, context, with_structure); }
 
+    bool withPartitionWildcard() const override { return getImpl().withPartitionWildcard(); }
+    bool withGlobsIgnorePartitionWildcard() const override { return getImpl().withGlobsIgnorePartitionWildcard(); }
+    bool isPathWithGlobs() const override { return getImpl().isPathWithGlobs(); }
+    bool isNamespaceWithGlobs() const override { return getImpl().isNamespaceWithGlobs(); }
     std::string getPathWithoutGlobs() const override { return getImpl().getPathWithoutGlobs(); }
 
     bool isArchive() const override { return getImpl().isArchive(); }
+    bool isPathInArchiveWithGlobs() const override { return getImpl().isPathInArchiveWithGlobs(); }
     std::string getPathInArchive() const override { return getImpl().getPathInArchive(); }
 
     void check(ContextPtr context) const override { getImpl().check(context); }
@@ -340,8 +346,19 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
     std::optional<ColumnsDescription> tryGetTableStructureFromMetadata() const override
         { return getImpl().tryGetTableStructureFromMetadata(); }
 
+    bool supportsFileIterator() const override { return getImpl().supportsFileIterator(); }
+    ObjectIterator iterate(
+        const ActionsDAG * filter_dag,
+        std::function<void(FileProgress)> callback,
+        size_t list_batch_size) override
+    {
+        return getImpl().iterate(filter_dag, callback, list_batch_size);
+    }
+
     void update(ObjectStoragePtr object_storage, ContextPtr local_context) override
         { return getImpl().update(object_storage, local_context); }
+    void updateIfRequired(ObjectStoragePtr object_storage, ContextPtr local_context) override
+        { return getImpl().updateIfRequired(object_storage, local_context); }
 
     void initialize(
         ASTs & engine_args,
@@ -366,7 +383,6 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
     void setCompressionMethod(const String & compression_method_) override { getImpl().setCompressionMethod(compression_method_); }
     void setStructure(const String & structure_) override { getImpl().setStructure(structure_); }
 
-protected:
     void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override
         { return getImpl().fromNamedCollection(collection, context); }
     void fromAST(ASTs & args, ContextPtr context, bool with_structure) override
@@ -449,6 +465,8 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
         createDynamicStorage(type);
     }
 
+    virtual void assertInitialized() const override { return getImpl().assertInitialized(); }
+
 private:
     inline StorageObjectStorage::Configuration & getImpl() const
     {

src/Storages/ObjectStorage/StorageObjectStorage.h

@@ -203,15 +203,15 @@ class StorageObjectStorage::Configuration
     virtual void addStructureAndFormatToArgsIfNeeded(
         ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) = 0;
 
-    bool withPartitionWildcard() const;
+    virtual bool withPartitionWildcard() const;
     bool withGlobs() const { return isPathWithGlobs() || isNamespaceWithGlobs(); }
-    bool withGlobsIgnorePartitionWildcard() const;
-    bool isPathWithGlobs() const;
-    bool isNamespaceWithGlobs() const;
+    virtual bool withGlobsIgnorePartitionWildcard() const;
+    virtual bool isPathWithGlobs() const;
+    virtual bool isNamespaceWithGlobs() const;
     virtual std::string getPathWithoutGlobs() const;
 
     virtual bool isArchive() const { return false; }
-    bool isPathInArchiveWithGlobs() const;
+    virtual bool isPathInArchiveWithGlobs() const;
     virtual std::string getPathInArchive() const;
 
     virtual void check(ContextPtr context) const;
@@ -256,7 +256,7 @@ class StorageObjectStorage::Configuration
     }
 
     virtual void update(ObjectStoragePtr object_storage, ContextPtr local_context);
-    void updateIfRequired(ObjectStoragePtr object_storage, ContextPtr local_context);
+    virtual void updateIfRequired(ObjectStoragePtr object_storage, ContextPtr local_context);
 
     const StorageObjectStorageSettings & getSettingsRef() const;
 
@@ -272,7 +272,7 @@ class StorageObjectStorage::Configuration
     virtual ObjectStorageType extractDynamicStorageType(ASTs & /* args */, ContextPtr /* context */, ASTPtr * /* type_arg */ = nullptr) const
     { return ObjectStorageType::None; }
 
-    void assertInitialized() const;
+    virtual void assertInitialized() const;
 
     virtual const String & getFormat() const { return format; }
     virtual const String & getCompressionMethod() const { return compression_method; }

src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp

@@ -304,7 +304,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded(
             {"s3", "s3Cluster"},
             {"azureBlobStorage", "azureBlobStorageCluster"},
             {"hdfs", "hdfsCluster"},
-            {"iceberg", "icebergS3Cluster"},
+            {"iceberg", "icebergCluster"},
             {"icebergS3", "icebergS3Cluster"},
             {"icebergAzure", "icebergAzureCluster"},
             {"icebergHDFS", "icebergHDFSCluster"},

src/Storages/ObjectStorage/StorageObjectStorageCluster.h

@@ -68,6 +68,22 @@ class StorageObjectStorageCluster : public IStorageCluster
 
     void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override;
 
+    std::optional<UInt64> totalRows(ContextPtr query_context) const override
+    {
+        if (pure_storage)
+            return pure_storage->totalRows(query_context);
+        configuration->update(object_storage, query_context);
+        return configuration->totalRows();
+    }
+
+    std::optional<UInt64> totalBytes(ContextPtr query_context) const override
+    {
+        if (pure_storage)
+            return pure_storage->totalBytes(query_context);
+        configuration->update(object_storage, query_context);
+        return configuration->totalBytes();
+    }
+
 private:
     void updateQueryToSendIfNeeded(
         ASTPtr & query,

tests/integration/compose/docker_compose_glue_catalog.yml

@@ -17,7 +17,7 @@ services:
     container_name: minio
     environment:
       - MINIO_ROOT_USER=minio
-      - MINIO_ROOT_PASSWORD=minio123
+      - MINIO_ROOT_PASSWORD=ClickHouse_Minio_P@ssw0rd
       - MINIO_DOMAIN=minio
     networks:
       default:
@@ -30,15 +30,15 @@ services:
   mc:
     depends_on:
       - minio
-    image: minio/mc
+    image: minio/mc:RELEASE.2025-04-16T18-13-26Z
     container_name: mc
     environment:
       - AWS_ACCESS_KEY_ID=minio
-      - AWS_SECRET_ACCESS_KEY=minio123
+      - AWS_SECRET_ACCESS_KEY=ClickHouse_Minio_P@ssw0rd
       - AWS_REGION=us-east-1
     entrypoint: >
       /bin/sh -c "
-      until (/usr/bin/mc config host add minio http://minio:9000 minio minio123) do echo '...waiting...' && sleep 1; done;
+      until (/usr/bin/mc config host add minio http://minio:9000 minio ClickHouse_Minio_P@ssw0rd) do echo '...waiting...' && sleep 1; done;
       /usr/bin/mc rm -r --force minio/warehouse;
       /usr/bin/mc mb minio/warehouse --ignore-existing;
       /usr/bin/mc policy set public minio/warehouse;

tests/integration/compose/docker_compose_iceberg_rest_catalog.yml

@@ -51,7 +51,7 @@ services:
   mc:
     depends_on:
       - minio
-    image: minio/mc
+    image: minio/mc:RELEASE.2025-04-16T18-13-26Z
     container_name: mc
     environment:
       - AWS_ACCESS_KEY_ID=minio

tests/integration/test_database_glue/test.py

@@ -30,6 +30,7 @@
 from helpers.cluster import ClickHouseCluster, ClickHouseInstance, is_arm
 from helpers.s3_tools import get_file_contents, list_s3_objects, prepare_s3_bucket
 from helpers.test_tools import TSV, csv_compare
+from helpers.config_cluster import minio_secret_key
 
 import boto3
 
@@ -86,7 +87,7 @@ def load_catalog_impl(started_cluster):
             "glue.region": "us-east-1",
             "s3.endpoint": "http://localhost:9002",
             "s3.access-key-id": "minio",
-            "s3.secret-access-key": "minio123",
+            "s3.secret-access-key": minio_secret_key,
         },)
 
 def create_table(
@@ -132,7 +133,7 @@ def create_clickhouse_glue_database(
         f"""
 DROP DATABASE IF EXISTS {name};
 SET allow_experimental_database_glue_catalog=true;
-CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}', 'minio', 'minio123')
+CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}', 'minio', '{minio_secret_key}')
 SETTINGS {",".join((k+"="+repr(v) for k, v in settings.items()))}
     """
     )
@@ -142,7 +143,7 @@ def print_objects():
     minio_client = Minio(
         f"minio:9002",
         access_key="minio",
-        secret_key="minio123",
+        secret_key=minio_secret_key,
         secure=False,
         http_client=urllib3.PoolManager(cert_reqs="CERT_NONE"),
     )