Merge pull request #761 from Altinity/feature/fix_integration_tests

Feature/fix integration tests

Author: Enmk

src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h

@@ -280,6 +280,7 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
     std::string getEngineName() const override { return getImpl().getEngineName(); }
     std::string getNamespaceType() const override { return getImpl().getNamespaceType(); }
 
+    Path getFullPath() const override { return getImpl().getFullPath(); }
     Path getPath() const override { return getImpl().getPath(); }
     void setPath(const Path & path) override { getImpl().setPath(path); }
 
@@ -296,9 +297,14 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
         ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) override
         { getImpl().addStructureAndFormatToArgsIfNeeded(args, structure_, format_, context, with_structure); }
 
+    bool withPartitionWildcard() const override { return getImpl().withPartitionWildcard(); }
+    bool withGlobsIgnorePartitionWildcard() const override { return getImpl().withGlobsIgnorePartitionWildcard(); }
+    bool isPathWithGlobs() const override { return getImpl().isPathWithGlobs(); }
+    bool isNamespaceWithGlobs() const override { return getImpl().isNamespaceWithGlobs(); }
     std::string getPathWithoutGlobs() const override { return getImpl().getPathWithoutGlobs(); }
 
     bool isArchive() const override { return getImpl().isArchive(); }
+    bool isPathInArchiveWithGlobs() const override { return getImpl().isPathInArchiveWithGlobs(); }
     std::string getPathInArchive() const override { return getImpl().getPathInArchive(); }
 
     void check(ContextPtr context) const override { getImpl().check(context); }
@@ -340,8 +346,19 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
     std::optional<ColumnsDescription> tryGetTableStructureFromMetadata() const override
         { return getImpl().tryGetTableStructureFromMetadata(); }
 
+    bool supportsFileIterator() const override { return getImpl().supportsFileIterator(); }
+    ObjectIterator iterate(
+        const ActionsDAG * filter_dag,
+        std::function<void(FileProgress)> callback,
+        size_t list_batch_size) override
+    {
+        return getImpl().iterate(filter_dag, callback, list_batch_size);
+    }
+
     void update(ObjectStoragePtr object_storage, ContextPtr local_context) override
         { return getImpl().update(object_storage, local_context); }
+    void updateIfRequired(ObjectStoragePtr object_storage, ContextPtr local_context) override
+        { return getImpl().updateIfRequired(object_storage, local_context); }
 
     void initialize(
         ASTs & engine_args,
@@ -366,7 +383,6 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
     void setCompressionMethod(const String & compression_method_) override { getImpl().setCompressionMethod(compression_method_); }
     void setStructure(const String & structure_) override { getImpl().setStructure(structure_); }
 
-protected:
     void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override
         { return getImpl().fromNamedCollection(collection, context); }
     void fromAST(ASTs & args, ContextPtr context, bool with_structure) override
@@ -449,6 +465,8 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
         createDynamicStorage(type);
     }
 
+    virtual void assertInitialized() const override { return getImpl().assertInitialized(); }
+
 private:
     inline StorageObjectStorage::Configuration & getImpl() const
     {

src/Storages/ObjectStorage/StorageObjectStorage.h

@@ -203,15 +203,15 @@ class StorageObjectStorage::Configuration
     virtual void addStructureAndFormatToArgsIfNeeded(
         ASTs & args, const String & structure_, const String & format_, ContextPtr context, bool with_structure) = 0;
 
-    bool withPartitionWildcard() const;
+    virtual bool withPartitionWildcard() const;
     bool withGlobs() const { return isPathWithGlobs() || isNamespaceWithGlobs(); }
-    bool withGlobsIgnorePartitionWildcard() const;
-    bool isPathWithGlobs() const;
-    bool isNamespaceWithGlobs() const;
+    virtual bool withGlobsIgnorePartitionWildcard() const;
+    virtual bool isPathWithGlobs() const;
+    virtual bool isNamespaceWithGlobs() const;
     virtual std::string getPathWithoutGlobs() const;
 
     virtual bool isArchive() const { return false; }
-    bool isPathInArchiveWithGlobs() const;
+    virtual bool isPathInArchiveWithGlobs() const;
     virtual std::string getPathInArchive() const;
 
     virtual void check(ContextPtr context) const;
@@ -256,7 +256,7 @@ class StorageObjectStorage::Configuration
     }
 
     virtual void update(ObjectStoragePtr object_storage, ContextPtr local_context);
-    void updateIfRequired(ObjectStoragePtr object_storage, ContextPtr local_context);
+    virtual void updateIfRequired(ObjectStoragePtr object_storage, ContextPtr local_context);
 
     const StorageObjectStorageSettings & getSettingsRef() const;
 
@@ -272,7 +272,7 @@ class StorageObjectStorage::Configuration
     virtual ObjectStorageType extractDynamicStorageType(ASTs & /* args */, ContextPtr /* context */, ASTPtr * /* type_arg */ = nullptr) const
     { return ObjectStorageType::None; }
 
-    void assertInitialized() const;
+    virtual void assertInitialized() const;
 
     virtual const String & getFormat() const { return format; }
     virtual const String & getCompressionMethod() const { return compression_method; }

src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp

@@ -304,7 +304,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded(
             {"s3", "s3Cluster"},
             {"azureBlobStorage", "azureBlobStorageCluster"},
             {"hdfs", "hdfsCluster"},
-            {"iceberg", "icebergS3Cluster"},
+            {"iceberg", "icebergCluster"},
             {"icebergS3", "icebergS3Cluster"},
             {"icebergAzure", "icebergAzureCluster"},
             {"icebergHDFS", "icebergHDFSCluster"},

src/Storages/ObjectStorage/StorageObjectStorageCluster.h

@@ -68,6 +68,22 @@ class StorageObjectStorageCluster : public IStorageCluster
 
     void addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const override;
 
+    std::optional<UInt64> totalRows(ContextPtr query_context) const override
+    {
+        if (pure_storage)
+            return pure_storage->totalRows(query_context);
+        configuration->update(object_storage, query_context);
+        return configuration->totalRows();
+    }
+
+    std::optional<UInt64> totalBytes(ContextPtr query_context) const override
+    {
+        if (pure_storage)
+            return pure_storage->totalBytes(query_context);
+        configuration->update(object_storage, query_context);
+        return configuration->totalBytes();
+    }
+
 private:
     void updateQueryToSendIfNeeded(
         ASTPtr & query,