Skip to content

Commit 420b584

Browse files
MyroTkMyroTk
authored
Merge branch 'antalya' into feature/rendezvous-hashing-filesystem-cache
2 parents cdfdd26 + 643c454 commit 420b584

File tree

14 files changed

+230
-84
lines changed

14 files changed

+230
-84
lines changed

‎.github/workflows/release_branches.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ jobs:
268268
secrets: inherit
269269
with:
270270
test_name: Stateless tests (aarch64)
271-
runner_type: func-tester-aarch64
271+
runner_type: altinity-func-tester-aarch64
272272
data: ${{ needs.RunConfig.outputs.data }}
273273
FunctionalStatelessTestAsan:
274274
needs: [RunConfig, BuilderDebAsan]
@@ -411,7 +411,7 @@ jobs:
411411
uses: ./.github/workflows/regression.yml
412412
secrets: inherit
413413
with:
414-
runner_type: altinity-on-demand, altinity-type-cx52, altinity-image-x86-app-docker-ce, altinity-setup-regression
414+
runner_type: altinity-on-demand, altinity-regression-tester
415415
commit: a170f32119a5c872e5ff209b8f39e13acc2d6626
416416
arch: release
417417
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
@@ -422,7 +422,7 @@ jobs:
422422
uses: ./.github/workflows/regression.yml
423423
secrets: inherit
424424
with:
425-
runner_type: altinity-on-demand, altinity-type-cax41, altinity-image-arm-app-docker-ce, altinity-setup-regression
425+
runner_type: altinity-on-demand, altinity-regression-tester-aarch64
426426
commit: a170f32119a5c872e5ff209b8f39e13acc2d6626
427427
arch: aarch64
428428
build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

‎src/Access/Common/AccessType.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ enum class AccessType : uint8_t
181181
M(SYSTEM_DROP_SCHEMA_CACHE, "SYSTEM DROP SCHEMA CACHE, DROP SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
182182
M(SYSTEM_DROP_FORMAT_SCHEMA_CACHE, "SYSTEM DROP FORMAT SCHEMA CACHE, DROP FORMAT SCHEMA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
183183
M(SYSTEM_DROP_S3_CLIENT_CACHE, "SYSTEM DROP S3 CLIENT, DROP S3 CLIENT CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
184+
M(SYSTEM_DROP_PARQUET_METADATA_CACHE, "SYSTEM DROP PARQUET METADATA CACHE", GLOBAL, SYSTEM_DROP_CACHE) \
184185
M(SYSTEM_DROP_CACHE, "DROP CACHE", GROUP, SYSTEM) \
185186
M(SYSTEM_RELOAD_CONFIG, "RELOAD CONFIG", GLOBAL, SYSTEM_RELOAD) \
186187
M(SYSTEM_RELOAD_USERS, "RELOAD USERS", GLOBAL, SYSTEM_RELOAD) \

‎src/Interpreters/InterpreterSelectQuery.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
#include <Processors/QueryPlan/TotalsHavingStep.h>
6969
#include <Processors/QueryPlan/WindowStep.h>
7070
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
71+
#include <Processors/QueryPlan/ObjectFilterStep.h>
7172
#include <Processors/Sources/NullSource.h>
7273
#include <Processors/Sources/SourceFromSingleChunk.h>
7374
#include <Processors/Transforms/AggregatingTransform.h>
@@ -189,6 +190,7 @@ namespace Setting
189190
extern const SettingsUInt64 min_count_to_compile_aggregate_expression;
190191
extern const SettingsBool enable_software_prefetch_in_aggregation;
191192
extern const SettingsBool optimize_group_by_constant_keys;
193+
extern const SettingsBool use_hive_partitioning;
192194
}
193195

194196
namespace ServerSetting
@@ -1965,6 +1967,22 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
19651967

19661968
if (expressions.second_stage || from_aggregation_stage)
19671969
{
1970+
if (settings[Setting::use_hive_partitioning]
1971+
&& !expressions.first_stage
1972+
&& expressions.hasWhere())
1973+
{
1974+
if (typeid_cast<ReadFromCluster *>(query_plan.getRootNode()->step.get()))
1975+
{
1976+
auto object_filter_step = std::make_unique<ObjectFilterStep>(
1977+
query_plan.getCurrentHeader(),
1978+
expressions.before_where->dag.clone(),
1979+
getSelectQuery().where()->getColumnName());
1980+
1981+
object_filter_step->setStepDescription("WHERE");
1982+
query_plan.addStep(std::move(object_filter_step));
1983+
}
1984+
}
1985+
19681986
if (from_aggregation_stage)
19691987
{
19701988
/// No need to aggregate anything, since this was done on remote shards.

‎src/Interpreters/InterpreterSystemQuery.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@
7474
#include <Formats/ProtobufSchemas.h>
7575
#endif
7676

77+
#if USE_PARQUET
78+
#include <Processors/Formats/Impl/ParquetFileMetaDataCache.h>
79+
#endif
80+
7781
#if USE_AWS_S3
7882
#include <IO/S3/Client.h>
7983
#endif
@@ -415,6 +419,16 @@ BlockIO InterpreterSystemQuery::execute()
415419
break;
416420
}
417421

422+
case Type::DROP_PARQUET_METADATA_CACHE:
423+
{
424+
#if USE_PARQUET
425+
getContext()->checkAccess(AccessType::SYSTEM_DROP_PARQUET_METADATA_CACHE);
426+
ParquetFileMetaDataCache::instance()->clear();
427+
break;
428+
#else
429+
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "The server was compiled without the support for Parquet");
430+
#endif
431+
}
418432
case Type::DROP_COMPILED_EXPRESSION_CACHE:
419433
#if USE_EMBEDDED_COMPILER
420434
getContext()->checkAccess(AccessType::SYSTEM_DROP_COMPILED_EXPRESSION_CACHE);
@@ -1445,6 +1459,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
14451459
case Type::DROP_PAGE_CACHE:
14461460
case Type::DROP_SCHEMA_CACHE:
14471461
case Type::DROP_FORMAT_SCHEMA_CACHE:
1462+
case Type::DROP_PARQUET_METADATA_CACHE:
14481463
case Type::DROP_S3_CLIENT_CACHE:
14491464
{
14501465
required_access.emplace_back(AccessType::SYSTEM_DROP_CACHE);

‎src/Parsers/ASTSystemQuery.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,7 @@ void ASTSystemQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & setti
431431
case Type::DROP_SKIPPING_INDEX_CACHE:
432432
case Type::DROP_COMPILED_EXPRESSION_CACHE:
433433
case Type::DROP_S3_CLIENT_CACHE:
434+
case Type::DROP_PARQUET_METADATA_CACHE:
434435
case Type::RESET_COVERAGE:
435436
case Type::RESTART_REPLICAS:
436437
case Type::JEMALLOC_PURGE:

‎src/Parsers/ASTSystemQuery.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class ASTSystemQuery : public IAST, public ASTQueryWithOnCluster
4040
DROP_SCHEMA_CACHE,
4141
DROP_FORMAT_SCHEMA_CACHE,
4242
DROP_S3_CLIENT_CACHE,
43+
DROP_PARQUET_METADATA_CACHE,
4344
STOP_LISTEN,
4445
START_LISTEN,
4546
RESTART_REPLICAS,

‎src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,13 @@ StorageObjectStorageCluster::StorageObjectStorageCluster(
9191
metadata.setColumns(columns);
9292
metadata.setConstraints(constraints_);
9393

94-
if (sample_path.empty() && context_->getSettingsRef()[Setting::use_hive_partitioning])
94+
if (sample_path.empty()
95+
&& context_->getSettingsRef()[Setting::use_hive_partitioning]
96+
&& !configuration->withPartitionWildcard())
9597
sample_path = getPathSample(metadata, context_);
9698

97-
setInMemoryMetadata(metadata);
9899
setVirtuals(VirtualColumnUtils::getVirtualsForFileLikeStorage(metadata.columns, context_, sample_path));
100+
setInMemoryMetadata(metadata);
99101

100102
pure_storage = std::make_shared<StorageObjectStorage>(
101103
configuration,

‎tests/integration/test_checking_s3_blobs_paranoid/test.py‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,18 +52,6 @@ def cluster():
5252
with_minio=True,
5353
stay_alive=True,
5454
)
55-
cluster.add_instance(
56-
"node_with_query_log_on_s3",
57-
main_configs=[
58-
"configs/storage_conf.xml",
59-
"configs/query_log_conf.xml",
60-
],
61-
user_configs=[
62-
"configs/setting.xml",
63-
"configs/no_s3_retries.xml",
64-
],
65-
with_minio=True,
66-
)
6755
logging.info("Starting cluster...")
6856
cluster.start()
6957

‎tests/integration/test_parquet_drop_metadata_cache/__init__.py‎

Whitespace-only changes.

‎tests/integration/test_parquet_drop_metadata_cache/configs/config.d/cluster.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<clickhouse>
2+
<remote_servers>
3+
<parquet_clear_cache_cluster>
4+
<shard>
5+
<replica>
6+
<host>node1</host>
7+
<port>9000</port>
8+
</replica>
9+
<replica>
10+
<host>node2</host>
11+
<port>9000</port>
12+
</replica>
13+
<replica>
14+
<host>node3</host>
15+
<port>9000</port>
16+
</replica>
17+
</shard>
18+
</parquet_clear_cache_cluster>
19+
</remote_servers>
20+
</clickhouse>

0 commit comments

Comments
 (0)