Skip to content

Commit 7f24459

Browse files
ianton-ruianton-ru
authored
Merge branch 'antalya' into feature/antalya-iceberg-table-all-storages
2 parents c3979a0 + bcf1266 commit 7f24459

20 files changed

+287
-117
lines changed

‎.github/workflows/regression.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,9 @@ name: Regression test workflow - Release
8888
env:
8989
# Force the stdout and stderr streams to be unbuffered
9090
PYTHONUNBUFFERED: 1
91-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
92-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
93-
AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
91+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
92+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
93+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
9494
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
9595
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
9696
CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }}

‎.github/workflows/release_branches.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ env:
77
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
88
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
99
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
10+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
11+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
12+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
13+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
14+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
1015

1116
on: # yamllint disable-line rule:truthy
1217
pull_request:

‎.github/workflows/reusable_build.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ env:
77
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
88
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
99
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
10+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
11+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
12+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
13+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
14+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
1015

1116
name: Build ClickHouse
1217
'on':
@@ -88,6 +93,11 @@ jobs:
8893
uses: ./.github/actions/common_setup
8994
with:
9095
job_type: build_check
96+
- name: Create source tar
97+
run: |
98+
mkdir -p "$TEMP_PATH/build_check/package_release"
99+
cd .. && tar czf $TEMP_PATH/build_source.src.tar.gz ClickHouse/
100+
cd $TEMP_PATH && tar xvzf $TEMP_PATH/build_source.src.tar.gz
91101
- name: Pre
92102
run: |
93103
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.build_name}}'

‎.github/workflows/reusable_sign.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
### For the pure soul wishes to move it to another place
2-
# https://github.com/orgs/community/discussions/9050
3-
4-
name: Testing workflow
1+
name: Sigining workflow
52
'on':
63
workflow_call:
74
inputs:
@@ -63,6 +60,11 @@ env:
6360
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
6461
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
6562
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
63+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
64+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
65+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
66+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
67+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
6668

6769
jobs:
6870
runner_labels_setup:

‎.github/workflows/reusable_test.yml‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,25 @@ name: Testing workflow
4444
description: if given, it's passed to the environments
4545
required: false
4646
AWS_SECRET_ACCESS_KEY:
47-
description: the access key to the aws param store.
47+
description: the access key to the aws s3 bucket.
4848
required: true
4949
AWS_ACCESS_KEY_ID:
50-
description: the access key id to the aws param store.
50+
description: the access key id to the aws s3 bucket.
51+
required: true
52+
CLICKHOUSE_TEST_STAT_LOGIN:
53+
description: username for ci db.
54+
required: true
55+
CLICKHOUSE_TEST_STAT_PASSWORD:
56+
description: password for ci db.
57+
required: true
58+
CLICKHOUSE_TEST_STAT_URL:
59+
description: url for ci db.
60+
required: true
61+
DOCKER_PASSWORD:
62+
description: token to upload docker images.
63+
required: true
64+
ROBOT_TOKEN:
65+
description: token to update ci status.
5166
required: true
5267

5368
env:
@@ -57,6 +72,11 @@ env:
5772
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
5873
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
5974
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
75+
CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
76+
CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
77+
CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
78+
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
79+
ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
6080

6181
jobs:
6282
runner_labels_setup:

‎.github/workflows/scheduled_runs.yml‎

Lines changed: 56 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,59 @@
1-
name: Scheduled Altinity Stable Builds
1+
# name: Scheduled Altinity Stable Builds
22

3-
on:
4-
schedule:
5-
- cron: '0 0 * * 6' #Weekly run for stable versions
6-
- cron: '0 0 * * *' #Daily run for antalya versions
7-
# Make sure that any changes to this file is actually tested with PRs
8-
pull_request:
9-
types:
10-
- synchronize
11-
- reopened
12-
- opened
13-
paths:
14-
- '**/scheduled_runs.yml'
3+
# on:
4+
# schedule:
5+
# - cron: '0 0 * * 6' #Weekly run for stable versions
6+
# - cron: '0 0 * * *' #Daily run for antalya versions
7+
# # Make sure that any changes to this file is actually tested with PRs
8+
# pull_request:
9+
# types:
10+
# - synchronize
11+
# - reopened
12+
# - opened
13+
# paths:
14+
# - '**/scheduled_runs.yml'
1515

16-
jobs:
17-
DailyRuns:
18-
strategy:
19-
fail-fast: false
20-
matrix:
21-
branch:
22-
- antalya
23-
name: ${{ matrix.branch }}
24-
if: github.event.schedule != '0 0 * * 6'
25-
runs-on: ubuntu-latest
26-
steps:
27-
- name: Run ${{ matrix.branch }} workflow
28-
run: |
29-
curl -L \
30-
-X POST \
31-
-H "Accept: application/vnd.github+json" \
32-
-H "Authorization: Bearer ${{ secrets.TOKEN }}" \
33-
-H "X-GitHub-Api-Version: 2022-11-28" \
34-
https://api.github.com/repos/Altinity/ClickHouse/actions/workflows/release_branches.yml/dispatches \
35-
-d '{"ref":"${{ matrix.branch }}"}'
16+
# jobs:
17+
# DailyRuns:
18+
# strategy:
19+
# fail-fast: false
20+
# matrix:
21+
# branch:
22+
# - antalya
23+
# name: ${{ matrix.branch }}
24+
# if: github.event.schedule != '0 0 * * 6'
25+
# runs-on: ubuntu-latest
26+
# steps:
27+
# - name: Run ${{ matrix.branch }} workflow
28+
# run: |
29+
# curl -L \
30+
# -X POST \
31+
# -H "Accept: application/vnd.github+json" \
32+
# -H "Authorization: Bearer ${{ secrets.TOKEN }}" \
33+
# -H "X-GitHub-Api-Version: 2022-11-28" \
34+
# https://api.github.com/repos/Altinity/ClickHouse/actions/workflows/release_branches.yml/dispatches \
35+
# -d '{"ref":"${{ matrix.branch }}"}'
3636

37-
WeeklyRuns:
38-
strategy:
39-
fail-fast: false
40-
matrix:
41-
branch:
42-
- customizations/22.8.21
43-
- customizations/23.3.19
44-
- customizations/23.8.16
45-
- customizations/24.3.14
46-
- customizations/24.8.11
47-
name: ${{ matrix.branch }}
48-
if: github.event.schedule != '0 0 * * *'
49-
runs-on: ubuntu-latest
50-
steps:
51-
- name: Run ${{ matrix.branch }} workflow
52-
run: |
53-
curl -L \
54-
-X POST \
55-
-H "Accept: application/vnd.github+json" \
56-
-H "Authorization: Bearer ${{ secrets.TOKEN }}" \
57-
-H "X-GitHub-Api-Version: 2022-11-28" \
58-
https://api.github.com/repos/Altinity/ClickHouse/actions/workflows/release_branches.yml/dispatches \
59-
-d '{"ref":"${{ matrix.branch }}"}'
37+
# WeeklyRuns:
38+
# strategy:
39+
# fail-fast: false
40+
# matrix:
41+
# branch:
42+
# - customizations/22.8.21
43+
# - customizations/23.3.19
44+
# - customizations/23.8.16
45+
# - customizations/24.3.14
46+
# - customizations/24.8.11
47+
# name: ${{ matrix.branch }}
48+
# if: github.event.schedule != '0 0 * * *'
49+
# runs-on: ubuntu-latest
50+
# steps:
51+
# - name: Run ${{ matrix.branch }} workflow
52+
# run: |
53+
# curl -L \
54+
# -X POST \
55+
# -H "Accept: application/vnd.github+json" \
56+
# -H "Authorization: Bearer ${{ secrets.TOKEN }}" \
57+
# -H "X-GitHub-Api-Version: 2022-11-28" \
58+
# https://api.github.com/repos/Altinity/ClickHouse/actions/workflows/release_branches.yml/dispatches \
59+
# -d '{"ref":"${{ matrix.branch }}"}'

‎docker/packager/packager‎

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@ import os
66
import subprocess
77
import sys
88
from pathlib import Path
9-
from typing import List, Optional
9+
from typing import Dict, List, Optional
1010

1111
SCRIPT_PATH = Path(__file__).absolute()
1212
IMAGE_TYPE = "binary-builder"
1313
IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}"
14-
14+
DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp'
15+
TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH))
1516

1617
class BuildException(Exception):
1718
pass
@@ -82,9 +83,22 @@ def run_docker_image_with_env(
8283
ch_root: Path,
8384
cargo_cache_dir: Path,
8485
ccache_dir: Optional[Path],
86+
aws_secrets : Optional[Dict[str,str]]
8587
) -> None:
8688
output_dir.mkdir(parents=True, exist_ok=True)
8789
cargo_cache_dir.mkdir(parents=True, exist_ok=True)
90+
extra_parts = ""
91+
92+
if aws_secrets:
93+
# Pass AWS credentials via file rather than via env to avoid leaking secrets
94+
env_part = {"AWS_CONFIG_FILE": "/home/clickhouse/.aws/credentials"}
95+
host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config'
96+
with open(host_aws_config_file_path, 'wt') as f:
97+
f.write("[default]")
98+
for key, value in aws_secrets.items():
99+
f.write(f"\n{key}={value}")
100+
101+
extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}"
88102

89103
env_part = " -e ".join(env_variables)
90104
if env_part:
@@ -107,6 +121,7 @@ def run_docker_image_with_env(
107121
cmd = (
108122
f"docker run --network=host --user={user} --rm {ccache_mount} "
109123
f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
124+
f" {extra_parts} "
110125
f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}"
111126
)
112127

@@ -130,11 +145,9 @@ def parse_env_variables(
130145
sanitizer: str,
131146
package_type: str,
132147
cache: str,
133-
s3_access_key_id: str,
134148
s3_bucket: str,
135149
s3_directory: str,
136150
s3_rw_access: bool,
137-
s3_secret_access_key: str,
138151
clang_tidy: bool,
139152
version: str,
140153
official: bool,
@@ -323,10 +336,6 @@ def parse_env_variables(
323336
result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}")
324337
if not s3_rw_access:
325338
result.append("SCCACHE_S3_NO_CREDENTIALS=true")
326-
if s3_access_key_id:
327-
result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}")
328-
if s3_secret_access_key:
329-
result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}")
330339

331340
if clang_tidy:
332341
# `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
@@ -544,11 +553,9 @@ def main() -> None:
544553
args.sanitizer,
545554
args.package_type,
546555
args.cache,
547-
args.s3_access_key_id,
548556
args.s3_bucket,
549557
args.s3_directory,
550558
args.s3_rw_access,
551-
args.s3_secret_access_key,
552559
args.clang_tidy,
553560
args.version,
554561
args.official,
@@ -567,6 +574,10 @@ def main() -> None:
567574
ch_root,
568575
args.cargo_cache_dir,
569576
args.ccache_dir,
577+
{
578+
"aws_access_key_id" : args.s3_access_key_id,
579+
"aws_secret_access_key" : args.s3_secret_access_key
580+
}
570581
)
571582
logging.info("Output placed into %s", args.output_dir)
572583

‎src/Core/FormatFactorySettings.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1251,7 +1251,7 @@ Set the quoting rule for identifiers in SHOW CREATE query
12511251
DECLARE(IdentifierQuotingStyle, show_create_query_identifier_quoting_style, IdentifierQuotingStyle::Backticks, R"(
12521252
Set the quoting style for identifiers in SHOW CREATE query
12531253
)", 0) \
1254-
DECLARE(Bool, input_format_parquet_use_metadata_cache, false, R"(Enable parquet file metadata caching)", 0) \
1254+
DECLARE(Bool, input_format_parquet_use_metadata_cache, true, R"(Enable parquet file metadata caching)", 0) \
12551255
// End of FORMAT_FACTORY_SETTINGS
12561256

12571257
#define OBSOLETE_FORMAT_SETTINGS(M, ALIAS) \

‎src/Core/SettingsChangesHistory.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@ String ClickHouseVersion::toString() const
5959
static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
6060
{
6161
{"24.12.2.20000",
62+
// Altinity Antalya modifications
6263
{
63-
{"input_format_parquet_use_metadata_cache", false, false, "New setting"}, // https://github.com/Altinity/ClickHouse/pull/586
64+
{"input_format_parquet_use_metadata_cache", true, true, "New setting, turned ON by default"}, // https://github.com/Altinity/ClickHouse/pull/586
6465
}
6566
},
6667
{"24.12",

0 commit comments

Comments
 (0)