Skip to content

Commit 9282174

Browse files
junhaoliaojunhaoliao
authored
Merge branch 'main' into batch-compression-tasks
2 parents 24edafc + cb2505d commit 9282174

File tree

33 files changed

+842
-115
lines changed

33 files changed

+842
-115
lines changed

‎.github/workflows/clp-lint.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ jobs:
2727
with:
2828
submodules: "recursive"
2929

30+
# Fetch all history for all branches; otherwise, `helm lint` would complain about not
31+
# finding the `origin/main` branch.
32+
fetch-depth: 0
33+
3034
- uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38"
3135
with:
3236
python-version: "3.11"
@@ -78,3 +82,7 @@ jobs:
7882
- name: "Lint .yaml files"
7983
shell: "bash"
8084
run: "task lint:check-yaml"
85+
86+
- name: "Lint Helm charts"
87+
shell: "bash"
88+
run: "task lint:check-helm"

‎components/clp-package-utils/clp_package_utils/controller.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,16 @@
1414
API_SERVER_COMPONENT_NAME,
1515
AwsAuthType,
1616
BundledService,
17+
CLP_DB_PASS_ENV_VAR_NAME,
18+
CLP_DB_ROOT_PASS_ENV_VAR_NAME,
19+
CLP_DB_ROOT_USER_ENV_VAR_NAME,
20+
CLP_DB_USER_ENV_VAR_NAME,
1721
ClpConfig,
22+
ClpDbUserType,
1823
COMPRESSION_JOBS_TABLE_NAME,
1924
COMPRESSION_SCHEDULER_COMPONENT_NAME,
2025
COMPRESSION_WORKER_COMPONENT_NAME,
26+
DatabaseEngine,
2127
DB_COMPONENT_NAME,
2228
DeploymentType,
2329
GARBAGE_COLLECTOR_COMPONENT_NAME,
@@ -150,7 +156,9 @@ def _set_up_env_for_database_bundling(self) -> EnvVarsDict:
150156
# Runtime config
151157
env_vars |= {
152158
"CLP_DB_CONTAINER_IMAGE_REF": (
153-
"mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy"
159+
"mysql:8.0.23"
160+
if self._clp_config.database.type == DatabaseEngine.MYSQL
161+
else "mariadb:10-jammy"
154162
),
155163
}
156164

@@ -175,9 +183,12 @@ def _set_up_env_for_database(self) -> EnvVarsDict:
175183
}
176184

177185
# Credentials
186+
credentials = self._clp_config.database.credentials
178187
env_vars |= {
179-
"CLP_DB_PASS": self._clp_config.database.password,
180-
"CLP_DB_USER": self._clp_config.database.username,
188+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
189+
CLP_DB_ROOT_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.ROOT].password,
190+
CLP_DB_ROOT_USER_ENV_VAR_NAME: credentials[ClpDbUserType.ROOT].username,
191+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
181192
}
182193

183194
return env_vars

‎components/clp-package-utils/clp_package_utils/general.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,12 @@ def load_config_file(
451451

452452
def generate_credentials_file(credentials_file_path: pathlib.Path):
453453
credentials = {
454-
DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)},
454+
DB_COMPONENT_NAME: {
455+
"username": "clp-user",
456+
"password": secrets.token_urlsafe(8),
457+
"root_username": "root",
458+
"root_password": secrets.token_urlsafe(8),
459+
},
455460
QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)},
456461
REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)},
457462
}

‎components/clp-package-utils/clp_package_utils/scripts/archive_manager.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
CLP_DB_USER_ENV_VAR_NAME,
1212
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1313
CLP_DEFAULT_DATASET_NAME,
14+
ClpDbUserType,
1415
StorageEngine,
1516
StorageType,
1617
)
@@ -226,9 +227,10 @@ def main(argv: list[str]) -> int:
226227
mounts.logs_dir,
227228
mounts.archives_output_dir,
228229
]
230+
credentials = clp_config.database.credentials
229231
extra_env_vars = {
230-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
231-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
232+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
233+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
232234
}
233235
container_start_cmd: list[str] = generate_container_start_cmd(
234236
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars

‎components/clp-package-utils/clp_package_utils/scripts/compress.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
CLP_DB_USER_ENV_VAR_NAME,
1212
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1313
CLP_DEFAULT_DATASET_NAME,
14+
ClpDbUserType,
1415
StorageEngine,
1516
StorageType,
1617
)
@@ -252,9 +253,10 @@ def main(argv):
252253
logger.error("No filesystem paths given for compression.")
253254
return -1
254255

256+
credentials = clp_config.database.credentials
255257
extra_env_vars = {
256-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
257-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
258+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
259+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
258260
}
259261
container_start_cmd = generate_container_start_cmd(
260262
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars

‎components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
CLP_DB_USER_ENV_VAR_NAME,
1212
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1313
CLP_DEFAULT_DATASET_NAME,
14+
ClpDbUserType,
1415
StorageEngine,
1516
StorageType,
1617
)
@@ -306,9 +307,10 @@ def main(argv):
306307
logger.error("No S3 URLs given for compression.")
307308
return -1
308309

310+
credentials = clp_config.database.credentials
309311
extra_env_vars = {
310-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
311-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
312+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
313+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
312314
}
313315
container_start_cmd = generate_container_start_cmd(
314316
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars

‎components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
CLP_DB_PASS_ENV_VAR_NAME,
1212
CLP_DB_USER_ENV_VAR_NAME,
1313
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
14+
ClpDbUserType,
1415
StorageEngine,
1516
StorageType,
1617
)
@@ -155,9 +156,10 @@ def main(argv: list[str]) -> int:
155156
if aws_mount:
156157
necessary_mounts.append(mounts.aws_config_dir)
157158

159+
credentials = clp_config.database.credentials
158160
extra_env_vars = {
159-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
160-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
161+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
162+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
161163
}
162164
container_start_cmd = generate_container_start_cmd(
163165
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars

‎components/clp-package-utils/clp_package_utils/scripts/decompress.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1212
CLP_DEFAULT_DATASET_NAME,
1313
ClpConfig,
14+
ClpDbUserType,
1415
StorageEngine,
1516
StorageType,
1617
)
@@ -132,9 +133,10 @@ def handle_extract_file_cmd(
132133
)
133134
)
134135

136+
credentials = clp_config.database.credentials
135137
extra_env_vars = {
136-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
137-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
138+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
139+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
138140
}
139141
container_start_cmd = generate_container_start_cmd(
140142
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
@@ -214,9 +216,10 @@ def handle_extract_stream_cmd(
214216
container_clp_config, clp_config, get_container_config_filename(container_name)
215217
)
216218
necessary_mounts = [mounts.logs_dir]
219+
credentials = clp_config.database.credentials
217220
extra_env_vars = {
218-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
219-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
221+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
222+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
220223
}
221224
container_start_cmd = generate_container_start_cmd(
222225
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars

‎components/clp-package-utils/clp_package_utils/scripts/native/decompress.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
CLP_DB_USER_ENV_VAR_NAME,
1414
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1515
ClpConfig,
16+
ClpDbUserType,
1617
Database,
1718
)
1819
from clp_py_utils.clp_metadata_db_utils import get_files_table_name
@@ -245,10 +246,11 @@ def handle_extract_file_cmd(
245246
"--db-table-prefix", clp_db_connection_params["table_prefix"],
246247
]
247248
# fmt: on
249+
credentials = clp_db_connection_params["credentials"]
248250
extract_env = {
249251
**os.environ,
250-
CLP_DB_USER_ENV_VAR_NAME: clp_db_connection_params["username"],
251-
CLP_DB_PASS_ENV_VAR_NAME: clp_db_connection_params["password"],
252+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
253+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
252254
}
253255

254256
files_to_extract_list_path = None

‎components/clp-package-utils/clp_package_utils/scripts/search.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
CLP_DB_USER_ENV_VAR_NAME,
1111
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1212
CLP_DEFAULT_DATASET_NAME,
13+
ClpDbUserType,
1314
StorageEngine,
1415
StorageType,
1516
)
@@ -134,9 +135,10 @@ def main(argv):
134135
container_clp_config, clp_config, get_container_config_filename(container_name)
135136
)
136137
necessary_mounts = [mounts.logs_dir]
138+
credentials = clp_config.database.credentials
137139
extra_env_vars = {
138-
CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username,
139-
CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
140+
CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password,
141+
CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username,
140142
}
141143
container_start_cmd = generate_container_start_cmd(
142144
container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars

0 commit comments

Comments
 (0)