From 6c9a87d92dad3b8def61c4810de7f118192d9543 Mon Sep 17 00:00:00 2001 From: Leah Cole Date: Fri, 25 Mar 2022 10:41:33 -0700 Subject: [PATCH 1/4] enable build specific projects for data science onramp Fixes #7493 Fixes #7489 --- data-science-onramp/data-ingestion/noxfile_config.py | 6 +++--- data-science-onramp/data-processing/noxfile_config.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/data-science-onramp/data-ingestion/noxfile_config.py b/data-science-onramp/data-ingestion/noxfile_config.py index 58a33dea032..5dd59678d76 100644 --- a/data-science-onramp/data-ingestion/noxfile_config.py +++ b/data-science-onramp/data-ingestion/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # There's no google-cloud-bigquery package for Python 3.9. - "ignored_versions": ["2.7", "3.6", "3.9"], + "ignored_versions": ["2.7", "3.6"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, @@ -31,8 +31,8 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": {}, diff --git a/data-science-onramp/data-processing/noxfile_config.py b/data-science-onramp/data-processing/noxfile_config.py index f3574c250c5..3d3b3bb93e8 100644 --- a/data-science-onramp/data-processing/noxfile_config.py +++ b/data-science-onramp/data-processing/noxfile_config.py @@ -23,7 +23,7 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to pyarrow compilation failure. - "ignored_versions": ["2.7", "3.6", "3.9"], + "ignored_versions": ["2.7", "3.6"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them "enforce_type_hints": False, @@ -31,8 +31,8 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + #"gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": {}, From a41c274b192db5094364c590c9143f5f0d0ade70 Mon Sep 17 00:00:00 2001 From: Leah Cole Date: Fri, 25 Mar 2022 10:47:21 -0700 Subject: [PATCH 2/4] run blacken --- data-science-onramp/data-ingestion/ingestion_test.py | 4 +++- data-science-onramp/data-ingestion/noxfile_config.py | 2 +- data-science-onramp/data-processing/noxfile_config.py | 4 ++-- data-science-onramp/data-processing/process_test.py | 4 +++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/data-science-onramp/data-ingestion/ingestion_test.py b/data-science-onramp/data-ingestion/ingestion_test.py index fdfbe7733f8..5f6545aaf37 100644 --- a/data-science-onramp/data-ingestion/ingestion_test.py +++ b/data-science-onramp/data-ingestion/ingestion_test.py @@ -93,7 +93,9 @@ def setup_and_teardown_cluster(): try: # Delete cluster operation = cluster_client.delete_cluster( - project_id=PROJECT_ID, region=CLUSTER_REGION, cluster_name=DATAPROC_CLUSTER + project_id=PROJECT_ID, + region=CLUSTER_REGION, + cluster_name=DATAPROC_CLUSTER, ) operation.result() except NotFound: diff --git a/data-science-onramp/data-ingestion/noxfile_config.py b/data-science-onramp/data-ingestion/noxfile_config.py index 5dd59678d76..b4668be1a62 100644 --- a/data-science-onramp/data-ingestion/noxfile_config.py +++ b/data-science-onramp/data-ingestion/noxfile_config.py @@ -32,7 +32,7 @@ # build specific Cloud project. You can also use your own string # to use your own Cloud project. # "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": {}, diff --git a/data-science-onramp/data-processing/noxfile_config.py b/data-science-onramp/data-processing/noxfile_config.py index 3d3b3bb93e8..b54eed03d1c 100644 --- a/data-science-onramp/data-processing/noxfile_config.py +++ b/data-science-onramp/data-processing/noxfile_config.py @@ -31,8 +31,8 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - #"gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + "gcloud_project_env": "BUILD_SPECIFIC_GCLOUD_PROJECT", # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. "envs": {}, diff --git a/data-science-onramp/data-processing/process_test.py b/data-science-onramp/data-processing/process_test.py index d6582c1effa..6ba292c9f7a 100644 --- a/data-science-onramp/data-processing/process_test.py +++ b/data-science-onramp/data-processing/process_test.py @@ -123,7 +123,9 @@ def setup_and_teardown_cluster(): try: # Delete cluster operation = cluster_client.delete_cluster( - project_id=PROJECT_ID, region=CLUSTER_REGION, cluster_name=DATAPROC_CLUSTER + project_id=PROJECT_ID, + region=CLUSTER_REGION, + cluster_name=DATAPROC_CLUSTER, ) operation.result() except NotFound: From bf435c5f4b14aa50166b7f67c5d235f3f234c66f Mon Sep 17 00:00:00 2001 From: "Leah E. Cole" <6719667+leahecole@users.noreply.github.com> Date: Fri, 25 Mar 2022 11:45:13 -0700 Subject: [PATCH 3/4] Update data-science-onramp/data-ingestion/noxfile_config.py --- data-science-onramp/data-ingestion/noxfile_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data-science-onramp/data-ingestion/noxfile_config.py b/data-science-onramp/data-ingestion/noxfile_config.py index b4668be1a62..a15bf87246d 100644 --- a/data-science-onramp/data-ingestion/noxfile_config.py +++ b/data-science-onramp/data-ingestion/noxfile_config.py @@ -22,7 +22,6 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # There's no google-cloud-bigquery package for Python 3.9. "ignored_versions": ["2.7", "3.6"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them From 338481d14b9da240ac6ca28bc0e1e304d16e2c22 Mon Sep 17 00:00:00 2001 From: "Leah E. Cole" <6719667+leahecole@users.noreply.github.com> Date: Fri, 25 Mar 2022 11:45:18 -0700 Subject: [PATCH 4/4] Update data-science-onramp/data-processing/noxfile_config.py --- data-science-onramp/data-processing/noxfile_config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data-science-onramp/data-processing/noxfile_config.py b/data-science-onramp/data-processing/noxfile_config.py index b54eed03d1c..a15bf87246d 100644 --- a/data-science-onramp/data-processing/noxfile_config.py +++ b/data-science-onramp/data-processing/noxfile_config.py @@ -22,7 +22,6 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - # Skipping for Python 3.9 due to pyarrow compilation failure. "ignored_versions": ["2.7", "3.6"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them