From f8f155236f9f6256f834bd29f1529919a0dd8a1e Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 11:10:33 -0700 Subject: [PATCH 01/22] print bucket_name --- geos_ats_package/geos_ats/baseline_io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 26b1920b..11afeeb6 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -128,7 +128,8 @@ def collect_baselines( bucket_name: str, archive_name = os.path.join( cache_directory, short_blob_tar ) else: archive_name = os.path.join( baseline_temporary_directory, short_blob_tar ) - + + print( bucket_name ) if 'https://' in bucket_name: # Download from URL try: From d258d78382b46a98725044a5b0de65b64c921ca8 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 11:39:10 -0700 Subject: [PATCH 02/22] log name of bucket. --- geos_ats_package/geos_ats/baseline_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 11afeeb6..e43f0099 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -129,7 +129,7 @@ def collect_baselines( bucket_name: str, else: archive_name = os.path.join( baseline_temporary_directory, short_blob_tar ) - print( bucket_name ) + logger.info( f'bucket_name {bucket_name}' ) if 'https://' in bucket_name: # Download from URL try: From 3241b4bf8133e54859a27de265e6d06a4afb0b6d Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 16:01:03 -0700 Subject: [PATCH 03/22] try specifying cert. --- geos_ats_package/geos_ats/baseline_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index e43f0099..ccb45884 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -28,7 +28,7 @@ def file_download_progress( headers: dict, url: str, filename: str ): path = pathlib.Path( filename ).expanduser().resolve() path.parent.mkdir( parents=True, exist_ok=True ) - r = requests.get( url, stream=True, allow_redirects=True, headers=headers ) + r = requests.get( url, stream=True, allow_redirects=True, headers=headers, cert='/usr/local/share/ca-certificates/ADPKI-11.the-lab.llnl.gov_ADPKI-11.crt.crt') if r.status_code != 200: r.raise_for_status() raise RuntimeError( f"Request to {url} returned status code {r.status_code}" ) From e546feb129e61cdbc567a0f0e283328ca73c98a9 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 16:31:13 -0700 Subject: [PATCH 04/22] removed hardcoed cert. --- geos_ats_package/geos_ats/baseline_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index ccb45884..e43f0099 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -28,7 +28,7 @@ def file_download_progress( headers: dict, url: str, filename: str ): path = pathlib.Path( filename ).expanduser().resolve() path.parent.mkdir( parents=True, exist_ok=True ) - r = requests.get( url, stream=True, allow_redirects=True, headers=headers, cert='/usr/local/share/ca-certificates/ADPKI-11.the-lab.llnl.gov_ADPKI-11.crt.crt') + r = requests.get( url, stream=True, allow_redirects=True, headers=headers ) if r.status_code != 200: r.raise_for_status() raise RuntimeError( f"Request to {url} returned status code {r.status_code}" ) From 60311356ff894ca0563aea69a99ecaf7797497af Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 16:41:58 -0700 Subject: [PATCH 05/22] try this cert. --- geos_ats_package/geos_ats/baseline_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index e43f0099..53f0a8bc 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -28,7 +28,7 @@ def file_download_progress( headers: dict, url: str, filename: str ): path = pathlib.Path( filename ).expanduser().resolve() path.parent.mkdir( parents=True, exist_ok=True ) - r = requests.get( url, stream=True, allow_redirects=True, headers=headers ) + r = requests.get( url, stream=True, allow_redirects=True, headers=headers, cert='/usr/local/share/ca-certificates/ADPKI_LLNLROOT.crt.crt') if r.status_code != 200: r.raise_for_status() raise RuntimeError( f"Request to {url} returned status code {r.status_code}" ) From 57374ee179717592f28983e07b17dc13ea3465e7 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 17:04:00 -0700 Subject: [PATCH 06/22] try all the certificates for google cloud. --- geos_ats_package/geos_ats/baseline_io.py | 26 +++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 53f0a8bc..d8a9424c 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -28,7 +28,7 @@ def file_download_progress( headers: dict, url: str, filename: str ): path = pathlib.Path( filename ).expanduser().resolve() path.parent.mkdir( parents=True, exist_ok=True ) - r = requests.get( url, stream=True, allow_redirects=True, headers=headers, cert='/usr/local/share/ca-certificates/ADPKI_LLNLROOT.crt.crt') + r = requests.get( url, stream=True, allow_redirects=True, headers=headers ) if r.status_code != 200: r.raise_for_status() raise RuntimeError( f"Request to {url} returned status code {r.status_code}" ) @@ -140,10 +140,26 @@ def collect_baselines( bucket_name: str, else: # Download from GCP try: - client = storage.Client.create_anonymous_client() - bucket = client.bucket( bucket_name ) - blob = bucket.blob( blob_tar ) - blob.download_to_filename( archive_name ) + certs = ["/usr/local/share/ca-certificates/ADPKI_LLNLROOT.crt.crt", + "/usr/local/share/ca-certificates/DigiCertGlobalCAG2.crt.crt", + "/usr/local/share/ca-certificates/cspca.crt.crt", + "usr/local/share/ca-certificates/ADPKI-11.the-lab.llnl.gov_ADPKI-11.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-12.the-lab.llnl.gov_ADPKI-12.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-13.the-lab.llnl.gov_ADPKI-13.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-14.the-lab.llnl.gov_ADPKI-14.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-15.the-lab.llnl.gov_ADPKI-15.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-16.the-lab.llnl.gov_ADPKI-16.crt.crt"] + for cert in certs: + try: + os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert + client = storage.Client() + os.environ + client = storage.Client.create_anonymous_client() + bucket = client.bucket( bucket_name ) + blob = bucket.blob( blob_tar ) + blob.download_to_filename( archive_name ) + except Exception as e: + logger.info(f"Failed to download using {cert}: {e}") except Exception as e: logger.error( f'Failed to download baseline from GCP ({bucket_name}/{blob_tar})' ) logger.error( repr( e ) ) From 1d5972caefcf523b1709a30f6efd6fd21d70aff2 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 17:04:25 -0700 Subject: [PATCH 07/22] fix typo. --- geos_ats_package/geos_ats/baseline_io.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index d8a9424c..061e0e96 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -152,8 +152,6 @@ def collect_baselines( bucket_name: str, for cert in certs: try: os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert - client = storage.Client() - os.environ client = storage.Client.create_anonymous_client() bucket = client.bucket( bucket_name ) blob = bucket.blob( blob_tar ) From 088811d91056434b5bd6a59f9ac3e1ea653509c9 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 17:37:04 -0700 Subject: [PATCH 08/22] try adding cert to authorized session --- geos_ats_package/geos_ats/baseline_io.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 061e0e96..dd080b06 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -9,6 +9,9 @@ from functools import partial from tqdm.auto import tqdm from google.cloud import storage +from google.auth.transport.requests import AuthorizedSession +from google.auth import default + logger = logging.getLogger( 'geos_ats' ) tmpdir = tempfile.TemporaryDirectory() @@ -48,6 +51,13 @@ def file_download_progress( headers: dict, url: str, filename: str ): f.write( chunk ) +# Create an anonymous client using the custom SSL context +def create_anonymous_client_with_custom_cert(cert_path): + ssl_context = ssl.create_default_context(cafile=cert_path) + transport = storage.Client()._http # Reuse default transport + transport._session.mount('https://', storage.AuthorizedSession(ssl_context)) + return storage.Client() + def collect_baselines( bucket_name: str, blob_name: str, baseline_path: str, @@ -152,7 +162,15 @@ def collect_baselines( bucket_name: str, for cert in certs: try: os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert - client = storage.Client.create_anonymous_client() + + + # Path to your self-signed certificate + + # Create a custom SSL context + ssl_context = ssl.create_default_context(cafile=cert) + + client = create_anonymous_client_with_custom_cert(cert_path) + bucket = client.bucket( bucket_name ) blob = bucket.blob( blob_tar ) blob.download_to_filename( archive_name ) From e374d6e896315d1a21e5ae1ae524711b957d8897 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 17:43:59 -0700 Subject: [PATCH 09/22] add missing module. --- geos_ats_package/geos_ats/baseline_io.py | 1 + geos_ats_package/setup.cfg | 1 + 2 files changed, 2 insertions(+) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index dd080b06..f7c29eb7 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -6,6 +6,7 @@ import time import requests import pathlib +import ssl from functools import partial from tqdm.auto import tqdm from google.cloud import storage diff --git a/geos_ats_package/setup.cfg b/geos_ats_package/setup.cfg index 6ee5dde7..f3c9b5c2 100644 --- a/geos_ats_package/setup.cfg +++ b/geos_ats_package/setup.cfg @@ -14,6 +14,7 @@ packages = geos_ats.machines install_requires = h5py + ssl mpi4py numpy lxml From d8fbda128f8cd9d10d48353287be9bea2ec70388 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 17:55:01 -0700 Subject: [PATCH 10/22] remove ssl requirment --- geos_ats_package/setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/geos_ats_package/setup.cfg b/geos_ats_package/setup.cfg index f3c9b5c2..6ee5dde7 100644 --- a/geos_ats_package/setup.cfg +++ b/geos_ats_package/setup.cfg @@ -14,7 +14,6 @@ packages = geos_ats.machines install_requires = h5py - ssl mpi4py numpy lxml From d94f2b1b12e35516eb0fb5bf1026ef1144251178 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 18:01:11 -0700 Subject: [PATCH 11/22] fix wrong var name. --- geos_ats_package/geos_ats/baseline_io.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index f7c29eb7..76959e13 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -164,13 +164,10 @@ def collect_baselines( bucket_name: str, try: os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert - - # Path to your self-signed certificate - # Create a custom SSL context ssl_context = ssl.create_default_context(cafile=cert) - client = create_anonymous_client_with_custom_cert(cert_path) + client = create_anonymous_client_with_custom_cert(cert) bucket = client.bucket( bucket_name ) blob = bucket.blob( blob_tar ) From 0c4c13c3272360840f076df492b942372b389539 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 18:22:33 -0700 Subject: [PATCH 12/22] try to add default credentials. --- geos_ats_package/geos_ats/baseline_io.py | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 76959e13..9ad04001 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -52,12 +52,28 @@ def file_download_progress( headers: dict, url: str, filename: str ): f.write( chunk ) -# Create an anonymous client using the custom SSL context +# # Create an anonymous client using the custom SSL context +# def create_anonymous_client_with_custom_cert(cert_path): +# ssl_context = ssl.create_default_context(cafile=cert_path) +# transport = storage.Client()._http # Reuse default transport +# transport._session.mount('https://', storage.AuthorizedSession(ssl_context)) +# return storage.Client() + def create_anonymous_client_with_custom_cert(cert_path): + # Create a custom SSL context ssl_context = ssl.create_default_context(cafile=cert_path) - transport = storage.Client()._http # Reuse default transport - transport._session.mount('https://', storage.AuthorizedSession(ssl_context)) - return storage.Client() + + # Obtain default credentials + credentials, project = default() + + # Create an authorized session with the custom SSL context + authed_session = AuthorizedSession(credentials, ssl_context=ssl_context) + + # Initialize the storage client with the custom session + client = storage.Client(credentials=credentials, _http=authed_session) + + return client + def collect_baselines( bucket_name: str, blob_name: str, @@ -163,6 +179,8 @@ def collect_baselines( bucket_name: str, for cert in certs: try: os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert + + logger.info(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) # Create a custom SSL context ssl_context = ssl.create_default_context(cafile=cert) From a7fea63bfac3fff77f4f7af19fd6c8cfbd79ed44 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 18:24:07 -0700 Subject: [PATCH 13/22] make it fail --- geos_ats_package/geos_ats/baseline_io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 9ad04001..93b69f24 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -195,6 +195,7 @@ def collect_baselines( bucket_name: str, except Exception as e: logger.error( f'Failed to download baseline from GCP ({bucket_name}/{blob_tar})' ) logger.error( repr( e ) ) + raise Exception (f'did not work') if os.path.isfile( archive_name ): # Unpack new baselines From 272f04753688f8df20122f355d89b04dbd2a6505 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 18:32:15 -0700 Subject: [PATCH 14/22] remove log. --- geos_ats_package/geos_ats/baseline_io.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 93b69f24..5aed3ac8 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -179,8 +179,6 @@ def collect_baselines( bucket_name: str, for cert in certs: try: os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert - - logger.info(os.environ['GOOGLE_APPLICATION_CREDENTIALS']) # Create a custom SSL context ssl_context = ssl.create_default_context(cafile=cert) From 4df956ede976b243465d96bd311b551953a3cd34 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 18:34:45 -0700 Subject: [PATCH 15/22] remove ex --- geos_ats_package/geos_ats/baseline_io.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 5aed3ac8..05e2a4b2 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -177,19 +177,17 @@ def collect_baselines( bucket_name: str, "/usr/local/share/ca-certificates/ADPKI-15.the-lab.llnl.gov_ADPKI-15.crt.crt", "/usr/local/share/ca-certificates/ADPKI-16.the-lab.llnl.gov_ADPKI-16.crt.crt"] for cert in certs: - try: - os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert + logger.info(f"try using {cert}") + os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert - # Create a custom SSL context - ssl_context = ssl.create_default_context(cafile=cert) + # Create a custom SSL context + ssl_context = ssl.create_default_context(cafile=cert) - client = create_anonymous_client_with_custom_cert(cert) + client = create_anonymous_client_with_custom_cert(cert) - bucket = client.bucket( bucket_name ) - blob = bucket.blob( blob_tar ) - blob.download_to_filename( archive_name ) - except Exception as e: - logger.info(f"Failed to download using {cert}: {e}") + bucket = client.bucket( bucket_name ) + blob = bucket.blob( blob_tar ) + blob.download_to_filename( archive_name ) except Exception as e: logger.error( f'Failed to download baseline from GCP ({bucket_name}/{blob_tar})' ) logger.error( repr( e ) ) From fe9f2e7bf9a1e758d72438dda4ce90e1ff7477a4 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Wed, 22 May 2024 19:54:17 -0700 Subject: [PATCH 16/22] try combining them and outputting credentials. --- geos_ats_package/geos_ats/baseline_io.py | 30 ++++++++++++++++-------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 05e2a4b2..e7279837 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -176,18 +176,28 @@ def collect_baselines( bucket_name: str, "/usr/local/share/ca-certificates/ADPKI-14.the-lab.llnl.gov_ADPKI-14.crt.crt", "/usr/local/share/ca-certificates/ADPKI-15.the-lab.llnl.gov_ADPKI-15.crt.crt", "/usr/local/share/ca-certificates/ADPKI-16.the-lab.llnl.gov_ADPKI-16.crt.crt"] - for cert in certs: - logger.info(f"try using {cert}") - os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = cert - - # Create a custom SSL context - ssl_context = ssl.create_default_context(cafile=cert) - client = create_anonymous_client_with_custom_cert(cert) + combined_cert_path = "/usr/local/share/ca-certificates/combined.crt" + + with open(combined_cert_path, 'w') as outputfile: + for cert in certs: + with open(cert) as infile: + outputfile.write(infile.read()) + outputfile.write("\n") + + os.environ['GRPC_DEFAULT_SSL_ROOTS_FILE_PATH'] = combined_cert_path + + # Obtain default credentials + credentials_path = os.getenv('GOOGLE_APPLICATION_CREDENTIALS') + + # Print the environment variable + logger.info(f"GOOGLE_APPLICATION_CREDENTIALS: {credentials_path}") + + client = create_anonymous_client_with_custom_cert(combined_cert_path) - bucket = client.bucket( bucket_name ) - blob = bucket.blob( blob_tar ) - blob.download_to_filename( archive_name ) + bucket = client.bucket( bucket_name ) + blob = bucket.blob( blob_tar ) + blob.download_to_filename( archive_name ) except Exception as e: logger.error( f'Failed to download baseline from GCP ({bucket_name}/{blob_tar})' ) logger.error( repr( e ) ) From 31eb67b60beb8c843b80a1207cf3748df4f9eae2 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Thu, 23 May 2024 09:31:29 -0700 Subject: [PATCH 17/22] specif all certificates for url too. --- geos_ats_package/geos_ats/baseline_io.py | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index e7279837..49357e6b 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -32,7 +32,25 @@ def file_download_progress( headers: dict, url: str, filename: str ): path = pathlib.Path( filename ).expanduser().resolve() path.parent.mkdir( parents=True, exist_ok=True ) - r = requests.get( url, stream=True, allow_redirects=True, headers=headers ) + certs = ["/usr/local/share/ca-certificates/ADPKI_LLNLROOT.crt.crt", + "/usr/local/share/ca-certificates/DigiCertGlobalCAG2.crt.crt", + "/usr/local/share/ca-certificates/cspca.crt.crt", + "usr/local/share/ca-certificates/ADPKI-11.the-lab.llnl.gov_ADPKI-11.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-12.the-lab.llnl.gov_ADPKI-12.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-13.the-lab.llnl.gov_ADPKI-13.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-14.the-lab.llnl.gov_ADPKI-14.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-15.the-lab.llnl.gov_ADPKI-15.crt.crt", + "/usr/local/share/ca-certificates/ADPKI-16.the-lab.llnl.gov_ADPKI-16.crt.crt"] + + combined_cert_path = "/usr/local/share/ca-certificates/combined.crt" + + with open(combined_cert_path, 'w') as outputfile: + for cert in certs: + with open(cert) as infile: + outputfile.write(infile.read()) + outputfile.write("\n") + + r = requests.get( url, stream=True, allow_redirects=True, headers=headers, cert=combined_cert_path ) if r.status_code != 200: r.raise_for_status() raise RuntimeError( f"Request to {url} returned status code {r.status_code}" ) @@ -51,14 +69,6 @@ def file_download_progress( headers: dict, url: str, filename: str ): for chunk in r.iter_content( chunk_size=128 ): f.write( chunk ) - -# # Create an anonymous client using the custom SSL context -# def create_anonymous_client_with_custom_cert(cert_path): -# ssl_context = ssl.create_default_context(cafile=cert_path) -# transport = storage.Client()._http # Reuse default transport -# transport._session.mount('https://', storage.AuthorizedSession(ssl_context)) -# return storage.Client() - def create_anonymous_client_with_custom_cert(cert_path): # Create a custom SSL context ssl_context = ssl.create_default_context(cafile=cert_path) From 04ad501d567cbd0610defadde5f665c616b49961 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Thu, 23 May 2024 09:32:53 -0700 Subject: [PATCH 18/22] Specify combined cert to requests. --- geos_ats_package/geos_ats/baseline_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 49357e6b..92655b9f 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -211,7 +211,6 @@ def collect_baselines( bucket_name: str, except Exception as e: logger.error( f'Failed to download baseline from GCP ({bucket_name}/{blob_tar})' ) logger.error( repr( e ) ) - raise Exception (f'did not work') if os.path.isfile( archive_name ): # Unpack new baselines From bc3ca22e357e0937ab3aeba4358421d34ea0c856 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Thu, 23 May 2024 10:01:06 -0700 Subject: [PATCH 19/22] add some output --- geos_ats_package/geos_ats/baseline_io.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 92655b9f..f67eb778 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -44,12 +44,15 @@ def file_download_progress( headers: dict, url: str, filename: str ): combined_cert_path = "/usr/local/share/ca-certificates/combined.crt" + logger.info("file name.") + with open(combined_cert_path, 'w') as outputfile: for cert in certs: with open(cert) as infile: outputfile.write(infile.read()) outputfile.write("\n") - + + r = requests.get( url, stream=True, allow_redirects=True, headers=headers, cert=combined_cert_path ) if r.status_code != 200: r.raise_for_status() From 481a88c028d28b988bc607c3d5a2a6f94e46f220 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Fri, 24 May 2024 09:14:25 -0700 Subject: [PATCH 20/22] add some logging. --- geos_ats_package/geos_ats/baseline_io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index f67eb778..05b56c64 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -111,7 +111,7 @@ def collect_baselines( bucket_name: str, short_blob_name = os.path.basename( blob_name ) # Check to see if the baselines are already downloaded - logger.info( 'Checking for existing baseline files...' ) + logger.info( f'Checking for existing baseline files in {baseline_path}' ) if os.path.isdir( baseline_path ): if os.listdir( baseline_path ): logger.info( f'Target baseline directory already exists: {baseline_path}' ) @@ -153,6 +153,7 @@ def collect_baselines( bucket_name: str, archive_name = '' blob_tar = f'{blob_name}.tar.gz' short_blob_tar = f'{short_blob_name}.tar.gz' + logger.info( f'Checking cache directory ({cache_directory}) for existing baseline...' ) if cache_directory and not force_redownload: cache_directory = os.path.abspath( os.path.expanduser( cache_directory ) ) logger.info( f'Checking cache directory ({cache_directory}) for existing baseline...' ) From 7b0ff3f868873e18dd90c32004fb956c3da14256 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Fri, 24 May 2024 09:37:13 -0700 Subject: [PATCH 21/22] comment rm --- geos_ats_package/geos_ats/baseline_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 05b56c64..707c545c 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -144,7 +144,7 @@ def collect_baselines( bucket_name: str, raise Exception( 'Failed to parse user options for old baselines' ) logger.info( 'Deleting old baselines...' ) - shutil.rmtree( baseline_path ) + # shutil.rmtree( baseline_path ) else: os.makedirs( os.path.dirname( baseline_path ), exist_ok=True ) From b43acd47e22216cd966269fc7f987da59efc97c9 Mon Sep 17 00:00:00 2001 From: Matteo Cusini <49037133+CusiniM@users.noreply.github.com> Date: Fri, 24 May 2024 18:10:54 -0700 Subject: [PATCH 22/22] add some output. --- geos_ats_package/geos_ats/baseline_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geos_ats_package/geos_ats/baseline_io.py b/geos_ats_package/geos_ats/baseline_io.py index 707c545c..88d5370a 100644 --- a/geos_ats_package/geos_ats/baseline_io.py +++ b/geos_ats_package/geos_ats/baseline_io.py @@ -144,7 +144,7 @@ def collect_baselines( bucket_name: str, raise Exception( 'Failed to parse user options for old baselines' ) logger.info( 'Deleting old baselines...' ) - # shutil.rmtree( baseline_path ) + shutil.rmtree( baseline_path ) else: os.makedirs( os.path.dirname( baseline_path ), exist_ok=True ) @@ -153,7 +153,7 @@ def collect_baselines( bucket_name: str, archive_name = '' blob_tar = f'{blob_name}.tar.gz' short_blob_tar = f'{short_blob_name}.tar.gz' - logger.info( f'Checking cache directory ({cache_directory}) for existing baseline...' ) + logger.info( f'Checking cache directory ({cache_directory}) for existing baseline named {short_blob_name}' ) if cache_directory and not force_redownload: cache_directory = os.path.abspath( os.path.expanduser( cache_directory ) ) logger.info( f'Checking cache directory ({cache_directory}) for existing baseline...' )