From 489c82df71a8a8ffd837278950ea3bec124e5246 Mon Sep 17 00:00:00 2001 From: Sai Shree Pradhan Date: Thu, 15 May 2025 11:04:13 +0530 Subject: [PATCH 1/6] PECOBLR-86 Improve logging for debug level Signed-off-by: Sai Shree Pradhan --- src/databricks/sql/client.py | 3 +++ src/databricks/sql/thrift_backend.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index ea901c3ae..46d710d93 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -214,6 +214,8 @@ def read(self) -> Optional[OAuthToken]: # use_cloud_fetch # Enable use of cloud fetch to extract large query results in parallel via cloud storage + logger.debug(f"Connection.__init__(server_hostname={server_hostname}, http_path={http_path})") + if access_token: access_token_kv = {"access_token": access_token} kwargs = {**kwargs, **access_token_kv} @@ -787,6 +789,7 @@ def execute( :returns self """ + logger.debug(f"Cursor.execute(operation={operation}, parameters={parameters})") param_approach = self._determine_parameter_approach(parameters) if param_approach == ParameterApproach.NONE: diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 2e3478d77..8b9dd9e88 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -131,6 +131,8 @@ def __init__( # max_download_threads # Number of threads for handling cloud fetch downloads. Defaults to 10 + logger.debug(f"ThriftBackend.__init__(server_hostname={server_hostname}, port={port}, http_path={http_path})") + port = port or 443 if kwargs.get("_connection_uri"): uri = kwargs.get("_connection_uri") @@ -390,6 +392,8 @@ def attempt_request(attempt): # TODO: don't use exception handling for GOS polling... + logger.debug(f"ThriftBackend.attempt_request: HTTPError: {err}") + gos_name = TCLIServiceClient.GetOperationStatus.__name__ if method.__name__ == gos_name: delay_default = ( @@ -404,6 +408,7 @@ def attempt_request(attempt): else: raise err except OSError as err: + logger.debug(f"ThriftBackend.attempt_request: OSError: {err}") error = err error_message = str(err) # fmt: off @@ -434,6 +439,7 @@ def attempt_request(attempt): else: logger.warning(log_string) except Exception as err: + logger.debug(f"ThriftBackend.attempt_request: Exception: {err}") error = err retry_delay = extract_retry_delay(attempt) error_message = ThriftBackend._extract_error_message_from_headers( @@ -1074,6 +1080,7 @@ def fetch_results( return queue, resp.hasMoreRows def close_command(self, op_handle): + logger.debug(f"ThriftBackend.close_command(op_handle={op_handle})") req = ttypes.TCloseOperationReq(operationHandle=op_handle) resp = self.make_request(self._client.CloseOperation, req) return resp.status From c8a8045ecefe5a5a62c9cbef82c22eb4f7fd4eb0 Mon Sep 17 00:00:00 2001 From: Sai Shree Pradhan Date: Thu, 15 May 2025 11:10:28 +0530 Subject: [PATCH 2/6] PECOBLR-86 Improve logging for debug level Signed-off-by: Sai Shree Pradhan --- src/databricks/sql/thrift_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 8b9dd9e88..0bcb18c70 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -894,6 +894,7 @@ def execute_command( ): assert session_handle is not None + logger.debug(f"ThriftBackend.execute_command(operation={operation}, session_handle={session_handle})") spark_arrow_types = ttypes.TSparkArrowTypes( timestampAsArrow=self._use_arrow_native_timestamps, decimalAsArrow=self._use_arrow_native_decimals, From 0adfeddd9d3133c786d2f039f553948ab0e8d257 Mon Sep 17 00:00:00 2001 From: saishreeeee Date: Thu, 15 May 2025 17:08:53 +0530 Subject: [PATCH 3/6] fixed format Signed-off-by: Sai Shree Pradhan --- src/databricks/sql/client.py | 4 +++- src/databricks/sql/thrift_backend.py | 15 ++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 46d710d93..08359d775 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -214,7 +214,9 @@ def read(self) -> Optional[OAuthToken]: # use_cloud_fetch # Enable use of cloud fetch to extract large query results in parallel via cloud storage - logger.debug(f"Connection.__init__(server_hostname={server_hostname}, http_path={http_path})") + logger.debug( + f"Connection.__init__(server_hostname={server_hostname}, http_path={http_path})" + ) if access_token: access_token_kv = {"access_token": access_token} diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 0bcb18c70..6e85392ab 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -131,7 +131,9 @@ def __init__( # max_download_threads # Number of threads for handling cloud fetch downloads. Defaults to 10 - logger.debug(f"ThriftBackend.__init__(server_hostname={server_hostname}, port={port}, http_path={http_path})") + logger.debug( + f"ThriftBackend.__init__(server_hostname={server_hostname}, port={port}, http_path={http_path})" + ) port = port or 443 if kwargs.get("_connection_uri"): @@ -392,7 +394,7 @@ def attempt_request(attempt): # TODO: don't use exception handling for GOS polling... - logger.debug(f"ThriftBackend.attempt_request: HTTPError: {err}") + logger.error(f"ThriftBackend.attempt_request: HTTPError: {err}") gos_name = TCLIServiceClient.GetOperationStatus.__name__ if method.__name__ == gos_name: @@ -408,7 +410,7 @@ def attempt_request(attempt): else: raise err except OSError as err: - logger.debug(f"ThriftBackend.attempt_request: OSError: {err}") + logger.error(f"ThriftBackend.attempt_request: OSError: {err}") error = err error_message = str(err) # fmt: off @@ -439,7 +441,7 @@ def attempt_request(attempt): else: logger.warning(log_string) except Exception as err: - logger.debug(f"ThriftBackend.attempt_request: Exception: {err}") + logger.error(f"ThriftBackend.attempt_request: Exception: {err}") error = err retry_delay = extract_retry_delay(attempt) error_message = ThriftBackend._extract_error_message_from_headers( @@ -894,7 +896,10 @@ def execute_command( ): assert session_handle is not None - logger.debug(f"ThriftBackend.execute_command(operation={operation}, session_handle={session_handle})") + logger.debug( + f"ThriftBackend.execute_command(operation={operation}, session_handle={session_handle})" + ) + spark_arrow_types = ttypes.TSparkArrowTypes( timestampAsArrow=self._use_arrow_native_timestamps, decimalAsArrow=self._use_arrow_native_decimals, From 511778711b6204b02a98e39a7939e206ae3c8054 Mon Sep 17 00:00:00 2001 From: Sai Shree Pradhan Date: Thu, 15 May 2025 18:09:40 +0530 Subject: [PATCH 4/6] used lazy logging Signed-off-by: Sai Shree Pradhan --- src/databricks/sql/client.py | 8 ++++++-- src/databricks/sql/thrift_backend.py | 15 ++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 08359d775..87f3bbb90 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -215,7 +215,9 @@ def read(self) -> Optional[OAuthToken]: # Enable use of cloud fetch to extract large query results in parallel via cloud storage logger.debug( - f"Connection.__init__(server_hostname={server_hostname}, http_path={http_path})" + "Connection.__init__(server_hostname=%s, http_path=%s)", + server_hostname, + http_path, ) if access_token: @@ -791,7 +793,9 @@ def execute( :returns self """ - logger.debug(f"Cursor.execute(operation={operation}, parameters={parameters})") + logger.debug( + "Cursor.execute(operation=%s, parameters=%s)", operation, parameters + ) param_approach = self._determine_parameter_approach(parameters) if param_approach == ParameterApproach.NONE: diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 6e85392ab..f8861c677 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -132,7 +132,10 @@ def __init__( # Number of threads for handling cloud fetch downloads. Defaults to 10 logger.debug( - f"ThriftBackend.__init__(server_hostname={server_hostname}, port={port}, http_path={http_path})" + "ThriftBackend.__init__(server_hostname=%s, port=%s, http_path=%s)", + server_hostname, + port, + http_path, ) port = port or 443 @@ -394,7 +397,7 @@ def attempt_request(attempt): # TODO: don't use exception handling for GOS polling... - logger.error(f"ThriftBackend.attempt_request: HTTPError: {err}") + logger.error("ThriftBackend.attempt_request: HTTPError: %s", err) gos_name = TCLIServiceClient.GetOperationStatus.__name__ if method.__name__ == gos_name: @@ -410,7 +413,7 @@ def attempt_request(attempt): else: raise err except OSError as err: - logger.error(f"ThriftBackend.attempt_request: OSError: {err}") + logger.error("ThriftBackend.attempt_request: OSError: %s", err) error = err error_message = str(err) # fmt: off @@ -441,7 +444,7 @@ def attempt_request(attempt): else: logger.warning(log_string) except Exception as err: - logger.error(f"ThriftBackend.attempt_request: Exception: {err}") + logger.error("ThriftBackend.attempt_request: Exception: %s", err) error = err retry_delay = extract_retry_delay(attempt) error_message = ThriftBackend._extract_error_message_from_headers( @@ -897,7 +900,9 @@ def execute_command( assert session_handle is not None logger.debug( - f"ThriftBackend.execute_command(operation={operation}, session_handle={session_handle})" + "ThriftBackend.execute_command(operation=%s, session_handle=%s)", + operation, + session_handle, ) spark_arrow_types = ttypes.TSparkArrowTypes( From a1940896e6ce6680abe2431bc52edbef1abed210 Mon Sep 17 00:00:00 2001 From: Sai Shree Pradhan Date: Thu, 15 May 2025 18:27:31 +0530 Subject: [PATCH 5/6] changed debug to error logs Signed-off-by: Sai Shree Pradhan --- src/databricks/sql/thrift_backend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index f8861c677..43f800ddc 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -413,7 +413,6 @@ def attempt_request(attempt): else: raise err except OSError as err: - logger.error("ThriftBackend.attempt_request: OSError: %s", err) error = err error_message = str(err) # fmt: off From 4e52dc2eaf00ab02092f330391a0fbc7e4171944 Mon Sep 17 00:00:00 2001 From: Sai Shree Pradhan Date: Thu, 22 May 2025 23:28:19 +0530 Subject: [PATCH 6/6] used lazy logging Signed-off-by: Sai Shree Pradhan --- src/databricks/sql/thrift_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 43f800ddc..e3dc38ad5 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -1090,7 +1090,7 @@ def fetch_results( return queue, resp.hasMoreRows def close_command(self, op_handle): - logger.debug(f"ThriftBackend.close_command(op_handle={op_handle})") + logger.debug("ThriftBackend.close_command(op_handle=%s)", op_handle) req = ttypes.TCloseOperationReq(operationHandle=op_handle) resp = self.make_request(self._client.CloseOperation, req) return resp.status