Skip to content

PECOBLR-86 improve logging on python driver #556

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+26 −0
Open

PECOBLR-86 improve logging on python driver #556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
489c82d
PECOBLR-86 Improve logging for debug level
saishreeeee May 15, 2025
c8a8045
PECOBLR-86 Improve logging for debug level
saishreeeee May 15, 2025
0adfedd
fixed format
saishreeeee May 15, 2025
5117787
used lazy logging
saishreeeee May 15, 2025
a194089
changed debug to error logs
saishreeeee May 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/databricks/sql/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,12 @@ def read(self) -> Optional[OAuthToken]:
# use_cloud_fetch
# Enable use of cloud fetch to extract large query results in parallel via cloud storage

logger.debug(
"Connection.__init__(server_hostname=%s, http_path=%s)",
server_hostname,
http_path,
)

if access_token:
access_token_kv = {"access_token": access_token}
kwargs = {**kwargs, **access_token_kv}
Expand Down Expand Up @@ -787,6 +793,9 @@ def execute(

:returns self
"""
logger.debug(
"Cursor.execute(operation=%s, parameters=%s)", operation, parameters
)

param_approach = self._determine_parameter_approach(parameters)
if param_approach == ParameterApproach.NONE:
Expand Down
17 changes: 17 additions & 0 deletions src/databricks/sql/thrift_backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,13 @@ def __init__(
# max_download_threads
# Number of threads for handling cloud fetch downloads. Defaults to 10

logger.debug(
"ThriftBackend.__init__(server_hostname=%s, port=%s, http_path=%s)",
server_hostname,
port,
http_path,
)

port = port or 443
if kwargs.get("_connection_uri"):
uri = kwargs.get("_connection_uri")
Expand Down Expand Up @@ -390,6 +397,8 @@ def attempt_request(attempt):

# TODO: don't use exception handling for GOS polling...

logger.error("ThriftBackend.attempt_request: HTTPError: %s", err)

gos_name = TCLIServiceClient.GetOperationStatus.__name__
if method.__name__ == gos_name:
delay_default = (
Expand Down Expand Up @@ -434,6 +443,7 @@ def attempt_request(attempt):
else:
logger.warning(log_string)
except Exception as err:
logger.error("ThriftBackend.attempt_request: Exception: %s", err)
error = err
retry_delay = extract_retry_delay(attempt)
error_message = ThriftBackend._extract_error_message_from_headers(
Expand Down Expand Up @@ -888,6 +898,12 @@ def execute_command(
):
assert session_handle is not None

logger.debug(
"ThriftBackend.execute_command(operation=%s, session_handle=%s)",
operation,
session_handle,
)

spark_arrow_types = ttypes.TSparkArrowTypes(
timestampAsArrow=self._use_arrow_native_timestamps,
decimalAsArrow=self._use_arrow_native_decimals,
Expand Down Expand Up @@ -1074,6 +1090,7 @@ def fetch_results(
return queue, resp.hasMoreRows

def close_command(self, op_handle):
logger.debug(f"ThriftBackend.close_command(op_handle={op_handle})")
req = ttypes.TCloseOperationReq(operationHandle=op_handle)
resp = self.make_request(self._client.CloseOperation, req)
return resp.status
Expand Down
Loading