Skip to content

Commit c67d4ae

Browse files
committed
clean up error handling for http_job_repository
1 parent cfc34a7 commit c67d4ae

File tree

1 file changed

+18
-9
lines changed

1 file changed

+18
-9
lines changed

airbyte_cdk/sources/declarative/requesters/http_job_repository.py

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -333,17 +333,26 @@ def _get_create_job_stream_slice(self, job: AsyncJob) -> StreamSlice:
333333

334334
def _get_download_targets(self, job: AsyncJob) -> Iterable[str]:
335335
"""Returns an iterable of strings to help target requests for downloading async jobs."""
336-
# If neither download_target_extractor nor download_target_requester are provided, return a single empty string
336+
# If neither download_target_extractor nor download_target_requester are provided, yield a single empty string
337337
# to express the need to make a single download request without any download_target value
338-
if not self.download_target_extractor and not self.download_target_requester:
339-
lazy_log(
340-
LOGGER,
341-
logging.DEBUG,
342-
lambda: "No download_target_extractor or download_target_requester provided. Using fallback behavior for single download request without download_target.",
343-
)
344-
return [""]
338+
if not self.download_target_extractor:
339+
if not self.download_target_requester:
340+
lazy_log(
341+
LOGGER,
342+
logging.DEBUG,
343+
lambda: "No download_target_extractor or download_target_requester provided. Will attempt a single download request without a `download_target`.",
344+
)
345+
yield ""
346+
return
347+
else:
348+
raise AirbyteTracedException(
349+
internal_message="Must define a `download_target_extractor` when using a `download_target_requester`.",
350+
failure_type=FailureType.config_error,
351+
)
345352

353+
# We have a download_target_extractor, use it to extract the donload_target
346354
if self.download_target_requester:
355+
# if a download_target_requester if defined, we extract from the response of a request specifically for download targets.
347356
stream_slice: StreamSlice = StreamSlice(
348357
partition={},
349358
cursor_slice={},
@@ -358,7 +367,7 @@ def _get_download_targets(self, job: AsyncJob) -> Iterable[str]:
358367
failure_type=FailureType.system_error,
359368
)
360369
else:
361-
# if no download_target_requester is provided, we extract directly from the polling response
370+
# if no download_target_requester is defined, we extract from the polling response
362371
url_response = self._polling_job_response_by_id[job.api_job_id()]
363372

364373
yield from self.download_target_extractor.extract_records(url_response) # type: ignore # we expect download_target_extractor to always return list of strings

0 commit comments

Comments
 (0)