diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index b4eef5f03..7bf5c4fa3 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -3744,6 +3744,16 @@ interpolation: - "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}" - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}" - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ', '%a, %d %b %Y %H:%M:%S %z') }}" + - title: str_to_datetime + description: Converts a string to a datetime object with UTC timezone. + arguments: + s: The string to convert. + return_type: datetime.datetime + examples: + - "{{ str_to_datetime('2022-01-14') }}" + - "{{ str_to_datetime('2022-01-01 13:45:30') }}" + - "{{ str_to_datetime('2022-01-01T13:45:30+00:00') }}" + - "{{ str_to_datetime('2022-01-01T13:45:30.123456Z') }}" filters: - title: hash description: Convert the specified value to a hashed string. diff --git a/airbyte_cdk/sources/declarative/interpolation/__init__.py b/airbyte_cdk/sources/declarative/interpolation/__init__.py index d721b99f1..84362e9ab 100644 --- a/airbyte_cdk/sources/declarative/interpolation/__init__.py +++ b/airbyte_cdk/sources/declarative/interpolation/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean diff --git a/airbyte_cdk/sources/declarative/interpolation/filters.py b/airbyte_cdk/sources/declarative/interpolation/filters.py index 52d76cab6..ffebe73da 100644 --- a/airbyte_cdk/sources/declarative/interpolation/filters.py +++ b/airbyte_cdk/sources/declarative/interpolation/filters.py @@ -1,6 +1,7 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # + import base64 import hashlib import json diff --git a/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py b/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py index 78569b350..04cc7e694 100644 --- a/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py +++ b/airbyte_cdk/sources/declarative/interpolation/interpolated_boolean.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # from dataclasses import InitVar, dataclass diff --git a/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py b/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py index 11b2dac97..b96a2a6b7 100644 --- a/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +++ b/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # diff --git a/airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py b/airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py index 82454919e..f441ba918 100644 --- a/airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py +++ b/airbyte_cdk/sources/declarative/interpolation/interpolated_nested_mapping.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # diff --git a/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py b/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py index 542fa8068..ef20a436f 100644 --- a/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py +++ b/airbyte_cdk/sources/declarative/interpolation/interpolated_string.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # from dataclasses import InitVar, dataclass diff --git a/airbyte_cdk/sources/declarative/interpolation/interpolation.py b/airbyte_cdk/sources/declarative/interpolation/interpolation.py index 5af61905e..021f96df6 100644 --- a/airbyte_cdk/sources/declarative/interpolation/interpolation.py +++ b/airbyte_cdk/sources/declarative/interpolation/interpolation.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # + from abc import ABC, abstractmethod from typing import Any, Optional diff --git a/airbyte_cdk/sources/declarative/interpolation/jinja.py b/airbyte_cdk/sources/declarative/interpolation/jinja.py index 8f8548aee..543fe9b46 100644 --- a/airbyte_cdk/sources/declarative/interpolation/jinja.py +++ b/airbyte_cdk/sources/declarative/interpolation/jinja.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # import ast diff --git a/airbyte_cdk/sources/declarative/interpolation/macros.py b/airbyte_cdk/sources/declarative/interpolation/macros.py index 1ca5b31f0..62b6904c8 100644 --- a/airbyte_cdk/sources/declarative/interpolation/macros.py +++ b/airbyte_cdk/sources/declarative/interpolation/macros.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # import builtins @@ -63,10 +63,24 @@ def timestamp(dt: Union[float, str]) -> Union[int, float]: if isinstance(dt, (int, float)): return int(dt) else: - return _str_to_datetime(dt).astimezone(pytz.utc).timestamp() + return str_to_datetime(dt).astimezone(pytz.utc).timestamp() -def _str_to_datetime(s: str) -> datetime.datetime: +def str_to_datetime(s: str) -> datetime.datetime: + """ + Converts a string to a datetime object with UTC timezone + + If the input string does not contain timezone information, UTC is assumed. + Supports both basic date strings like "2022-01-14" and datetime strings with optional timezone + like "2022-01-01T13:45:30+00:00". + + Usage: + `"{{ str_to_datetime('2022-01-14') }}"` + + :param s: string to parse as datetime + :return: datetime object in UTC timezone + """ + parsed_date = parser.isoparse(s) if not parsed_date.tzinfo: # Assume UTC if the input does not contain a timezone @@ -155,7 +169,7 @@ def format_datetime( if isinstance(dt, datetime.datetime): return dt.strftime(format) dt_datetime = ( - datetime.datetime.strptime(dt, input_format) if input_format else _str_to_datetime(dt) + datetime.datetime.strptime(dt, input_format) if input_format else str_to_datetime(dt) ) if format == "%s": return str(int(dt_datetime.timestamp())) @@ -172,5 +186,6 @@ def format_datetime( duration, format_datetime, today_with_timezone, + str_to_datetime, ] macros = {f.__name__: f for f in _macros_list} diff --git a/unit_tests/sources/declarative/interpolation/test_macros.py b/unit_tests/sources/declarative/interpolation/test_macros.py index 3fcad5d15..c531a9811 100644 --- a/unit_tests/sources/declarative/interpolation/test_macros.py +++ b/unit_tests/sources/declarative/interpolation/test_macros.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2025 Airbyte, Inc., all rights reserved. # import datetime @@ -120,3 +120,47 @@ def test_utc_datetime_to_local_timestamp_conversion(): This test ensures correct timezone handling independent of the timezone of the system on which the sync is running. """ assert macros["format_datetime"](dt="2020-10-01T00:00:00Z", format="%s") == "1601510400" + + +@pytest.mark.parametrize( + "test_name, input_value, expected_output", + [ + ( + "test_basic_date", + "2022-01-14", + datetime.datetime(2022, 1, 14, tzinfo=datetime.timezone.utc), + ), + ( + "test_datetime_with_time", + "2022-01-01 13:45:30", + datetime.datetime(2022, 1, 1, 13, 45, 30, tzinfo=datetime.timezone.utc), + ), + ( + "test_datetime_with_timezone", + "2022-01-01T13:45:30+00:00", + datetime.datetime(2022, 1, 1, 13, 45, 30, tzinfo=datetime.timezone.utc), + ), + ( + "test_datetime_with_timezone_offset", + "2022-01-01T13:45:30+05:30", + datetime.datetime(2022, 1, 1, 8, 15, 30, tzinfo=datetime.timezone.utc), + ), + ( + "test_datetime_with_microseconds", + "2022-01-01T13:45:30.123456Z", + datetime.datetime(2022, 1, 1, 13, 45, 30, 123456, tzinfo=datetime.timezone.utc), + ), + ], +) +def test_give_valid_date_str_to_datetime_returns_datetime_object( + test_name, input_value, expected_output +): + str_to_datetime_fn = macros["str_to_datetime"] + actual_output = str_to_datetime_fn(input_value) + assert actual_output == expected_output + + +def test_given_invalid_date_str_to_datetime_raises_value_error(): + str_to_datetime_fn = macros["str_to_datetime"] + with pytest.raises(ValueError): + str_to_datetime_fn("invalid-date")