Skip to content

Commit fb21cd4

Browse files
darynaishchenkodarynaishchenko
committed
added unit tests
1 parent 18f948d commit fb21cd4

File tree

1 file changed

+216
-1
lines changed

1 file changed

+216
-1
lines changed

‎unit_tests/sources/declarative/resolvers/test_http_components_resolver.py‎

Lines changed: 216 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,131 @@
197197
],
198198
}
199199

200-
200+
_MANIFEST_WITH_HTTP_COMPONENT_RESOLVER_WITH_RETRIEVER_WITH_PARENT_STREAM = {
201+
"version": "6.7.0",
202+
"type": "DeclarativeSource",
203+
"check": {"type": "CheckStream", "stream_names": ["Rates"]},
204+
"dynamic_streams": [
205+
{
206+
"type": "DynamicDeclarativeStream",
207+
"stream_template": {
208+
"type": "DeclarativeStream",
209+
"name": "",
210+
"primary_key": [],
211+
"schema_loader": {
212+
"type": "InlineSchemaLoader",
213+
"schema": {
214+
"$schema": "http://json-schema.org/schema#",
215+
"properties": {
216+
"ABC": {"type": "number"},
217+
"AED": {"type": "number"},
218+
},
219+
"type": "object",
220+
},
221+
},
222+
"retriever": {
223+
"type": "SimpleRetriever",
224+
"requester": {
225+
"type": "HttpRequester",
226+
"url_base": "https://api.test.com",
227+
"path": "",
228+
"http_method": "GET",
229+
"authenticator": {
230+
"type": "ApiKeyAuthenticator",
231+
"header": "apikey",
232+
"api_token": "{{ config['api_key'] }}",
233+
},
234+
},
235+
"record_selector": {
236+
"type": "RecordSelector",
237+
"extractor": {"type": "DpathExtractor", "field_path": []},
238+
},
239+
"paginator": {"type": "NoPagination"},
240+
},
241+
},
242+
"components_resolver": {
243+
"type": "HttpComponentsResolver",
244+
"retriever": {
245+
"type": "SimpleRetriever",
246+
"requester": {
247+
"type": "HttpRequester",
248+
"url_base": "https://api.test.com",
249+
"path": "parent/{{ stream_partition.parent_id }}/items",
250+
"http_method": "GET",
251+
"authenticator": {
252+
"type": "ApiKeyAuthenticator",
253+
"header": "apikey",
254+
"api_token": "{{ config['api_key'] }}",
255+
},
256+
},
257+
"record_selector": {
258+
"type": "RecordSelector",
259+
"extractor": {"type": "DpathExtractor", "field_path": []},
260+
},
261+
"paginator": {"type": "NoPagination"},
262+
"partition_router": {
263+
"type": "SubstreamPartitionRouter",
264+
"parent_stream_configs": [
265+
{
266+
"type": "ParentStreamConfig",
267+
"parent_key": "id",
268+
"partition_field": "parent_id",
269+
"stream": {
270+
"type": "DeclarativeStream",
271+
"name": "parent",
272+
"retriever": {
273+
"type": "SimpleRetriever",
274+
"requester": {
275+
"type": "HttpRequester",
276+
"url_base": "https://api.test.com",
277+
"path": "/parents",
278+
"http_method": "GET",
279+
"authenticator": {
280+
"type": "ApiKeyAuthenticator",
281+
"header": "apikey",
282+
"api_token": "{{ config['api_key'] }}",
283+
},
284+
},
285+
"record_selector": {
286+
"type": "RecordSelector",
287+
"extractor": {"type": "DpathExtractor", "field_path": []},
288+
},
289+
},
290+
"schema_loader": {
291+
"type": "InlineSchemaLoader",
292+
"schema": {
293+
"$schema": "http://json-schema.org/schema#",
294+
"properties": {
295+
"id": {"type": "integer"}
296+
},
297+
"type": "object",
298+
},
299+
},
300+
}
301+
}
302+
]
303+
}
304+
},
305+
"components_mapping": [
306+
{
307+
"type": "ComponentMappingDefinition",
308+
"field_path": ["name"],
309+
"value": "parent_{{stream_slice['parent_id']}}_{{components_values['name']}}",
310+
},
311+
{
312+
"type": "ComponentMappingDefinition",
313+
"field_path": [
314+
"retriever",
315+
"requester",
316+
"path",
317+
],
318+
"value": "{{ stream_slice['parent_id'] }}/{{ components_values['id'] }}",
319+
},
320+
],
321+
},
322+
}
323+
],
324+
}
201325
@pytest.mark.parametrize(
202326
"components_mapping, retriever_data, stream_template_config, expected_result",
203327
[
@@ -234,6 +358,41 @@ def test_http_components_resolver(
234358
result = list(resolver.resolve_components(stream_template_config=stream_template_config))
235359
assert result == expected_result
236360

361+
@pytest.mark.parametrize(
362+
"components_mapping, retriever_data, stream_template_config, expected_result",
363+
[
364+
(
365+
[
366+
ComponentMappingDefinition(
367+
field_path=[InterpolatedString.create("path", parameters={})],
368+
value="{{stream_slice['parent_id']}}/{{components_values['id']}}",
369+
value_type=str,
370+
parameters={},
371+
)
372+
],
373+
[{"id": "1", "field1": "data1"}, {"id": "2", "field1": "data2"}],
374+
{"path": None},
375+
[{"path": "1/1"}, {"path": "1/2"}, {"path": "2/1"}, {"path": "2/2"}],
376+
)
377+
],
378+
)
379+
def test_http_components_resolver_with_stream_slices(
380+
components_mapping, retriever_data, stream_template_config, expected_result
381+
):
382+
mock_retriever = MagicMock()
383+
mock_retriever.read_records.return_value = retriever_data
384+
mock_retriever.stream_slices.return_value = [{"parent_id": 1}, {"parent_id": 2}]
385+
config = {}
386+
387+
resolver = HttpComponentsResolver(
388+
retriever=mock_retriever,
389+
config=config,
390+
components_mapping=components_mapping,
391+
parameters={},
392+
)
393+
394+
result = list(resolver.resolve_components(stream_template_config=stream_template_config))
395+
assert result == expected_result
237396

238397
def test_dynamic_streams_read_with_http_components_resolver():
239398
expected_stream_names = ["item_1", "item_2"]
@@ -306,3 +465,59 @@ def test_duplicated_dynamic_streams_read_with_http_components_resolver():
306465
str(exc_info.value)
307466
== "Dynamic streams list contains a duplicate name: item_2. Please contact Airbyte Support."
308467
)
468+
469+
def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_stream():
470+
expected_stream_names = [
471+
"parent_1_item_1", "parent_1_item_2", "parent_2_item_1", "parent_2_item_2"
472+
]
473+
with HttpMocker() as http_mocker:
474+
http_mocker.get(
475+
HttpRequest(url="https://api.test.com/parents"),
476+
HttpResponse(
477+
body=json.dumps([{"id": 1}, {"id": 2}])
478+
),
479+
)
480+
parent_ids = [1, 2]
481+
for parent_id in parent_ids:
482+
http_mocker.get(
483+
HttpRequest(url=f"https://api.test.com/parent/{parent_id}/items"),
484+
HttpResponse(
485+
body=json.dumps(
486+
[
487+
{"id": 1, "name": "item_1"},
488+
{"id": 2, "name": "item_2"},
489+
]
490+
)
491+
),
492+
)
493+
dynamic_stream_paths = ["1/1", "2/1", "1/2", "2/2"]
494+
for dynamic_stream_path in dynamic_stream_paths:
495+
http_mocker.get(
496+
HttpRequest(url=f"https://api.test.com/{dynamic_stream_path}"),
497+
HttpResponse(
498+
body=json.dumps([{"ABC": 1, "AED": 2}])
499+
),
500+
)
501+
502+
source = ConcurrentDeclarativeSource(
503+
source_config=_MANIFEST_WITH_HTTP_COMPONENT_RESOLVER_WITH_RETRIEVER_WITH_PARENT_STREAM, config=_CONFIG, catalog=None, state=None
504+
)
505+
506+
actual_catalog = source.discover(logger=source.logger, config=_CONFIG)
507+
508+
configured_streams = [
509+
to_configured_stream(stream, primary_key=stream.source_defined_primary_key)
510+
for stream in actual_catalog.streams
511+
]
512+
configured_catalog = to_configured_catalog(configured_streams)
513+
514+
records = [
515+
message.record
516+
for message in source.read(MagicMock(), _CONFIG, configured_catalog)
517+
if message.type == Type.RECORD
518+
]
519+
520+
assert len(actual_catalog.streams) == 4
521+
assert [stream.name for stream in actual_catalog.streams] == expected_stream_names
522+
assert len(records) == 4
523+
assert [record.stream for record in records] == expected_stream_names

0 commit comments

Comments
 (0)