Merge branch 'main' into aj/tests/improve-fast-test-outputs

Author: aaronsteers

.github/pr-welcome-community.md

@@ -0,0 +1,32 @@
+## 👋 Welcome to the Airbyte Python CDK!
+
+Thank you for your contribution from **{{ .repo_name }}**! We're excited to have you in the Airbyte community.
+
+### Testing This CDK Version
+
+You can test this version of the CDK using the following:
+
+```bash
+# Run the CLI from this branch:
+uvx 'git+https://github.com/airbytehq/airbyte-python-cdk.git@{{ .branch_name }}#egg=airbyte-python-cdk[dev]' --help
+
+# Update a connector to use the CDK from this branch ref:
+cd airbyte-integrations/connectors/source-example
+poe use-cdk-branch {{ .branch_name }}
+```
+
+### Helpful Resources
+
+- [Contributing Guidelines](https://docs.airbyte.com/contributing-to-airbyte/)
+- [CDK API Reference](https://airbytehq.github.io/airbyte-python-cdk/)
+
+### PR Slash Commands
+
+As needed or by request, Airbyte Maintainers can execute the following slash commands on your PR:
+
+- `/autofix` - Fixes most formatting and linting issues
+- `/poetry-lock` - Updates poetry.lock file
+- `/test` - Runs connector tests with the updated CDK
+
+If you have any questions, feel free to ask in the PR comments or join our [Slack community](https://airbytehq.slack.com/).
+

.github/pr-welcome-internal.md

@@ -0,0 +1,31 @@
+## 👋 Greetings, Airbyte Team Member!
+
+Here are some helpful tips and reminders for your convenience.
+
+### Testing This CDK Version
+
+You can test this version of the CDK using the following:
+
+```bash
+# Run the CLI from this branch:
+uvx 'git+https://github.com/airbytehq/airbyte-python-cdk.git@{{ .branch_name }}#egg=airbyte-python-cdk[dev]' --help
+
+# Update a connector to use the CDK from this branch ref:
+cd airbyte-integrations/connectors/source-example
+poe use-cdk-branch {{ .branch_name }}
+```
+
+### Helpful Resources
+
+- [CDK API Reference](https://airbytehq.github.io/airbyte-python-cdk/)
+
+### PR Slash Commands
+
+Airbyte Maintainers can execute the following slash commands on your PR:
+
+- `/autofix` - Fixes most formatting and linting issues
+- `/poetry-lock` - Updates poetry.lock file
+- `/test` - Runs connector tests with the updated CDK
+- `/poe <command>` - Runs any poe command in the CDK environment
+
+[📝 _Edit this welcome message._](https://github.com/airbytehq/airbyte-python-cdk/blob/main/.github/pr-welcome-internal.md)

.github/workflows/connector-tests.yml

@@ -77,8 +77,10 @@ jobs:
           - connector: destination-motherduck
             # For now, we mark as 'n/a' to always test this connector
             cdk_extra: n/a  # change to 'sql' to test less often
-          - connector: source-amplitude
-            cdk_extra: n/a
+          # source-amplitude failing for unrelated issue "date too far back"
+          # e.g. https://github.com/airbytehq/airbyte-python-cdk/actions/runs/16053716569/job/45302638848?pr=639
+          # - connector: source-amplitude
+          #   cdk_extra: n/a
           - connector: source-intercom
             cdk_extra: n/a
           - connector: source-pokeapi

.github/workflows/welcome-message.yml

@@ -0,0 +1,46 @@
+name: Community PR Welcome Message
+
+# This workflow automatically adds a welcome message to PRs from community contributors (forks)
+# It includes CDK usage instructions and other helpful resources for testing changes
+#
+# MANUAL TESTING INSTRUCTIONS:
+# To manually test this workflow, temporarily uncomment the "synchronize" event type as a workflow trigger.
+# Then the workflow will run for all new commits.
+#
+# Before merging, remember to again comment-out the "synchronize" clause and uncomment the `if:` condition.
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      # Toggle this line, uncommenting for testing:
+      # - synchronize
+
+jobs:
+  welcome-contributor:
+    name: PR Welcome Message
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Render template
+        id: template
+        uses: chuhlomin/[email protected]
+        with:
+          # Use a different template for internal vs forks (community)
+          template: ${{ github.event.pull_request.head.repo.fork == true && '.github/pr-welcome-community.md' || '.github/pr-welcome-internal.md' }}
+          vars: |
+            repo_name: ${{ github.event.pull_request.head.repo.full_name }}
+            branch_name: ${{ github.event.pull_request.head.ref }}
+
+      - name: Create comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: ${{ steps.template.outputs.result }}

airbyte_cdk/manifest_migrations/migrations/http_requester_request_body_json_data_to_request_body.py

@@ -54,7 +54,7 @@ def _migrate_body_json(self, manifest: ManifestType, key: str) -> None:
         if isinstance(manifest[key], str):
             self._migrate_value(manifest, key, text_type)
         elif isinstance(manifest[key], dict):
-            if manifest[key].get(query_key) is not None:
+            if isinstance(manifest[key].get(query_key), str):
                 self._migrate_value(manifest, key, graph_ql_type)
             else:
                 self._migrate_value(manifest, key, json_object_type)

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2140,30 +2140,6 @@ definitions:
           - stream_interval
           - stream_partition
           - stream_slice
-        examples:
-          - type: RequestBodyJsonObject
-            value:
-              sort_order: "ASC"
-              sort_field: "CREATED_AT"
-          - type: RequestBodyJsonObject
-            value:
-              key: "{{ config['value'] }}"
-          - type: RequestBodyJsonObject
-            value:
-              sort:
-                field: "updated_at"
-                order: "ascending"
-          - type: RequestBodyPlainText
-            value: "plain_text_body"
-          - type: RequestBodyUrlEncodedForm
-            value:
-              param1: "value1"
-              param2: "{{ config['param2_value'] }}"
-          - type: RequestBodyGraphQL
-            value:
-              query:
-                param1: "value1"
-                param2: "{{ config['param2_value'] }}"
       error_handler:
         title: Error Handler
         description: Error handler component that defines how to handle errors.
@@ -3145,28 +3121,19 @@ definitions:
       type:
         type: string
         enum: [ParentStreamConfig]
-      lazy_read_pointer:
-        title: Lazy Read Pointer
-        description: If set, this will enable lazy reading, using the initial read of parent records to extract child records.
-        type: array
-        default: []
-        items:
-          type: string
-        interpolation_context:
-          - config
+      stream:
+        title: Parent Stream
+        description: Reference to the parent stream.
+        anyOf:
+          - "$ref": "#/definitions/DeclarativeStream"
+          - "$ref": "#/definitions/StateDelegatingStream"
       parent_key:
         title: Parent Key
         description: The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.
         type: string
         examples:
           - "id"
           - "{{ config['parent_record_id'] }}"
-      stream:
-        title: Parent Stream
-        description: Reference to the parent stream.
-        anyOf:
-          - "$ref": "#/definitions/DeclarativeStream"
-          - "$ref": "#/definitions/StateDelegatingStream"
       partition_field:
         title: Current Parent Key Value Identifier
         description: While iterating over parent records during a sync, the parent_key value can be referenced by using this field.
@@ -3183,6 +3150,15 @@ definitions:
         description: Indicates whether the parent stream should be read incrementally based on updates in the child stream.
         type: boolean
         default: false
+      lazy_read_pointer:
+        title: Lazy Read Pointer
+        description: If set, this will enable lazy reading, using the initial read of parent records to extract child records.
+        type: array
+        default: []
+        items:
+          type: string
+        interpolation_context:
+          - config
       extra_fields:
         title: Extra Fields
         description: Array of field paths to include as additional fields in the stream slice. Each path is an array of strings representing keys to access fields in the respective parent record. Accessible via `stream_slice.extra_fields`. Missing fields are set to `None`.
@@ -3616,18 +3592,18 @@ definitions:
         default: false
       partition_router:
         title: Partition Router
-        description: PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.
-        default: []
+        description: Used to iteratively execute requests over a set of values, such as a parent stream's records or a list of constant values.
         anyOf:
-          - "$ref": "#/definitions/ListPartitionRouter"
           - "$ref": "#/definitions/SubstreamPartitionRouter"
+          - "$ref": "#/definitions/ListPartitionRouter"
           - "$ref": "#/definitions/GroupingPartitionRouter"
           - "$ref": "#/definitions/CustomPartitionRouter"
           - type: array
+            title: Multiple Partition Routers
             items:
               anyOf:
-                - "$ref": "#/definitions/ListPartitionRouter"
                 - "$ref": "#/definitions/SubstreamPartitionRouter"
+                - "$ref": "#/definitions/ListPartitionRouter"
                 - "$ref": "#/definitions/GroupingPartitionRouter"
                 - "$ref": "#/definitions/CustomPartitionRouter"
       $parameters:
@@ -4259,6 +4235,11 @@ definitions:
           - "$ref": "#/definitions/HttpComponentsResolver"
           - "$ref": "#/definitions/ConfigComponentsResolver"
           - "$ref": "#/definitions/ParametrizedComponentsResolver"
+      use_parent_parameters:
+        title: Use Parent Parameters
+        description: Whether or not to prioritize parent parameters over component parameters when constructing dynamic streams. Defaults to true for backward compatibility.
+        type: boolean
+        default: true
     required:
       - type
       - stream_template
@@ -4326,10 +4307,9 @@ definitions:
       - query
     properties:
       query:
-        type: object
-        additionalProperties: true
+        type: string
         description: The GraphQL query to be executed
-        default: {}
+        default: "query {\n    \n}"
     additionalProperties: true
   DpathValidator:
     title: Dpath Validator

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -4,6 +4,7 @@
 
 import builtins
 import datetime
+import re
 import typing
 from typing import Optional, Union
 from urllib.parse import quote_plus
@@ -194,6 +195,18 @@ def sanitize_url(value: str) -> str:
     return sanitization_strategy(value)
 
 
+def camel_case_to_snake_case(value: str) -> str:
+    """
+     Converts CamelCase strings to snake_case format
+
+     Usage:
+    `"{{ camel_case_to_snake_case('CamelCase') }}"`
+     :param value: string to convert from CamelCase to snake_case
+     :return: snake_case formatted string
+    """
+    return re.sub(r"(?<!^)(?=[A-Z])", "_", value).lower()
+
+
 _macros_list = [
     now_utc,
     today_utc,
@@ -206,5 +219,6 @@ def sanitize_url(value: str) -> str:
     today_with_timezone,
     str_to_datetime,
     sanitize_url,
+    camel_case_to_snake_case,
 ]
 macros = {f.__name__: f for f in _macros_list}

airbyte_cdk/sources/declarative/manifest_declarative_source.py

@@ -553,9 +553,13 @@ def _dynamic_stream_configs(
             for dynamic_stream in components_resolver.resolve_components(
                 stream_template_config=stream_template_config
             ):
+                # Get the use_parent_parameters configuration from the dynamic definition
+                # Default to True for backward compatibility, since connectors were already using it by default when this param was added
+                use_parent_parameters = dynamic_definition.get("use_parent_parameters", True)
+
                 dynamic_stream = {
                     **ManifestComponentTransformer().propagate_types_and_parameters(
-                        "", dynamic_stream, {}, use_parent_parameters=True
+                        "", dynamic_stream, {}, use_parent_parameters=use_parent_parameters
                     )
                 }