Skip to content

Commit 776e0c9

Browse files
brianjlaibrianjlai
committed
Merge branch 'main' into brian/fix_state_migration_to_use_declarative_stream_model
2 parents 26b4244 + 9a075a1 commit 776e0c9

File tree

153 files changed

+10343
-5823
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

153 files changed

+10343
-5823
lines changed

‎.github/actions/check-docker-tag/action.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: 'Check Docker Tag Exists'
2+
description: 'Check if a Docker tag exists on DockerHub to prevent overwrites'
3+
inputs:
4+
image_name:
5+
description: 'Docker image name (e.g. airbyte/source-declarative-manifest)'
6+
required: true
7+
tag:
8+
description: 'Docker tag to check'
9+
required: true
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: "Check for existing tag (${{ inputs.image_name }}:${{ inputs.tag }})"
14+
shell: bash
15+
run: |
16+
image="${{ inputs.image_name }}"
17+
tag_input="${{ inputs.tag }}"
18+
if [ -z "$image" ] || [ -z "$tag_input" ]; then
19+
echo "Error: image_name and tag are required."
20+
exit 1
21+
fi
22+
tag="${image}:${tag_input}"
23+
echo "Checking if tag '$tag' exists on DockerHub..."
24+
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
25+
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
26+
exit 1
27+
fi
28+
echo "No existing tag '$tag' found. Proceeding with publish."

‎.github/workflows/autofix-command.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,20 @@ jobs:
2727
runs-on: "${{ matrix.os }}-latest"
2828
steps:
2929
# Custom steps to fetch the PR and checkout the code:
30+
- name: Authenticate as GitHub App
31+
uses: actions/create-github-app-token@v2
32+
id: get-app-token
33+
with:
34+
owner: "airbytehq"
35+
repositories: "airbyte-python-cdk"
36+
app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }}
37+
private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }}
3038
- name: Checkout Airbyte
3139
uses: actions/checkout@v4
3240
with:
3341
# Important that this is set so that CI checks are triggered again
3442
# Without this we would be forever waiting on required checks to pass
35-
token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
43+
token: ${{ steps.get-app-token.outputs.token }}
3644

3745
- name: Checkout PR (${{ github.event.inputs.pr }})
3846
uses: dawidd6/action-checkout-pr@v1
@@ -73,9 +81,9 @@ jobs:
7381
[1]: ${{ steps.vars.outputs.run-url }}
7482
7583
- name: Set up Poetry
76-
uses: Gr1N/setup-poetry@v9
84+
uses: snok/install-poetry@v1
7785
with:
78-
poetry-version: "2.0.1"
86+
version: "2.0.1"
7987
- name: Set up Python
8088
uses: actions/setup-python@v5
8189
with:

‎.github/workflows/connector-tests.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,9 +141,9 @@ jobs:
141141
142142
- name: Set up Poetry
143143
if: steps.no_changes.outputs.status != 'cancelled'
144-
uses: Gr1N/setup-poetry@v9
144+
uses: snok/install-poetry@v1
145145
with:
146-
poetry-version: "2.0.1"
146+
version: "2.0.1"
147147

148148
- name: Get Connector Language
149149
if: steps.no_changes.outputs.status != 'cancelled'

‎.github/workflows/docker-build-check.yml‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
name: Docker Build Check
2+
permissions:
3+
contents: read
24

35
on:
46
pull_request:
57
branches:
68
- main
79

810
jobs:
9-
docker-build-check:
10-
name: SDM Docker Image Build # Renamed job to be more descriptive
11+
sdm-docker-build-check:
12+
name: SDM Docker Image Build
1113
runs-on: ubuntu-24.04
1214
steps:
1315
- name: Checkout code
@@ -42,3 +44,29 @@ jobs:
4244
push: false
4345
tags: airbyte/source-declarative-manifest:pr-${{ github.event.pull_request.number }}
4446
outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=SDM Docker image for PR ${{ github.event.pull_request.number }}
47+
48+
manifest-server-docker-build-check:
49+
name: Manifest Server Docker Image Build
50+
runs-on: ubuntu-24.04
51+
steps:
52+
- name: Checkout code
53+
uses: actions/checkout@v4
54+
with:
55+
fetch-depth: 0
56+
57+
- name: Set up QEMU for multi-platform builds
58+
uses: docker/setup-qemu-action@v3
59+
60+
- name: Set up Docker Buildx
61+
uses: docker/setup-buildx-action@v3
62+
63+
- name: Build Manifest Server Docker image for multiple platforms
64+
id: manifest-server-build
65+
uses: docker/build-push-action@v5
66+
with:
67+
context: .
68+
file: airbyte_cdk/manifest_server/Dockerfile
69+
platforms: linux/amd64,linux/arm64
70+
push: false
71+
tags: airbyte/manifest-server:pr-${{ github.event.pull_request.number }}
72+
outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=Manifest Server Docker image for PR ${{ github.event.pull_request.number }}

‎.github/workflows/pdoc_preview.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ jobs:
1414
- name: Checkout code
1515
uses: actions/checkout@v4
1616
- name: Set up Poetry
17-
uses: Gr1N/setup-poetry@v9
17+
uses: snok/install-poetry@v1
1818
with:
19-
poetry-version: "2.0.1"
19+
version: "2.0.1"
2020
- name: Set up Python
2121
uses: actions/setup-python@v5
2222
with:

‎.github/workflows/pdoc_publish.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ jobs:
3131
- name: Checkout code
3232
uses: actions/checkout@v4
3333
- name: Set up Poetry
34-
uses: Gr1N/setup-poetry@v9
34+
uses: snok/install-poetry@v1
3535
with:
36-
poetry-version: "2.0.1"
36+
version: "2.0.1"
3737
- name: Set up Python
3838
uses: actions/setup-python@v5
3939
with:

‎.github/workflows/poe-command.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,17 @@ jobs:
2020
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
2121
runs-on: ubuntu-latest
2222
steps:
23+
- name: Authenticate as GitHub App
24+
uses: actions/create-github-app-token@v2
25+
id: get-app-token
26+
with:
27+
owner: "airbytehq"
28+
repositories: "airbyte-python-cdk"
29+
app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }}
30+
private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }}
2331
- name: Run Poe Slash Command Processor
2432
uses: aaronsteers/poe-command-processor@v1
2533
with:
2634
pr: ${{ github.event.inputs.pr }}
2735
comment-id: ${{ github.event.inputs.comment-id }}
28-
github-token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
36+
github-token: ${{ steps.get-app-token.outputs.token }}

‎.github/workflows/poetry-lock-command.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,20 @@ jobs:
2424
runs-on: "${{ matrix.os }}-latest"
2525
steps:
2626
# Custom steps to fetch the PR and checkout the code:
27+
- name: Authenticate as GitHub App
28+
uses: actions/create-github-app-token@v2
29+
id: get-app-token
30+
with:
31+
owner: "airbytehq"
32+
repositories: "airbyte-python-cdk"
33+
app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }}
34+
private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }}
2735
- name: Checkout Airbyte
2836
uses: actions/checkout@v4
2937
with:
3038
# Important that this is set so that CI checks are triggered again
3139
# Without this we would be forever waiting on required checks to pass
32-
token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
40+
token: ${{ steps.get-app-token.outputs.token }}
3341

3442
- name: Checkout PR (${{ github.event.inputs.pr }})
3543
uses: dawidd6/action-checkout-pr@v1
@@ -81,9 +89,9 @@ jobs:
8189
[1]: ${{ steps.vars.outputs.run-url }}
8290
8391
- name: Set up Poetry
84-
uses: Gr1N/setup-poetry@v9
92+
uses: snok/install-poetry@v1
8593
with:
86-
poetry-version: "2.0.1"
94+
version: "2.0.1"
8795
- name: Set up Python
8896
uses: actions/setup-python@v5
8997
with:

‎.github/workflows/pypi_publish.yml‎ renamed to ‎.github/workflows/publish.yml‎

Lines changed: 107 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
# we have to also update the Trusted Publisher settings on PyPI.
77

88
name: CDK Publish
9+
permissions:
10+
contents: read
911

1012
on:
1113
push:
@@ -31,6 +33,11 @@ on:
3133
type: boolean
3234
required: true
3335
default: true
36+
publish_manifest_server:
37+
description: "Publish Manifest Server to DockerHub. If true, the workflow will publish the Manifest Server to DockerHub."
38+
type: boolean
39+
required: true
40+
default: true
3441
update_connector_builder:
3542
description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
3643
type: boolean
@@ -204,18 +211,10 @@ jobs:
204211

205212
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
206213
if: env.VERSION != ''
207-
run: |
208-
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
209-
if [ -z "$tag" ]; then
210-
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
211-
exit 1
212-
fi
213-
echo "Checking if tag '$tag' exists on DockerHub..."
214-
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
215-
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
216-
exit 1
217-
fi
218-
echo "No existing tag '$tag' found. Proceeding with publish."
214+
uses: ./.github/actions/check-docker-tag
215+
with:
216+
image_name: airbyte/source-declarative-manifest
217+
tag: ${{ env.VERSION }}
219218

220219
- name: "Build and push (sha tag: '${{ github.sha }}')"
221220
# Only run if the version is not set
@@ -250,6 +249,90 @@ jobs:
250249
tags: |
251250
airbyte/source-declarative-manifest:latest
252251
252+
publish_manifest_server:
253+
name: Publish Manifest Server to DockerHub
254+
if: >
255+
(github.event_name == 'push' &&
256+
startsWith(github.ref, 'refs/tags/v')) ||
257+
(github.event_name == 'workflow_dispatch' &&
258+
github.event.inputs.publish_manifest_server == 'true'
259+
)
260+
runs-on: ubuntu-24.04
261+
needs: [build]
262+
environment:
263+
name: DockerHub
264+
url: https://hub.docker.com/r/airbyte/manifest-server/tags
265+
env:
266+
VERSION: ${{ needs.build.outputs.VERSION }}
267+
IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
268+
269+
steps:
270+
- uses: actions/checkout@v4
271+
with:
272+
fetch-depth: 0
273+
274+
# We need to download the build artifact again because the previous job was on a different runner
275+
- name: Download Build Artifact
276+
uses: actions/download-artifact@v4
277+
with:
278+
name: Packages-${{ github.run_id }}
279+
path: dist
280+
281+
- name: Set up QEMU for multi-platform builds
282+
uses: docker/setup-qemu-action@v3
283+
284+
- name: Set up Docker Buildx
285+
uses: docker/setup-buildx-action@v3
286+
287+
- name: Login to Docker Hub
288+
uses: docker/login-action@v3
289+
with:
290+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
291+
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
292+
293+
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
294+
if: env.VERSION != ''
295+
uses: ./.github/actions/check-docker-tag
296+
with:
297+
image_name: airbyte/manifest-server
298+
tag: ${{ env.VERSION }}
299+
300+
- name: "Build and push (sha tag: '${{ github.sha }}')"
301+
# Only run if the version is not set
302+
if: env.VERSION == ''
303+
uses: docker/build-push-action@v5
304+
with:
305+
context: .
306+
file: airbyte_cdk/manifest_server/Dockerfile
307+
platforms: linux/amd64,linux/arm64
308+
push: true
309+
tags: |
310+
airbyte/manifest-server:${{ github.sha }}
311+
312+
- name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
313+
# Only run if the version is set
314+
if: env.VERSION != ''
315+
uses: docker/build-push-action@v5
316+
with:
317+
context: .
318+
file: airbyte_cdk/manifest_server/Dockerfile
319+
platforms: linux/amd64,linux/arm64
320+
push: true
321+
tags: |
322+
airbyte/manifest-server:${{ env.VERSION }}
323+
324+
- name: Build and push ('latest' tag)
325+
# Only run if version is set and IS_PRERELEASE is false
326+
if: env.VERSION != '' && env.IS_PRERELEASE == 'false'
327+
uses: docker/build-push-action@v5
328+
with:
329+
context: .
330+
file: airbyte_cdk/manifest_server/Dockerfile
331+
platforms: linux/amd64,linux/arm64
332+
push: true
333+
tags: |
334+
airbyte/manifest-server:latest
335+
253336
update-connector-builder:
254337
# Create a PR against the Builder, to update the CDK version that it uses.
255338
# In the future, Builder may use the SDM docker image instead of the Python CDK package.
@@ -275,11 +358,19 @@ jobs:
275358
- uses: actions/setup-python@v5
276359
with:
277360
python-version: "3.10"
361+
- name: Authenticate as GitHub App
362+
uses: actions/create-github-app-token@v2
363+
id: get-app-token
364+
with:
365+
owner: "airbytehq"
366+
repositories: "airbyte-platform-internal"
367+
app-id: ${{ secrets.OCTAVIA_BOT_APP_ID }}
368+
private-key: ${{ secrets.OCTAVIA_BOT_PRIVATE_KEY }}
278369
- name: Checkout Airbyte Platform Internal
279370
uses: actions/checkout@v4
280371
with:
281372
repository: airbytehq/airbyte-platform-internal
282-
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
373+
token: ${{ steps.get-app-token.outputs.token }}
283374
- name: Update Builder's CDK version to ${{ env.VERSION }}
284375
# PyPI servers aren't immediately updated so we may need to retry a few times.
285376
uses: nick-fields/retry@v3
@@ -293,6 +384,8 @@ jobs:
293384
PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
294385
sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile"
295386
sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in
387+
sed -i "s/tag: ${PREVIOUS_VERSION}/tag: ${VERSION}/g" "oss/charts/v2/airbyte/values.yaml"
388+
sed -i "s/refs\/tags\/v${PREVIOUS_VERSION}/refs\/tags\/v${VERSION}/g" "oss/airbyte-api/manifest-server-api/build.gradle.kts"
296389
echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
297390
cd oss/airbyte-connector-builder-server
298391
python -m pip install --no-cache-dir pip-tools
@@ -301,7 +394,7 @@ jobs:
301394
id: create-pull-request
302395
uses: peter-evans/create-pull-request@v7
303396
with:
304-
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
397+
token: ${{ steps.get-app-token.outputs.token }}
305398
commit-message: "chore: update CDK version following release"
306399
title: "chore: update CDK version following release"
307400
body: This is an automatically generated PR triggered by a CDK release

‎.github/workflows/pytest_fast.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ jobs:
1515
uses: actions/checkout@v4
1616

1717
- name: Set up Poetry
18-
uses: Gr1N/setup-poetry@v9
18+
uses: snok/install-poetry@v1
1919
with:
20-
poetry-version: "2.0.1"
20+
version: "2.0.1"
2121

2222
- name: Check Poetry lock file is current
2323
run: poetry check
@@ -42,9 +42,9 @@ jobs:
4242
- name: Checkout code
4343
uses: actions/checkout@v4
4444
- name: Set up Poetry
45-
uses: Gr1N/setup-poetry@v9
45+
uses: snok/install-poetry@v1
4646
with:
47-
poetry-version: "2.0.1"
47+
version: "2.0.1"
4848
- name: Set up Python
4949
uses: actions/setup-python@v5
5050
with:

0 commit comments

Comments
 (0)