Skip to content

Commit 6460550

Browse files
aaronsteersaaronsteers
authored
Merge branch 'main' into aj/chore/add-ruff-lint-rules-2
2 parents 2a2f543 + 3ae6bd7 commit 6460550

File tree

10 files changed

+169
-105
lines changed

10 files changed

+169
-105
lines changed

‎.github/dependabot.yml‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,29 @@
55

66
version: 2
77
updates:
8-
- package-ecosystem: "pip"
8+
- package-ecosystem: pip
99
directory: "/" # Location of package manifests
10+
commit-message:
11+
prefix: "chore(deps): "
1012
schedule:
11-
interval: "weekly"
13+
interval: daily
14+
labels:
15+
- chore
16+
open-pull-requests-limit: 8 # default is 5
1217

13-
- package-ecosystem: "github-actions"
14-
# Workflow files stored in the default location of `.github/workflows`. (You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.)
18+
- package-ecosystem: github-actions
19+
open-pull-requests-limit: 5 # default is 5
1520
directory: "/"
21+
commit-message:
22+
prefix: "ci(deps): "
1623
schedule:
17-
interval: "weekly"
24+
interval: daily
25+
labels:
26+
- ci
27+
groups:
28+
# allow combining github-actions updates into a single PR
29+
minor-and-patch:
30+
applies-to: version-updates
31+
update-types:
32+
- patch
33+
- minor

‎.github/workflows/connector-tests.yml‎

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,15 @@
1-
name: Connectors Tests
1+
name: Test Connectors
2+
on:
3+
workflow_dispatch:
4+
pull_request:
5+
types:
6+
- opened
7+
- synchronize
8+
# TODO: Consider moving to run these only after the "PyTest (Fast)" workflow is successful.
9+
# workflow_run:
10+
# workflows: [PyTest (Fast)]
11+
# types:
12+
# - completed
213

314
concurrency:
415
# This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
@@ -11,12 +22,6 @@ concurrency:
1122
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
1223
cancel-in-progress: true
1324

14-
on:
15-
workflow_dispatch:
16-
pull_request:
17-
types:
18-
- opened
19-
- synchronize
2025
jobs:
2126
cdk_changes:
2227
name: Get Changes
@@ -51,21 +56,6 @@ jobs:
5156
vector-db-based: ${{ steps.changes.outputs.vector-db-based }}
5257
sql: ${{ steps.changes.outputs.sql }}
5358

54-
# # The Connector CI Tests is a status check emitted by airbyte-ci
55-
# # We make it pass once we have determined that there are no changes to the connectors
56-
# - name: "Skip Connectors CI tests"
57-
# if: steps.changes.outputs.src != 'true' && github.event_name == 'pull_request'
58-
# run: |
59-
# curl --request POST \
60-
# --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
61-
# --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
62-
# --header 'content-type: application/json' \
63-
# --data '{
64-
# "state": "success",
65-
# "context": "CDK Changes - Connectors Tests",
66-
# "target_url": "${{ github.event.workflow_run.html_url }}"
67-
# }' \
68-
6959
connectors_ci:
7060
needs: cdk_changes
7161
# We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
@@ -75,22 +65,21 @@ jobs:
7565
runs-on: ubuntu-latest
7666
timeout-minutes: 360 # 6 hours
7767
strategy:
78-
fail-fast: true # Save resources by aborting if one connector fails
68+
fail-fast: false
7969
matrix:
8070
include:
8171
- connector: source-shopify
8272
cdk_extra: n/a
83-
- connector: source-zendesk-support
84-
cdk_extra: n/a
73+
# Currently not passing CI (unrelated)
74+
# - connector: source-zendesk-support
75+
# cdk_extra: n/a
8576
- connector: source-s3
8677
cdk_extra: file-based
8778
- connector: destination-pinecone
8879
cdk_extra: vector-db-based
8980
- connector: destination-motherduck
9081
cdk_extra: sql
91-
if: >
92-
( github.event_name == 'pull_request' && needs.cdk_changes.outputs.src == 'true' && github.event.pull_request.head.repo.fork != true
93-
) || github.event_name == 'workflow_dispatch'
82+
9483
name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
9584
steps:
9685
- name: Abort if extra not changed (${{matrix.cdk_extra}})
@@ -102,18 +91,25 @@ jobs:
10291
exit 1
10392
continue-on-error: true
10493
# Get the monorepo so we can test the connectors
94+
- name: Checkout CDK
95+
if: steps.no_changes.outcome != 'failure'
96+
uses: actions/checkout@v4
97+
with:
98+
path: airbyte-python-cdk
10599
- name: Checkout Airbyte Monorepo
106100
uses: actions/checkout@v4
107101
if: steps.no_changes.outcome != 'failure'
108102
with:
109103
repository: airbytehq/airbyte
110104
ref: master
105+
path: airbyte
111106
- name: Test Connector
112107
if: steps.no_changes.outcome != 'failure'
113108
timeout-minutes: 90
114109
env:
115110
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
116111
run: |
112+
cd airbyte
117113
make tools.airbyte-ci-binary.install
118114
airbyte-ci connectors \
119115
--name ${{matrix.connector}} \

‎.github/workflows/pypi_publish.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Build and/or Publish
1+
name: Python Packaging
22

33
on:
44
push:
@@ -7,6 +7,7 @@ on:
77

88
jobs:
99
build:
10+
name: Build
1011
runs-on: ubuntu-latest
1112
steps:
1213
- uses: actions/checkout@v4
@@ -22,15 +23,15 @@ jobs:
2223
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
2324
contents: write # Needed to upload artifacts to the release
2425
environment:
25-
name: PyPi
26-
url: https://pypi.org/p/airbyte
27-
if: startsWith(github.ref, 'refs/tags/')
26+
name: PyPI
27+
url: "https://pypi.org/p/airbyte-cdk"
28+
if: startsWith(github.ref, 'refs/tags/v')
2829
steps:
2930
- uses: actions/download-artifact@v4
3031
with:
3132
name: Packages
3233
path: dist
33-
- name: Upload wheel to release
34+
- name: Attach Wheel to GitHub Release
3435
uses: svenstaro/upload-release-action@v2
3536
with:
3637
repo_token: ${{ secrets.GITHUB_TOKEN }}
@@ -39,5 +40,8 @@ jobs:
3940
overwrite: true
4041
file_glob: true
4142

42-
- name: Publish
43+
- name: Publish to PyPI (${{vars.PYPI_PUBLISH_URL}})
4344
uses: pypa/[email protected]
45+
with:
46+
# Can be toggled at the repository level between `https://upload.pypi.org/legacy/` and `https://test.pypi.org/legacy/`
47+
repository-url: ${{vars.PYPI_PUBLISH_URL}}

‎.github/workflows/pytest_fast.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Pytest (Fast)
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'airbyte_cdk/**'
9+
- 'poetry.lock'
10+
- 'pyproject.toml'
11+
pull_request:
12+
paths:
13+
- 'airbyte_cdk/**'
14+
- 'poetry.lock'
15+
- 'pyproject.toml'
16+
17+
jobs:
18+
pytest-fast:
19+
name: Pytest (Fast)
20+
runs-on: ubuntu-latest
21+
steps:
22+
# Common steps:
23+
- name: Checkout code
24+
uses: actions/checkout@v4
25+
- name: Set up Poetry
26+
uses: Gr1N/setup-poetry@v9
27+
with:
28+
poetry-version: "1.7.1"
29+
- name: Set up Python
30+
uses: actions/setup-python@v5
31+
with:
32+
python-version: "3.10"
33+
cache: "poetry"
34+
- name: Install dependencies
35+
run: poetry install --all-extras
36+
37+
- name: Run Pytest with Coverage (Fast Tests Only)
38+
timeout-minutes: 20
39+
run: >
40+
poetry run coverage run -m pytest
41+
--durations=5 --exitfirst
42+
-m "not slow"
43+
44+
- name: Print Coverage Report
45+
if: always()
46+
run: poetry run coverage report
47+
48+
- name: Create Coverage Artifacts
49+
if: always()
50+
run: |
51+
poetry run coverage html -d htmlcov
52+
poetry run coverage xml -o htmlcov/coverage.xml
53+
54+
- name: Upload coverage to GitHub Artifacts
55+
if: always()
56+
uses: actions/upload-artifact@v4
57+
with:
58+
name: fasttest-coverage
59+
path: htmlcov/

‎.github/workflows/python_pytest.yml‎ renamed to ‎.github/workflows/pytest_matrix.yml‎

Lines changed: 10 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -5,70 +5,23 @@
55
# 2. `pytest`: Run all tests, across multiple Python versions.
66
#
77
# Note that `pytest-fast` also skips tests that require credentials, allowing it to run on forks.
8-
name: PyTest
8+
name: PyTest Matrix
99

1010
on:
1111
push:
1212
branches:
1313
- main
14-
pull_request: {}
14+
paths:
15+
- 'airbyte_cdk/**'
16+
- 'poetry.lock'
17+
- 'pyproject.toml'
18+
pull_request:
19+
paths:
20+
- 'airbyte_cdk/**'
21+
- 'poetry.lock'
22+
- 'pyproject.toml'
1523

1624
jobs:
17-
pytest-fast:
18-
name: Pytest (Fast)
19-
runs-on: ubuntu-latest
20-
steps:
21-
# Common steps:
22-
- name: Checkout code
23-
uses: actions/checkout@v4
24-
- name: Set up Poetry
25-
uses: Gr1N/setup-poetry@v9
26-
with:
27-
poetry-version: "1.7.1"
28-
- name: Set up Python
29-
uses: actions/setup-python@v5
30-
with:
31-
python-version: "3.10"
32-
cache: "poetry"
33-
- name: Install dependencies
34-
run: poetry install --all-extras
35-
36-
- name: Run Pytest with Coverage (Fast Tests Only)
37-
timeout-minutes: 60
38-
env:
39-
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
40-
run: >
41-
poetry run coverage run -m pytest
42-
--durations=5 --exitfirst
43-
-m "not slow and not requires_creds and not linting and not flaky"
44-
45-
- name: Run Pytest with Coverage (Flaky Tests Only)
46-
timeout-minutes: 60
47-
continue-on-error: true
48-
env:
49-
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
50-
run: >
51-
poetry run coverage run -m pytest
52-
--durations=5 --exitfirst
53-
-m "flaky and not slow and not requires_creds"
54-
55-
- name: Print Coverage Report
56-
if: always()
57-
run: poetry run coverage report
58-
59-
- name: Create Coverage Artifacts
60-
if: always()
61-
run: |
62-
poetry run coverage html -d htmlcov
63-
poetry run coverage xml -o htmlcov/coverage.xml
64-
65-
- name: Upload coverage to GitHub Artifacts
66-
if: always()
67-
uses: actions/upload-artifact@v4
68-
with:
69-
name: fasttest-coverage
70-
path: htmlcov/
71-
7225
pytest:
7326
name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})
7427
# Don't run on forks. Run on pushes to main, and on PRs that are not from forks.

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# Release Notes
2+
3+
Newer updates can be found here: [GitHub Release Notes](https://github.com/airbytehq/airbyte-python-cdk/releases)
4+
15
# Changelog
26

37
## 6.5.2

‎airbyte_cdk/__init__.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
# Imports should also be placed in `if TYPE_CHECKING` blocks if they are only used as type
4747
# hints - again, to avoid circular dependencies.
4848
# Once those issues are resolved, the below can be sorted with isort.
49-
from importlib import metadata
49+
import dunamai as _dunamai
5050

5151
from .destinations import Destination
5252
from .models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type, FailureType, AirbyteStream, AdvancedAuth, DestinationSyncMode, ConnectorSpecification, OAuthConfigSpecification, OrchestratorType, ConfiguredAirbyteStream, SyncMode, AirbyteLogMessage, Level, AirbyteRecordMessage
@@ -281,4 +281,13 @@
281281
"Source",
282282
"StreamSlice",
283283
]
284-
__version__ = metadata.version("airbyte_cdk")
284+
285+
__version__ = _dunamai.get_version(
286+
"airbyte-cdk",
287+
third_choice=_dunamai.Version.from_any_vcs,
288+
).serialize()
289+
"""Version generated by poetry dynamic versioning during publish.
290+
291+
When running in development, dunamai will calculate a new prerelease version
292+
from existing git release tag info.
293+
"""

‎airbyte_cdk/sources/declarative/manifest_declarative_source.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,10 @@ def _validate_source(self) -> None:
256256
manifest_version, "manifest"
257257
)
258258

259-
if cdk_major < manifest_major or (
259+
if cdk_version.startswith("0.0.0"):
260+
# Skipping version compatibility check on unreleased dev branch
261+
pass
262+
elif cdk_major < manifest_major or (
260263
cdk_major == manifest_major and cdk_minor < manifest_minor
261264
):
262265
raise ValidationError(

‎poetry.lock‎

Lines changed: 15 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)