Skip to content

Commit 90d4364

Browse files
bitnergadomski
bitner
and
gadomski
authored
Rust Hydration (#201)
* Update Docker with Rust tooling, split into database and pypgstac as separate images * update pypgstac to add rust hydration module * cleanup * add private json object to items and collections. add geometry, datetime, and end_datetime generated fields to collections * update changelog * update ci * update workflow for release * fix test on incremental migrations * switch from serde to pyo3 * fixes from PR review * fix: remove cargo registry This fixes a permissions error when running the test scripts as non-root. * deps: track Cargo.lock * fix: small rust cleanups --------- Co-authored-by: Pete Gadomski <[email protected]>
1 parent 45b1f96 commit 90d4364

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+6296
-514
lines changed

‎.dockerignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
venv/*
77
*/.direnv/*
88
*/.ruff_cache/*
9+
*/.pytest_cache/*
910
*/.vscode/*
1011
*/.mypy_cache/*
1112
*/.pgadmin/*
@@ -14,5 +15,5 @@ venv/*
1415
*/.github/*
1516
*/env/*
1617
Dockerfile
17-
docker-compose.yml
18+
docker compose.yml
1819
*/.devcontainer/*

‎.github/workflows/continuous-integration.yml‎

Lines changed: 212 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,224 @@ env:
1212
DOCKER_BUILDKIT: 1
1313

1414
jobs:
15-
test:
16-
name: test
15+
changes:
1716
runs-on: ubuntu-latest
17+
permissions:
18+
pull-requests: read
19+
outputs:
20+
pgdocker: ${{ steps.check.outputs.pgtag }}
21+
buildpgdocker: ${{ steps.check.outputs.buildpg }}
22+
pyrustdocker: ${{ steps.check.outputs.pytag }}
23+
buildpyrustdocker: ${{ steps.check.outputs.buildpy }}
24+
25+
steps:
26+
- uses: actions/checkout@v3
27+
- uses: dorny/paths-filter@v2
28+
id: filter
29+
with:
30+
filters: |
31+
pgstac:
32+
- 'docker/pgstac/**'
33+
pypgstac:
34+
- 'docker/pypgstac/**'
35+
- id: check
36+
run: |
37+
buildpg=false;
38+
ref=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}};
39+
[[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || ref=main;
40+
echo "pgtag=${{ env.REGISTRY }}/stac-utils/pgstac-postgres:$ref" >>$GITHUB_OUTPUT;
41+
echo "buildpg=$buildpg" >>$GITHUB_OUTPUT;
42+
buildy=false;
43+
ref=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}};
44+
[[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || ref=main;
45+
echo "pytag=${{ env.REGISTRY }}/stac-utils/pgstac-pyrust:$ref" >>$GITHUB_OUTPUT;
46+
echo "buildpy=$buildpg" >>$GITHUB_OUTPUT;
47+
48+
buildpg:
49+
name: Build and push base postgres
50+
if: ${{ needs.changes.outputs.buildpgdocker == 'true' }}
51+
runs-on: ubuntu-latest
52+
needs: [changes]
1853
steps:
1954
- uses: actions/checkout@v3
20-
- uses: docker/setup-buildx-action@v1
21-
- name: builder
22-
id: builder
23-
uses: docker/build-push-action@v2
55+
- uses: docker/setup-buildx-action@v2
56+
- name: Log in to the Container registry
57+
uses: docker/login-action@v2
58+
with:
59+
registry: ${{ env.REGISTRY }}
60+
username: ${{ github.actor }}
61+
password: ${{ secrets.GITHUB_TOKEN }}
62+
- name: Build and Push Base Postgres
63+
uses: docker/build-push-action@v4
2464
with:
2565
context: .
26-
load: true
27-
push: false
66+
target: pgstacbase-plrust
67+
file: docker/pgstac/Dockerfile
68+
tags: ${{ needs.changes.outputs.pgdocker }}
69+
push: true
2870
cache-from: type=gha
2971
cache-to: type=gha, mode=max
3072

73+
buildpyrust:
74+
name: Build and push base pyrust
75+
if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' }}
76+
runs-on: ubuntu-latest
77+
needs: [changes]
78+
steps:
79+
- uses: actions/checkout@v3
80+
- uses: docker/setup-buildx-action@v2
81+
- name: Log in to the Container registry
82+
uses: docker/login-action@v2
83+
with:
84+
registry: ${{ env.REGISTRY }}
85+
username: ${{ github.actor }}
86+
password: ${{ secrets.GITHUB_TOKEN }}
87+
- name: Build and Push Base pyrust
88+
uses: docker/build-push-action@v4
89+
with:
90+
context: .
91+
target: pyrustbase
92+
file: docker/pypgstac/Dockerfile
93+
tags: ${{ needs.changes.outputs.pyrustdocker }}
94+
push: true
95+
cache-from: type=gha
96+
cache-to: type=gha, mode=max
97+
98+
test:
99+
name: test
100+
needs: [changes, buildpg, buildpyrust]
101+
runs-on: ubuntu-latest
102+
container:
103+
image: ${{ needs.changes.outputs.pyrustdocker }}
104+
env:
105+
PGPASSWORD: postgres
106+
PGHOST: postgres
107+
PGDATABASE: postgres
108+
PGUSER: postgres
109+
110+
services:
111+
postgres:
112+
env:
113+
POSTGRES_PASSWORD: postgres
114+
image: ${{ needs.changes.outputs.pgdocker }}
115+
options: >-
116+
--health-cmd pg_isready
117+
--health-interval 10s
118+
--health-timeout 5s
119+
--health-retries 5
120+
121+
steps:
122+
- uses: actions/checkout@v3
123+
- name: Install pypgstac
124+
working-directory: /__w/pgstac/pgstac/src/pypgstac
125+
run: |
126+
pip install .[dev,test,psycopg];
127+
31128
- name: Run tests
32-
run: docker run --rm ${{ steps.builder.outputs.imageid }} test
129+
working-directory: /__w/pgstac/pgstac/docker/pypgstac/bin
130+
run: |
131+
./test
132+
133+
linux:
134+
runs-on: ubuntu-latest
135+
strategy:
136+
matrix:
137+
target: [x86_64, x86, aarch64, armv7, s390x, ppc64le]
138+
steps:
139+
- uses: actions/checkout@v3
140+
- uses: actions/setup-python@v4
141+
with:
142+
python-version: '3.10'
143+
- name: Build wheels
144+
uses: PyO3/maturin-action@v1
145+
with:
146+
working-directory: src/pypgstac
147+
target: ${{ matrix.target }}
148+
args: --release --out /home/runner/work/pgstac/pgstac/dist
149+
sccache: 'true'
150+
manylinux: auto
151+
- name: Upload wheels
152+
uses: actions/upload-artifact@v3
153+
with:
154+
name: wheels
155+
path: /home/runner/work/pgstac/pgstac/dist/*
156+
157+
windows:
158+
runs-on: windows-latest
159+
strategy:
160+
matrix:
161+
target: [x64, x86]
162+
steps:
163+
- uses: actions/checkout@v3
164+
- uses: actions/setup-python@v4
165+
with:
166+
python-version: '3.10'
167+
architecture: ${{ matrix.target }}
168+
- name: Build wheels
169+
uses: PyO3/maturin-action@v1
170+
with:
171+
working-directory: src/pypgstac
172+
target: ${{ matrix.target }}
173+
args: --release --out /home/runner/work/pgstac/pgstac/dist
174+
sccache: 'true'
175+
- name: Upload wheels
176+
uses: actions/upload-artifact@v3
177+
with:
178+
name: wheels
179+
path: /home/runner/work/pgstac/pgstac/dist/*
180+
181+
macos:
182+
runs-on: macos-latest
183+
strategy:
184+
matrix:
185+
target: [x86_64, aarch64]
186+
steps:
187+
- uses: actions/checkout@v3
188+
- uses: actions/setup-python@v4
189+
with:
190+
python-version: '3.10'
191+
- name: Build wheels
192+
uses: PyO3/maturin-action@v1
193+
with:
194+
working-directory: src/pypgstac
195+
target: ${{ matrix.target }}
196+
args: --release --out /tmp/dist
197+
sccache: 'true'
198+
- name: Upload wheels
199+
uses: actions/upload-artifact@v3
200+
with:
201+
name: wheels
202+
path: /tmp/dist/*
203+
204+
sdist:
205+
runs-on: ubuntu-latest
206+
steps:
207+
- uses: actions/checkout@v3
208+
- name: Build sdist
209+
uses: PyO3/maturin-action@v1
210+
with:
211+
working-directory: src/pypgstac
212+
command: sdist
213+
args: --out /home/runner/work/pgstac/pgstac/dist
214+
- name: Upload sdist
215+
uses: actions/upload-artifact@v3
216+
with:
217+
name: wheels
218+
path: /home/runner/work/pgstac/pgstac/dist/*
219+
220+
# release:
221+
# name: Release
222+
# runs-on: ubuntu-latest
223+
# if: "startsWith(github.ref, 'refs/tags/')"
224+
# needs: [linux, windows, macos, sdist]
225+
# steps:
226+
# - uses: actions/download-artifact@v3
227+
# with:
228+
# name: wheels
229+
# - name: Publish to PyPI
230+
# uses: PyO3/maturin-action@v1
231+
# env:
232+
# MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
233+
# with:
234+
# command: upload
235+
# args: --skip-existing *

0 commit comments

Comments
 (0)