Switch to uv (#1213)

* Switch to uv
* Run tests in parallel

Author: leplatrem

.dockerignore

@@ -5,7 +5,7 @@
 !/bin
 !/config
 !/jbi
-!/poetry.lock
+!/uv.lock
 !/pyproject.toml
 !/version.json
 

.github/workflows/build-publish.yaml

@@ -102,7 +102,8 @@ jobs:
 
       - name: Check that container is running
         run: |
-          docker exec ${{ env.TEST_CONTAINER_NAME }} python bin/healthcheck.py
+          docker exec ${{ env.TEST_CONTAINER_NAME }} curl --retry 10 --retry-delay 1 --retry-connrefused http://127.0.0.1:8000/__heartbeat__
+
 
       - name: Spin down container
         run: |

.github/workflows/ci.yaml

@@ -7,39 +7,26 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: actions/setup-python@v6
-        id: setup-python
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.13"
-      - name: Install poetry
-        run: pipx install poetry
-      - uses: actions/cache@v4
-        with:
-          path: .venv
-          key: poetry-${{ steps.setup-python.outputs.python-version}}-${{ hashFiles('poetry.lock') }}
-      - name: Install dependencies
-        run: make install
-      - run: bin/lint.sh
+          enable-cache: true
+          python-version: 3.13
+      - name: Run lint
+        run: make lint
   run_test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5
-      - uses: actions/setup-python@v6
-        id: setup-python
-        with:
-          python-version: "3.13"
       - name: Install pandoc
         run: sudo apt-get install -y pandoc
-      - name: Install poetry
-        run: pipx install poetry
-      - uses: actions/cache@v4
+      - uses: actions/checkout@v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
         with:
-          path: .venv
-          key: poetry-${{ steps.setup-python.outputs.python-version}}-${{ hashFiles('poetry.lock') }}
-      - name: Install dependencies
-        run: make install
+          enable-cache: true
+          python-version: 3.13
       - name: Run tests
-        run: bin/test.sh
+        run: make tests
       - name: Run retry
         env:
           JBI_API_KEY: key  # pragma: allowlist secret
@@ -48,7 +35,7 @@ jobs:
           BUGZILLA_API_KEY: key  # pragma: allowlist secret
           DL_QUEUE_CONSTANT_RETRY: false
           DL_QUEUE_DSN: "file:///tmp/dlqueue"
-        run: .venv/bin/python -m jbi.retry
+        run: uv run python -m jbi.retry
   review-dependabot-pr:
     permissions:
       contents: write

Dockerfile

@@ -1,50 +1,39 @@
-# Creating a python base with shared environment variables
-FROM python:3.14.0 AS base
-ENV PIP_NO_CACHE_DIR=off \
-    PIP_DEFAULT_TIMEOUT=100 \
-    PIP_DISABLE_PIP_VERSION_CHECK=on \
-    POETRY_HOME="/opt/poetry" \
-    POETRY_NO_INTERACTION=1 \
-    POETRY_VIRTUALENVS_IN_PROJECT=true \
-    PYSETUP_PATH="/opt/pysetup"
-
-ENV PATH="$POETRY_HOME/bin:$PATH"
-
-# Install Poetry - respects $POETRY_VERSION & $POETRY_HOME
-RUN python3 -m venv $POETRY_HOME && \
-    $POETRY_HOME/bin/pip install poetry==1.7.1 && \
-    $POETRY_HOME/bin/poetry --version
-
-# We copy our Python requirements here to cache them
-# and install only runtime deps using poetry
-WORKDIR $PYSETUP_PATH
-COPY ./poetry.lock ./pyproject.toml ./
-RUN $POETRY_HOME/bin/poetry install --without dev --no-root
-
-# `production` stage uses the dependencies downloaded in the `base` stage
-FROM python:3.14.0-slim AS production
+FROM python:3.14.0-slim
 
-# Install pandoc for markdown to Jira conversions.
-RUN apt-get -y update && \
-    apt-get -y install --no-install-recommends pandoc
-
-ENV PORT=8000 \
-    PYTHONUNBUFFERED=1 \
+ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
-    VIRTUAL_ENV=/opt/pysetup/.venv
-
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-COPY --from=base $VIRTUAL_ENV $VIRTUAL_ENV
+    HOST=0.0.0.0 \
+    PORT=8000
 
 ARG userid=10001
 ARG groupid=10001
 RUN groupadd --gid $groupid app && \
     useradd -g app --uid $userid --shell /usr/sbin/nologin --create-home app
-USER app
+
+# Install pandoc for markdown to Jira conversions.
+RUN apt-get -y update && \
+    apt-get -y install --no-install-recommends pandoc curl
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+COPY ./pyproject.toml /app/
+COPY ./uv.lock /app/
+
+ENV UV_CACHE_DIR=/opt/uv-cache
+RUN mkdir -p "${UV_CACHE_DIR}" && \
+    chown app:app "${UV_CACHE_DIR}"
 
 WORKDIR /app
-COPY . .
+COPY . /app
+
+# Install dependencies
+RUN --mount=type=cache,target="${UV_CACHE_DIR}" \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --no-dev --locked --no-install-project --no-editable
+
+# run as non priviledged user
+USER app
 
 EXPOSE $PORT
-CMD ["python", "-m", "asgi"]
+CMD ["uv", "run", "--no-dev", "--frozen", "python", "-m", "asgi"]

Makefile

@@ -8,69 +8,56 @@ VENV := $(shell echo $${VIRTUAL_ENV-.venv})
 INSTALL_STAMP = $(VENV)/.install.stamp
 DOTENV_FILE = .env
 
-.PHONY: help
+
+INSTALL_STAMP := .install.stamp
+UV := $(shell command -v uv 2> /dev/null)
+
 help:
-	@echo "Usage: make RULE"
-	@echo ""
-	@echo "JBI make rules:"
-	@echo ""
-	@echo "Local"
-	@echo "  clean         - clean local cache folders"
-	@echo "  format        - run linting checks, fix in place"
-	@echo "  lint          - run linting checks"
-	@echo "  start         - run the API service locally"
-	@echo "  test          - run test suite"
-	@echo ""
-	@echo "Docker"
-	@echo "  build         - build docker container"
-	@echo "  docker-start  - run the API service through docker"
-	@echo "  docker-shell  - open a shell in the web container"
-	@echo ""
-	@echo "  help          - see this text"
+	@echo "Please use 'make <target>' where <target> is one of the following commands.\n"
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@echo "\nCheck the Makefile to know exactly what each target is doing."
 
 .PHONY: clean
-clean:
+clean: ## Delete cache files
 	find . -name "__pycache__" | xargs rm -rf
 	rm -rf .mypy_cache .ruff_cache .coverage .venv
 
-$(VENV)/bin/python:
-	python3 -m venv $(VENV)
-
-install: $(VENV)/bin/python $(INSTALL_STAMP)
-$(INSTALL_STAMP): poetry.lock
-	@if [ -z $(shell command -v poetry 2> /dev/null) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
-	POETRY_VIRTUALENVS_IN_PROJECT=1 poetry install --with dev --no-root
+install: $(INSTALL_STAMP)
+$(INSTALL_STAMP): pyproject.toml uv.lock  ## Install dependencies
+	@if [ -z $(UV) ]; then echo "uv could not be found. See https://docs.astral.sh/uv/"; exit 2; fi
+	$(UV) --version
+	$(UV) sync --frozen --verbose
 	touch $(INSTALL_STAMP)
 
 .PHONY: build
-build:
+build: ## Build docker container
 	docker-compose build \
 		--build-arg userid=${_UID} --build-arg groupid=${_GID}
 
 .PHONY: format
-format: $(INSTALL_STAMP)
+format: $(INSTALL_STAMP)  ## Format code base
 	bin/lint.sh lint --fix
 	bin/lint.sh format --fix
 
 .PHONY: lint
-lint: $(INSTALL_STAMP)
+lint: $(INSTALL_STAMP)  ## Analyze code base
 	bin/lint.sh
 
 .PHONY: start
-start: $(INSTALL_STAMP) $(DOTENV_FILE)
-	poetry run python -m asgi
+start: $(INSTALL_STAMP) $(DOTENV_FILE) ## Start local
+	$(UV) run python -m asgi
 
-$(DOTENV_FILE):
+$(DOTENV_FILE):  ## Initialize default configuration
 	cp .env.example $(DOTENV_FILE)
 
 .PHONY: docker-shell
-docker-shell: $(DOTENV_FILE)
+docker-shell: $(DOTENV_FILE) ## Run shell from container
 	docker compose run --rm web /bin/sh
 
 .PHONY: docker-start
-docker-start: $(DOTENV_FILE)
+docker-start: $(DOTENV_FILE) ## Start container
 	docker compose up
 
 .PHONY: test
-test: $(INSTALL_STAMP)
-	bin/test.sh
+test: $(INSTALL_STAMP) ## Run unit tests
+	$(UV) run pytest tests -n auto --cov-report term-missing --cov-fail-under 75 --cov jbi --cov checks

README.md

@@ -85,7 +85,7 @@ We use [pandoc](https://pandoc.org) to convert markdown to the Jira syntax. Make
 In order to pass arguments to `pytest`:
 
 ```
-poetry run pytest -vv -k test_bugzilla_list_webhooks
+PYTEST_ADDOPTS="--last-failed -vv" make test
 ```
 
 You may consider:

bin/healthcheck.py

@@ -2,23 +2,23 @@
 
 set -e
 
-POETRY_RUN="poetry run"
+UV_RUN="uv run"
 
-BANDIT_CMD="$POETRY_RUN bandit -lll --recursive jbi"
+BANDIT_CMD="$UV_RUN bandit -lll --recursive jbi"
 
-FORMAT_CMD="$POETRY_RUN ruff format ."
+FORMAT_CMD="$UV_RUN ruff format ."
 
-# Scan only files fixed into the repo, omit poetry.lock
-DETECT_SECRETS_FILES="$(git ls-tree --full-tree -r --name-only HEAD | grep -v poetry.lock)"
-DETECT_SECRETS_CMD="$POETRY_RUN detect-secrets-hook $DETECT_SECRETS_FILES --baseline .secrets.baseline"
+# Scan only files fixed into the repo, omit uv.lock
+DETECT_SECRETS_FILES="$(git ls-tree --full-tree -r --name-only HEAD | grep -v uv.lock)"
+DETECT_SECRETS_CMD="$UV_RUN detect-secrets-hook $DETECT_SECRETS_FILES --baseline .secrets.baseline"
 
-LINT_CMD="$POETRY_RUN ruff check ."
+LINT_CMD="$UV_RUN ruff check ."
 
-MYPY_CMD="$POETRY_RUN mypy jbi"
+MYPY_CMD="$UV_RUN mypy jbi"
 
-YAMLLINT_CMD="$POETRY_RUN yamllint -c .yamllint config/*.yaml"
+YAMLLINT_CMD="$UV_RUN yamllint -c .yamllint config/*.yaml"
 
-ACTIONS_LINT_CMD="$POETRY_RUN jbi lint"
+ACTIONS_LINT_CMD="$UV_RUN python -m jbi lint"
 
 all () {
   echo "running bandit"

bin/lint.sh

@@ -2,7 +2,7 @@ version: "3.8"
 services:
   web:
     build: .
-    command: python -m asgi
+    command: uv run python -m asgi
     env_file:
       - .env
     # Let the init system handle signals for us.
@@ -14,7 +14,7 @@ services:
       - .:/app
   retry:
     build: .
-    command: python -m jbi.retry
+    command: uv run python -m jbi.retry
     env_file:
       - .env
     volumes: