From f89ce892a1efbae959e8301b7b7dafbbb96a6bb1 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 02:33:32 +0530 Subject: [PATCH 01/19] Fixes #768 --- .github/buildkitd.toml | 4 +++ .github/workflows/test-library.yml | 39 +++++++++++++++--------------- Makefile | 14 ++++++----- 3 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 .github/buildkitd.toml diff --git a/.github/buildkitd.toml b/.github/buildkitd.toml new file mode 100644 index 0000000000..59b12d17bd --- /dev/null +++ b/.github/buildkitd.toml @@ -0,0 +1,4 @@ +[worker.oci] + max-parallelism = 4 +[registry."docker.io"] + mirrors = ["mirror.gcr.io"] diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 2d983cc55e..5730b3c8da 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -609,7 +609,7 @@ jobs: matrix: os: [ubuntu, windows, macOS] node: ['10.x', '11.x', '12.x'] - max-parallel: 4 + # max-parallel: 4 fail-fast: false steps: - uses: actions/checkout@v2 @@ -629,37 +629,36 @@ jobs: cd .. docker: - # To build our docker container, we must wait for check, + # TODO: To build our docker container, we must wait for check, # so that we can use the same distribution available. - # - # TL;DR -- Docker must be packaged with package from - # test.pypi.org or from pypi.org. - # needs: - # - publish-pypi runs-on: ${{ matrix.os }}-latest name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.os }} strategy: matrix: os: [ubuntu] python: ['3.10'] - max-parallel: 1 + # max-parallel: 1 fail-fast: false steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 + - name: Checkout + uses: actions/checkout@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 with: - python-version: ${{ matrix.python }} - - name: Install dependencies + buildkitd-flags: --debug + config: .github/buildkitd.toml + install: true + - name: Inspect builder run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-release.txt - pip install -r requirements-testing.txt - pip install -r requirements-tunnel.txt - - name: Build + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + - name: Build container run: | - make container + make container-buildx check: # This job does nothing and is only used for the branch protection needs: diff --git a/Makefile b/Makefile index 418fd8b91e..ccaf59e413 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ SHELL := /bin/bash NS ?= abhinavsingh IMAGE_NAME ?= proxy.py -LATEST_TAG := $(NS)/$(IMAGE_NAME):latest -IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh) +DOCKER_LATEST_TAG := $(NS)/$(IMAGE_NAME):latest +DOCKER_IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh) HTTPS_KEY_FILE_PATH := https-key.pem HTTPS_CERT_FILE_PATH := https-cert.pem @@ -151,12 +151,14 @@ dashboard: dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi +container-buildx: + docker build -t $(DOCKER_IMAGE_TAG) --platform linux/amd64,linux/arm/v7,linux/arm64 . + container: - docker build -t $(LATEST_TAG) -t $(IMAGE_TAG) . + docker build -t $(DOCKER_IMAGE_TAG) . container-release: - docker push $(IMAGE_TAG) - docker push $(LATEST_TAG) + docker push $(DOCKER_IMAGE_TAG) container-run: - docker run -it -p 8899:8899 --rm $(LATEST_TAG) + docker run -it -p 8899:8899 --rm $(DOCKER_LATEST_TAG) From c2c5d1ef83dc1ee67943bf2bd25decf838673fbe Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 02:35:57 +0530 Subject: [PATCH 02/19] buildx build --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ccaf59e413..ea4fbbe012 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ endif .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest .PHONY: lib-release-test lib-release lib-profile lib-doc .PHONY: lib-dep lib-flake8 lib-mypy -.PHONY: container container-run container-release +.PHONY: container container-run container-release container-buildx .PHONY: devtools dashboard dashboard-clean all: lib-test @@ -152,7 +152,7 @@ dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi container-buildx: - docker build -t $(DOCKER_IMAGE_TAG) --platform linux/amd64,linux/arm/v7,linux/arm64 . + docker buildx build -t $(DOCKER_IMAGE_TAG) --platform linux/amd64,linux/arm/v7,linux/arm64 . container: docker build -t $(DOCKER_IMAGE_TAG) . From e60d80d5835ec26edbaf669d5844bac1f03c8f40 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 02:40:57 +0530 Subject: [PATCH 03/19] We need `setuptools_scm` to run `Makefile` --- .github/workflows/test-library.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 5730b3c8da..21acfa443b 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -656,6 +656,9 @@ jobs: echo "Status: ${{ steps.buildx.outputs.status }}" echo "Flags: ${{ steps.buildx.outputs.flags }}" echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + - name: Install makefile dependency + run: | + pip install setuptools_scm - name: Build container run: | make container-buildx From 005fe57042d9a9e58a893e59b37016a371f72036 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 04:35:25 +0530 Subject: [PATCH 04/19] Build `docker` container using `wheel` --- .dockerignore | 10 +------- .github/workflows/test-library.yml | 5 +++- Dockerfile | 38 +++++++++++++----------------- Makefile | 20 ++++++++++++---- README.md | 2 +- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3483deb2b5..3bd432ac85 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,5 @@ # Ignore everything ** -# Except proxy -!.git -!proxy -!requirements.txt -!pyproject.toml -!setup.cfg +!dist/*.whl !README.md - -# Ignore __pycache__ directory -proxy/__pycache__ diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 21acfa443b..8ff3c14a7e 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -632,6 +632,9 @@ jobs: # TODO: To build our docker container, we must wait for check, # so that we can use the same distribution available. runs-on: ${{ matrix.os }}-latest + needs: + - build + - pre-setup # transitive, for accessing settings name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.os }} strategy: matrix: @@ -661,7 +664,7 @@ jobs: pip install setuptools_scm - name: Build container run: | - make container-buildx + make container-buildx -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' check: # This job does nothing and is only used for the branch protection needs: diff --git a/Dockerfile b/Dockerfile index 0f37fda244..e98f0a6095 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,24 @@ FROM python:3.10-alpine as base -RUN apk add git +LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ + com.abhinavsingh.description="⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \ + Network monitoring, controls & Application development, testing, debugging." \ + com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ + com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" -FROM base as builder +ARG PROXYPY_PKG_PATH -COPY .git /app/.git -COPY requirements.txt /app/ -COPY pyproject.toml /app/ -COPY setup.cfg /app/ -COPY README.md /app/ -COPY proxy/ /app/proxy/ -WORKDIR /app +COPY README.md / +COPY $PROXYPY_PKG_PATH / RUN pip install --upgrade pip && \ - pip install --prefix=/deps . - -FROM base - -LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \ - Network monitoring, controls & Application development, testing, debugging." \ - com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ - com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" - -COPY --from=builder /deps /usr/local + pip install \ + --no-index \ + --find-links file:/// \ + proxy.py && \ + rm *.whl -# Install openssl to enable TLS interception within container +# Install openssl to enable TLS interception & HTTPS proxy options within container +# NOTE: You can comment out this line if you don't intend to use those features. RUN apk update && apk add openssl EXPOSE 8899/tcp diff --git a/Makefile b/Makefile index ea4fbbe012..b5269a3b83 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,9 @@ CA_KEY_FILE_PATH := ca-key.pem CA_CERT_FILE_PATH := ca-cert.pem CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem +# Dummy invalid hardcoded value +PROXYPY_PKG_PATH := dist/proxy.py.whl + OPEN=$(shell which open) UNAME := $(shell uname) ifeq ($(UNAME), Linux) @@ -24,7 +27,7 @@ endif .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest .PHONY: lib-release-test lib-release lib-profile lib-doc .PHONY: lib-dep lib-flake8 lib-mypy -.PHONY: container container-run container-release container-buildx +.PHONY: container container-run container-release container-build container-buildx .PHONY: devtools dashboard dashboard-clean all: lib-test @@ -152,10 +155,19 @@ dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi container-buildx: - docker buildx build -t $(DOCKER_IMAGE_TAG) --platform linux/amd64,linux/arm/v7,linux/arm64 . + docker buildx build \ + --platform linux/amd64,linux/arm/v7,linux/arm64 \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ + -t $(DOCKER_IMAGE_TAG) . + +container-build: + docker build \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ + -t $(DOCKER_IMAGE_TAG) . + +container: lib-package + $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) -container: - docker build -t $(DOCKER_IMAGE_TAG) . container-release: docker push $(DOCKER_IMAGE_TAG) diff --git a/README.md b/README.md index 4d611de448..82ace22c5f 100644 --- a/README.md +++ b/README.md @@ -2038,7 +2038,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.3.2.dev183+g808caa1.d20211124 +proxy.py v2.3.2.dev190+ge60d80d.d20211124 options: -h, --help show this help message and exit From 5843b72145fc36e90fb2a162c5886b7cf8296cb2 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 04:43:17 +0530 Subject: [PATCH 05/19] Download package distribution --- .github/workflows/test-library.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 8ff3c14a7e..4f612d4575 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -662,6 +662,11 @@ jobs: - name: Install makefile dependency run: | pip install setuptools_scm + - name: Download all the dists + uses: actions/download-artifact@v2 + with: + name: python-package-distributions + path: dist/ - name: Build container run: | make container-buildx -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' From e408701f4ea15ce00883de70378343cd193ac73a Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 04:49:09 +0530 Subject: [PATCH 06/19] Adjust supported platforms --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b5269a3b83..0dc3c2acaf 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ dashboard-clean: container-buildx: docker buildx build \ - --platform linux/amd64,linux/arm/v7,linux/arm64 \ + --platform linux/386,linux/amd64,linux/arm/v5,linux/arm64 \ --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ -t $(DOCKER_IMAGE_TAG) . From 1e7d816d8e481f1c309f5ee25c9bab4962ff6747 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 04:54:35 +0530 Subject: [PATCH 07/19] Alpine only supports v6 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0dc3c2acaf..8ddd856301 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ dashboard-clean: container-buildx: docker buildx build \ - --platform linux/386,linux/amd64,linux/arm/v5,linux/arm64 \ + --platform linux/386,linux/amd64,linux/arm/v6,linux/arm64 \ --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ -t $(DOCKER_IMAGE_TAG) . From ee65da51cf4f45fa338538294774d2432e7b6851 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 04:58:04 +0530 Subject: [PATCH 08/19] . --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8ddd856301..4fee6580a4 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ dashboard-clean: container-buildx: docker buildx build \ - --platform linux/386,linux/amd64,linux/arm/v6,linux/arm64 \ + --platform linux/386,linux/amd64,linux/arm64 \ --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ -t $(DOCKER_IMAGE_TAG) . From 74f83ee91af62ebbf99be5e6e2abf4a58bc6f486 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 16:07:29 +0530 Subject: [PATCH 09/19] May-be fix `lsb_release` --- .github/workflows/test-library.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 4f612d4575..a19b7d7e76 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -667,6 +667,9 @@ jobs: with: name: python-package-distributions path: dist/ + - name: Fix lsb_release + run: | + sudo apt install -y lsb-release - name: Build container run: | make container-buildx -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' From ede25417abf549133f9c06dd36161e561e62a820 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 17:09:39 +0530 Subject: [PATCH 10/19] Enable `multiarch` env for docker buildx --- .github/workflows/test-library.yml | 5 +++-- Dockerfile | 2 +- Makefile | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index a19b7d7e76..ed68ccd3e2 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -659,6 +659,7 @@ jobs: echo "Status: ${{ steps.buildx.outputs.status }}" echo "Flags: ${{ steps.buildx.outputs.flags }}" echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + docker buildx ls - name: Install makefile dependency run: | pip install setuptools_scm @@ -667,9 +668,9 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Fix lsb_release + - name: Enable Multiarch # This slows down container build time by 4-5x run: | - sudo apt install -y lsb-release + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: Build container run: | make container-buildx -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' diff --git a/Dockerfile b/Dockerfile index e98f0a6095..65bdb7e094 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" - +ENV PYTHONUNBUFFERED 1 ARG PROXYPY_PKG_PATH COPY README.md / diff --git a/Makefile b/Makefile index 4fee6580a4..af6e821c0b 100644 --- a/Makefile +++ b/Makefile @@ -154,6 +154,8 @@ dashboard: dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi +# Usage: +# make container-buildx -e PROXYPY_PKG_PATH=$(ls dist/*.whl) container-buildx: docker buildx build \ --platform linux/386,linux/amd64,linux/arm64 \ From ec6cfd778665951327dcbe836646e200293d327e Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 17:43:18 +0530 Subject: [PATCH 11/19] Deprecate `DOCKER_IMAGE_TAG` which removes `write-scm-version.py` dependency. Also pass targetplatform as an argument to `container-buildx` make target --- .github/workflows/test-library.yml | 30 +++++++++++++++++++----------- Makefile | 16 +++++----------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index ed68ccd3e2..9eb621a873 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -635,11 +635,19 @@ jobs: needs: - build - pre-setup # transitive, for accessing settings - name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.os }} + name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.targetplatform }} strategy: matrix: - os: [ubuntu] - python: ['3.10'] + os: + - Ubuntu + python: + - '3.10' + targetplatform: + - 'linux/386' + - 'linux/amd64' + - 'linux/arm/v5' + - 'linux/arm/v7' + - 'linux/arm64/v8' # max-parallel: 1 fail-fast: false steps: @@ -652,13 +660,11 @@ jobs: buildkitd-flags: --debug config: .github/buildkitd.toml install: true - - name: Inspect builder + - name: Create builder run: | - echo "Name: ${{ steps.buildx.outputs.name }}" - echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" - echo "Status: ${{ steps.buildx.outputs.status }}" - echo "Flags: ${{ steps.buildx.outputs.flags }}" - echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + docker buildx create --name proxypybuilder + docker buildx use proxypybuilder + docker buildx inspect docker buildx ls - name: Install makefile dependency run: | @@ -668,12 +674,14 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Enable Multiarch # This slows down container build time by 4-5x + - name: Enable Multiarch # This slows down arm build by 4-5x run: | docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: Build container run: | - make container-buildx -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' + make container-buildx \ + -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ + -e BUILDX_TARGET_PLATFORM='{{ matrix.targetplatform }}' check: # This job does nothing and is only used for the branch protection needs: diff --git a/Makefile b/Makefile index af6e821c0b..c07c9bfab1 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ SHELL := /bin/bash NS ?= abhinavsingh IMAGE_NAME ?= proxy.py DOCKER_LATEST_TAG := $(NS)/$(IMAGE_NAME):latest -DOCKER_IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh) HTTPS_KEY_FILE_PATH := https-key.pem HTTPS_CERT_FILE_PATH := https-cert.pem @@ -16,6 +15,7 @@ CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem # Dummy invalid hardcoded value PROXYPY_PKG_PATH := dist/proxy.py.whl +BUILDX_TARGET_PLATFORM := linux/amd64 OPEN=$(shell which open) UNAME := $(shell uname) @@ -155,24 +155,18 @@ dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi # Usage: -# make container-buildx -e PROXYPY_PKG_PATH=$(ls dist/*.whl) +# make container-buildx -e PROXYPY_PKG_PATH=$(ls dist/*.whl) -e BUILDX_TARGET_PLATFORM=linux/arm64 container-buildx: docker buildx build \ - --platform linux/386,linux/amd64,linux/arm64 \ - --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ - -t $(DOCKER_IMAGE_TAG) . + --platform $(BUILDX_TARGET_PLATFORM) \ + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . container-build: docker build \ - --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) \ - -t $(DOCKER_IMAGE_TAG) . + --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . container: lib-package $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) - -container-release: - docker push $(DOCKER_IMAGE_TAG) - container-run: docker run -it -p 8899:8899 --rm $(DOCKER_LATEST_TAG) From 8f31d1dbbee95963a3a999ad898df63271fe18ef Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 17:43:58 +0530 Subject: [PATCH 12/19] Remove `setuptools_scm` dep for docker workflow step --- .github/workflows/test-library.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 9eb621a873..de3ed4e4b6 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -666,9 +666,6 @@ jobs: docker buildx use proxypybuilder docker buildx inspect docker buildx ls - - name: Install makefile dependency - run: | - pip install setuptools_scm - name: Download all the dists uses: actions/download-artifact@v2 with: From ebe41b3886e2cf1d358e216c4f3ae3265d8486a5 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 17:47:11 +0530 Subject: [PATCH 13/19] woof --- .github/workflows/test-library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index de3ed4e4b6..a0892d794e 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -678,7 +678,7 @@ jobs: run: | make container-buildx \ -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ - -e BUILDX_TARGET_PLATFORM='{{ matrix.targetplatform }}' + -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' check: # This job does nothing and is only used for the branch protection needs: From 6afbfeacd676583e2669b002962f7c27ce381e0b Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 17:54:27 +0530 Subject: [PATCH 14/19] Match all target platforms to match `python-alpine` docker --- .github/workflows/test-library.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index a0892d794e..bce6be9685 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -645,9 +645,11 @@ jobs: targetplatform: - 'linux/386' - 'linux/amd64' - - 'linux/arm/v5' + - 'linux/arm/v6' - 'linux/arm/v7' - 'linux/arm64/v8' + - 'linux/ppc64le' + - 'linux/s390x' # max-parallel: 1 fail-fast: false steps: @@ -660,12 +662,6 @@ jobs: buildkitd-flags: --debug config: .github/buildkitd.toml install: true - - name: Create builder - run: | - docker buildx create --name proxypybuilder - docker buildx use proxypybuilder - docker buildx inspect - docker buildx ls - name: Download all the dists uses: actions/download-artifact@v2 with: @@ -674,6 +670,12 @@ jobs: - name: Enable Multiarch # This slows down arm build by 4-5x run: | docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Create builder + run: | + docker buildx create --name proxypybuilder + docker buildx use proxypybuilder + docker buildx inspect + docker buildx ls - name: Build container run: | make container-buildx \ From e6812a2ee641e34c11cf76fc11e31386cc7751fa Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 18:02:51 +0530 Subject: [PATCH 15/19] yamllint --- .github/workflows/test-library.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index bce6be9685..398ac4ddb3 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -679,7 +679,8 @@ jobs: - name: Build container run: | make container-buildx \ - -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ + -e PROXYPY_PKG_PATH=\ + 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' check: # This job does nothing and is only used for the branch protection From f06ab7090e5d9de82b6d14b4542c039cb49f6a69 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 18:08:51 +0530 Subject: [PATCH 16/19] Use `yamllint disable rule:line-length` for `Dockerfile` --- .github/workflows/test-library.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 398ac4ddb3..84366e77e6 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -1,4 +1,5 @@ --- +# yamllint disable rule:line-length name: lib on: # yamllint disable-line rule:truthy @@ -679,8 +680,7 @@ jobs: - name: Build container run: | make container-buildx \ - -e PROXYPY_PKG_PATH=\ - 'dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ + -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' check: # This job does nothing and is only used for the branch protection From cc2439768c4eff2e84b9c7e384643c879447e743 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 19:38:01 +0530 Subject: [PATCH 17/19] Tag the container using `dist-version` --- .github/workflows/test-library.yml | 3 ++- Dockerfile | 6 ++++-- Makefile | 21 +++++++++++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 84366e77e6..7f2b45915a 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -681,7 +681,8 @@ jobs: run: | make container-buildx \ -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ - -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' + -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \ + -e PROXYPY_CONTAINER_VERSION='${{ needs.pre-setup.outputs.dist-version }}' check: # This job does nothing and is only used for the branch protection needs: diff --git a/Dockerfile b/Dockerfile index 65bdb7e094..0d59cd5bf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ FROM python:3.10-alpine as base LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \ - com.abhinavsingh.description="⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \ - Network monitoring, controls & Application development, testing, debugging." \ + com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \ + 😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \ + 👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \ + 👷 \"Work\" acceptor & executor framework" \ com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \ com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \ com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py" diff --git a/Makefile b/Makefile index c07c9bfab1..d54ab9242f 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,10 @@ SHELL := /bin/bash NS ?= abhinavsingh IMAGE_NAME ?= proxy.py -DOCKER_LATEST_TAG := $(NS)/$(IMAGE_NAME):latest +# Override to target specific versions of proxy.py +PROXYPY_CONTAINER_VERSION := latest +# Used by container build and run targets +PROXYPY_CONTAINER_TAG := $(NS)/$(IMAGE_NAME):$(PROXYPY_CONTAINER_VERSION) HTTPS_KEY_FILE_PATH := https-key.pem HTTPS_CERT_FILE_PATH := https-cert.pem @@ -154,19 +157,25 @@ dashboard: dashboard-clean: if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi +container: lib-package + $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) + # Usage: -# make container-buildx -e PROXYPY_PKG_PATH=$(ls dist/*.whl) -e BUILDX_TARGET_PLATFORM=linux/arm64 +# +# make container-buildx \ +# -e PROXYPY_PKG_PATH=$(ls dist/*.whl) \ +# -e BUILDX_TARGET_PLATFORM=linux/arm64 \ +# -e PROXYPY_CONTAINER_VERSION=latest container-buildx: docker buildx build \ --platform $(BUILDX_TARGET_PLATFORM) \ + -t $(PROXYPY_CONTAINER_TAG) \ --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . container-build: docker build \ + -t $(PROXYPY_CONTAINER_TAG) \ --build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) . -container: lib-package - $(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl) - container-run: - docker run -it -p 8899:8899 --rm $(DOCKER_LATEST_TAG) + docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG) From 07d077186aa06a24f2d74e9555de56d9dddd2583 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 20:07:32 +0530 Subject: [PATCH 18/19] Replace + with . for dev version tag --- .github/workflows/test-library.yml | 3 ++- README.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index 7f2b45915a..d1699877ed 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -679,10 +679,11 @@ jobs: docker buildx ls - name: Build container run: | + PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .) \ make container-buildx \ -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \ - -e PROXYPY_CONTAINER_VERSION='${{ needs.pre-setup.outputs.dist-version }}' + -e PROXYPY_CONTAINER_VERSION=$PROXYPY_CONTAINER_VERSION check: # This job does nothing and is only used for the branch protection needs: diff --git a/README.md b/README.md index 82ace22c5f..e9ba838e9f 100644 --- a/README.md +++ b/README.md @@ -257,12 +257,30 @@ or from GitHub `master` branch ## Using Docker +Stable version container releases are available for following platforms: + +- `linux/386` +- `linux/amd64` +- `linux/arm/v6` +- `linux/arm/v7` +- `linux/arm64/v8` +- `linux/ppc64le` +- `linux/s390x` + ### Stable Version from Docker Hub +Run `proxy.py` latest container: + ```console ❯ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest ``` +To run specific target platform container on multi-platform supported servers: + +```console +❯ docker run -it -p 8899:8899 --rm --platform linux/arm64/v8 abhinavsingh/proxy.py:latest +``` + ### Build Development Version Locally ```console @@ -394,6 +412,16 @@ To start `proxy.py` from source code follow these instructions: ❯ make lib-dep ``` +- Generate `proxy/common/_scm_version.py` + + NOTE: *Following step is not necessary for editable installs.* + + This file writes SCM detected version to `proxy/common/_scm_version.py` file. + + ```console + ❯ ./write-scm-version.sh + ``` + - Optionally, run tests ```console From f059f82a97c1c64de48c4a408d38740394e95678 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Thu, 25 Nov 2021 20:15:06 +0530 Subject: [PATCH 19/19] Add `PROXYPY_CONTAINER_VERSION` step --- .github/workflows/test-library.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-library.yml b/.github/workflows/test-library.yml index d1699877ed..924f4735c6 100644 --- a/.github/workflows/test-library.yml +++ b/.github/workflows/test-library.yml @@ -677,13 +677,15 @@ jobs: docker buildx use proxypybuilder docker buildx inspect docker buildx ls + - name: Set PROXYPY_CONTAINER_VERSION + run: | + echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV - name: Build container run: | - PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .) \ make container-buildx \ -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \ -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \ - -e PROXYPY_CONTAINER_VERSION=$PROXYPY_CONTAINER_VERSION + -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}' check: # This job does nothing and is only used for the branch protection needs: