Build docker container for all linux target architectures386, amd64, arm/v6, arm/v7, arm64/v8, ppc64le, s390x (#797)

* Fixes #768

* buildx build

* We need `setuptools_scm` to run `Makefile`

* Build `docker` container using `wheel`

* Download package distribution

* Adjust supported platforms

* Alpine only supports v6

* .

* May-be fix `lsb_release`

* Enable `multiarch` env for docker buildx

* Deprecate `DOCKER_IMAGE_TAG` which removes `write-scm-version.py` dependency.  Also pass targetplatform as an argument to `container-buildx` make target

* Remove `setuptools_scm` dep for docker workflow step

* woof

* Match all target platforms to match `python-alpine` docker

* yamllint

* Use `yamllint disable rule:line-length` for `Dockerfile`

* Tag the container using `dist-version`

* Replace + with . for dev version tag

* Add `PROXYPY_CONTAINER_VERSION` step

Author: abhinavsingh

.dockerignore

@@ -1,13 +1,5 @@
 # Ignore everything
 **
 
-# Except proxy
-!.git
-!proxy
-!requirements.txt
-!pyproject.toml
-!setup.cfg
+!dist/*.whl
 !README.md
-
-# Ignore __pycache__ directory
-proxy/__pycache__

.github/buildkitd.toml

@@ -0,0 +1,4 @@
+[worker.oci]
+  max-parallelism = 4
+[registry."docker.io"]
+  mirrors = ["mirror.gcr.io"]

.github/workflows/test-library.yml

@@ -1,4 +1,5 @@
 ---
+# yamllint disable rule:line-length
 name: lib
 
 on:   # yamllint disable-line rule:truthy
@@ -609,7 +610,7 @@ jobs:
       matrix:
         os: [ubuntu, windows, macOS]
         node: ['10.x', '11.x', '12.x']
-      max-parallel: 4
+      # max-parallel: 4
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
@@ -629,37 +630,62 @@ jobs:
         cd ..
 
   docker:
-    # To build our docker container, we must wait for check,
+    # TODO: To build our docker container, we must wait for check,
     # so that we can use the same distribution available.
-    #
-    # TL;DR -- Docker must be packaged with package from
-    # test.pypi.org or from pypi.org.
-    # needs:
-    # - publish-pypi
     runs-on: ${{ matrix.os }}-latest
-    name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.os }}
+    needs:
+    - build
+    - pre-setup  # transitive, for accessing settings
+    name: 🐳 🐍${{ matrix.python }} @ ${{ matrix.targetplatform }}
     strategy:
       matrix:
-        os: [ubuntu]
-        python: ['3.10']
-      max-parallel: 1
+        os:
+        - Ubuntu
+        python:
+        - '3.10'
+        targetplatform:
+        - 'linux/386'
+        - 'linux/amd64'
+        - 'linux/arm/v6'
+        - 'linux/arm/v7'
+        - 'linux/arm64/v8'
+        - 'linux/ppc64le'
+        - 'linux/s390x'
+      # max-parallel: 1
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v1
       with:
-        python-version: ${{ matrix.python }}
-    - name: Install dependencies
+        buildkitd-flags: --debug
+        config: .github/buildkitd.toml
+        install: true
+    - name: Download all the dists
+      uses: actions/download-artifact@v2
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Enable Multiarch    # This slows down arm build by 4-5x
+      run: |
+        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+    - name: Create builder
+      run: |
+        docker buildx create --name proxypybuilder
+        docker buildx use proxypybuilder
+        docker buildx inspect
+        docker buildx ls
+    - name: Set PROXYPY_CONTAINER_VERSION
       run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-release.txt
-        pip install -r requirements-testing.txt
-        pip install -r requirements-tunnel.txt
-    - name: Build
+        echo "PROXYPY_CONTAINER_VERSION=$(echo '${{ needs.pre-setup.outputs.dist-version }}' | tr + .)" > $GITHUB_ENV
+    - name: Build container
       run: |
-        make container
+        make container-buildx \
+          -e PROXYPY_PKG_PATH='dist/${{ needs.pre-setup.outputs.wheel-artifact-name }}' \
+          -e BUILDX_TARGET_PLATFORM='${{ matrix.targetplatform }}' \
+          -e PROXYPY_CONTAINER_VERSION='${{ env.PROXYPY_CONTAINER_VERSION }}'
 
   check:  # This job does nothing and is only used for the branch protection
     needs:

Dockerfile

@@ -1,30 +1,26 @@
 FROM python:3.10-alpine as base
-RUN apk add git
-
-FROM base as builder
-
-COPY .git /app/.git
-COPY requirements.txt /app/
-COPY pyproject.toml /app/
-COPY setup.cfg /app/
-COPY README.md /app/
-COPY proxy/ /app/proxy/
-WORKDIR /app
-RUN pip install --upgrade pip && \
-    pip install --prefix=/deps .
-
-FROM base
-
 LABEL com.abhinavsingh.name="abhinavsingh/proxy.py" \
-      com.abhinavsingh.description="⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on \
-        Network monitoring, controls & Application development, testing, debugging." \
-      com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \
-      com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \
-      com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py"
+  com.abhinavsingh.description="⚡ Fast • 🪶 Lightweight • 0️⃣ Dependency • 🔌 Pluggable • \
+  😈 TLS interception • 🔒 DNS-over-HTTPS • 🔥 Poor Man's VPN • ⏪ Reverse & ⏩ Forward • \
+  👮🏿 \"Proxy Server\" framework • 🌐 \"Web Server\" framework • ➵ ➶ ➷ ➠ \"PubSub\" framework • \
+  👷 \"Work\" acceptor & executor framework" \
+  com.abhinavsingh.url="https://github.com/abhinavsingh/proxy.py" \
+  com.abhinavsingh.vcs-url="https://github.com/abhinavsingh/proxy.py" \
+  com.abhinavsingh.docker.cmd="docker run -it --rm -p 8899:8899 abhinavsingh/proxy.py"
+ENV PYTHONUNBUFFERED 1
+ARG PROXYPY_PKG_PATH
 
-COPY --from=builder /deps /usr/local
+COPY README.md /
+COPY $PROXYPY_PKG_PATH /
+RUN pip install --upgrade pip && \
+  pip install \
+  --no-index \
+  --find-links file:/// \
+  proxy.py && \
+  rm *.whl
 
-# Install openssl to enable TLS interception within container
+# Install openssl to enable TLS interception & HTTPS proxy options within container
+# NOTE: You can comment out this line if you don't intend to use those features.
 RUN apk update && apk add openssl
 
 EXPOSE 8899/tcp

Makefile

@@ -2,8 +2,10 @@ SHELL := /bin/bash
 
 NS ?= abhinavsingh
 IMAGE_NAME ?= proxy.py
-LATEST_TAG := $(NS)/$(IMAGE_NAME):latest
-IMAGE_TAG := $(NS)/$(IMAGE_NAME):$(shell ./write-scm-version.sh)
+# Override to target specific versions of proxy.py
+PROXYPY_CONTAINER_VERSION := latest
+# Used by container build and run targets
+PROXYPY_CONTAINER_TAG := $(NS)/$(IMAGE_NAME):$(PROXYPY_CONTAINER_VERSION)
 
 HTTPS_KEY_FILE_PATH := https-key.pem
 HTTPS_CERT_FILE_PATH := https-cert.pem
@@ -14,6 +16,10 @@ CA_KEY_FILE_PATH := ca-key.pem
 CA_CERT_FILE_PATH := ca-cert.pem
 CA_SIGNING_KEY_FILE_PATH := ca-signing-key.pem
 
+# Dummy invalid hardcoded value
+PROXYPY_PKG_PATH := dist/proxy.py.whl
+BUILDX_TARGET_PLATFORM := linux/amd64
+
 OPEN=$(shell which open)
 UNAME := $(shell uname)
 ifeq ($(UNAME), Linux)
@@ -24,7 +30,7 @@ endif
 .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
 .PHONY: lib-release-test lib-release lib-profile lib-doc
 .PHONY: lib-dep lib-flake8 lib-mypy
-.PHONY: container container-run container-release
+.PHONY: container container-run container-release container-build container-buildx
 .PHONY: devtools dashboard dashboard-clean
 
 all: lib-test
@@ -151,12 +157,25 @@ dashboard:
 dashboard-clean:
 	if [[ -d dashboard/public ]]; then rm -rf dashboard/public; fi
 
-container:
-	docker build -t $(LATEST_TAG) -t $(IMAGE_TAG) .
-
-container-release:
-	docker push $(IMAGE_TAG)
-	docker push $(LATEST_TAG)
+container: lib-package
+	$(MAKE) container-build -e PROXYPY_PKG_PATH=$$(ls dist/*.whl)
+
+# Usage:
+#
+# make container-buildx \
+#	-e PROXYPY_PKG_PATH=$(ls dist/*.whl) \
+#	-e BUILDX_TARGET_PLATFORM=linux/arm64 \
+#	-e PROXYPY_CONTAINER_VERSION=latest
+container-buildx:
+	docker buildx build \
+		--platform $(BUILDX_TARGET_PLATFORM) \
+		-t $(PROXYPY_CONTAINER_TAG) \
+		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
+
+container-build:
+	docker build \
+		-t $(PROXYPY_CONTAINER_TAG) \
+		--build-arg PROXYPY_PKG_PATH=$(PROXYPY_PKG_PATH) .
 
 container-run:
-	docker run -it -p 8899:8899 --rm $(LATEST_TAG)
+	docker run -it -p 8899:8899 --rm $(PROXYPY_CONTAINER_TAG)

README.md

@@ -257,12 +257,30 @@ or from GitHub `master` branch
 
 ## Using Docker
 
+Stable version container releases are available for following platforms:
+
+- `linux/386`
+- `linux/amd64`
+- `linux/arm/v6`
+- `linux/arm/v7`
+- `linux/arm64/v8`
+- `linux/ppc64le`
+- `linux/s390x`
+
 ### Stable Version from Docker Hub
 
+Run `proxy.py` latest container:
+
 ```console
 ❯ docker run -it -p 8899:8899 --rm abhinavsingh/proxy.py:latest
 ```
 
+To run specific target platform container on multi-platform supported servers:
+
+```console
+❯ docker run -it -p 8899:8899 --rm --platform linux/arm64/v8 abhinavsingh/proxy.py:latest
+```
+
 ### Build Development Version Locally
 
 ```console
@@ -394,6 +412,16 @@ To start `proxy.py` from source code follow these instructions:
   ❯ make lib-dep
   ```
 
+- Generate `proxy/common/_scm_version.py`
+
+  NOTE: *Following step is not necessary for editable installs.*
+
+  This file writes SCM detected version to `proxy/common/_scm_version.py` file.
+
+  ```console
+  ❯ ./write-scm-version.sh
+  ```
+
 - Optionally, run tests
 
   ```console
@@ -2038,7 +2066,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.3.2.dev183+g808caa1.d20211124
+proxy.py v2.3.2.dev190+ge60d80d.d20211124
 
 options:
   -h, --help            show this help message and exit