27 recieve merge of karmacomputingserver bootstrap ncl

.github/workflows/git-auto-issue-branch-creation.yml

@@ -6,6 +6,10 @@ on:
 jobs:
   create-issue-branch:
     runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: write
+      issues: write
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -22,7 +26,7 @@ jobs:
           # Given an issue title: "Fix the 2'nd bug in UI where there's a # in the form"
           # Becomes:
           #
-                                                             # Note there is a space here, to keep both '-' and spaces ' '
+          # Note there is a space here, to keep both '-' and spaces ' '
           echo ISSUE_BRANCH_NAME=`echo "${{ github.event.issue.title }}" | \
           # Remove non alpha/numeric chars
           tr -cd '[:alnum:]- ' | \
@@ -41,3 +45,12 @@ jobs:
           git checkout -b "${{ steps.issue.outputs.number }}-${{ env.ISSUE_BRANCH_NAME }}"
           git push -u origin HEAD
 
+      - name: Creating PR based on branch name ${{ github.ref_name }}
+        run: |
+          curl -L \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/Subscribie/subscribie/pulls \
+          -d '{"title": "#${{ steps.issue.outputs.number }} ${{ env.ISSUE_BRANCH_NAME }}","body":"Pull request related issue: #${{ steps.issue.outputs.number }}. Please pull these awesome changes in!","head":"${{ steps.issue.outputs.number }}-${{ env.ISSUE_BRANCH_NAME }}","base":"master"}'

.gitignore

@@ -7,3 +7,6 @@ node_modules
 test-results/
 playwright-report/
 playwright/.cache/
+key*
+deploy/serve/www/*
+!deploy/serve/www/bootfile

repo-server-bootstrap-ncl-issue-20/README.md

@@ -0,0 +1,50 @@
+# Overview
+
+> [!NOTE]
+> This repo & docs is very much in flux.
+> For project purpose see [Server Bootstrap Project Concise project summary & deliverables](https://docs.google.com/document/d/15YR0hAkHfq8g_2rFzpKjpKf1mytyygREyR7a1ZWfzhE/edit?usp=sharing)
+> For background reading also see [https://github.com/KarmaComputing/server-bootstrap](https://github.com/KarmaComputing/server-bootstrap)
+
+![diagram](docs/diagram.drawio.png)
+
+
+## Table of contents
+
+- [Alpine Mirroring](./deploy/alpine-mirror/README.md)
+
+
+
+## Run for debugging
+
+- In `internal/runner`
+
+    ```sh
+    URL=https://192.168.0.230 USERNAME=Administrator PASSWORD=A0F7HKUU VALIDCERT=false WIPEINTERVAL=300 go run .
+    ```
+
+## Full deploy
+
+1. Build
+  - Building ipxe.iso image
+    - Use new/existing iPXE config file in `deploy/build/ipxe/scripts`
+    - Input its name as FILE variable in `deploy/scripts/build-ipxe-iso.sh`
+    - Run `deploy/scripts/build-ipxe-iso.sh`
+    - `ipxe.iso` is placed in `deploy/serve/www`
+  - Building alpine-netboot image
+    - Run `deploy/scripts/build-alpine.sh`
+    - Files are placed in `deploy/serve/www/iso`
+  - SSH keys
+    - An SSH keypair is automatically generated upon building an `ipxe.iso` image with the above command
+      - The **private** key is placed at `deploy/ssh/key`
+      - The **public** key is placed at `deploy/serve/www/ssh/key.pub`
+    - Up **re**building the `ipxe.iso`, the script will prompt to replace these keys or not
+
+2. Run stack
+  - Ensure the files are correctly placed from step 1
+  - `podman compose up -d` in repository root
+
+3. !! VM FOR TESTING !!
+  - Ensure `qemu` is installed and runnable
+  - Ensure web server is accessible at whatever address is defined in the iPXE boot script
+  - `qemu-system-x86_64 -cdrom <ipxe.iso> -net nic -net user,hostfwd=tcp::2223-:22 -m 3072 -smp $(nproc)`
+  - VM can be accessed over SSH at `localhost:2223`

repo-server-bootstrap-ncl-issue-20/compose.yaml

@@ -0,0 +1,21 @@
+services:
+  controller:
+    build: ./internal/runner
+    # ports:
+    # - 8081:8080
+    volumes:
+      - ./deploy/ssh/key:/app/key:z
+      - ./deploy/ansible:/app/ansible:z
+    environment:
+      - LC_ALL=en_US.UTF-8
+      - USERNAME=Administrator
+      - PASSWORD=A0F7HKUU
+      - URL=https://192.168.0.230
+      - VALIDCERT=false
+      - WIPEINTERVAL=300
+  apache:
+    build: ./deploy/serve
+    volumes:
+      - ./deploy/serve/www:/usr/local/apache2/htdocs:z
+    ports:
+      - 8080:80

repo-server-bootstrap-ncl-issue-20/deploy/alpine-mirror/Containerfile

@@ -0,0 +1,10 @@
+FROM docker.io/library/alpine:3.20
+
+RUN apk add rsync
+
+COPY ./synchroniser /app
+RUN cat /app/crontab >> /etc/crontabs/root
+
+RUN chmod +x /app/mirror.sh
+
+CMD ["crond", "-f"]

repo-server-bootstrap-ncl-issue-20/deploy/alpine-mirror/README.md

@@ -0,0 +1,23 @@
+# How to Use
+
+- Adjust `synchroniser/crontab` for desired schedule of synchronising
+- Adjust `synchroniser/mirror.sh` for which repos to exclude, etc.
+- The container image must be re-built when `crontab` or `mirror.sh` are changed
+- Run by `<podman|docker> compose up`
+
+# Notes
+
+- [Alpine Wiki - How to setup a Mirror](https://wiki.alpinelinux.org/wiki/How_to_setup_a_Alpine_Linux_mirror)
+- You can trigger the syncs manually via
+  - `<podman|docker> exec -it alpine-mirror-sync /app/mirror.sh`
+
+# How I've Tested It
+
+1. `git clone [email protected]:KarmaComputing/server-bootstrap-ncl.git`
+2. `cd server-bootstrap-ncl/deploy/alpine-mirror`
+3. `sudo docker compose up`
+4. `sudo docker run --rm -it --network host alpine:3.21 sh`
+  - `echo -e "http://localhost/v3.21/main\nhttp://localhost/v3.21/community" > /etc/apk/repositories`
+  - `apk update`
+  - Feel free to install anything to test, with `apk add`
+    - e.g. `apk add fastfetch` and `fastfetch`

repo-server-bootstrap-ncl-issue-20/deploy/alpine-mirror/compose.yaml

@@ -0,0 +1,13 @@
+services:
+  sync:
+    container_name: alpine-mirror-sync
+    build: .
+    volumes:
+      - ./repo:/site:z
+  server:
+    container_name: alpine-mirror-server
+    image: docker.io/library/httpd:alpine
+    volumes:
+      - ./repo:/usr/local/apache2/htdocs:z
+    ports:
+      - 8080:80

repo-server-bootstrap-ncl-issue-20/deploy/alpine-mirror/repo/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

repo-server-bootstrap-ncl-issue-20/deploy/alpine-mirror/synchroniser/crontab

@@ -0,0 +1,2 @@
+# every 5 days
+* * */5 * * sh /app/mirror.sh

repo-server-bootstrap-ncl-issue-20/deploy/alpine-mirror/synchroniser/mirror.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# make sure we never run 2 rsync at the same time
+lockfile="/tmp/alpine-mirror.lock"
+if [ -z "$flock" ] ; then
+  exec env flock=1 flock -n $lockfile "$0" "$@"
+fi
+
+src=rsync://rsync.alpinelinux.org/alpine/ 
+dest=/site
+
+exclude="--exclude v2.* --exclude v3.0 --exclude v3.1 --exclude v3.2 --exclude v3.3 --exclude v3.4 --exclude v3.5 --exclude v3.6 --exclude v3.7 --exclude v3.8 --exclude v3.9 --exclude v3.10 --exclude v3.11 --exclude v3.12 --exclude v3.13 --exclude v3.14 --exclude v3.15 --exclude v3.16 --exclude v3.17"
+
+echo "--- Starting Sync ---"
+
+mkdir -p "$dest"
+/usr/bin/rsync \
+        --archive \
+        --update \
+        --hard-links \
+        --delete \
+        --delete-after \
+        --delay-updates \
+        --timeout=600 \
+        $exclude \
+        "$src" "$dest"
+
+echo "--- Finished Sync ---"

repo-server-bootstrap-ncl-issue-20/deploy/ansible/playbook.yml

@@ -0,0 +1,13 @@
+---
+- name: Test
+  hosts: all
+  gather_facts: false
+
+  tasks:
+    - name: Install python
+      raw: test -e /usr/bin/python3 || apk add --no-cache python3
+
+    - name: Gather facts
+      setup:
+      vars:
+        ansible_python_interpreter: /usr/bin/python3

repo-server-bootstrap-ncl-issue-20/deploy/build/alpine/Containerfile

@@ -0,0 +1,21 @@
+FROM alpine:latest
+
+RUN apk update && apk add --no-cache alpine-sdk \
+    build-base \
+    apk-tools \
+    busybox \
+    fakeroot \
+    syslinux \
+    xorriso \
+    squashfs-tools \
+    sudo \
+    git \
+    grub \
+    grub-efi
+
+WORKDIR /build
+COPY build.sh .
+
+RUN chmod +x build.sh
+
+CMD ["sh", "/build/build.sh"]

repo-server-bootstrap-ncl-issue-20/deploy/build/alpine/build.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+
+#
+# !!!!!
+#  THIS SCRIPT IS NOT MEANT TO BE MANUALLY RUN, USE THE ADJACENT CONTAINERFILE
+# !!!!!
+#
+
+# https://github.com/KarmaComputing/server-bootstrap/blob/main/build-alpine-netboot-zfs.sh
+# Purpose:
+#   Build netboot image with zfs kernel module included
+
+set -x
+
+# Note we now build alpine-conf from source (rather than doing apk add alpine-conf)
+# due to issue https://github.com/KarmaComputing/server-bootstrap/issues/20
+
+# Clone and build latest alpine-conf
+git clone https://gitlab.alpinelinux.org/alpine/alpine-conf.git
+cd alpine-conf
+make
+make install
+cd -
+
+# Start build
+adduser build --disabled-password -G abuild
+# Set password non interactively
+echo -e "password\npassword" | passwd build
+echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/abuild
+
+su - build << 'EOF'
+set -x
+SUDO=sudo abuild-keygen -n -i -a
+# aports contains build utilities such as mkimage.sh
+git clone --depth 1 https://gitlab.alpinelinux.org/alpine/aports.git
+cd aports
+
+# Create & build alpine netboot profile with zfs kernel module enabled
+cat > ./scripts/mkimg.zfsnetboot.sh << 'EOFINNER'
+
+profile_zfsnetboot() {
+        profile_standard
+        kernel_cmdline="overlay_size=0 console=tty0 console=ttyS0,115200"
+        syslinux_serial="0 115200"
+        kernel_addons="zfs"
+        apks="$apks zfs-scripts zfs zfs-utils-py python3 mkinitfs syslinux util-linux linux-firmware"
+        initfs_features="base network squashfs usb virtio"
+        output_format="netboot"
+        image_ext="tar.gz"
+}
+EOFINNER
+
+cat ./scripts/mkimg.zfsnetboot.sh
+echo Running mkimage.sh
+./scripts/mkimage.sh --arch x86_64 --repository https://dl-cdn.alpinelinux.org/alpine/v3.20/main --profile zfsnetboot
+EOF
+
+ls -la /home/build/aports
+cp /home/build/aports/alpine-zfsnetboot-*.tar.gz /output --force
+tar -xvf /home/build/aports/alpine-zfsnetboot-*.tar.gz --directory /output
+chmod 0644 /output/boot/* # fix permissions, specifically for initramfs for some reason
+exit

repo-server-bootstrap-ncl-issue-20/deploy/build/ipxe/Containerfile

@@ -0,0 +1,34 @@
+FROM alpine:latest
+
+RUN apk update && apk add --no-cache build-base \
+    git \
+    gcc \
+    binutils \
+    make \
+    perl \
+    xz-dev \
+    mtools \
+    syslinux \
+    xorriso \
+    curl \
+    openssl \
+    coreutils
+
+WORKDIR /build
+
+RUN git clone https://github.com/ipxe/ipxe.git
+WORKDIR /build/ipxe/src
+
+RUN curl -s http://ca.ipxe.org/ca.crt > ca.pem &&\
+    curl -s https://letsencrypt.org/certs/isrgrootx1.pem > isrgrootx1.pem &&\
+    curl -s https://letsencrypt.org/certs/lets-encrypt-r3.pem > lets-encrypt-r3.pem
+
+RUN sed -i 's$//#define PING_CMD$#define PING_CMD$g' config/general.h &&\
+    sed -i 's$//#define NET_PROTO_IPV6$#define NET_PROTO_IPV6$g' config/general.h &&\
+    sed -i 's/undef.*DOWNLOAD_PROTO_HTTPS/define DOWNLOAD_PROTO_HTTPS/g' config/general.h
+
+CMD make -j${ISO_MAKE_THREADS} bin/ipxe.iso EMBED=/input/${FILE} \
+        DEBUG=tls,httpcore,x509,certstore \
+        CERT=ca.pem,isrgrootx1.pem,lets-encrypt-r3.pem \
+        TRUST=ca.pem,isrgrootx1.pem,lets-encrypt-r3.pem \
+    && mv bin/ipxe.iso /output/ipxe.iso

repo-server-bootstrap-ncl-issue-20/deploy/build/ipxe/scripts/alpinebooter.ipxe

@@ -0,0 +1,8 @@
+#!ipxe
+
+echo Setting temporary DHCP address to chain into bootfile
+dhcp
+
+set chain-url http://192.168.0.170:8080/bootfile
+echo Chaining to ${chain_url}
+chain ${chain-url}?uuid=${uuid}

repo-server-bootstrap-ncl-issue-20/deploy/build/ipxe/scripts/vm-test.ipxe

@@ -0,0 +1,23 @@
+#!ipxe
+
+# The iPXE script below is based on:
+# https://boot.alpinelinux.org/boot.ipxe
+
+dhcp
+
+set console console=tty0
+set cmdline modules=loop,squashfs nomodeset
+set branch v3.20
+set flavor lts
+set arch x86_64
+
+set serverip 192.168.1.29:8080
+set server-url http://${serverip}
+set base-url ${server-url}/iso/alpine-netboot/boot
+set repo-url http://dl-cdn.alpinelinux.org/alpine/${branch}/main
+set sshkey-url ${server-url}/ssh/key.pub
+
+imgfree
+kernel ${base-url}/vmlinuz-${flavor} ${cmdline} ${console} ip=dhcp alpine_repo=${repo-url} modloop=${base-url}/modloop-${flavor} alpine_dev=tmpfs ssh_key=${sshkey-url}
+initrd ${base-url}/initramfs-${flavor}
+boot

repo-server-bootstrap-ncl-issue-20/deploy/scripts/build-alpine.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+cd "$(dirname "$0")"
+
+set -exu
+
+PODMAN_IMAGE_NAME="alpine_builder"
+WWW_DIR="../serve/www/iso/alpine-netboot"
+BUILD_DIR="../build/alpine"
+
+echo "--- Creating directory at ${WWW_DIR} ---"
+mkdir -p ${WWW_DIR}
+
+echo "--- Building ${PODMAN_IMAGE_NAME} ---"
+podman build --tag ${PODMAN_IMAGE_NAME} ${BUILD_DIR}
+
+echo "--- Running ${PODMAN_IMAGE_NAME} ---"
+podman run --rm -v ${WWW_DIR}:/output:z \
+    localhost/${PODMAN_IMAGE_NAME}:latest