From 9df906ad9c35f3ef63574f95fe5d8501933db4bc Mon Sep 17 00:00:00 2001 From: ChinYikMing Date: Sat, 22 Feb 2025 22:27:58 +0800 Subject: [PATCH 1/2] Eliminate Linux SUBLEVEL bump Add SHA256SUM as a new checksum verifier for downloading Linux tarballs since the latest Linux tarball is available at https://cdn.kernel.org/pub/linux/kernel/v6.x/ and the official SHA256 values can be found at https://cdn.kernel.org/pub/linux/kernel/v6.x/sha256sums.asc. To maintain compatibility with SHA1SUM, the verify function in mk/external.mk has been modified to support multiple SHA commands. Additionally, strip-components option has been added to ensure correct extraction when using tar to address variations in top-level directory structures such as the Linux tarball including a top-level directory while the Timidity tarball does not. With these changes, the Linux kernel SUBLEVEL bump is no longer necessary, and builds can be triggered via a workflow dispatch event. Afterwards, version updates of Linux will only apply to the VERSION and PATCHLEVEL. --- mk/artifact.mk | 10 +++--- mk/common.mk | 11 +++++++ mk/external.mk | 89 +++++++++++++++++++++++++++++++------------------- 3 files changed, 72 insertions(+), 38 deletions(-) diff --git a/mk/artifact.mk b/mk/artifact.mk index 2963ff62..fed9cdb0 100644 --- a/mk/artifact.mk +++ b/mk/artifact.mk @@ -84,7 +84,7 @@ ifeq ($(call has, SYSTEM), 1) $(Q)$(eval PREBUILT_LINUX_IMAGE_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-linux-image | awk '{ print $$2 };')) $(Q)$(eval $(foreach FILE,$(PREBUILT_LINUX_IMAGE_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-image | awk '{ print $$1 };'),$(BIN_DIR)/linux-image/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-image | awk '{ print $$1 };'),$(BIN_DIR)/linux-image/$(FILE),RES) \ )) $(Q)$(eval RV32EMU_PREBUILT_TARBALL := rv32emu-linux-image-prebuilt.tar.gz) @@ -92,7 +92,7 @@ else ifeq ($(call has, ARCH_TEST), 1) $(Q)$(eval PREBUILT_SAIL_FILENAME := $(shell cat $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha | awk '{ print $$2 };')) $(Q)$(eval $(foreach FILE,$(PREBUILT_SAIL_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha | awk '{ print $$1 };'),$(BIN_DIR)/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha | awk '{ print $$1 };'),$(BIN_DIR)/$(FILE),RES) \ )) $(Q)$(eval RV32EMU_PREBUILT_TARBALL := rv32emu-prebuilt-sail-$(HOST_PLATFORM)) @@ -101,10 +101,10 @@ else $(Q)$(eval PREBUILT_RV32_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$2 };')) $(Q)$(eval $(foreach FILE,$(PREBUILT_X86_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$1 };'),$(BIN_DIR)/linux-x86-softfp/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$1 };'),$(BIN_DIR)/linux-x86-softfp/$(FILE),RES) \ )) $(Q)$(eval $(foreach FILE,$(PREBUILT_RV32_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$1 };'),$(BIN_DIR)/riscv32/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$1 };'),$(BIN_DIR)/riscv32/$(FILE),RES) \ )) $(Q)$(eval RV32EMU_PREBUILT_TARBALL := rv32emu-prebuilt.tar.gz) @@ -199,7 +199,7 @@ ifeq ($(call has, PREBUILT), 0) ifeq ($(call has, SYSTEM), 0) $(Q)$(call prologue,"scimark2") $(Q)$(call download,$(SCIMARK2_URL)) - $(Q)$(call verify,$(SCIMARK2_SHA1),$(notdir $(SCIMARK2_URL))) + $(Q)$(call verify,$(SHA1SUM),$(SCIMARK2_SHA1),$(notdir $(SCIMARK2_URL))) $(Q)$(call extract,"./tests/scimark2",$(notdir $(SCIMARK2_URL))) $(Q)$(call epilogue,$(notdir $(SCIMARK2_URL)),$(SHA1_FILE1),$(SHA1_FILE2)) $(Q)$(PRINTF) "Building scimark2 ...\n" diff --git a/mk/common.mk b/mk/common.mk index 6418974f..6935f1c9 100644 --- a/mk/common.mk +++ b/mk/common.mk @@ -64,3 +64,14 @@ ifndef SHA1SUM SHA1SUM := echo endif endif + +SHA256SUM = sha256sum +SHA256SUM := $(shell which $(SHA256SUM)) +ifndef SHA256SUM + SHA256SUM = shasum -a 256 + SHA256SUM := $(shell which shasum) + ifndef SHA256SUM + $(warning No sha256sum found. Disable checksums) + SHA256SUM := echo + endif +endif diff --git a/mk/external.mk b/mk/external.mk index c9f39b91..16f9fa44 100644 --- a/mk/external.mk +++ b/mk/external.mk @@ -4,10 +4,10 @@ COMPRESSED_IS_DIR := EXTRACTOR := VERIFIER := -# temporarily files to store correct SHA1 value and computed SHA1 value +# temporarily files to store correct SHA value and computed SHA value # respectively for verification of directory source -$(eval SHA1_FILE1 := $(shell mktemp)) -$(eval SHA1_FILE2 := $(shell mktemp)) +$(eval SHA_FILE1 := $(shell mktemp)) +$(eval SHA_FILE2 := $(shell mktemp)) # $(1): compressed source define prologue @@ -32,6 +32,7 @@ endef # $(1): destination directory # $(2): compressed source(.zip or.gz) +# $(3): strip component level (only for tar) # For Buildroot and Linux kernel, no need to extract define extract $(eval COMPRESSED_SUFFIX := $(suffix $(2))) @@ -42,85 +43,107 @@ define extract ($(eval _ := \ $(if $(COMPRESSED_IS_ZIP), \ ($(eval EXTRACTOR := unzip -d $(1) $(2))), \ - ($(eval EXTRACTOR := tar -xf $(2) -C $(1))) \ + ($(eval EXTRACTOR := tar -xf $(2) --strip-components=$(3) -C $(1))) \ )) \ $(eval _ := $(shell $(EXTRACTOR)))) ) \ ) endef -# $(1): correct SHA1 value -# $(2): filename or directory path -# $(3): (optional) returned result +# $(1): SHA algorithm command +# $(2): correct SHA value +# $(3): filename or directory path +# $(4): (optional) returned result # # Note: -# 1. for regular file, $(SHA1SUM) command's -c option generates keyword "FAILED" for indicating an unmatch +# 1. for regular file, $(1) command's -c option generates keyword "FAILED" for indicating an unmatch # 2. for directory, cmp command outputs keyword "differ" for indicating an unmatch define verify - $(eval COMPRESSED_IS_DIR := $(if $(wildcard $(2)/*),1,0)) + $(eval COMPRESSED_IS_DIR := $(if $(wildcard $(3)/*),1,0)) $(eval _ := \ $(if $(filter 1,$(COMPRESSED_IS_DIR)), \ ($(eval VERIFIER := \ - echo $(1) > $(SHA1_FILE1) \ - | find $(2) -type f -not -path '*/.git/*' -print0 \ + echo $(2) > $(SHA_FILE1) \ + | find $(3) -type f -not -path '*/.git/*' -print0 \ | sort -z \ - | xargs -0 $(SHA1SUM) \ + | xargs -0 $(1) \ | sort \ - | $(SHA1SUM) \ - | cut -f 1 -d ' ' > $(SHA1_FILE2) && cmp $(SHA1_FILE1) $(SHA1_FILE2))), \ - ($(eval VERIFIER := (ls $(2) >/dev/null 2>&1 || echo FAILED) && echo "$(strip $(1)) $(strip $(2))" | $(SHA1SUM) -c -)) \ + | $(1) \ + | cut -f 1 -d ' ' > $(SHA_FILE2) && cmp $(SHA_FILE1) $(SHA_FILE2))), \ + ($(eval VERIFIER := (ls $(3) >/dev/null 2>&1 || echo FAILED) && echo "$(strip $(2)) $(strip $(3))" | $(1) -c -)) \ )) $(eval _ := $(shell $(VERIFIER) 2>&1)) $(eval _ := \ $(if $(filter FAILED differ:,$(_)), \ - ($(if $(3), \ - $(eval $(3) := 1), \ + ($(if $(4), \ + $(eval $(4) := 1), \ $(error $(_)) \ )), \ - (# SHA1 value match, do nothing) \ + (# SHA value match, do nothing) \ )) endef # For each external target, the following must be defined in advance: # _DATA_URL : the hyperlink which points to archive. +# _DATA_DEST : the extract destination of the file (only for compressed file) # _DATA : the file to be read by specific executable. -# _DATA_SHA1 : the checksum of the content in _DATA +# _DATA_SKIP_DIR_LEVEL : the strip component level when using tar +# _DATA_SHA : the SHA value of the content in _DATA +# _DATA_SHA_CMD : the SHA command # Doom # https://tipsmake.com/how-to-run-doom-on-raspberry-pi-without-emulator DOOM_DATA_URL = http://www.doomworld.com/3ddownloads/ports/shareware_doom_iwad.zip -DOOM_DATA = $(OUT)/DOOM1.WAD -DOOM_DATA_SHA1 = 5b2e249b9c5133ec987b3ea77596381dc0d6bc1d +DOOM_DATA_DEST = $(OUT) +DOOM_DATA = $(DOOM_DATA_DEST)/DOOM1.WAD +DOOM_DATA_SHA = 5b2e249b9c5133ec987b3ea77596381dc0d6bc1d +DOOM_DATA_SHA_CMD = $(SHA1SUM) # Quake QUAKE_DATA_URL = https://www.libsdl.org/projects/quake/data/quakesw-1.0.6.zip -QUAKE_DATA = $(OUT)/id1/pak0.pak -QUAKE_DATA_SHA1 = 36b42dc7b6313fd9cabc0be8b9e9864840929735 +QUAKE_DATA_DEST = $(OUT) +QUAKE_DATA = $(QUAKE_DATA_DEST)/id1/pak0.pak +QUAKE_DATA_SHA = 36b42dc7b6313fd9cabc0be8b9e9864840929735 +QUAKE_DATA_SHA_CMD = $(SHA1SUM) # Timidity software synthesizer configuration for SDL2_mixer TIMIDITY_DATA_URL = http://www.libsdl.org/projects/mixer/timidity/timidity.tar.gz -TIMIDITY_DATA = $(OUT)/timidity -TIMIDITY_DATA_SHA1 = cf6217a5d824b717ec4a07e15e6c129a4657ca25 +TIMIDITY_DATA_DEST = $(OUT) +TIMIDITY_DATA = $(TIMIDITY_DATA_DEST)/timidity +TIMIDITY_DATA_SKIP_DIR_LEVEL = 0 +TIMIDITY_DATA_SHA = cf6217a5d824b717ec4a07e15e6c129a4657ca25 +TIMIDITY_DATA_SHA_CMD = $(SHA1SUM) # Buildroot BUILDROOT_VERSION = 2024.11 BUILDROOT_DATA = /tmp/buildroot BUILDROOT_DATA_URL = git clone https://github.com/buildroot/buildroot $(BUILDROOT_DATA) -b $(BUILDROOT_VERSION) --depth=1 -BUILDROOT_DATA_SHA1 = e678801287ab68369af1731dcf1acc39e4adccff +BUILDROOT_DATA_SHA = e678801287ab68369af1731dcf1acc39e4adccff +BUILDROOT_DATA_SHA_CMD = $(SHA1SUM) # Linux kernel -LINUX_VERSION = linux-6.1.y -LINUX_DATA = /tmp/linux -LINUX_DATA_URL = git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git $(LINUX_DATA) -b $(LINUX_VERSION) --depth=1 -LINUX_DATA_SHA1 = 43b6b7fbf9231656d7b65f118445996172250fc0 +LINUX_VERSION = 6 +LINUX_PATCHLEVEL = 1 +LINUX_CDN_BASE_URL = https://cdn.kernel.org/pub/linux/kernel +LINUX_CDN_VERSION_URL = $(LINUX_CDN_BASE_URL)/v$(LINUX_VERSION).x +$(shell mkdir -p /tmp/linux) +LINUX_DATA_DEST = /tmp/linux +LINUX_DATA = $(shell wget -q -O- $(LINUX_CDN_VERSION_URL) | \ + grep -o 'linux-$(LINUX_VERSION).$(LINUX_PATCHLEVEL).[0-9]\+\.tar.gz' | \ + sort -V | tail -n 1) +LINUX_DATA_URL = $(LINUX_CDN_VERSION_URL)/$(LINUX_DATA) +LINUX_DATA_SKIP_DIR_LEVEL = 1 +LINUX_DATA_SHA = $(shell wget -q -O- $(LINUX_CDN_VERSION_URL)/sha256sums.asc | \ + grep $(LINUX_DATA) | awk '{print $$1}') +LINUX_DATA_SHA_CMD = $(SHA256SUM) define download-extract-verify $($(T)_DATA): $(Q)$$(call prologue,$$@) $(Q)$$(call download,$(strip $($(T)_DATA_URL))) - $(Q)$$(call extract,$(OUT),$(notdir $($(T)_DATA_URL))) - $(Q)$$(call verify,$($(T)_DATA_SHA1),$($(T)_DATA)) - $(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA1_FILE1),$(SHA1_FILE2)) + $(Q)$$(call extract,$($(T)_DATA_DEST),$(notdir $($(T)_DATA_URL)),$($(T)_DATA_SKIP_DIR_LEVEL)) + $(Q)$$(call verify,$($(T)_DATA_SHA_CMD),$($(T)_DATA_SHA),$($(T)_DATA)) + $(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA_FILE1),$(SHA_FILE2)) endef EXTERNAL_DATA = DOOM QUAKE TIMIDITY BUILDROOT LINUX From ec4a794d41723f57a1496f7844c206e845bae582 Mon Sep 17 00:00:00 2001 From: ChinYikMing Date: Sat, 22 Feb 2025 22:28:29 +0800 Subject: [PATCH 2/2] CI: Decouple Linux image build in build-artifact Separate the Linux image build process from the build-artifact CI. The Linux image is now majorly built via a workflow dispatch event while ELF artifacts do not need to be built in the same event, thus decoupling these processes ensures that ELF artifacts are not unnecessarily rebuilt. This modular approach improves the flexibility of the artifact building workflow. --- .github/workflows/build-artifact.yml | 58 --------------- .github/workflows/build-linux-artifacts.yml | 78 +++++++++++++++++++++ 2 files changed, 78 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/build-linux-artifacts.yml diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index c2d4b15a..59c2503c 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -27,12 +27,6 @@ jobs: tests/quake/** tests/scimark2/** tests/*.c - - name: Test file change of Linux image - id: test-linux-image-version-change - uses: tj-actions/changed-files@v45 - with: - files: | - mk/external.mk - name: Set alias id: has_changed_files run: | @@ -41,60 +35,8 @@ jobs: else echo "has_changed_files=false" >> $GITHUB_OUTPUT fi - if [[ ${{ steps.test-linux-image-version-change.outputs.any_modified }} == true ]]; then - # Determine if the changes are from Buildroot or the Linux version (The Linux might have several patches, so also need to check the SHA value) - echo -n $(git --no-pager diff HEAD^ HEAD | grep -e "+BUILDROOT_VERSION" -e "+LINUX_VERSION" -e "+LINUX_DATA_SHA1") >> linux-image-version-change - if [[ -s linux-image-version-change ]]; then - echo "has_changed_linux_image_version=true" >> $GITHUB_OUTPUT - else - echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT - fi - else - echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT - fi outputs: has_changed_files: ${{ steps.has_changed_files.outputs.has_changed_files }} - has_changed_linux_image_version: ${{ steps.has_changed_files.outputs.has_changed_linux_image_version }} - - build-linux-image-artifact: - needs: [detect-file-change] - if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get update -q=2 - sudo apt-get upgrade -q=2 - sudo apt-get install -q=2 build-essential git - - name: Build Linux image - run: | - make build-linux-image - make artifact ENABLE_PREBUILT=0 ENABLE_SYSTEM=1 - mkdir -p /tmp/rv32emu-linux-image-prebuilt/linux-image - mv build/linux-image/Image /tmp/rv32emu-linux-image-prebuilt/linux-image - mv build/linux-image/rootfs.cpio /tmp/rv32emu-linux-image-prebuilt/linux-image - mv build/sha1sum-linux-image /tmp - - name: Create tarball - run: | - cd /tmp - tar -zcvf rv32emu-linux-image-prebuilt.tar.gz rv32emu-linux-image-prebuilt - - name: Create GitHub Release - env: - GH_TOKEN: ${{ secrets.RV32EMU_PREBUILT_TOKEN }} - run: | - RELEASE_TAG=$(date +'%Y.%m.%d'-Linux-Image) - cd /tmp - gh release create $RELEASE_TAG \ - --repo sysprog21/rv32emu-prebuilt \ - --title "$RELEASE_TAG""-nightly" - gh release upload $RELEASE_TAG \ - rv32emu-linux-image-prebuilt.tar.gz \ - sha1sum-linux-image \ - --repo sysprog21/rv32emu-prebuilt build-artifact: needs: [detect-file-change] diff --git a/.github/workflows/build-linux-artifacts.yml b/.github/workflows/build-linux-artifacts.yml new file mode 100644 index 00000000..1ea70d2f --- /dev/null +++ b/.github/workflows/build-linux-artifacts.yml @@ -0,0 +1,78 @@ +name: Build Linux artifact + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + detect-file-change: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Test file change of Linux image + id: test-linux-image-version-change + uses: tj-actions/changed-files@v45 + with: + files: | + mk/external.mk + - name: Set alias + id: has_changed_files + run: | + if [[ ${{ steps.test-linux-image-version-change.outputs.any_modified }} == true ]]; then + # Determine if the changes are from Buildroot or the Linux version (The Linux might have several patches, so also need to check the SHA value) + echo -n $(git --no-pager diff HEAD^ HEAD | grep -e "+BUILDROOT_VERSION" -e "+LINUX_VERSION" -e "+LINUX_PATCHLEVEL") >> linux-image-version-change + if [[ -s linux-image-version-change ]]; then + echo "has_changed_linux_image_version=true" >> $GITHUB_OUTPUT + else + echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT + fi + else + echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT + fi + outputs: + has_changed_linux_image_version: ${{ steps.has_changed_files.outputs.has_changed_linux_image_version }} + + build-linux-image-artifact: + needs: [detect-file-change] + if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Install dependencies + run: | + sudo apt-get update -q=2 + sudo apt-get upgrade -q=2 + sudo apt-get install -q=2 build-essential git + - name: Build Linux image + run: | + make build-linux-image + make artifact ENABLE_PREBUILT=0 ENABLE_SYSTEM=1 + mkdir -p /tmp/rv32emu-linux-image-prebuilt/linux-image + mv build/linux-image/Image /tmp/rv32emu-linux-image-prebuilt/linux-image + mv build/linux-image/rootfs.cpio /tmp/rv32emu-linux-image-prebuilt/linux-image + mv build/sha1sum-linux-image /tmp + - name: Create tarball + run: | + cd /tmp + tar -zcvf rv32emu-linux-image-prebuilt.tar.gz rv32emu-linux-image-prebuilt + - name: Create GitHub Release + env: + GH_TOKEN: ${{ secrets.RV32EMU_PREBUILT_TOKEN }} + run: | + RELEASE_TAG=$(date +'%Y.%m.%d'-Linux-Image) + cd /tmp + gh release create $RELEASE_TAG \ + --repo sysprog21/rv32emu-prebuilt \ + --title "$RELEASE_TAG""-nightly" + gh release upload $RELEASE_TAG \ + rv32emu-linux-image-prebuilt.tar.gz \ + sha1sum-linux-image \ + --repo sysprog21/rv32emu-prebuilt