Skip to content

Commit b78b168

Browse files
me-no-devme-no-dev
authored
Merge pull request espressif#7 from me-no-dev/master
Build system overhaul
2 parents 7dbfa98 + 5fdfb88 commit b78b168

12 files changed

+364
-52
lines changed

‎.github/workflows/cron.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Cron Build
2+
3+
on:
4+
schedule:
5+
# ┌───────────── minute (0 - 59)
6+
# │ ┌───────────── hour (0 - 23)
7+
# │ │ ┌───────────── day of the month (1 - 31)
8+
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
9+
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
10+
# │ │ │ │ │
11+
# │ │ │ │ │
12+
# │ │ │ │ │
13+
# * * * * *
14+
- cron: '0 */6 * * *'
15+
16+
jobs:
17+
run:
18+
name: Build with IDF ${{ matrix.idf_branch }}
19+
runs-on: ubuntu-latest
20+
21+
strategy:
22+
matrix:
23+
idf_branch: [release/v3.2]
24+
steps:
25+
- uses: actions/checkout@v1
26+
- name: Install dependencies
27+
run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
28+
- name: Install Python Wheel
29+
run: pip install wheel
30+
- name: Build
31+
env:
32+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
33+
IDF_BRANCH: ${{ matrix.idf_branch }}
34+
run: bash ./tools/cron.sh
35+
- name: Upload archive
36+
uses: actions/upload-artifact@v1
37+
with:
38+
name: artifacts
39+
path: dist

‎.github/workflows/main.yml renamed to ‎.github/workflows/push.yml

+3-6
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ on:
66
- master
77
- release/*
88
pull_request:
9-
repository_dispatch:
109

1110
jobs:
1211

@@ -20,11 +19,9 @@ jobs:
2019
- name: Install Python Wheel
2120
run: pip install wheel
2221
- name: Build Arduino Libs
23-
env:
24-
TRAVIS_BUILD_DIR: ${{ github.workspace }}
25-
run: bash $TRAVIS_BUILD_DIR/build.sh
26-
- name: Archive libs
22+
run: bash ./build.sh
23+
- name: Upload archive
2724
uses: actions/upload-artifact@v1
2825
with:
29-
name: arduino-libs
26+
name: artifacts
3027
path: dist

‎.github/workflows/repository_dispatch.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Remote Trigger
2+
3+
on: repository_dispatch
4+
5+
jobs:
6+
run:
7+
name: Dispatch Event
8+
runs-on: ubuntu-latest
9+
steps:
10+
- uses: actions/checkout@v1
11+
- name: Install dependencies
12+
run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
13+
- name: Install Python Wheel
14+
run: pip install wheel
15+
- name: Handle Event
16+
env:
17+
GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
18+
run: bash ./tools/repository_dispatch.sh
19+
- name: Upload archive
20+
uses: actions/upload-artifact@v1
21+
with:
22+
name: artifacts
23+
path: dist

‎build.sh

+5-3
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,16 @@ if ! [ -x "$(command -v stat)" ]; then
3535
exit 1
3636
fi
3737

38-
# install esp-idf and gcc toolchain
39-
source ./tools/install-esp-idf.sh
40-
if [ $? -ne 0 ]; then exit 1; fi
38+
mkdir -p dist
4139

4240
# update components from git
4341
./tools/update-components.sh
4442
if [ $? -ne 0 ]; then exit 1; fi
4543

44+
# install esp-idf and gcc toolchain
45+
source ./tools/install-esp-idf.sh
46+
if [ $? -ne 0 ]; then exit 1; fi
47+
4648
# build and prepare libs
4749
./tools/build-libs.sh
4850
if [ $? -ne 0 ]; then exit 1; fi

‎tools/archive-build.sh

+11-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
#!/bin/bash
22

3+
IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
4+
IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD)
5+
36
idf_version_string=${IDF_BRANCH//\//_}"-$IDF_COMMIT"
47
archive_path="dist/arduino-esp32-libs-$idf_version_string.tar.gz"
8+
build_archive_path="dist/arduino-esp32-build-$idf_version_string.tar.gz"
59

6-
mkdir -p dist && \
7-
rm -rf $archive_path && \
8-
cd out && \
9-
tar zcf ../$archive_path * \
10-
&& cd ..
10+
mkdir -p dist && rm -rf "$archive_path" "$build_archive_path"
11+
if [ -d "out" ]; then
12+
cd out && tar zcf "../$archive_path" * && cd ..
13+
fi
14+
if [ -d "build" ]; then
15+
cd build && tar zcf "../$build_archive_path" * && cd ..
16+
fi

‎tools/config.sh

+54-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/bin/bash
2-
IDF_REPO="https://github.com/espressif/esp-idf.git"
3-
IDF_BRANCH="release/v3.2"
2+
43
IDF_COMPS="$IDF_PATH/components"
54
IDF_TOOLCHAIN="xtensa-esp32-elf"
65
IDF_TOOLCHAIN_LINUX_ARMEL="https://dl.espressif.com/dl/xtensa-esp32-elf-linux-armel-1.22.0-87-gb57bad3-5.2.0.tar.gz"
@@ -9,10 +8,25 @@ IDF_TOOLCHAIN_LINUX64="https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22
98
IDF_TOOLCHAIN_WIN32="https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
109
IDF_TOOLCHAIN_MACOS="https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz"
1110

12-
CAMERA_REPO="https://github.com/espressif/esp32-camera.git"
13-
FACE_REPO="https://github.com/espressif/esp-face.git"
11+
if [ -z $IDF_BRANCH ]; then
12+
IDF_BRANCH="release/v3.2"
13+
fi
14+
15+
# Owner of the target ESP32 Arduino repository
16+
AR_USER="espressif"
17+
18+
# The full name of the repository
19+
AR_REPO="$AR_USER/arduino-esp32"
20+
21+
IDF_REPO_URL="https://github.com/espressif/esp-idf.git"
22+
CAMERA_REPO_URL="https://github.com/espressif/esp32-camera.git"
23+
FACE_REPO_URL="https://github.com/espressif/esp-face.git"
24+
AR_REPO_URL="https://github.com/$AR_REPO.git"
25+
26+
if [ -n $GITHUB_TOKEN ]; then
27+
AR_REPO_URL="https://$GITHUB_TOKEN@github.com/$AR_REPO.git"
28+
fi
1429

15-
AR_REPO="https://github.com/espressif/arduino-esp32.git"
1630
AR_ROOT="$PWD"
1731
AR_COMPS="$AR_ROOT/components"
1832
AR_OUT="$AR_ROOT/out"
@@ -56,3 +70,38 @@ if [[ "$AR_OS" == "macos" ]]; then
5670
export SED="gsed"
5771
export SSTAT="stat -f %z"
5872
fi
73+
74+
function git_commit_exists(){ #git_commit_exists <repo-path> <commit-message>
75+
local repo_path="$1"
76+
local commit_message="$2"
77+
local commits_found=`git -C "$repo_path" log --all --grep="$commit_message" | grep commit`
78+
if [ -n "$commits_found" ]; then echo 1; else echo 0; fi
79+
}
80+
81+
function git_branch_exists(){ # git_branch_exists <repo-path> <branch-name>
82+
local repo_path="$1"
83+
local branch_name="$2"
84+
local branch_found=`git -C "$repo_path" ls-remote --heads origin "$branch_name"`
85+
if [ -n "$branch_found" ]; then echo 1; else echo 0; fi
86+
}
87+
88+
function git_pr_exists(){ # git_pr_exists <branch-name>
89+
local pr_num=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$AR_REPO/pulls?head=$AR_USER:$1&state=open" | jq -r '.[].number'`
90+
if [ ! "$pr_num" == "" ] && [ ! "$pr_num" == "null" ]; then echo 1; else echo 0; fi
91+
}
92+
93+
function git_create_pr(){ # git_create_pr <branch> <title>
94+
local pr_branch="$1"
95+
local pr_title="$2"
96+
local pr_body=""
97+
for component in `ls "$AR_COMPS"`; do
98+
if [ ! $component == "arduino" ] && [ -d "$AR_COMPS/$component/.git" ]; then
99+
pr_body+="$component: "$(git -C "$AR_COMPS/$component" symbolic-ref --short HEAD)" "$(git -C "$AR_COMPS/$component" rev-parse --short HEAD)"\r\n"
100+
fi
101+
done
102+
local pr_data="{\"title\": \"$pr_title\", \"body\": \"$pr_body\", \"head\": \"$AR_USER:$pr_branch\", \"base\": \"master\"}"
103+
git_create_pr_res=`echo "$pr_data" | curl -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" --data @- "https://api.github.com/repos/$AR_REPO/pulls"`
104+
local done_pr=`echo "$git_create_pr_res" | jq -r '.title'`
105+
if [ ! "$done_pr" == "" ] && [ ! "$done_pr" == "null" ]; then echo 1; else echo 0; fi
106+
}
107+

‎tools/cron.sh

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
4+
echo "Wrong event '$GITHUB_EVENT_NAME'!"
5+
exit 1
6+
fi
7+
8+
git checkout "$IDF_BRANCH" #local branches should match what the matrix wants to build
9+
source ./build.sh
10+
bash ./tools/push-to-arduino.sh

‎tools/install-esp-idf.sh

+86-10
Original file line numberDiff line numberDiff line change
@@ -7,26 +7,102 @@ if ! [ -x "$(command -v $SED)" ]; then
77
exit 1
88
fi
99

10+
#
11+
# CLONE ESP-IDF
12+
#
13+
1014
if [ -z "$IDF_PATH" ]; then
1115
echo "ESP-IDF is not installed! Installing local copy"
16+
idf_was_installed="1"
1217
if ! [ -d esp-idf ]; then
13-
git clone $IDF_REPO -b $IDF_BRANCH
18+
git clone $IDF_REPO_URL -b $IDF_BRANCH
1419
fi
1520
export IDF_PATH="$AR_ROOT/esp-idf"
16-
cd $IDF_PATH
17-
git fetch origin && git pull origin $IDF_BRANCH
18-
git submodule update --init --recursive
19-
python -m pip install -r requirements.txt
20-
cd "$AR_ROOT"
2121
fi
2222

2323
if [ "$IDF_COMMIT" ]; then
24-
git -C $IDF_PATH checkout $IDF_COMMIT
25-
git -C $IDF_PATH submodule update
24+
git -C "$IDF_PATH" checkout "$IDF_COMMIT"
25+
commit_predefined="1"
26+
fi
27+
28+
export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
29+
export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD)
30+
31+
#
32+
# SETUP ARDUINO DEPLOY
33+
#
34+
35+
if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
36+
# format new branch name and pr title
37+
if [ -x $commit_predefined ]; then #commit was not specified at build time
38+
AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH"
39+
AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT"
40+
AR_NEW_PR_TITLE="IDF $IDF_BRANCH"
41+
else
42+
AR_NEW_BRANCH_NAME="idf-$IDF_COMMIT"
43+
AR_NEW_COMMIT_MESSAGE="IDF $IDF_COMMIT"
44+
AR_NEW_PR_TITLE="$AR_NEW_COMMIT_MESSAGE"
45+
fi
46+
47+
AR_HAS_COMMIT=`git_commit_exists "$AR_COMPS/arduino" "$AR_NEW_COMMIT_MESSAGE"`
48+
AR_HAS_BRANCH=`git_branch_exists "$AR_COMPS/arduino" "$AR_NEW_BRANCH_NAME"`
49+
AR_HAS_PR=`git_pr_exists "$AR_NEW_BRANCH_NAME"`
50+
51+
if [ "$AR_HAS_COMMIT" == "1" ]; then
52+
echo "Commit '$AR_NEW_COMMIT_MESSAGE' Already Exists"
53+
exit 0
54+
fi
55+
56+
if [ "$AR_HAS_BRANCH" == "1" ]; then
57+
echo "Branch '$AR_NEW_BRANCH_NAME' Already Exists"
58+
fi
59+
60+
if [ "$AR_HAS_PR" == "1" ]; then
61+
echo "PR '$AR_NEW_PR_TITLE' Already Exists"
62+
fi
63+
64+
# setup git for pushing
65+
git config --global github.user "$GITHUB_ACTOR"
66+
git config --global user.name "$GITHUB_ACTOR"
67+
git config --global user.email "$GITHUB_ACTOR@github.com"
68+
69+
# create or checkout the branch
70+
if [ ! $AR_HAS_BRANCH == "0" ]; then
71+
echo "Switching to arduino branch '$AR_NEW_BRANCH_NAME'..."
72+
git -C "$AR_COMPS/arduino" checkout $AR_NEW_BRANCH_NAME
73+
else
74+
echo "Creating arduino branch '$AR_NEW_BRANCH_NAME'..."
75+
git -C "$AR_COMPS/arduino" checkout -b $AR_NEW_BRANCH_NAME
76+
fi
77+
if [ $? -ne 0 ]; then
78+
echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed"
79+
exit 1
80+
fi
81+
82+
export AR_NEW_BRANCH_NAME
83+
export AR_NEW_COMMIT_MESSAGE
84+
export AR_NEW_PR_TITLE
85+
86+
export AR_HAS_COMMIT
87+
export AR_HAS_BRANCH
88+
export AR_HAS_PR
89+
fi
90+
91+
#
92+
# UPDATE IDF MODULES
93+
#
94+
95+
if [ -x $idf_was_installed ]; then
96+
git -C $IDF_PATH fetch origin && git -C $IDF_PATH pull origin $IDF_BRANCH
97+
git -C $IDF_PATH submodule update --init --recursive
98+
else
99+
git -C $IDF_PATH submodule update --init --recursive
100+
cd $IDF_PATH && python -m pip install -r requirements.txt && cd "$AR_ROOT"
26101
fi
27102

28-
export IDF_COMMIT=$(git -C $IDF_PATH rev-parse --short HEAD)
29-
export IDF_BRANCH=$(git -C $IDF_PATH symbolic-ref --short HEAD)
103+
#
104+
# INSTALL TOOLCHAIN
105+
#
30106

31107
if ! [ -x "$(command -v $IDF_TOOLCHAIN-gcc)" ]; then
32108
echo "GCC toolchain is not installed! Installing local copy"

‎tools/prepare-libs.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ cat pio_start.txt > "$AR_PLATFORMIO_PY"
1616
rm pio_end.txt 1pio_start.txt 2pio_start.txt pio_start.txt
1717

1818
# include dirs
19-
AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H \"-I{compiler.sdk.path}/include/config\""
19+
AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H -DGCC_NOT_5_2_0=0 -DWITH_POSIX \"-I{compiler.sdk.path}/include/config\""
2020
echo " CPPPATH=[" >> "$AR_PLATFORMIO_PY" && echo " join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"include\", \"config\")," >> "$AR_PLATFORMIO_PY"
2121
while [ "$1" != "" ]; do
2222
cpath=$1

‎tools/push-to-arduino.sh

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
source ./tools/config.sh
3+
4+
if [ -x $GITHUB_TOKEN ]; then
5+
echo "ERROR: GITHUB_TOKEN was not defined"
6+
exit 1
7+
fi
8+
9+
if ! [ -d "$AR_COMPS/arduino" ]; then
10+
echo "ERROR: Target arduino folder does not exist!"
11+
exit 1
12+
fi
13+
14+
#
15+
# UPDATE FILES
16+
#
17+
18+
if [ $AR_HAS_COMMIT == "0" ]; then
19+
cd $AR_COMPS/arduino
20+
21+
# make changes to the files
22+
echo "Patching files in branch '$AR_NEW_BRANCH_NAME'..."
23+
rm -rf $AR_COMPS/arduino/tools/sdk
24+
cp -Rf $AR_SDK $AR_COMPS/arduino/tools/sdk
25+
cp -f $AR_ESPTOOL_PY $AR_COMPS/arduino/tools/esptool.py
26+
cp -f $AR_GEN_PART_PY $AR_COMPS/arduino/tools/gen_esp32part.py
27+
cp -f $AR_PLATFORMIO_PY $AR_COMPS/arduino/tools/platformio-build.py
28+
cp -f $AR_PLATFORM_TXT $AR_COMPS/arduino/platform.txt
29+
30+
# did any of the files change?
31+
if [ -n "$(git status --porcelain)" ]; then
32+
echo "Pushing changes to branch '$AR_NEW_BRANCH_NAME'..."
33+
git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
34+
if [ $? -ne 0 ]; then
35+
echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed"
36+
exit 1
37+
fi
38+
else
39+
echo "No changes in branch '$AR_NEW_BRANCH_NAME'"
40+
if [ $AR_HAS_BRANCH == "0" ]; then
41+
echo "Delete created branch '$AR_NEW_BRANCH_NAME'"
42+
git branch -d $AR_NEW_BRANCH_NAME
43+
fi
44+
exit 0
45+
fi
46+
fi
47+
48+
#
49+
# CREATE PULL REQUEST
50+
#
51+
52+
if [ "$AR_HAS_PR" == "0" ]; then
53+
pr_created=`git_create_pr "$AR_NEW_BRANCH_NAME" "$AR_NEW_PR_TITLE"`
54+
if [ $pr_created == "0" ]; then
55+
echo "ERROR: Failed to create PR '$AR_NEW_PR_TITLE': "`echo "$git_create_pr_res" | jq -r '.message'`": "`echo "$git_create_pr_res" | jq -r '.errors[].message'`
56+
exit 1
57+
fi
58+
fi
59+
exit 0

0 commit comments

Comments
 (0)