From 935809e834a64840a4e5770160b7c82f05d83402 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Wed, 8 Jul 2020 10:23:27 -0700 Subject: [PATCH 01/12] Transition from TravisCI to GitHub CI Convert the continuous integration process to use GitHub's internal CI cloud. Allows us to run up to 20 jobs in parallel, speeding up CI immensely. Translate all Travis CI configuration into event-driven GH YAML files. --- .github/workflows/push.yml | 242 +++++++++++++++++++++ .github/workflows/release-to-publish.yml | 61 ++++++ .github/workflows/tag-to-draft-release.yml | 35 +++ .travis.yml | 162 -------------- doc/filesystem.rst | 16 +- doc/libraries.rst | 2 +- doc/ota_updates/readme.rst | 2 +- package/deploy_package_index.sh | 19 +- package/esp8266_github_io_deploy.enc | Bin 432 -> 2608 bytes package/upload_release.py | 30 +++ tests/ci/install_astyle.sh | 16 -- tests/common.sh | 27 +-- tests/debug6.sh | 21 ++ 13 files changed, 416 insertions(+), 217 deletions(-) create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release-to-publish.yml create mode 100644 .github/workflows/tag-to-draft-release.yml delete mode 100644 .travis.yml create mode 100644 package/upload_release.py delete mode 100755 tests/ci/install_astyle.sh create mode 100644 tests/debug6.sh diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000000..2aff3ff2f8 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,242 @@ +# Run whenever a PR is generated or updated. Do the normal suite of tests +# and report success/fail in the PR. + +name: ESP8266 Arduino CI + +on: + pull_request: + + +jobs: + + build-linux: + name: Build ${{ matrix.chunk }} + runs-on: ubuntu-latest + strategy: + matrix: + chunk: [0, 1, 2, 3, 4, 5, 6, 7] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Cache Linux toolchain + id: cache-linux + uses: actions/cache@v2 + with: + path: ./tools/dist + key: key-linux-toolchain + - name: Build Sketches + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + BUILD_PARITY: custom + mod: 8 + rem: ${{ matrix.chunk }} + run: | + bash ./tests/build.sh + + + build-debug-ipv6: + name: Debug IPv6 ${{ matrix.chunk }} + runs-on: ubuntu-latest + strategy: + matrix: + chunk: [0, 1, 2, 3, 4, 5, 6, 7] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Cache Linux toolchain + id: cache-linux + uses: actions/cache@v2 + with: + path: ./tools/dist + key: key-linux-toolchain + - name: Build Sketches + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + BUILD_PARITY: custom + mod: 8 + rem: ${{ matrix.chunk }} + run: | + bash ./tests/debug6.sh + + + build-windows: + name: Windows + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Cache Windows toolchain + id: cache-windows + uses: actions/cache@v2 + with: + path: ./tools/dist + key: key-windows-toolchain + - name: Build Sketch + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + WINDOWS: 1 + BUILD_PARITY: custom + mod: 500 + rem: 1 + run: | + copy (get-command python).source (get-command python).source.Replace("python.exe", "python3.exe") + bash ./tests/build.sh + + + build-mac: + name: Mac + runs-on: macOS-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Cache Mac toolchain + id: cache-mac + uses: actions/cache@v2 + with: + path: ./tools/dist + key: key-mac-toolchain + - name: Build Sketch + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + MACOSX: 1 + BUILD_PARITY: custom + mod: 500 + rem: 1 + run: | + bash ./tests/build.sh + + + build-pio: + name: Build Platform.IO + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Build subset on Platform.IO + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + BUILD_PARITY: custom + mod: 42 + rem: 13 + run: | + sudo apt-get install python3-pip python3-setuptools + PATH=/home/runner/.local/bin:$PATH bash ./tests/platformio.sh + + + host-tests: + name: Host tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Run host tests + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + run: | + sudo apt-get install valgrind lcov + bash ./tests/ci/host_test.sh + + + documentation: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Build documentation + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + run: | + sudo apt-get install python3-pip python3-setuptools + PATH=/home/runner/.local/bin:$PATH pip3 install --user -r doc/requirements.txt + PATH=/home/runner/.local/bin:$PATH bash ./tests/ci/build_docs.sh + + + style-check: + name: Style and formatting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Style check + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + run: | + sudo apt-get install astyle + bash ./tests/ci/style_check.sh + + + mock-check: + name: Mock trivial test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Mock build + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + run: | + bash ./tests/buildm.sh + + + boards-check: + name: Boards.txt check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Boards.txt diff + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_TAG: ${{ github.ref }} + run: | + bash ./tests/ci/build_boards.sh diff --git a/.github/workflows/release-to-publish.yml b/.github/workflows/release-to-publish.yml new file mode 100644 index 0000000000..943bb4c086 --- /dev/null +++ b/.github/workflows/release-to-publish.yml @@ -0,0 +1,61 @@ +# Whenever a release is published from a draft, this will update the +# master Arduino JSON file to add its new entry. + +# We keep the master JSON file in another repo, so we need to use a pre-set GH +# SSH key to be able to push a change to the repo. + +# At some point this should really be moved to the main repo to avoid this grief + +#### Steps to follow when you need to make a new SSH key for upload (not +#### normally needed!) + +# We encrypt the id_rda (private key) using a key and IV generated randomly: +# openssl enc -nosalt -aes-256-cbc -pbkdf2 -k "$(openssl rand -base64 100 | sed s/=//)" -P +# > key = +# > iv = + +# Set that key and IV strings are set as secrets in the *Arduino repo* GitHub +# CI Secrets object as GHKEY and GHIV. + +# Now make a SSH key using ssh-keygen (do not use your own, make a new one!) +# ssh-keygen ... + +# Upload the id_rsa.pub (public key) to GitHub +# Settings->SSH and GPG Keys->New SSH Key + +# Encrypt the private part of the key using the key and IV above and commit the change +# openssl enc -aes-256-cbc -K -iv -in id_rsa -out package/esp8266_github_io_deploy.enc +# git add package/esp8266_github_io_deploy.enc + + +name: ESP8266 Arduino Release Publisher + +on: + release: + types: [published] + +jobs: + package: + name: Update master JSON file + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Set GIT tag name + run: | + echo "::set-env name=TRAVIS_TAG::$(git describe --exact-match --tags)" + - name: Deploy updated JSON + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + BUILD_TYPE: package + CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} + GHKEY: ${{ secrets.GHKEY }} + GHIV: ${{ secrets.GHIV }} + run: | + bash ./tests/ci/build_package.sh + # Only the regenerated JSON file will be used, but it's simpler than looking for it in a GH release + bash ./package/deploy_package_index.sh \ No newline at end of file diff --git a/.github/workflows/tag-to-draft-release.yml b/.github/workflows/tag-to-draft-release.yml new file mode 100644 index 0000000000..1301e24bfa --- /dev/null +++ b/.github/workflows/tag-to-draft-release.yml @@ -0,0 +1,35 @@ +# Whenever a tag of the form #.xxxx is pushed against master, generate a +# draft release and upload the ZIP and JSON file to it. Maintainers then +# will manually add the changelist and publish it. + +name: ESP8266 Arduino Draft Release + +on: + push: + tags: + - '[0-9]+.[0-9]+.[0-9]+*' + +jobs: + package: + name: Package + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Set GIT tag name + run: | + echo "::set-env name=TRAVIS_TAG::$(git describe --exact-match --tags)" + - name: Build package JSON + env: + TRAVIS_BUILD_DIR: ${{ github.workspace }} + BUILD_TYPE: package + CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} + run: | + export TRAVIS_TAG=$(git describe --exact-match --tags) + bash ./tests/ci/build_package.sh + pip3 install PyGithub + python3 ./package/upload_release.py --user "$GITHUB_ACTOR" --repo "$GITHUB_REPOSITORY" --token "$CI_GITHUB_API_KEY" --tag "$TRAVIS_TAG" --name "Release $TRAVIS_TAG" --msg "Text goes here" package/versions/*/*.zip package/versions/*/package_esp8266com_index.json diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c187c27123..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,162 +0,0 @@ -language: bash -os: linux -dist: bionic - -git: - depth: 1 - submodules: false - -before_install: - - git submodule update --init # no recursive update - -cache: - directories: - - $HOME/astyle - -stages: - - build - - deploy - -jobs: - include: - # Build stage. To save time, run all kinds of builds and tests in parallel. - - - name: "Platformio (1)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/platformio.sh - install: - - sudo apt-get install python3-pip python3-setuptools - env: - - BUILD_PARITY=even - - name: "Platformio (2)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/platformio.sh - install: - - sudo apt-get install python3-pip python3-setuptools - env: - - BUILD_PARITY=odd - - - name: "Build (1)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/build.sh - env: - - BUILD_PARITY=even - - name: "Build (2)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/build.sh - env: - - BUILD_PARITY=odd - - - name: "Debug (1)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/debug.sh - env: - - BUILD_PARITY=even - - name: "Debug (2)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/debug.sh - env: - - BUILD_PARITY=odd - - - name: "Build IPv6 (1)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/build6.sh - env: - - BUILD_PARITY=even - - name: "Build IPv6 (2)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/build6.sh - env: - - BUILD_PARITY=odd - - - name: "Build lwIP-v1.4 (1)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/build1.sh - env: - - BUILD_PARITY=even - - name: "Build lwIP-v1.4 (2)" - stage: build - script: $TRAVIS_BUILD_DIR/tests/build1.sh - env: - - BUILD_PARITY=odd - - - name: "Mac OSX can build sketches" - os: osx - stage: build - script: $TRAVIS_BUILD_DIR/tests/build.sh - env: MACOSX=1 BUILD_PARITY=custom mod=500 rem=1 - - - name: "Windows can build sketches" - os: windows - stage: build - script: $TRAVIS_BUILD_DIR/tests/build.sh - env: WINDOWS=1 BUILD_PARITY=custom mod=500 rem=1 - - - name: "Host tests" - stage: build - script: $TRAVIS_BUILD_DIR/tests/ci/host_test.sh - install: - - sudo apt-get install valgrind lcov - - - name: "Docs" - stage: build - script: $TRAVIS_BUILD_DIR/tests/ci/build_docs.sh - install: - - sudo apt-get install python3-pip python3-setuptools - - pip3 install --user -r doc/requirements.txt; - - - name: "Style check" - stage: build - script: $TRAVIS_BUILD_DIR/tests/ci/style_check.sh - install: tests/ci/install_astyle.sh - - - name: "Mock trivial test" - stage: build - script: $TRAVIS_BUILD_DIR/tests/buildm.sh - - - name: "Boards" - stage: build - script: $TRAVIS_BUILD_DIR/tests/ci/build_boards.sh - - # Deploy stage. - # Here we build the package JSON (always) and do the deployments - - name: "Package / deploy" - stage: deploy - script: tests/ci/build_package.sh - env: BUILD_TYPE=package - before_deploy: git submodule update --init - deploy: - # Create Github release, upload artifacts - - provider: releases - draft: true - skip_cleanup: true - api_key: - secure: kYsxX/N21fwLSTLpbb0c96PnQHn1CIMqZstm02hfUhCX83FygWSh4vs3gzW28DMpjQMZ6vC4g+jtfosYU2tUhht/bynurDH4edpEyGeMyK+fzCI9pAr4JT0RbKQI84EC18ScpgP/UP0jTc1LJ+xl8UMwSiDE0mzHx7xJ4mMNQbA= - file_glob: true - tag_name: $TRAVIS_TAG - target_commitish: $TRAVIS_COMMIT - file: - - package/versions/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip - - package/versions/$TRAVIS_TAG/package_esp8266com_index.json - on: - repo: esp8266/Arduino - tags: true - - # Update the package index URL to point to the new version - - provider: script - skip_cleanup: true - script: bash package/deploy_package_index.sh - on: - repo: esp8266/Arduino - tags: true - -notifications: - email: - on_success: change - on_failure: change - webhooks: - urls: - - secure: "dnSY+KA7NK+KD+Z71copmANDUsyVePrZ0iXvXxmqMEQv+lp3j2Z87G5pHn7j0WNcNZrejJqOdbElJ9Q4QESRaAYxTR7cA6ameJeEKHiFJrQtN/4abvoXb9E1CxpL8aNON/xgnqCk+fycOK3nbWWXlJBodzBm7KN64vrcHO7et+M=" - on_success: change # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: false # default: false diff --git a/doc/filesystem.rst b/doc/filesystem.rst index a5a3cffbbe..cc6c51280c 100644 --- a/doc/filesystem.rst +++ b/doc/filesystem.rst @@ -474,8 +474,8 @@ Performs the same operation as ``info`` but allows for reporting greater than 4GB for filesystem size/used/etc. Should be used with the SD and SDFS filesystems since most SD cards today are greater than 4GB in size. -setTimeCallback(time_t (*cb)(void)) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +setTimeCallback(time_t (\*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: cpp @@ -574,8 +574,8 @@ rewind Resets the internal pointer to the start of the directory. -setTimeCallback(time_t (*cb)(void)) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +setTimeCallback(time_t (\*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sets the time callback for any files accessed from this Dir object via openNextFile. Note that the SD and SDFS filesystems only support a filesystem-wide callback and @@ -693,7 +693,7 @@ Close the file. No other operations should be performed on *File* object after ``close`` function was called. openNextFile (compatibiity method, not recommended for new code) -~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: cpp @@ -705,7 +705,7 @@ Opens the next file in the directory pointed to by the File. Only valid when ``File.isDirectory() == true``. rewindDirectory (compatibiity method, not recommended for new code) -~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: cpp @@ -718,8 +718,8 @@ rewindDirectory (compatibiity method, not recommended for new code) Resets the ``openNextFile`` pointer to the top of the directory. Only valid when ``File.isDirectory() == true``. -setTimeCallback(time_t (*cb)(void)) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +setTimeCallback(time_t (\*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sets the time callback for this specific file. Note that the SD and SDFS filesystems only support a filesystem-wide callback and calls to diff --git a/doc/libraries.rst b/doc/libraries.rst index cf860231ec..22422b834b 100644 --- a/doc/libraries.rst +++ b/doc/libraries.rst @@ -87,7 +87,7 @@ Some ESP-specific APIs related to deep sleep, RTC and flash memories are availab ``ESP.getHeapFragmentation()`` returns the fragmentation metric (0% is clean, more than ~50% is not harmless) -``ESP.getMaxFreeBlockSize()`` returns the largest contiguous free RAM block in the heap, useful for checking heap fragmentation. **NOTE:** Maximum ``malloc()``able block will be smaller due to memory manager overheads. +``ESP.getMaxFreeBlockSize()`` returns the largest contiguous free RAM block in the heap, useful for checking heap fragmentation. **NOTE:** Maximum ``malloc()`` block will be smaller due to memory manager overheads. ``ESP.getChipId()`` returns the ESP8266 chip ID as a 32-bit integer. diff --git a/doc/ota_updates/readme.rst b/doc/ota_updates/readme.rst index 35eb931507..b422a2cd8b 100755 --- a/doc/ota_updates/readme.rst +++ b/doc/ota_updates/readme.rst @@ -161,7 +161,7 @@ If signing is desired, sign the gzip compressed file *after* compression. Updating apps in the field to support compression ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you have applications deployed in the field and wish to update them to support compressed OTA uploads, you will need to first recompile the application, then _upload the uncompressed `.bin` file once_. Attempting to upload a `gzip` compressed binary to a legacy app will result in the Updater rejecting the upload as it does not understand the `gzip` format. After this initial upload, which will include the new bootloader and `Updater` class with compression support, compressed updates can then be used. +If you have applications deployed in the field and wish to update them to support compressed OTA uploads, you will need to first recompile the application, then __upload the uncompressed `.bin` file once__. Attempting to upload a `gzip` compressed binary to a legacy app will result in the Updater rejecting the upload as it does not understand the `gzip` format. After this initial upload, which will include the new bootloader and `Updater` class with compression support, compressed updates can then be used. Safety diff --git a/package/deploy_package_index.sh b/package/deploy_package_index.sh index f8b5cedff0..60635601f2 100644 --- a/package/deploy_package_index.sh +++ b/package/deploy_package_index.sh @@ -1,32 +1,35 @@ #!/bin/bash -# # This script updates package index hosted on esp8266.github.io (aka arduino.esp8266.com). -# Normally is run by Travis CI for tagged versions, as a deploy step. tag=`git describe --tags` cd $(dirname "$0") +set -e # Abort with error if anything here does not go as expected! + # Decrypt and install SSH private key. # "encrypted_xxx_key" and "encrypted_xxx_iv" are environment variables # known to Travis CI builds. -openssl aes-256-cbc -K $encrypted_3a94a4db7dec_key -iv $encrypted_3a94a4db7dec_iv -in esp8266_github_io_deploy.enc -out esp8266_github_io_deploy -d +openssl enc -d -aes-256-cbc -K $GHKEY -iv $GHIV -in esp8266_github_io_deploy.enc -out esp8266_github_io_deploy eval "$(ssh-agent -s)" chmod 600 esp8266_github_io_deploy ssh-add esp8266_github_io_deploy +mkdir -p ~/.ssh +chmod go-w ~/.ssh echo -e "Host github.com\nStrictHostKeyChecking no\n" >> ~/.ssh/config +chmod go-w ~/.ssh/config # Clone the Github pages repository -git clone git@github.com:esp8266/esp8266.github.io.git +git clone git@github.com:earlephilhower/esp8266.github.io.git pushd esp8266.github.io # Update the package index -cp ../versions/$tag/package_esp8266com_index.json stable/package_esp8266com_index.json +cp ../versions/*/package_esp8266com_index.json stable/package_esp8266com_index.json git add stable/package_esp8266com_index.json # Commit and push the changes -git config user.email "travis@travis-ci.org" -git config user.name "Travis CI" -git commit -m "update package index for release $tag" +git config user.email "github-ci-action@github.com" +git config user.name "GitHub CI Action" +git commit -m "Update package index for release $tag" git push origin master popd diff --git a/package/esp8266_github_io_deploy.enc b/package/esp8266_github_io_deploy.enc index b9eed57cc6c7f1820fd26c127585c468d0e444e1..e97e928ae1825a382231c0cdd6ecdc73d07ff74b 100644 GIT binary patch literal 2608 zcmV-03eWXn5j9Tq8#y0)T5K@S=F0 zYsPyFfByF=s0Q-tLe|k{lm7m<)-90BPjJSyVXXOfAqk2oJwm+hBX6_ig@`8y+y)pQ z&!s^Ft4>GGA1N7VnWRPc_1439ZZ=`FaRQBeV18{Yo>Sln2%I-jwnBa4h6^`sWFKwi z!1_futYL_x_)~y;(^yFR`eKAr#tj~;GQvU{Se7i4SoBayb;>tMZv=V0T;E3P;RrEP zpA$MB5Gus|!DvZb!R@JCt_QH#V(8IUp{{=A(W+g#XsA~oi7^75g?U&bfX`v=0{8H{ z`ux9fR)giJSGEP1q<<_c^;Gi*`!6}U5r;as8te5C^nxD}eS7&vsLO_55C-$qg73+* zp{hLsK*VIJQkWr*>Ifb?w|yBkp3poj!4I@>xdQ>|{*(HU@0rIuH@4Kt#kK$Tgm@ZSK55Y_5n$(f)j^qC zw63pWd$~9x;Hx`E*%r$c_d^IKn9B-hi^#S_zt1|%cgRq93>PW7Ik3?MgJ``Yd^JtM zIxseRL~C8-;ez;0`{223e91bKO1=5g75p z>Kv~MH@tB(U~+ssjV`?tw|OO2h~HtlV@+7i@CIG;<%U_G;}IM*+S85J8{``;pJ?ne zm)eOuBHB%z8q%N^ejC1R*LpfLFsrRj6T?+5TwE_?y_G+?VV;TPl^NsrBWj&l<~ zAmIcCQa%eYfl#r%GZTkTK!S!b!WdTfznsta5E`z1O%q__34@>Gn?MGY^xo#|CwvaY zU~$oDtPt|wlZ5c9rUPMLQ%4r=kCR#6TfJ#NOVu8sJ_4wXgA~E28o1^-8O$W{WpM{v?PH*kk9TxVfdfuB@m2z2wg=1| znBxi-zbb|3wE~fPzmB5M;P!!cs4M$^yjy^0%Y5`@Wigw<3(paXift~@1m_m!COcXT@u{MoqV-0yc+FcBvy*GAF2jl?bxB$hvz*w4E=k@|ynTU}pw;_C z5mAE562;j=Q)SZ5Qu}O0U{zxoAFb2;2CvYIdK%X+`r&u_cHWce-lq=E;p{zCp>6H-5ukITkswBZ`a*$(JEPr=jv z2l24-{{}*gjC=^6V|tDO=wE*GM$M|$7IsNnZu60BD!ynqZ&GlU^z1Xh`Rb-J_Kc>& z{0n|Ql6t17X$ej9Qc4epYPz^Ns;!(7J`2wFn1>qhhq(w+F(orBMq*8!utky7+7*VM zISAHW;QnT9r~9q^CpLw|T|S%fU#{ohFaLju zieX|1`}F5x!txv8z2#W@c<1-YFHZ}R`IZRFv9f7+QzTs~9lfL@@%3%5`*iu^G!r4% zy%=DxVU6(XQ$NY~C=jfUR<#TIALi0fR+9&&g??si@UUBaBL}Od}&kH)GKy`|8WU~D^fdm@}b%6p;aQO^RCK;dm3S`@ZGalM_dHOLLh~_T~ zJARe+aOB6o>RCQ}SZB+KfEhETzjK)<(s96pdo_;+I~mFiqKObqYM(;;m5n$7<=Ks% zJj>WPHb%&d71`)ZPo=i4-u&y?Eff-R10XVr<1Zh-{pzfx@d+Q#Zz7!mEN;`<3=Ro$I1^-mVoO}OOWoE?R<8+C7FWxiw8N+ZW{e>AXO)_@BfKkRoK$`6 zc$-VY%ky(ltiot9RiDS+2OZrU7O;GiDV`J`hSorO9(VE7vI#7jE619$x!EZ1)otZk zNGBO=g#6&-hnw(1KDe_p41%Z4?J1B@oYsP^NRWZn1F6qFK7S+LCn)xTpwYzfR#2#( zORSi(vEoYrEdR++q@nRQ1#%iU0t>p#<&~Xs99EQu?as3~g;RZ!X5s-v3Q*2KUw?Hc zQ2uve^X^nxnR;v?U21v&qzt$zLCMY@Hb)}npWb{4(pELiTn_%2yGU$59GCy7uG?V^ z{Uxv;Aok3k{&?hXys?qeh=rux%-?PX6}ypbZh+pa$BG;rEFy6;&0o4fNaCqlf zC>=1Fw061aXbJ^l<&6 z&)hiwpcHr3Kj%OlWs7GfdF3spNrk-&Q-8ev)xhG3@4tGQj-+_rW`l*Gez)cq?K8(LjgZz=+&uk2m9+Oy9*Jaf z3f^^BYrCPgUClLlAENdkR=P@kGdI-6!o`VAg*%M9Uly+NnY{?Pxq{t}#`E>-lgs$| SEA;M~3L9jAYIS*mv31RO;Rwk9 literal 432 zcmV;h0Z;x|!oZim#`hG0J@flq^9Q3y?Oig(dwZ0v1@zT}UacaIL-_Vq{e%7eoQ&39 z_P~A;w?m909toET0oGo11kvW~_luiG2oG(3M8F58jX$WX$P$i(aXquUkk2D7~xJXw}E ziE%m&mL8Gr z;#A!D6yl|F>KK^_k>nZGk4pHwBGn>4YdWh>jmHXI^GF@ z_|Xm#4j%a=t1kHI`St09J)u Date: Fri, 10 Jul 2020 11:28:41 -0700 Subject: [PATCH 02/12] Remove non-CI changes --- doc/filesystem.rst | 16 ++++++++-------- doc/ota_updates/readme.rst | 2 +- tests/debug6.sh | 1 + 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/filesystem.rst b/doc/filesystem.rst index cc6c51280c..a5a3cffbbe 100644 --- a/doc/filesystem.rst +++ b/doc/filesystem.rst @@ -474,8 +474,8 @@ Performs the same operation as ``info`` but allows for reporting greater than 4GB for filesystem size/used/etc. Should be used with the SD and SDFS filesystems since most SD cards today are greater than 4GB in size. -setTimeCallback(time_t (\*cb)(void)) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +setTimeCallback(time_t (*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: cpp @@ -574,8 +574,8 @@ rewind Resets the internal pointer to the start of the directory. -setTimeCallback(time_t (\*cb)(void)) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +setTimeCallback(time_t (*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sets the time callback for any files accessed from this Dir object via openNextFile. Note that the SD and SDFS filesystems only support a filesystem-wide callback and @@ -693,7 +693,7 @@ Close the file. No other operations should be performed on *File* object after ``close`` function was called. openNextFile (compatibiity method, not recommended for new code) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~ .. code:: cpp @@ -705,7 +705,7 @@ Opens the next file in the directory pointed to by the File. Only valid when ``File.isDirectory() == true``. rewindDirectory (compatibiity method, not recommended for new code) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~ .. code:: cpp @@ -718,8 +718,8 @@ rewindDirectory (compatibiity method, not recommended for new code) Resets the ``openNextFile`` pointer to the top of the directory. Only valid when ``File.isDirectory() == true``. -setTimeCallback(time_t (\*cb)(void)) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +setTimeCallback(time_t (*cb)(void)) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sets the time callback for this specific file. Note that the SD and SDFS filesystems only support a filesystem-wide callback and calls to diff --git a/doc/ota_updates/readme.rst b/doc/ota_updates/readme.rst index b422a2cd8b..35eb931507 100755 --- a/doc/ota_updates/readme.rst +++ b/doc/ota_updates/readme.rst @@ -161,7 +161,7 @@ If signing is desired, sign the gzip compressed file *after* compression. Updating apps in the field to support compression ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you have applications deployed in the field and wish to update them to support compressed OTA uploads, you will need to first recompile the application, then __upload the uncompressed `.bin` file once__. Attempting to upload a `gzip` compressed binary to a legacy app will result in the Updater rejecting the upload as it does not understand the `gzip` format. After this initial upload, which will include the new bootloader and `Updater` class with compression support, compressed updates can then be used. +If you have applications deployed in the field and wish to update them to support compressed OTA uploads, you will need to first recompile the application, then _upload the uncompressed `.bin` file once_. Attempting to upload a `gzip` compressed binary to a legacy app will result in the Updater rejecting the upload as it does not understand the `gzip` format. After this initial upload, which will include the new bootloader and `Updater` class with compression support, compressed updates can then be used. Safety diff --git a/tests/debug6.sh b/tests/debug6.sh index b0a1aeaaba..bf2d4f5197 100644 --- a/tests/debug6.sh +++ b/tests/debug6.sh @@ -19,3 +19,4 @@ install_arduino debug build_sketches_with_arduino "$mod" "$rem" lm6f rm -rf "$cache_dir" + From 2e82f33b69e0dca4f0cc778df2f4983920f7e9ed Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 12 Jul 2020 09:27:19 -0700 Subject: [PATCH 03/12] Change docs to use a GH Deploy Key The original comments suggested adding a new SSH key to a user account which would have allowed complete access to all repos and all other global user settings. A really bad thing if it got into the wild. Instead, a single deploy key can be added to a single repo. This only grants git push access/etc. for that single repo and not for the entire user account. Update the docs accordingly. Also rename push.yml->pull-request.yml since it runs on a pull request, not a master push. --- .github/workflows/{push.yml => pull-request.yml} | 0 .github/workflows/release-to-publish.yml | 7 +++++-- 2 files changed, 5 insertions(+), 2 deletions(-) rename .github/workflows/{push.yml => pull-request.yml} (100%) diff --git a/.github/workflows/push.yml b/.github/workflows/pull-request.yml similarity index 100% rename from .github/workflows/push.yml rename to .github/workflows/pull-request.yml diff --git a/.github/workflows/release-to-publish.yml b/.github/workflows/release-to-publish.yml index 943bb4c086..459e9687b9 100644 --- a/.github/workflows/release-to-publish.yml +++ b/.github/workflows/release-to-publish.yml @@ -20,8 +20,11 @@ # Now make a SSH key using ssh-keygen (do not use your own, make a new one!) # ssh-keygen ... -# Upload the id_rsa.pub (public key) to GitHub -# Settings->SSH and GPG Keys->New SSH Key +# Upload the id_rsa.pub (public key) to the GH.IO GitHub as a *deploy key* +# (https://github.com/esp8266/esp8266.github.io->Settings->Deploy Keys->Add Key) +# DO NOT USE Settings->SSH and GPG Keys->New SSH Key because that allows +# full access to all repos owned by the user, while a Deploy Key only allows +# access to a single repo. # Encrypt the private part of the key using the key and IV above and commit the change # openssl enc -aes-256-cbc -K -iv -in id_rsa -out package/esp8266_github_io_deploy.enc From 7a07644a4337ee71bd1999dcd579ca89b7cbce4c Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 12 Jul 2020 09:54:11 -0700 Subject: [PATCH 04/12] Undo add'l documentation change --- .github/workflows/release-to-publish.yml | 3 ++- doc/libraries.rst | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-to-publish.yml b/.github/workflows/release-to-publish.yml index 459e9687b9..0d01a678dd 100644 --- a/.github/workflows/release-to-publish.yml +++ b/.github/workflows/release-to-publish.yml @@ -61,4 +61,5 @@ jobs: run: | bash ./tests/ci/build_package.sh # Only the regenerated JSON file will be used, but it's simpler than looking for it in a GH release - bash ./package/deploy_package_index.sh \ No newline at end of file + bash ./package/deploy_package_index.sh + diff --git a/doc/libraries.rst b/doc/libraries.rst index 22422b834b..cf860231ec 100644 --- a/doc/libraries.rst +++ b/doc/libraries.rst @@ -87,7 +87,7 @@ Some ESP-specific APIs related to deep sleep, RTC and flash memories are availab ``ESP.getHeapFragmentation()`` returns the fragmentation metric (0% is clean, more than ~50% is not harmless) -``ESP.getMaxFreeBlockSize()`` returns the largest contiguous free RAM block in the heap, useful for checking heap fragmentation. **NOTE:** Maximum ``malloc()`` block will be smaller due to memory manager overheads. +``ESP.getMaxFreeBlockSize()`` returns the largest contiguous free RAM block in the heap, useful for checking heap fragmentation. **NOTE:** Maximum ``malloc()``able block will be smaller due to memory manager overheads. ``ESP.getChipId()`` returns the ESP8266 chip ID as a 32-bit integer. From 81d9689569cad976efacb15d088b5ea19e808f12 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 12 Jul 2020 12:18:10 -0700 Subject: [PATCH 05/12] Update upload_release.py --- package/upload_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/upload_release.py b/package/upload_release.py index 0da1cf4c59..20390c2b11 100644 --- a/package/upload_release.py +++ b/package/upload_release.py @@ -8,7 +8,7 @@ import mimetypes import os -parser = argparse.ArgumentParser(description='Patch in a section of the Arduino tools JSON') +parser = argparse.ArgumentParser(description='Upload a set of files to a new draft release') parser.add_argument('--user', help="Github username", type=str, required=True) parser.add_argument('--token', help="Github Personal Access Token (PAT)", type=str, required=True) parser.add_argument('--repo', help="Repository", type=str, required=True) From 42d4a9788d680ebdc554f56c5a8c838eec5d76cf Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 12 Jul 2020 20:18:11 -0700 Subject: [PATCH 06/12] Add copious documentation to YML files --- .github/workflows/pull-request.yml | 23 +++++++++++++++++++--- .github/workflows/release-to-publish.yml | 3 ++- .github/workflows/tag-to-draft-release.yml | 9 +++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 2aff3ff2f8..926cb65d89 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,5 +1,7 @@ -# Run whenever a PR is generated or updated. Do the normal suite of tests -# and report success/fail in the PR. +# Run whenever a PR is generated or updated. + +# Most jobs check out the code, ensure Python3 is installed, and for build +# tests the ESP8266 toolchain is cached when possible to speed up execution. name: ESP8266 Arduino CI @@ -9,6 +11,7 @@ on: jobs: +# Run 8 parallel jobs for the default build of all examples. build-linux: name: Build ${{ matrix.chunk }} runs-on: ubuntu-latest @@ -39,6 +42,8 @@ jobs: bash ./tests/build.sh +# Cover the debug and IPv6 cases by enabling both and running 8 parallel jobs +# over all example code. build-debug-ipv6: name: Debug IPv6 ${{ matrix.chunk }} runs-on: ubuntu-latest @@ -69,6 +74,7 @@ jobs: bash ./tests/debug6.sh +# Single build under Windows to ensure the Win toolchain is good. build-windows: name: Windows runs-on: windows-latest @@ -94,10 +100,13 @@ jobs: mod: 500 rem: 1 run: | + # Windows has python3 already installed, but it's called "python". + # Copy python.exe to the proper name so scripts "just work". copy (get-command python).source (get-command python).source.Replace("python.exe", "python3.exe") bash ./tests/build.sh +# Single build under macOS to ensure the Mac toolchain is good. build-mac: name: Mac runs-on: macOS-latest @@ -126,6 +135,7 @@ jobs: bash ./tests/build.sh +# Run a few Platform.IO jobs (not full suite) to check PIO integration. build-pio: name: Build Platform.IO runs-on: ubuntu-latest @@ -141,13 +151,14 @@ jobs: TRAVIS_BUILD_DIR: ${{ github.workspace }} TRAVIS_TAG: ${{ github.ref }} BUILD_PARITY: custom - mod: 42 + mod: 42 # Picked at random to give 4-5 builds and exit. rem: 13 run: | sudo apt-get install python3-pip python3-setuptools PATH=/home/runner/.local/bin:$PATH bash ./tests/platformio.sh +# Run host test suite under valgrind for runtime checking of code. host-tests: name: Host tests runs-on: ubuntu-latest @@ -167,6 +178,7 @@ jobs: bash ./tests/ci/host_test.sh +# Ensure Sphinx can build the documentation properly. documentation: name: Documentation runs-on: ubuntu-latest @@ -183,10 +195,13 @@ jobs: TRAVIS_TAG: ${{ github.ref }} run: | sudo apt-get install python3-pip python3-setuptools + # GitHub CI installs pip3 and setuptools outside the path. + # Update the path to include them and run. PATH=/home/runner/.local/bin:$PATH pip3 install --user -r doc/requirements.txt PATH=/home/runner/.local/bin:$PATH bash ./tests/ci/build_docs.sh +# Standard Arduino formatting in all the examples style-check: name: Style and formatting runs-on: ubuntu-latest @@ -206,6 +221,7 @@ jobs: bash ./tests/ci/style_check.sh +# Quick test that the mocking builds succeed mock-check: name: Mock trivial test runs-on: ubuntu-latest @@ -224,6 +240,7 @@ jobs: bash ./tests/buildm.sh +# Ensure no manual edits to boards.txt boards-check: name: Boards.txt check runs-on: ubuntu-latest diff --git a/.github/workflows/release-to-publish.yml b/.github/workflows/release-to-publish.yml index 0d01a678dd..88f2cf13b0 100644 --- a/.github/workflows/release-to-publish.yml +++ b/.github/workflows/release-to-publish.yml @@ -60,6 +60,7 @@ jobs: GHIV: ${{ secrets.GHIV }} run: | bash ./tests/ci/build_package.sh - # Only the regenerated JSON file will be used, but it's simpler than looking for it in a GH release + # Only the regenerated JSON file will be used, but it's simpler + # than looking for it in a GH release. bash ./package/deploy_package_index.sh diff --git a/.github/workflows/tag-to-draft-release.yml b/.github/workflows/tag-to-draft-release.yml index 1301e24bfa..265b709f7a 100644 --- a/.github/workflows/tag-to-draft-release.yml +++ b/.github/workflows/tag-to-draft-release.yml @@ -7,6 +7,7 @@ name: ESP8266 Arduino Draft Release on: push: tags: + # Run for tags of the x.x.x* form (i.e. 3.0.0, 3.0.0-beta, etc.). - '[0-9]+.[0-9]+.[0-9]+*' jobs: @@ -22,6 +23,7 @@ jobs: python-version: '3.x' - name: Set GIT tag name run: | + # Sets an environment variable used in the next steps echo "::set-env name=TRAVIS_TAG::$(git describe --exact-match --tags)" - name: Build package JSON env: @@ -29,7 +31,10 @@ jobs: BUILD_TYPE: package CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} run: | - export TRAVIS_TAG=$(git describe --exact-match --tags) bash ./tests/ci/build_package.sh pip3 install PyGithub - python3 ./package/upload_release.py --user "$GITHUB_ACTOR" --repo "$GITHUB_REPOSITORY" --token "$CI_GITHUB_API_KEY" --tag "$TRAVIS_TAG" --name "Release $TRAVIS_TAG" --msg "Text goes here" package/versions/*/*.zip package/versions/*/package_esp8266com_index.json + # Create a draft release and upload the ZIP and JSON files. + # This draft is not visible to normal users and needs to be + # updated manually with release notes and published from the + # GitHub web interface. + python3 ./package/upload_release.py --user "$GITHUB_ACTOR" --repo "$GITHUB_REPOSITORY" --token "$CI_GITHUB_API_KEY" --tag "$TRAVIS_TAG" --name "Release $TRAVIS_TAG" --msg "Update the draft with release notes before publishing." package/versions/*/*.zip package/versions/*/package_esp8266com_index.json From fcf2a6b13f7efdf5eb78870bd4bb92ca62ff98a4 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 12 Jul 2020 20:19:16 -0700 Subject: [PATCH 07/12] Update debug6.sh to remove dead line --- tests/debug6.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/debug6.sh b/tests/debug6.sh index bf2d4f5197..b0a1aeaaba 100644 --- a/tests/debug6.sh +++ b/tests/debug6.sh @@ -19,4 +19,3 @@ install_arduino debug build_sketches_with_arduino "$mod" "$rem" lm6f rm -rf "$cache_dir" - From b1b08e8961c39e13fbe29036b54e657a49ddee40 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Mon, 13 Jul 2020 09:54:07 -0700 Subject: [PATCH 08/12] Re-add basic TravisCI testing Add basic, fast TravisCI script to keep the TravisCI jobs alive even if GitHub CI goes down. Belt-and-braces, but it's free. Limit the runs on the Travis jobs to 5 short ones in total to match the runtime of the GitHubCI. --- .travis.yml | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..27636ada5b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,70 @@ +# TravisCI left in repo as a backup CI solution in case GitHub CI suffers a +# major disruption. Only a few, quick tests are run in order to keep the +# TravisCI runtime equal to the GitHub CI time (to avoid bottlenecking on +# TravisCI). +# +# If GitHub CI goes away, it would make sense to drop this .YML file and +# use the complete one from release tag 2.7.2 + + +language: bash +os: linux +dist: bionic + +git: + depth: 1 + submodules: false + +before_install: + - git submodule update --init # no recursive update + +stages: + - build + - deploy + + +# Run only 5 jobs since TravisCI only allows 5 in parallel +jobs: + include: + - name: "Platform.IO" + stage: build + script: $TRAVIS_BUILD_DIR/tests/platformio.sh + install: + - sudo apt-get install python3-pip python3-setuptools + env: + - BUILD_PARITY=custom mod=10 rem=1 + + - name: "Build 1/4" + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: + - BUILD_PARITY=custom mod=10 rem=1 + + - name: "Build 2/4" + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: + - BUILD_PARITY=custom mod=10 rem=2 + + - name: "Build 3/4" + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: + - BUILD_PARITY=custom mod=10 rem=3 + + - name: "Build 4/4" + stage: build + script: $TRAVIS_BUILD_DIR/tests/build.sh + env: + - BUILD_PARITY=custom mod=10 rem=4 + +notifications: + email: + on_success: change + on_failure: change + webhooks: + urls: + - secure: "dnSY+KA7NK+KD+Z71copmANDUsyVePrZ0iXvXxmqMEQv+lp3j2Z87G5pHn7j0WNcNZrejJqOdbElJ9Q4QESRaAYxTR7cA6ameJeEKHiFJrQtN/4abvoXb9E1CxpL8aNON/xgnqCk+fycOK3nbWWXlJBodzBm7KN64vrcHO7et+M=" + on_success: change # options: [always|never|change] default: always + on_failure: always # options: [always|never|change] default: always + on_start: false # default: false From 08c1ce806886b30e4bb3eea08db2bcc6e7cf034a Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Mon, 13 Jul 2020 10:05:52 -0700 Subject: [PATCH 09/12] Halve PIO runs, they're 2x slower than Arduino --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27636ada5b..858c284bc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,8 @@ jobs: install: - sudo apt-get install python3-pip python3-setuptools env: - - BUILD_PARITY=custom mod=10 rem=1 + # PIO is very slow, so do 1/2 as many builds as Arduino + - BUILD_PARITY=custom mod=20 rem=5 - name: "Build 1/4" stage: build From 2d08c0922b537516aaaa04f10a9d73218d16effc Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Tue, 14 Jul 2020 12:44:41 -0700 Subject: [PATCH 10/12] Use single GitHub Secret for deployment Testing with my local repo first --- .github/workflows/release-to-publish.yml | 34 ++++++++--------------- package/arduino-esp8266-travis.enc | Bin 432 -> 0 bytes package/deploy_package_index.sh | 8 ++---- 3 files changed, 15 insertions(+), 27 deletions(-) delete mode 100644 package/arduino-esp8266-travis.enc diff --git a/.github/workflows/release-to-publish.yml b/.github/workflows/release-to-publish.yml index 88f2cf13b0..a2a27deb76 100644 --- a/.github/workflows/release-to-publish.yml +++ b/.github/workflows/release-to-publish.yml @@ -1,35 +1,26 @@ # Whenever a release is published from a draft, this will update the # master Arduino JSON file to add its new entry. -# We keep the master JSON file in another repo, so we need to use a pre-set GH -# SSH key to be able to push a change to the repo. - -# At some point this should really be moved to the main repo to avoid this grief +# We keep the master JSON file in another repo, so we need to use a pre-set +# Deployment SSH key to be able to push a change to the repo. #### Steps to follow when you need to make a new SSH key for upload (not #### normally needed!) -# We encrypt the id_rda (private key) using a key and IV generated randomly: -# openssl enc -nosalt -aes-256-cbc -pbkdf2 -k "$(openssl rand -base64 100 | sed s/=//)" -P -# > key = -# > iv = +# Generate a new SSH key private/public pair -# Set that key and IV strings are set as secrets in the *Arduino repo* GitHub -# CI Secrets object as GHKEY and GHIV. +# ssh-keygen -t rsa -b 4096 -C "your@email.com" -f ./deploy_rsa -# Now make a SSH key using ssh-keygen (do not use your own, make a new one!) -# ssh-keygen ... +# Upload deploy_rsa.pub to the *ESP8266.GITHUB.IO* repo as a deployment key -# Upload the id_rsa.pub (public key) to the GH.IO GitHub as a *deploy key* -# (https://github.com/esp8266/esp8266.github.io->Settings->Deploy Keys->Add Key) -# DO NOT USE Settings->SSH and GPG Keys->New SSH Key because that allows -# full access to all repos owned by the user, while a Deploy Key only allows -# access to a single repo. +# Convert the private key to base64 (to remove line breaks and allow easier +# usage in the script as an environment variable) -# Encrypt the private part of the key using the key and IV above and commit the change -# openssl enc -aes-256-cbc -K -iv -in id_rsa -out package/esp8266_github_io_deploy.enc -# git add package/esp8266_github_io_deploy.enc +# base64.exe -w 0 < deploy_rsa > deploy_rsa.b64 +# Copy the contents of the .b64 file to the clipboard, make a new GitHub +# secret in the ESP8266/Arduino repo called "GHCI_DEPLOY_KEY" and paste +# the B64 code into the variable. name: ESP8266 Arduino Release Publisher @@ -56,8 +47,7 @@ jobs: TRAVIS_BUILD_DIR: ${{ github.workspace }} BUILD_TYPE: package CI_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }} - GHKEY: ${{ secrets.GHKEY }} - GHIV: ${{ secrets.GHIV }} + GHCI_DEPLOY_KEY: ${{ secrets.GHCI_DEPLOY_KEY }} run: | bash ./tests/ci/build_package.sh # Only the regenerated JSON file will be used, but it's simpler diff --git a/package/arduino-esp8266-travis.enc b/package/arduino-esp8266-travis.enc deleted file mode 100644 index 66cb5e1768cc89447cf19dee9129229abda8b22a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 432 zcmV;h0Z;xPthMAb{45{^7IrES0;%@-caH;cv&lQG0q$({B=)HsKPL@mHSDS^4pA*G zEcOg0q}^!#zf;VdvQwz=`^vU({d+E+H_*3vBun@UW)R_+y}Mwba9x^&5zEM-H}C+7 z%+eG}_;dwyw>0IEwIKSvdk34pmS2y+`lPJ0NYMI>S?jh+4}IP*|E?6^HT_V@n*7th z4IZnc+e|rGvQUB(+{IX^UB>^K4pPdDy#EA-MT_B0bSXVmX52=XEOC7Q3 zZ7!jTf(s*xV5N esp8266_github_io_deploy eval "$(ssh-agent -s)" chmod 600 esp8266_github_io_deploy ssh-add esp8266_github_io_deploy @@ -20,7 +18,7 @@ echo -e "Host github.com\nStrictHostKeyChecking no\n" >> ~/.ssh/config chmod go-w ~/.ssh/config # Clone the Github pages repository -git clone git@github.com:esp8266/esp8266.github.io.git +git clone git@github.com:earlephilhower/esp8266.github.io.git pushd esp8266.github.io # Update the package index From a5aa3d6131759d75eb293ae21ed0f0387d37602b Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Tue, 14 Jul 2020 21:26:39 -0700 Subject: [PATCH 11/12] Update package and update script --- package/build_boards_manager_package.sh | 36 ++++++++++--------------- package/deploy_package_index.sh | 5 +++- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index 372fa2c63f..5f459581df 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -1,33 +1,25 @@ #!/bin/bash -#set -x +# Extract the release name from a release -ver=`git describe --tag` +# Default to draft tag name +ver=$(basename $(jq -e -r '.ref' "$GITHUB_EVENT_PATH")) +# If not available, try the publish tag name +if [ "$ver" == "null" ]; then + ver=$(jq -e -r '.release.tag_name' "$GITHUB_EVENT_PATH") +fi +# Fall back to the git description OTW (i.e. interactive) +if [ "$ver" == "null" ]; then + ver=$(git describe --tag) +fi visiblever=$ver -# match 0.0.* -if [ "${ver%.*}" = 0.0 ]; then +plainver=$ver +# Match 0.0.* as special-case early-access builds +if [ "${ver%.*}" = 0.0 ]; then git tag -d ${ver} ver=`git describe --tag HEAD` plain_ver=$ver - -else - - # Extract next version from platform.txt - next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt` - - # Figure out how will the package be called - ver=`git describe --exact-match` - if [ $? -ne 0 ]; then - # not tagged version; generate nightly package - date_str=`date +"%Y%m%d"` - is_nightly=1 - plain_ver="${next}-nightly" - ver="${plain_ver}+${date_str}" - else - plain_ver=$ver - fi - visiblever=$ver fi set -e diff --git a/package/deploy_package_index.sh b/package/deploy_package_index.sh index 0bc2433867..6e5cbc09d7 100644 --- a/package/deploy_package_index.sh +++ b/package/deploy_package_index.sh @@ -1,7 +1,10 @@ #!/bin/bash # This script updates package index hosted on esp8266.github.io (aka arduino.esp8266.com). -tag=`git describe --tags` +tag=$(jq -r '.release.tag_name' "$GITHUB_EVENT_PATH") +if [ "$tag" == "" ]; then + tag=`git describe --tags` +fi cd $(dirname "$0") From 21ed7eeb97dc95ff593f4d6427fb12d62d72774e Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Tue, 14 Jul 2020 21:44:47 -0700 Subject: [PATCH 12/12] Remove my local repo from testing, validated naming Testing now gets the proper release name for package(draft) and release. Go back to the esp8266,github.io repo for release updates. --- package/deploy_package_index.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/deploy_package_index.sh b/package/deploy_package_index.sh index 6e5cbc09d7..edfb27c230 100644 --- a/package/deploy_package_index.sh +++ b/package/deploy_package_index.sh @@ -21,7 +21,7 @@ echo -e "Host github.com\nStrictHostKeyChecking no\n" >> ~/.ssh/config chmod go-w ~/.ssh/config # Clone the Github pages repository -git clone git@github.com:earlephilhower/esp8266.github.io.git +git clone git@github.com:esp8266/esp8266.github.io.git pushd esp8266.github.io # Update the package index