Skip to content

Commit 8912f94

Browse files
committed
Run KNOWNBUG and THOROUGH regression tests in CI
Copy the check-ubuntu-20_04-cmake-gcc job to run KNOWNBUG (any test reported as failure will tell us that a bug has unexpectedly been fixed) an THOROUGH (tests that are expected to pass, but take longer to do so) tests, each as a separate GitHub action. Also run all tests tagged broken-smt-backend to confirm they haven't been fixed.
1 parent f92881c commit 8912f94

File tree

1 file changed

+84
-0
lines changed

1 file changed

+84
-0
lines changed

.github/workflows/pull-request-checks.yaml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,90 @@ jobs:
170170
- name: Run tests
171171
run: cd build; ctest . -V -L CORE -j2
172172

173+
check-ubuntu-20_04-cmake-gcc-KNOWNBUG:
174+
runs-on: ubuntu-20.04
175+
steps:
176+
- uses: actions/checkout@v2
177+
with:
178+
submodules: recursive
179+
- name: Fetch dependencies
180+
env:
181+
# This is needed in addition to -yq to prevent apt-get from asking for
182+
# user input
183+
DEBIAN_FRONTEND: noninteractive
184+
run: |
185+
sudo apt-get update
186+
sudo apt-get install --no-install-recommends -yq cmake ninja-build gcc g++ maven flex bison libxml2-utils ccache z3
187+
- name: Prepare ccache
188+
uses: actions/cache@v2
189+
with:
190+
path: .ccache
191+
key: ${{ runner.os }}-20.04-Release-${{ github.ref }}-${{ github.sha }}-PR
192+
restore-keys: |
193+
${{ runner.os }}-20.04-Release-${{ github.ref }}
194+
${{ runner.os }}-20.04-Release
195+
- name: ccache environment
196+
run: |
197+
echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
198+
echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
199+
- name: Configure using CMake
200+
run: cmake -H. -Bbuild -G Ninja
201+
- name: Zero ccache stats and limit in size
202+
run: ccache -z --max-size=500M
203+
- name: Build with Ninja
204+
run: ninja -C build -j2
205+
- name: Print ccache stats
206+
run: ccache -s
207+
- name: Run tests
208+
run: |
209+
cd build
210+
ctest . -V -L KNOWNBUG -j2
211+
export PATH=$PWD/bin:$PATH
212+
cd ../regression/cbmc
213+
sed -i '1s/^CORE\(.*\)broken-smt-backend/KNOWNBUG\1broken-smt-backend/' */*.desc
214+
# the following tests fail on Windows only
215+
git checkout -- memory_allocation_1 printf1 union12 va_list3
216+
# the following tests fail on Ubuntu 18.04 only
217+
git checkout -- byte_update12 gcc_vector1
218+
../test.pl -c "cbmc --cprover-smt2" -I broken-smt-backend -K
219+
220+
check-ubuntu-20_04-cmake-gcc-THOROUGH:
221+
runs-on: ubuntu-20.04
222+
steps:
223+
- uses: actions/checkout@v2
224+
with:
225+
submodules: recursive
226+
- name: Fetch dependencies
227+
env:
228+
# This is needed in addition to -yq to prevent apt-get from asking for
229+
# user input
230+
DEBIAN_FRONTEND: noninteractive
231+
run: |
232+
sudo apt-get update
233+
sudo apt-get install --no-install-recommends -yq cmake ninja-build gcc g++ maven flex bison libxml2-utils ccache z3
234+
- name: Prepare ccache
235+
uses: actions/cache@v2
236+
with:
237+
path: .ccache
238+
key: ${{ runner.os }}-20.04-Release-${{ github.ref }}-${{ github.sha }}-PR
239+
restore-keys: |
240+
${{ runner.os }}-20.04-Release-${{ github.ref }}
241+
${{ runner.os }}-20.04-Release
242+
- name: ccache environment
243+
run: |
244+
echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
245+
echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
246+
- name: Configure using CMake
247+
run: cmake -H. -Bbuild -G Ninja
248+
- name: Zero ccache stats and limit in size
249+
run: ccache -z --max-size=500M
250+
- name: Build with Ninja
251+
run: ninja -C build -j2
252+
- name: Print ccache stats
253+
run: ccache -s
254+
- name: Run tests
255+
run: cd build; ctest . -V -L THOROUGH -j2
256+
173257
check-macos-10_15-make-clang:
174258
runs-on: macos-10.15
175259
steps:

0 commit comments

Comments
 (0)