Skip to content

Commit f84b250

Browse files
committed
Run KNOWNBUG and THOROUGH regression tests in CI
Copy the check-ubuntu-20_04-cmake-gcc job to run KNOWNBUG (any test reported as failure will tell us that a bug has unexpectedly been fixed) an THOROUGH (tests that are expected to pass, but take longer to do so) tests, each as a separate GitHub action. Also run all tests tagged broken-smt-backend to confirm they haven't been fixed.
1 parent 3614e77 commit f84b250

File tree

1 file changed

+80
-0
lines changed

1 file changed

+80
-0
lines changed

.github/workflows/pull-request-checks.yaml

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,86 @@ jobs:
170170
- name: Run tests
171171
run: cd build; ctest . -V -L CORE -j2
172172

173+
check-ubuntu-20_04-cmake-gcc-KNOWNBUG:
174+
runs-on: ubuntu-20.04
175+
steps:
176+
- uses: actions/checkout@v2
177+
with:
178+
submodules: recursive
179+
- name: Fetch dependencies
180+
env:
181+
# This is needed in addition to -yq to prevent apt-get from asking for
182+
# user input
183+
DEBIAN_FRONTEND: noninteractive
184+
run: |
185+
sudo apt-get update
186+
sudo apt-get install --no-install-recommends -yq cmake ninja-build gcc g++ maven flex bison libxml2-utils ccache z3
187+
- name: Prepare ccache
188+
uses: actions/cache@v2
189+
with:
190+
path: .ccache
191+
key: ${{ runner.os }}-20.04-Release-${{ github.ref }}-${{ github.sha }}-PR
192+
restore-keys: |
193+
${{ runner.os }}-20.04-Release-${{ github.ref }}
194+
${{ runner.os }}-20.04-Release
195+
- name: ccache environment
196+
run: |
197+
echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
198+
echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
199+
- name: Configure using CMake
200+
run: cmake -H. -Bbuild -G Ninja
201+
- name: Zero ccache stats and limit in size
202+
run: ccache -z --max-size=500M
203+
- name: Build with Ninja
204+
run: ninja -C build -j2
205+
- name: Print ccache stats
206+
run: ccache -s
207+
- name: Run tests
208+
run: |
209+
cd build
210+
ctest . -V -L KNOWNBUG -j2
211+
export PATH=$PWD/bin:$PATH
212+
cd ../regression/cbmc
213+
sed -i '1s/^CORE\(.*\)broken-smt-backend/KNOWNBUG\1broken-smt-backend/' */*.desc
214+
../test.pl -c "cbmc --cprover-smt2" -I broken-smt-backend -K
215+
216+
check-ubuntu-20_04-cmake-gcc-THOROUGH:
217+
runs-on: ubuntu-20.04
218+
steps:
219+
- uses: actions/checkout@v2
220+
with:
221+
submodules: recursive
222+
- name: Fetch dependencies
223+
env:
224+
# This is needed in addition to -yq to prevent apt-get from asking for
225+
# user input
226+
DEBIAN_FRONTEND: noninteractive
227+
run: |
228+
sudo apt-get update
229+
sudo apt-get install --no-install-recommends -yq cmake ninja-build gcc g++ maven flex bison libxml2-utils ccache z3
230+
- name: Prepare ccache
231+
uses: actions/cache@v2
232+
with:
233+
path: .ccache
234+
key: ${{ runner.os }}-20.04-Release-${{ github.ref }}-${{ github.sha }}-PR
235+
restore-keys: |
236+
${{ runner.os }}-20.04-Release-${{ github.ref }}
237+
${{ runner.os }}-20.04-Release
238+
- name: ccache environment
239+
run: |
240+
echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
241+
echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
242+
- name: Configure using CMake
243+
run: cmake -H. -Bbuild -G Ninja
244+
- name: Zero ccache stats and limit in size
245+
run: ccache -z --max-size=500M
246+
- name: Build with Ninja
247+
run: ninja -C build -j2
248+
- name: Print ccache stats
249+
run: ccache -s
250+
- name: Run tests
251+
run: cd build; ctest . -V -L THOROUGH -j2
252+
173253
check-macos-10_15-make-clang:
174254
runs-on: macos-10.15
175255
steps:

0 commit comments

Comments
 (0)