Skip to content

Commit c26aab2

Browse files
zhiyuan1izhiyuan1i
authored
[CI] Refract GPU CIs (#487)
1 parent c1eb9d4 commit c26aab2

File tree

5 files changed

+344
-664
lines changed

5 files changed

+344
-664
lines changed

‎.github/workflows/intel-a770.yml‎

Lines changed: 9 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -15,107 +15,13 @@ on:
1515
- '*'
1616

1717
jobs:
18-
test-ops:
18+
test-a770-pytorch-2-7:
19+
name: Test A770 (PyTorch 2.7)
1920
if: github.event_name != 'pull_request' || github.event.action != 'closed'
20-
runs-on: 'intel-a770'
21-
env:
22-
FLA_CI_ENV: 1
23-
steps:
24-
- name: Check out repo
25-
uses: actions/checkout@v4
26-
27-
- name: Check skip keyword in LATEST commit (Push only)
28-
id: check_skip
29-
run: |
30-
if [ "${{ github.event_name }}" = "push" ] && ! [[ "${{ github.ref }}" =~ ^refs/tags/ ]]; then
31-
COMMIT_MSG=$(jq -r '.head_commit.message' <<< '${{ toJSON(github.event) }}')
32-
echo "Latest commit message: $COMMIT_MSG"
33-
if echo "$COMMIT_MSG" | grep -qF "[skip test]"; then
34-
echo "::notice::Tests skipped by commit message"
35-
echo "skip_tests=true" >> $GITHUB_OUTPUT
36-
else
37-
echo "skip_tests=false" >> $GITHUB_OUTPUT
38-
fi
39-
else
40-
echo "skip_tests=false" >> $GITHUB_OUTPUT
41-
fi
42-
43-
- name: Get changed files
44-
if: |
45-
(github.event_name == 'pull_request' ||
46-
(github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/'))) &&
47-
(steps.check_skip.outputs.skip_tests != 'true' || github.event_name != 'push')
48-
id: changed-files
49-
uses: tj-actions/[email protected]
50-
51-
- name: Find dependent test files
52-
if: |
53-
(github.event_name == 'pull_request' ||
54-
(github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/'))) &&
55-
(steps.check_skip.outputs.skip_tests != 'true' || github.event_name != 'push')
56-
id: find-dependent-tests
57-
run: |
58-
# Run the Python script to find dependent test files
59-
TEST_FILES=$(TEST_SCOPE=EXCLUDE_MODELS python scripts/find_dependent_tests.py "${{ steps.changed-files.outputs.all_changed_files }}")
60-
echo "test_files=$TEST_FILES" >> $GITHUB_OUTPUT
61-
62-
- name: Setup python
63-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
64-
run: |
65-
pip install -U uv
66-
pip uninstall -y flash-linear-attention
67-
uv pip install -U pytest setuptools wheel ninja
68-
uv pip install -U torch~=2.7.0 pytorch-triton-xpu --index-url https://download.pytorch.org/whl/xpu
69-
pip install .
70-
71-
- name: Check GPU status
72-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
73-
run: |
74-
python scripts/check_gpu.py
75-
if [ $? -ne 0 ]; then
76-
echo "GPU is occupied. Stopping the workflow."
77-
exit 1
78-
fi
79-
80-
- name: Test compiling on changed test files
81-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
82-
run: |
83-
FLA_COMPILER_MODE=1 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
84-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
85-
86-
- name: Run pytest on test files
87-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
88-
continue-on-error: true
89-
run: |
90-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
91-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
92-
93-
- name: Run pytest on varlen test files
94-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
95-
continue-on-error: true
96-
run: |
97-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=0 \
98-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }} || \
99-
echo "Varlen tests failed (non-critical)"
100-
101-
- name: Test full compiling on all test files
102-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
103-
run: |
104-
FLA_COMPILER_MODE=1 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
105-
pytest tests/
106-
107-
# skip full pytest because it takes too long
108-
- name: Run full pytest on test files
109-
if: false && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
110-
continue-on-error: true
111-
run: |
112-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
113-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
114-
115-
- name: Run full pytest on varlen test files
116-
if: false && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
117-
continue-on-error: true
118-
run: |
119-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=0 \
120-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }} || \
121-
echo "Varlen tests failed (non-critical)"
21+
uses: ./.github/workflows/reusable-ci-tests.yml
22+
with:
23+
runner: 'intel-a770'
24+
gpu_type: 'intel'
25+
conda_env_name: 'pytorch_2_7'
26+
pytorch_version: '2.7.0'
27+
skip_gpu_check: false

‎.github/workflows/nvidia-4090.yml‎

Lines changed: 9 additions & 186 deletions
Original file line numberDiff line numberDiff line change
@@ -15,190 +15,13 @@ on:
1515
- '*'
1616

1717
jobs:
18-
test-ops:
18+
test-4090-pytorch-2-7:
19+
name: Test 4090 (PyTorch 2.7)
1920
if: github.event_name != 'pull_request' || github.event.action != 'closed'
20-
runs-on: 'nvidia-4090'
21-
env:
22-
FLA_CI_ENV: 1
23-
steps:
24-
- name: Check out repo
25-
uses: actions/checkout@v4
26-
27-
- name: Check skip keyword in LATEST commit (Push only)
28-
id: check_skip
29-
run: |
30-
if [ "${{ github.event_name }}" = "push" ] && ! [[ "${{ github.ref }}" =~ ^refs/tags/ ]]; then
31-
COMMIT_MSG=$(jq -r '.head_commit.message' <<< '${{ toJSON(github.event) }}')
32-
echo "Latest commit message: $COMMIT_MSG"
33-
if echo "$COMMIT_MSG" | grep -qF "[skip test]"; then
34-
echo "::notice::Tests skipped by commit message"
35-
echo "skip_tests=true" >> $GITHUB_OUTPUT
36-
else
37-
echo "skip_tests=false" >> $GITHUB_OUTPUT
38-
fi
39-
else
40-
echo "skip_tests=false" >> $GITHUB_OUTPUT
41-
fi
42-
43-
- name: Get changed files
44-
if: |
45-
(github.event_name == 'pull_request' ||
46-
(github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/'))) &&
47-
(steps.check_skip.outputs.skip_tests != 'true' || github.event_name != 'push')
48-
id: changed-files
49-
uses: tj-actions/[email protected]
50-
51-
- name: Find dependent test files
52-
if: |
53-
(github.event_name == 'pull_request' ||
54-
(github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/'))) &&
55-
(steps.check_skip.outputs.skip_tests != 'true' || github.event_name != 'push')
56-
id: find-dependent-tests
57-
run: |
58-
# Run the Python script to find dependent test files
59-
TEST_FILES=$(TEST_SCOPE=EXCLUDE_MODELS python scripts/find_dependent_tests.py "${{ steps.changed-files.outputs.all_changed_files }}")
60-
echo "test_files=$TEST_FILES" >> $GITHUB_OUTPUT
61-
62-
- name: Setup python
63-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
64-
run: |
65-
# Installed by hand to avoid issues with pip
66-
# pip install -U pytest setuptools wheel ninja torch triton
67-
# MAX_JOBS=4 pip install -U flash-attn --no-build-isolation
68-
# pip install git+https://github.com/Dao-AILab/causal-conv1d.git@a4b40 --no-build-isolation
69-
pip install -U uv
70-
pip uninstall -y flash-linear-attention
71-
uv pip install -U pytest setuptools wheel ninja
72-
uv pip install torch~=2.7.0 triton -U --index-url https://download.pytorch.org/whl/cu128
73-
uv pip install flash-attn -U --no-cache-dir --no-build-isolation
74-
pip install .
75-
76-
- name: Check GPU status
77-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
78-
run: |
79-
python scripts/check_gpu.py
80-
if [ $? -ne 0 ]; then
81-
echo "GPU is occupied. Stopping the workflow."
82-
exit 1
83-
fi
84-
85-
- name: Test compiling on changed test files
86-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
87-
run: |
88-
FLA_COMPILER_MODE=1 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
89-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
90-
91-
- name: Run pytest on test files
92-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
93-
run: |
94-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
95-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
96-
97-
- name: Run pytest on varlen test files
98-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
99-
run: |
100-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=0 \
101-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }} || \
102-
echo "Varlen tests failed (non-critical)"
103-
104-
- name: Test full compiling on all test files
105-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
106-
run: |
107-
FLA_COMPILER_MODE=1 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
108-
pytest tests/
109-
110-
- name: Run full pytest on test files
111-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
112-
run: |
113-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
114-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
115-
116-
- name: Run full pytest on varlen test files
117-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
118-
run: |
119-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=0 \
120-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }} || \
121-
echo "Varlen tests failed (non-critical)"
122-
123-
test-models:
124-
needs: test-ops
125-
if: always() && github.event_name != 'pull_request' || github.event.action != 'closed'
126-
runs-on: 'nvidia-4090'
127-
env:
128-
FLA_CI_ENV: 1
129-
steps:
130-
- name: Check out repo
131-
uses: actions/checkout@v4
132-
133-
- name: Check skip keyword in LATEST commit (Push only)
134-
id: check_skip
135-
run: |
136-
if [ "${{ github.event_name }}" = "push" ] && ! [[ "${{ github.ref }}" =~ ^refs/tags/ ]]; then
137-
COMMIT_MSG=$(jq -r '.head_commit.message' <<< '${{ toJSON(github.event) }}')
138-
echo "Latest commit message: $COMMIT_MSG"
139-
if echo "$COMMIT_MSG" | grep -qF "[skip test]"; then
140-
echo "::notice::Tests skipped by commit message"
141-
echo "skip_tests=true" >> $GITHUB_OUTPUT
142-
else
143-
echo "skip_tests=false" >> $GITHUB_OUTPUT
144-
fi
145-
else
146-
echo "skip_tests=false" >> $GITHUB_OUTPUT
147-
fi
148-
149-
- name: Get changed files
150-
if: |
151-
(github.event_name == 'pull_request' ||
152-
(github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/'))) &&
153-
(steps.check_skip.outputs.skip_tests != 'true' || github.event_name != 'push')
154-
id: changed-files
155-
uses: tj-actions/[email protected]
156-
157-
- name: Find dependent test files
158-
if: |
159-
(github.event_name == 'pull_request' ||
160-
(github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/'))) &&
161-
(steps.check_skip.outputs.skip_tests != 'true' || github.event_name != 'push')
162-
id: find-dependent-tests
163-
run: |
164-
# Run the Python script to find dependent test files
165-
TEST_FILES=$(TEST_SCOPE=MODELS_ONLY python scripts/find_dependent_tests.py "${{ steps.changed-files.outputs.all_changed_files }}")
166-
echo "test_files=$TEST_FILES" >> $GITHUB_OUTPUT
167-
168-
- name: Test compiling on changed test files
169-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
170-
run: |
171-
FLA_COMPILER_MODE=1 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
172-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
173-
174-
- name: Run pytest on test files
175-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
176-
run: |
177-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
178-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
179-
180-
- name: Run pytest on varlen test files
181-
if: steps.find-dependent-tests.outputs.test_files && steps.check_skip.outputs.skip_tests == 'false'
182-
run: |
183-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=0 \
184-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }} || \
185-
echo "Varlen tests failed (non-critical)"
186-
187-
- name: Test full compiling on all test files
188-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
189-
run: |
190-
FLA_COMPILER_MODE=1 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
191-
pytest tests/
192-
193-
- name: Run full pytest on test files
194-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
195-
run: |
196-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=1 \
197-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }}
198-
199-
- name: Run full pytest on varlen test files
200-
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.check_skip.outputs.skip_tests == 'false'
201-
run: |
202-
FLA_COMPILER_MODE=0 TRITON_PRINT_AUTOTUNING=0 SKIP_TEST_CHUNK_VARLEN=0 \
203-
pytest -s -v ${{ steps.find-dependent-tests.outputs.test_files }} || \
204-
echo "Varlen tests failed (non-critical)"
21+
uses: ./.github/workflows/reusable-ci-tests.yml
22+
with:
23+
runner: 'nvidia-4090'
24+
gpu_type: 'nvidia'
25+
conda_env_name: 'fla'
26+
pytorch_version: '2.7.0'
27+
skip_gpu_check: false

0 commit comments

Comments
 (0)