Skip to content

Commit 66b7597

Browse files
phymbertphymbert
committed
server: bench: init
1 parent 43139cc commit 66b7597

File tree

5 files changed

+439
-9
lines changed

5 files changed

+439
-9
lines changed

‎.github/workflows/bench.yml

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# Benchmark
2+
name: Benchmark
3+
4+
on:
5+
workflow_dispatch:
6+
inputs:
7+
gpu-series:
8+
description: 'Azure GPU series to run with'
9+
required: true
10+
type: choice
11+
options:
12+
- Standard_NC4as_T4_v3
13+
- Standard_NC64as_T4_v3
14+
- Standard_NC24ads_A100_v4
15+
- Standard_NC48ads_A100_v4
16+
- Standard_ND96asr_A100_v4
17+
- Standard_NC40ads_H100_v5
18+
- Standard_NC80adis_H100_v5
19+
push:
20+
branches:
21+
- master
22+
paths: ['.github/workflows/bench.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/bench/**.*']
23+
pull_request:
24+
types: [opened, synchronize, reopened]
25+
paths: ['.github/workflows/bench.yml', '**/CMakeLists.txt', '**/Makefile', '**/*.h', '**/*.hpp', '**/*.c', '**/*.cpp', '**/*.cu', '**/*.swift', '**/*.m', 'examples/server/bench/**.*']
26+
schedule:
27+
- cron: '04 2 * * *'
28+
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.ref }}
31+
cancel-in-progress: true
32+
33+
jobs:
34+
bench-server-baseline:
35+
runs-on: Standard_NC4as_T4_v3
36+
env:
37+
RUNNER_LABEL: Standard_NC4as_T4_v3 # FIXME Do not find a way to not duplicate it
38+
#if: ${{ github.event.inputs.gpu-series == 'Standard_NC4as_T4_v3' || github.event.schedule || github.event.pull_request || github.event.push.ref == 'refs/heads/master' }}
39+
steps:
40+
- name: Clone
41+
id: checkout
42+
uses: actions/checkout@v3
43+
with:
44+
fetch-depth: 0
45+
46+
- name: Install python env
47+
id: pipenv
48+
run: |
49+
cd examples/server/bench
50+
python3 -m venv venv
51+
source venv/bin/activate
52+
pip install -r requirements.txt
53+
54+
- name: Prometheus
55+
id: install_prometheus
56+
run: |
57+
wget --quiet https://github.com/prometheus/prometheus/releases/download/v2.51.0/prometheus-2.51.0.linux-amd64.tar.gz
58+
tar xzf prometheus*.tar.gz --strip-components=1
59+
./prometheus --config.file=examples/server/bench/prometheus.yml &
60+
while ! nc -z localhost 9090; do
61+
sleep 0.1
62+
done
63+
64+
- name: Install k6
65+
id: k6_installation
66+
run: |
67+
cd examples/server/bench
68+
wget --quiet https://github.com/grafana/k6/releases/download/v0.49.0/k6-v0.49.0-linux-amd64.tar.gz
69+
tar xzf k6*.tar.gz --strip-components=1
70+
71+
- name: Build
72+
id: cmake_build
73+
run: |
74+
set -eux
75+
mkdir build
76+
cd build
77+
cmake .. \
78+
-DLLAMA_NATIVE=OFF \
79+
-DLLAMA_BUILD_SERVER=ON \
80+
-DLLAMA_CURL=ON \
81+
-DLLAMA_CUBLAS=ON \
82+
-DCUDAToolkit_ROOT=/usr/local/cuda \
83+
-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
84+
-DCMAKE_CUDA_ARCHITECTURES=75 \
85+
-DLLAMA_FATAL_WARNINGS=OFF \
86+
-DLLAMA_ALL_WARNINGS=OFF \
87+
-DCMAKE_BUILD_TYPE=Release;
88+
cmake --build . --config Release -j $(nproc) --target server
89+
90+
- name: Download the dataset
91+
id: download_dataset
92+
run: |
93+
cd examples/server/bench
94+
wget --quiet https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
95+
96+
- name: Server bench
97+
id: server_bench
98+
run: |
99+
set -eux
100+
101+
cd examples/server/bench
102+
source venv/bin/activate
103+
BENCH_K6_BIN_PATH=./k6 python bench.py \
104+
--runner-label ${{ env.RUNNER_LABEL }} \
105+
--name ${{ github.job }} \
106+
--branch ${{ github.ref_name }} \
107+
--commit ${{ github.sha }} \
108+
--scenario script.js \
109+
--duration 30s \
110+
--hf-repo ggml-org/models \
111+
--hf-file phi-2/ggml-model-q4_0.gguf \
112+
--model-path-prefix /models \
113+
--parallel 8 \
114+
-ngl 33 \
115+
--batch-size 2048 \
116+
--ubatch-size 256 \
117+
--ctx-size 16384 \
118+
--n-prompts 1000 \
119+
--max-prompt-tokens 1024 \
120+
--max-tokens 2048
121+
122+
cat results.github.env >> $GITHUB_ENV
123+
124+
# - name: Comment PR
125+
# uses: mshick/add-pr-comment@v2
126+
# id: comment_pr
127+
# if: ${{ github.event.pull_request != '' }}
128+
# with:
129+
# message-id: bench-${{ github.job }}-${{ env.RUNNER_LABEL }}
130+
# message: |
131+
# $BENCH_PR_COMMENT
132+
133+
- name: Commit status
134+
uses: Sibz/github-status-action@v1
135+
with:
136+
authToken: ${{secrets.GITHUB_TOKEN}}
137+
context: ${{ github.job }}
138+
description: |
139+
${{ env.BENCH_RESULTS }}
140+
state: 'success'
141+
142+
- name: Upload results
143+
if: ${{ github.event.pull_request != '' }}
144+
uses: edunad/[email protected]
145+
with:
146+
path: '*.png'
147+
title: |
148+
llama.cpp server benchmark results for ${{ github.job }} on ${{ env.RUNNER_LABEL }}: ${{ env.LLAMACPP_TOKENS_SECOND_AVG}}tk/s
149+
annotationLevel: 'success'

0 commit comments

Comments
 (0)