From 6f16887451992a1390d1b51cd1879adbff7fbb24 Mon Sep 17 00:00:00 2001
From: Jialei A Wang <jialei.a.wang@intel.com>
Date: Sat, 12 Oct 2024 01:32:00 +0000
Subject: [PATCH] gpu nightly test

---
 .github/workflows/ov-gpu-nightly.yml | 82 ++++++++++++++++++++++++++++
 scripts/compile.sh                   |  2 +-
 2 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/ov-gpu-nightly.yml

diff --git a/.github/workflows/ov-gpu-nightly.yml b/.github/workflows/ov-gpu-nightly.yml
new file mode 100644
index 00000000..aefa64b7
--- /dev/null
+++ b/.github/workflows/ov-gpu-nightly.yml
@@ -0,0 +1,82 @@
+name: OpenVino GPU Nightly Test
+
+on:
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  nightly:
+    name: OpenVino GPU Nightly
+    runs-on: [self-hosted, l0]
+
+    env:
+      GH_TOKEN: ${{ github.token }}    
+
+    steps:
+      - name: "Checkout Graph Compiler"
+        uses: actions/checkout@v4
+        with:
+          repository: 'intel/graph-compiler'
+          path: 'graph-compiler'
+
+      - name: Setup MLIR Python bindings
+        id: setup-mlir-python-bindings
+        uses: ./graph-compiler/.github/actions/setup-mlir-python-bindings
+
+      - name: Build LLVM with IMEX
+        working-directory: graph-compiler
+        run: |
+          scripts/compile.sh --dev --llvm --imex
+          echo LLVM_INST_PATH=$(pwd)/externals/llvm-project/build >>$GITHUB_ENV
+
+      - name: "Checkout OpenVino"
+        uses: actions/checkout@v4
+        with:
+          repository: 'dchigarev/openvino'
+          ref: 'gc-gpu'
+          path: 'openvino'
+          submodules: true
+          fetch-depth: 0
+          recursive: true
+      
+      - name: Build OpenVino
+        working-directory: openvino
+        run: |
+          cmake -B build -G Ninja -DLLVM_DIR=${LLVM_INST_PATH}/lib/cmake/llvm -DMLIR_DIR=${LLVM_INST_PATH}/lib/cmake/mlir -DENABLE_GRAPH_COMPILER=ON -DENABLE_INTEL_GPU=ON -DENABLE_TESTS=ON
+          cmake --build build --target all
+      
+      - name: Benchmark
+        working-directory: openvino
+        run: |
+          pip install openvino torch
+          for param in 'matmul[512,512,512]' 'matmul[1024,1024,1024]' 'matmul[2048,2048,2048]', 'matmul[4096,4096,4096]' 'matmul[8192,8192,8192]' 'matmul[4096,512,4096]'; do
+            python3 ./tools/mlir_bench/ov_model_gen.py -l=$param -t f16 -n test.xml
+            OV_MLIR_MODE=GC_GPU ./bin/intel64/Release/benchmark_app -m ./test.xml -d GPU -use_device_mem -ip f16 -infer_precision f16 -niter 100 -hint none -nstreams 1 -nthreads 1 >tmp
+            echo 'gc-gpu: ' $param
+            cat tmp
+            COMPILE_TIME=$(grep "Compile model took" tmp|tr -cd '0-9.')
+            FIRST_INFERENCE_TIME=$(grep "First inference took" tmp|tr -cd '0-9.')
+            THROUGHPUT=$(grep "Throughput" tmp|tr -cd '0-9.')
+            echo "openvino,$param,$COMPILE_TIME,$FIRST_INFERENCE_TIME,$THROUGHPUT" >> perf.csv
+
+            OV_MLIR=0 ./bin/intel64/Release/benchmark_app -m ./test.xml -d GPU -use_device_mem -ip f16 -infer_precision f16 -niter 100 -hint none -nstreams 1 -nthreads 1 >tmp
+            echo 'baseline: ' $param
+            cat tmp
+            COMPILE_TIME=$(grep "Compile model took" tmp|tr -cd '0-9.')
+            FIRST_INFERENCE_TIME=$(grep "First inference took" tmp|tr -cd '0-9.')
+            THROUGHPUT=$(grep "Throughput" tmp|tr -cd '0-9.')
+            echo "openvino,$param,$COMPILE_TIME,$FIRST_INFERENCE_TIME,$THROUGHPUT" >> perf-baseline.csv
+          done
+
+      - name: Upload Performance Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf.csv
+          path: openvino/perf.csv     
+
+      - name: Upload Baseline Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: perf-baseline.csv
+          path: openvino/perf-baseline.csv     
diff --git a/scripts/compile.sh b/scripts/compile.sh
index 8d5709e0..2ce44d5c 100755
--- a/scripts/compile.sh
+++ b/scripts/compile.sh
@@ -114,7 +114,6 @@ build_llvm() {
     local llvm_dir="$EXTERNALS_DIR/llvm-project"
     LLVM_BUILD_DIR="$llvm_dir/build$BUILD_DIR_SFX"
     MLIR_DIR="$LLVM_BUILD_DIR/lib/cmake/mlir"
-    mkdir -p "$LLVM_BUILD_DIR"
 
     if ! [ -d "$llvm_dir" ]; then
         git clone https://github.com/llvm/llvm-project.git
@@ -126,6 +125,7 @@ build_llvm() {
         # build that would break 'git checkout ${LLVM_HASH}')
         git checkout -- .
     fi
+    mkdir -p "$LLVM_BUILD_DIR"
 
     git checkout ${LLVM_HASH}
     [ -z "$CLEANUP" ] || rm -rf "$LLVM_BUILD_DIR"