[CI] Enhance pytest xml files collection (#2380)

1. fix the xml files collection issue
2. modify xml file name
3. remove run_test_with_only.py which is duplicated with
run_test_with_skip.py after test_decomp enabled

Author: mengfei25

.github/actions/linux-uttest/action.yml

@@ -74,34 +74,7 @@ runs:
         python run_test_with_skip.py \
           2> ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test_error.log | \
           tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test.log
-        ls -al
-        cp *.xml ${{ github.workspace }}/ut_log
-        find op_ut_with_skip.nn op_ut_with_skip.quantization/core op_ut_with_all.functorch -type f -exec sh -c '
-            dir_path=$(dirname "$1");
-            case "$dir_path" in
-                *"op_ut_with_skip.quantization/core"*)
-                    dir_name="op_ut_with_skip.quantization.core";;
-                *"op_ut_with_all.functorch"*)
-                    dir_name="op_ut_with_all.functorch";;
-                *)
-                    dir_name="op_ut_with_skip.nn";;
-            esac;
-            mv "$1" "$dir_path/${dir_name}.$(basename "$1")"
-        ' _ {} \;
-        ls -al op_ut_with_skip.nn op_ut_with_skip.quantization/core op_ut_with_all.functorch
-        cp op_ut_with_skip.nn/*.xml ${{ github.workspace }}/ut_log
-        cp op_ut_with_skip.quantization/core/*.xml ${{ github.workspace }}/ut_log
-        cp op_ut_with_all.functorch/*.xml ${{ github.workspace }}/ut_log
-        # Cases run with a on-demand white list, since some suites are too
-        # slow to go through all operators on CPU. So add cases on-demand
-        # when XPU implementatoin is done.
-        # test_foreach, test_decomp
-        # Run with only
-        python run_test_with_only.py \
-          2> ${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test_error.log | \
-          tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test.log
-        ls -al
-        cp *.xml ${{ github.workspace }}/ut_log
+        find . -type f -name "op_ut_with_*.xml" -exec mv {} ${{ github.workspace }}/ut_log/ \; || true
         echo -e "File Path: cd pytorch/third_party/torch-xpu-ops/test/xpu" | tee -a ${{ github.workspace }}/ut_log/reproduce_op_ut.log
         echo -e "Reproduce Command: pytest -sv failed_case" | tee -a ${{ github.workspace }}/ut_log/reproduce_op_ut.log
     - name: skipped_ut
@@ -114,9 +87,9 @@ runs:
         python run_test_with_skip.py --test-cases skipped \
           2> ${{ github.workspace }}/ut_log/skipped_ut/skipped_ut_with_skip_test_error.log | \
           tee ${{ github.workspace }}/ut_log/skipped_ut/skipped_ut_with_skip_test.log
+        find . -type f -name "op_ut_with_*.xml" -exec mv {} ${{ github.workspace }}/ut_log/ \; || true
         echo -e "File Path: cd pytorch/third_party/torch-xpu-ops/test/xpu" | tee -a ${{ github.workspace }}/ut_log/reproduce_skipped_ut.log
         echo -e "Reproduce Command: pytest -sv failed_case" | tee -a ${{ github.workspace }}/ut_log/reproduce_skipped_ut.log
-        cp *.xml ${{ github.workspace }}/ut_log
     - name: torch_xpu
       shell: timeout 3600 bash -xe {0}
       if: ${{ inputs.ut_name == 'torch_xpu' }}

.github/scripts/e2e_summary.sh

@@ -213,7 +213,7 @@ process_model() {
     done
 
     local row
-    row=$(generate_html_row "$suite" "$model" "${results[@]}")
+    row=$(generate_html_row "$suite" "$model")
 
     if [[ "$row" =~ ${RED}|${GREEN}|${YELLOW} ]]; then
         echo "$row" | tee -a accuracy.details.html >> accuracy.regression.html
@@ -258,15 +258,20 @@ determine_color() {
 
 generate_html_row() {
     local suite="$1" model="$2"
-    shift 2
-    local results=("$@")
-
     cat << EOF
 <tr>
     <td>$suite</td>
     <td>$model</td>
-    <td>${results[0]}</td><td>${results[1]}</td><td>${results[2]}</td><td>${results[3]}</td><td>${results[4]}</td>
-    <td>${results[5]}</td><td>${results[6]}</td><td>${results[7]}</td><td>${results[8]}</td><td>${results[9]}</td>
+    <td>${results["training_float32"]}</td>
+    <td>${results["training_bfloat16"]}</td>
+    <td>${results["training_float16"]}</td>
+    <td>${results["training_amp_bf16"]}</td>
+    <td>${results["training_amp_fp16"]}</td>
+    <td>${results["inference_float32"]}</td>
+    <td>${results["inference_bfloat16"]}</td>
+    <td>${results["inference_float16"]}</td>
+    <td>${results["inference_amp_bf16"]}</td>
+    <td>${results["inference_amp_fp16"]}</td>
 </tr>
 EOF
 }

.github/scripts/get_issue.py

@@ -8,7 +8,7 @@
 import re
 import json
 import argparse
-from github import Github
+from github import Github, Auth
 
 
 def get_github_issues(repo_owner, repo_name, labels, state='open'):
@@ -91,8 +91,9 @@ def main():
 
     # Quick token check
     token = os.getenv('GH_TOKEN')
+    auth = Auth.Token(token)
     global g
-    g = Github(token)  # Increase page size for fewer API calls
+    g = Github(auth=auth)
 
     print(f"Fetching known issues from {args.repo_owner}/{args.repo_name}...")
 

.github/scripts/ut_result_check.sh

@@ -11,10 +11,10 @@ readonly REPO="intel/torch-xpu-ops"
 # Used to detect significant test case reductions (>5%)
 declare -A EXPECTED_CASES=(
     ["op_extended"]=5349
-    ["op_regression"]=244
+    ["op_regression"]=268
     ["op_regression_dev1"]=1
-    ["op_transformers"]=237
-    ["op_ut"]=120408
+    ["op_transformers"]=262
+    ["op_ut"]=178548
     ["test_xpu"]=69
 )
 

test/xpu/xpu_test_utils.py

@@ -1092,15 +1092,15 @@ def copy_tests(
 
 def launch_test(test_case, skip_list=None, exe_list=None):
     os.environ["PYTORCH_TEST_WITH_SLOW"] = "1"
+    module_name = test_case.replace(".py", "").replace("/", ".").replace("\\", ".")
     if skip_list is not None:
         skip_options = ' -k "not ' + skip_list[0]
         for skip_case in skip_list[1:]:
             skip_option = " and not " + skip_case
             skip_options += skip_option
         skip_options += '"'
         test_command = (
-            f"pytest --junit-xml=./op_ut_with_skip.{test_case}.xml --max-worker-restart=1000 "
-            + test_case
+            f"pytest --junit-xml=./op_ut_with_skip.{module_name}.xml " + test_case
         )
         test_command += skip_options
     elif exe_list is not None:
@@ -1110,13 +1110,11 @@ def launch_test(test_case, skip_list=None, exe_list=None):
             exe_options += exe_option
         exe_options += '"'
         test_command = (
-            f"pytest --junit-xml=./op_ut_with_exe.{test_case}.xml --max-worker-restart=1000 "
-            + test_case
+            f"pytest --junit-xml=./op_ut_with_exe.{module_name}.xml " + test_case
         )
         test_command += exe_options
     else:
         test_command = (
-            f"pytest --junit-xml=./op_ut_with_all.{test_case}.xml --max-worker-restart=1000 "
-            + test_case
+            f"pytest --junit-xml=./op_ut_with_all.{module_name}.xml " + test_case
         )
     return os.system(test_command)