Backmerging with Msft commits

.github/actions/install-onnxruntime-wheel/action.yml

@@ -0,0 +1,18 @@
+name: 'Install ONNX Runtime Wheel'
+description: 'Uninstalls existing ONNX Runtime packages and installs a wheel from a specified directory.'
+inputs:
+  whl-directory:
+    description: 'The directory containing the ONNX Runtime wheel files.'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Uninstall onnxruntime packages
+      shell: pwsh
+      run: |
+        python -m pip uninstall -y onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml -qq
+
+    - name: Install onnxruntime wheel from specified directory
+      shell: pwsh
+      run: |
+        Get-ChildItem -Path ${{ inputs.whl-directory }}/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}

.github/actions/locate-vcvarsall-and-setup-env/action.yml

@@ -0,0 +1,63 @@
+name: 'Locate vcvarsall and Setup Environment'
+description: 'Locates vcvarsall.bat, sets up the environment, and handles PATH updates.'
+inputs:
+  architecture:
+    description: 'Target architecture (x64 or x86)'
+    required: true
+    default: 'x64'
+outputs:
+  vcvarsall_path:
+    description: "Path to vcvarsall.bat"
+    value: ${{ steps.find-vcvarsall.outputs.vcvarsall_path }}
+runs:
+  using: "composite"
+  steps:
+    - name: Find vcvarsall.bat
+      id: find-vcvarsall
+      shell: python  # Use Python shell
+      run: |
+        import os
+        import subprocess
+
+        vswhere_path = os.path.join(os.environ["ProgramFiles(x86)"], "Microsoft Visual Studio", "Installer", "vswhere.exe")
+
+        try:
+            process = subprocess.run([vswhere_path, "-latest", "-property", "installationPath"], capture_output=True, text=True, check=True)
+            vs_install_path = process.stdout.strip()
+            vcvarsall_path = os.path.join(vs_install_path, "VC", "Auxiliary", "Build", "vcvarsall.bat")
+
+            if os.path.exists(vcvarsall_path):
+                print(f"vcvarsall found at: {vcvarsall_path}")
+                # Use GITHUB_OUTPUT environment variable
+                with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
+                    f.write(f"vcvarsall_path={vcvarsall_path}\n")
+            else:
+                print(f"vcvarsall.bat not found at expected path: {vcvarsall_path}")
+                # Use 'exit(1)' for Python to properly signal failure to GitHub Actions
+                exit(1)
+
+
+        except subprocess.CalledProcessError as e:
+            print(f"Error running vswhere.exe: {e}")
+            print(f"vswhere output: {e.stdout}")
+            print(f"vswhere stderr: {e.stderr}")
+            exit(1) # Exit with a non-zero code on error
+        except FileNotFoundError:
+            print(f"vswhere.exe not found at: {vswhere_path}")
+            exit(1)
+
+
+    - name: Setup Environment
+      shell: cmd
+      run: |
+        REM Get initial environment variables
+        set > initial_env.txt
+
+        REM Call vcvarsall.bat using the output from the previous step
+        call "${{ steps.find-vcvarsall.outputs.vcvarsall_path }}" ${{ inputs.architecture }}
+
+        REM Get environment variables after calling vcvarsall.bat
+        set > final_env.txt
+
+        REM Call the Python script to update the GitHub Actions environment
+        python ${{ github.action_path }}\update_environment.py

.github/actions/locate-vcvarsall-and-setup-env/locate_vcvarsall.bat

@@ -0,0 +1,30 @@
+@echo off
+setlocal
+
+set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
+set vcvarsall_arch=%1
+if "%vcvarsall_arch%" == "x86" (
+  set vcvarsall_arch=x86
+) else (
+  set vcvarsall_arch=x64
+)
+
+for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
+  if exist "%%i\VC\Auxiliary\Build\vcvars%vcvarsall_arch%.bat" (
+    set "vcvarsall=%%i\VC\Auxiliary\Build\vcvars%vcvarsall_arch%.bat"
+  )
+)
+
+echo "Get initial environment variables"
+set > initial_env.txt
+
+echo "Call vcvarsall.bat"
+call "%vcvarsall%"
+
+echo "Get environment variables after calling vcvarsall.bat"
+set > final_env.txt
+
+echo "Call the Python script to update the GitHub Actions environment"
+python "%~dp0\update_environment.py"
+
+endlocal

.github/actions/locate-vcvarsall-and-setup-env/update_environment.py

@@ -0,0 +1,37 @@
+import os
+import re
+
+
+def read_env_file(filepath):
+    env_vars = {}
+    with open(filepath) as f:
+        for line in f:
+            match = re.match(r"^(.*?)=(.*)$", line.strip())
+            if match:
+                env_vars[match.group(1).upper()] = match.group(2)
+    return env_vars
+
+
+initial_env = read_env_file("initial_env.txt")
+final_env = read_env_file("final_env.txt")
+
+for key, value in final_env.items():
+    if key not in initial_env or initial_env[key] != value:
+        if key.startswith("_"):
+            continue
+        if key.upper() == "PATH":
+            new_paths = value.split(";")
+            initial_paths = initial_env.get("PATH", "").split(";")
+            added_paths = [p for p in new_paths if p not in initial_paths and p]
+
+            if added_paths:
+                print("Adding paths")
+                with open(os.environ["GITHUB_PATH"], "a") as f:
+                    for path in added_paths:
+                        print(f"Adding PATH: {path}")
+                        f.write(path + os.linesep)
+        else:
+            # Use GITHUB_ENV
+            with open(os.environ["GITHUB_ENV"], "a") as f:
+                print(f"Setting {key}={value}\n")
+                f.write(f"{key}={value}\n")

.github/workflows/ios.yml

@@ -0,0 +1,64 @@
+name: iOS_CI_on_Mac
+
+on:
+  push:
+    branches:
+      - main
+      - rel-*
+  pull_request:
+    branches:
+      - main
+      - rel-*
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  iOS_CI_on_Mac:
+    runs-on: macos-13
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        submodules: false
+    - name: Use Xcode ${{ env.XCODE_VERSION }}
+      shell: bash
+      run: |
+        set -e -x
+        XCODE_DEVELOPER_DIR="/Applications/Xcode_${{ env.XCODE_VERSION }}.app/Contents/Developer"
+        sudo xcode-select --switch "${XCODE_DEVELOPER_DIR}"
+
+    - name: Export GitHub Actions cache environment variables
+      uses: actions/github-script@v7
+      with:
+        script: |
+          core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+          core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+    - name: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator
+      shell: bash
+      run: |
+        python3 ${{ github.workspace }}/tools/ci_build/build.py \
+          --skip_submodule_sync \
+          --build_dir ${{ github.workspace }}/iOS \
+          --build_shared_lib \
+          --use_coreml \
+          --use_xnnpack \
+          --ios \
+          --apple_sysroot iphonesimulator \
+          --osx_arch x86_64 \
+          --apple_deploy_target=15.1 \
+          --use_xcode \
+          --config RelWithDebInfo \
+          --build_apple_framework \
+          --parallel \
+          --use_binskim_compliant_compile_flags
+      env:
+        ORT_GET_SIMULATOR_DEVICE_INFO_REQUESTED_RUNTIME_VERSION: ${{ env.IOS_SIMULATOR_RUNTIME_VERSION }}
+
+    timeout-minutes: 150
+    env:
+      XCODE_VERSION: 14.3.1
+      IOS_SIMULATOR_RUNTIME_VERSION: 16.4

.github/workflows/mac.yml

@@ -1,4 +1,4 @@
-name: Mac_CI
+name: "MacOS CI Pipeline"
 
 on:
   push:
@@ -19,6 +19,100 @@ env:
   python_version: 3.11
 
 jobs:
+  MacOS_C_API_Packaging_CPU_x86_64:
+    runs-on: macos-13
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        submodules: false
+
+    - name: Use Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.python_version }}
+
+    - name: Use Node.js 20.x
+      uses: actions/setup-node@v4
+      with:
+        node-version: '20.x'
+
+    - name: Install Java 17
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '17'
+        architecture: x64
+
+    - name: Set version number variables for Unix
+      shell: bash
+      run: |
+        # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+        set +x
+
+        _OnnxRuntimeVersion=$(head -1 ${{ github.workspace }}/VERSION_NUMBER)
+        echo "OnnxRuntimeVersion=$_OnnxRuntimeVersion"
+
+        _OnnxRuntimeGitCommitHash=$(git rev-parse HEAD)
+        echo "OnnxRuntimeGitCommitHash=$_OnnxRuntimeGitCommitHash"
+
+        _OnnxRuntimeGitCommitHash=$(git rev-parse --short=8 HEAD)
+        echo "OnnxRuntimeGitCommitHashShort=$_OnnxRuntimeGitCommitHash"
+      working-directory: ${{ github.workspace }}
+
+    - name: Use Xcode 14.3.1
+      shell: bash
+      run: |
+        set -e -x
+        XCODE_DEVELOPER_DIR="/Applications/Xcode_14.3.1.app/Contents/Developer"
+        sudo xcode-select --switch "${XCODE_DEVELOPER_DIR}"
+
+    - name: Setup environment variables
+      shell: bash
+      run: |
+        set -e -x
+        export PATH=${{ github.workspace }}/installed/bin:$PATH
+        export ONNX_ML=1
+        export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=ON -DONNX_WERROR=OFF"
+        python3 -m pip install -r '${{ github.workspace }}/tools/ci_build/github/linux/docker/scripts/requirements.txt'
+
+    - name: Export GitHub Actions cache environment variables
+      uses: actions/github-script@v7
+      with:
+        script: |
+          core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+          core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+    - name: Configure Build (build.py --update)
+      shell: bash
+      run: |
+        set -e -x
+        rm -rf ${{ github.workspace }}/Release
+        python3 ${{ github.workspace }}/tools/ci_build/build.py --update --build_objc --build_wheel --use_xnnpack --build_nodejs --build_java --use_coreml --use_webgpu --build_dir ${{ github.workspace }} --skip_submodule_sync --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --build_shared_lib --config Release --use_vcpkg --use_vcpkg_ms_internal_asset_cache
+
+    - name: Build (build.py --build)
+      shell: bash
+      run: |
+        set -e -x
+        python3 ${{ github.workspace }}/tools/ci_build/build.py --build --build_objc --build_wheel --use_xnnpack --build_nodejs --build_java --use_coreml --use_webgpu --build_dir ${{ github.workspace }} --skip_submodule_sync --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --build_shared_lib --config Release --use_vcpkg --use_vcpkg_ms_internal_asset_cache
+
+    - name: Install
+      shell: bash
+      run: |
+        set -e -x
+        cd ${{ github.workspace }}/Release
+        make install DESTDIR=${{ github.workspace }}/installed
+
+    - name: Running Tests (build.py --test)
+      shell: bash
+      run: |
+        set -e -x
+        python3 ${{ github.workspace }}/tools/ci_build/build.py --test --build_objc --build_wheel --use_xnnpack --build_nodejs --build_java --use_coreml --use_webgpu --build_dir ${{ github.workspace }} --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --config Release --use_vcpkg --use_vcpkg_ms_internal_asset_cache
+    timeout-minutes: 300
+    env:
+      MACOSX_DEPLOYMENT_TARGET: '13.3'
+      ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
+
   ARM64-Xcode16:
     runs-on: macos-15
 
@@ -70,7 +164,7 @@ jobs:
       matrix:
         target_arch: [x86_64, arm64]
 
-    timeout-minutes: 60
+    timeout-minutes: 75
 
     steps:
     - uses: actions/setup-python@v5

.github/workflows/macos_coreml.yml

@@ -0,0 +1,51 @@
+name: "CoreML CI Pipeline"
+
+on:
+  push:
+    branches: [ main, 'rel-*']
+  pull_request:
+    branches: [ main ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build_with_coreml:
+    runs-on: macos-13
+    strategy:
+      matrix:
+        use_coreml: [true, false]
+
+    env:
+      MACOSX_DEPLOYMENT_TARGET: '13.3'
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install coreutils and ninja
+      run: brew install coreutils ninja
+
+    - name: Use Xcode 14.3.1
+      run: |
+        XCODE_DEVELOPER_DIR="/Applications/Xcode_14.3.1.app/Contents/Developer"
+        sudo xcode-select --switch "${XCODE_DEVELOPER_DIR}"
+
+    - name: Export GitHub Actions cache environment variables
+      uses: actions/github-script@v7
+      with:
+        script: |
+          core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+          core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+    - name: CoreML EP, Build and Test on macOS
+      run: |
+        python3 tools/ci_build/build.py \
+        --build_dir build \
+        --skip_submodule_sync \
+        --cmake_generator=Ninja \
+        --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags \
+        --build_shared_lib \
+        --config Debug \
+        ${{ matrix.use_coreml && '--use_coreml' || '' }}