diff --git a/.github/actions/validate-binary/action.yml b/.github/actions/validate-binary/action.yml
deleted file mode 100644
index baa041b7e..000000000
--- a/.github/actions/validate-binary/action.yml
+++ /dev/null
@@ -1,110 +0,0 @@
-name: 'validate-binary'
-description: 'Binary Conda or Wheel Validation for Linux and MacOS'
-inputs:
-  gpu_arch_type:
-    description: 'GPU arch type'
-    required: true
-    default: 'cpu'
-  gpu_arch_ver:
-    description: 'GPU arch version'
-    required: true
-    default: 'cpu'
-  installation:
-    description: 'Installation instructions'
-    required: true
-    default: ''
-  installation_pypi:
-    description: 'Installation instructions for pypi'
-    required: false
-    default: ''
-  python_version:
-    description: 'Python version'
-    required: true
-    default: '3.9'
-  desired_cuda:
-    description: 'Desired CUDA'
-    required: true
-    default: 'cpu'
-  dev_toolset:
-    description: 'Dev Toolset'
-    required: false
-    default: ''
-  package_type:
-    description: 'Package Type'
-    required: true
-    default: 'conda'
-  target_os:
-    description: 'Target OS linux or macos'
-    required: false
-    default: 'linux'
-runs:
-  using: "composite"
-  steps:
-    - name: Checkout PyTorch builder
-      uses: actions/checkout@v2
-    - name: Check nvidia smi
-      if: ${{ inputs.gpu_arch_type == 'cuda' }}
-      shell: bash
-      run: |
-        nvidia-smi
-    - name: Install Conda Linux
-      if: ${{ inputs.target_os == 'linux' && inputs.python_version != '3.11' }}
-      uses: conda-incubator/setup-miniconda@v2
-      with:
-        python-version: ${{ inputs.python_version }}
-        auto-update-conda: true
-        miniconda-version: "latest"
-        activate-environment: testenv
-    - name: Install Conda MacOS
-      if: ${{ inputs.target_os == 'macos' }}
-      uses: pytorch/test-infra/.github/actions/setup-miniconda@main
-    - name: Install PyTorch and run tests
-      shell: bash
-      env:
-        GPU_ARCH_VER: ${{ inputs.gpu_arch_ver }}
-        GPU_ARCH_TYPE: ${{ inputs.gpu_arch_type }}
-        INSTALLATION: ${{ inputs.installation }}
-        INSTALLATION_PYPI: ${{ inputs.installation_pypi }}
-        ENV_NAME: conda-env-${{ github.run_id }}
-        DESIRED_PYTHON: ${{ inputs.python_version }}
-        DESIRED_CUDA: ${{ inputs.desired_cuda }}
-        DESIRED_DEVTOOLSET: ${{ inputs.dev_toolset }}
-        PACKAGE_TYPE: ${{ inputs.package_type }}
-        TARGET_OS: ${{ inputs.target_os }}
-      run: |
-
-        # Special case Python 3.11 wheels
-        if [ $DESIRED_PYTHON == '3.11' ]; then
-          set -ex
-          export CPYTHON_VERSIONS=3.11.0
-          sudo apt-get install build-essential gdb lcov libbz2-dev libffi-dev \
-          libgdbm-dev liblzma-dev libncurses5-dev libreadline6-dev \
-          libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g zlib1g-dev -y
-
-          export PYTHON_PATH="/opt/_internal/cpython-3.11.0rc2/bin/"
-          export PIP_PATH="${PYTHON_PATH}/pip"
-          export PIP_INSTALLATION="${INSTALLATION/pip3/"$PIP_PATH"}"
-          ./common/install_cpython.sh
-
-          eval ${PYTHON_PATH}/python --version
-          eval ${PIP_INSTALLATION}
-          eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly
-        else
-          set -ex
-
-          # Special case Pypi installation instructions
-          if [ ! -z "${INSTALLATION_PYPI}" ]; then
-            conda create -yp ${ENV_NAME}_pypi python=${{ inputs.python_version }} numpy
-            conda run -p ${ENV_NAME}_pypi $INSTALLATION_PYPI
-            conda run -p ${ENV_NAME}_pypi python3 ./test/smoke_test/smoke_test.py --package torchonly
-            conda env remove -p ${ENV_NAME}_pypi
-          fi
-
-          conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy
-          conda run -p ${ENV_NAME} $INSTALLATION
-          conda run -p ${ENV_NAME} python3  ./test/smoke_test/smoke_test.py
-          export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
-          export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
-          conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh
-          conda env remove -p ${ENV_NAME}
-        fi
diff --git a/.github/workflows/validate-macos-binaries.yml b/.github/workflows/validate-macos-binaries.yml
index 61c91e6b4..8cfac3f50 100644
--- a/.github/workflows/validate-macos-binaries.yml
+++ b/.github/workflows/validate-macos-binaries.yml
@@ -7,6 +7,11 @@ on:
         description: "Channel to use (nightly, test, release, all)"
         required: true
         type: string
+      ref:
+        description: 'Reference to checkout, defaults to empty'
+        default: ""
+        required: false
+        type: string
   workflow_dispatch:
     inputs:
       channel:
@@ -18,105 +23,92 @@ on:
           - nightly
           - test
           - all
+      ref:
+        description: 'Reference to checkout, defaults to empty'
+        default: ""
+        required: false
+        type: string
 
 jobs:
-  generate-macos-arm64-conda-matrix:
-    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
-    with:
-      package-type: conda
-      os: macos-arm64
-      channel: ${{ inputs.channel }}
-  generate-macos-arm64-wheel-matrix:
-    uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
-    with:
-      package-type: wheel
-      os: macos-arm64
-      channel: ${{ inputs.channel }}
-  generate-macos-x86_64-conda-matrix:
+  generate-macos-matrix:
     uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
     with:
-      package-type: conda
+      package-type: all
       os: macos
       channel: ${{ inputs.channel }}
-  generate-macos-x86_64-wheel-matrix:
+  generate-macos-arm64-matrix:
     uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
     with:
-      package-type: wheel
-      os: macos
+      package-type: all
+      os: macos-arm64
       channel: ${{ inputs.channel }}
 
-  mac-arm64-conda:
-    needs: generate-macos-arm64-conda-matrix
+  macos:
+    needs: generate-macos-matrix
     strategy:
-      matrix:
-        ${{ fromJson(needs.generate-macos-arm64-conda-matrix.outputs.matrix) }}
+      matrix: ${{ fromJson(needs.generate-macos-matrix.outputs.matrix) }}
       fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Validate binary conda
-        uses: pytorch/builder/.github/actions/validate-binary@main
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
-          desired_cuda: ${{ matrix.desired_cuda }}
-          package_type: conda
-          target_os: macos
+    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
+    name: ${{ matrix.build_name }}
+    with:
+      runner: ${{ matrix.validation_runner }}
+      repository: "pytorch/builder"
+      ref: ${{ inputs.ref || github.ref }}
+      job-name: ${{ matrix.build_name }}
+      script: |
+        set -ex
+        export ENV_NAME="conda-env-${{ github.run_id }}"
+        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
+        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
+        export INSTALLATION="${{ matrix.installation }}"
+        export CUDA_VER="${{ matrix.desired_cuda }}"
+        export DESIRED_PYTHON="${{ matrix.python_version }}"
+        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
+        export DESIRED_DEVTOOLSET="${{ matrix.devtoolset }}"
+        export PACKAGE_TYPE="${{ matrix.package_type }}"
+        export TARGET_OS="macos"
+        export LD_LIBRARY_PATH="$(dirname $(which python))/../lib"
+        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
+        conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow
+        conda activate ${ENV_NAME}
 
-  mac-arm64-wheel:
-    needs: generate-macos-arm64-wheel-matrix
-    strategy:
-      matrix:
-        ${{ fromJson(needs.generate-macos-arm64-wheel-matrix.outputs.matrix) }}
-      fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Validate binary wheel
-        uses: pytorch/builder/.github/actions/validate-binary@main
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
-          desired_cuda: ${{ matrix.desired_cuda }}
-          package_type: wheel
-          target_os: macos
 
-  mac-x64-conda:
-    needs: generate-macos-x86_64-conda-matrix
-    strategy:
-      matrix:
-        ${{ fromJson(needs.generate-macos-x86_64-conda-matrix.outputs.matrix) }}
-      fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Validate binary conda
-        uses: pytorch/builder/.github/actions/validate-binary@main
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
-          desired_cuda: ${{ matrix.desired_cuda }}
-          package_type: conda
-          target_os: macos
+        if [[ ${{ matrix.package_type }} == "libtorch" ]]; then
+          curl ${{ matrix.installation }} -o libtorch.zip
+          unzip libtorch.zip
+        else
+          eval $INSTALLATION
+          python  ./test/smoke_test/smoke_test.py
+          ${PWD}/check_binary.sh
+        fi
 
-  mac-x64-wheel:
-    needs: generate-macos-x86_64-wheel-matrix
+  macos-arm64:
+    needs: generate-macos-arm64-matrix
     strategy:
-      matrix:
-        ${{ fromJson(needs.generate-macos-x86_64-wheel-matrix.outputs.matrix) }}
+      matrix: ${{ fromJson(needs.generate-macos-arm64-matrix.outputs.matrix) }}
       fail-fast: false
-    runs-on: ${{ matrix.validation_runner }}
-    steps:
-      - name: Validate binary wheel
-        uses: pytorch/builder/.github/actions/validate-binary@main
-        with:
-          gpu_arch_type: ${{ matrix.gpu_arch_type }}
-          gpu_arch_ver: ${{ matrix.gpu_arch_version }}
-          installation: ${{ matrix.installation }}
-          python_version: ${{ matrix.python_version }}
-          desired_cuda: ${{ matrix.desired_cuda }}
-          package_type: wheel
-          target_os: macos
+    uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
+    name: ${{ matrix.build_name }}
+    with:
+      runner: ${{ matrix.validation_runner }}
+      repository: "pytorch/builder"
+      ref: ${{ inputs.ref || github.ref }}
+      job-name: ${{ matrix.build_name }}
+      script: |
+        set -ex
+        export ENV_NAME="conda-env-${{ github.run_id }}"
+        export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
+        export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
+        export INSTALLATION="${{ matrix.installation }}"
+        export CUDA_VER="${{ matrix.desired_cuda }}"
+        export DESIRED_PYTHON="${{ matrix.python_version }}"
+        export DESIRED_CUDA="${{ matrix.desired_cuda }}"
+        export PACKAGE_TYPE="${{ matrix.package_type }}"
+        export TARGET_OS="macos"
+        export LD_LIBRARY_PATH="$(dirname $(which python))/../lib"
+        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
+        conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow
+        conda activate ${ENV_NAME}
+        eval $INSTALLATION
+        python  ./test/smoke_test/smoke_test.py
+        ${PWD}/check_binary.sh