diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh deleted file mode 100755 index 2e838613a..000000000 --- a/.github/scripts/validate_binaries.sh +++ /dev/null @@ -1,112 +0,0 @@ -if [[ ${MATRIX_PACKAGE_TYPE} == "libtorch" ]]; then - curl ${MATRIX_INSTALLATION} -o libtorch.zip - unzip libtorch.zip -else - - export PYTHON_RUN="python3" - if [[ ${TARGET_OS} == 'windows' ]]; then - export PYTHON_RUN="python" - # Currently xpu env need a helper script to activate - if [[ ${MATRIX_GPU_ARCH_TYPE} == "xpu" ]]; then - export PYTHON_RUN="${PWD}/.github/scripts/xpu_env_helper.bat python" - fi - fi - - if [[ ${TARGET_OS} == 'macos-arm64' ]]; then - conda update -y -n base -c defaults conda - elif [[ ${TARGET_OS} != 'linux-aarch64' ]]; then - # Conda pinned see issue: https://github.com/ContinuumIO/anaconda-issues/issues/13350 - conda install -y conda=23.11.0 - fi - # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159 - conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg - conda activate ${ENV_NAME} - - # Remove when https://github.com/pytorch/builder/issues/1985 is fixed - if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then - pip3 install numpy --force-reinstall - fi - - INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"} - TEST_SUFFIX="" - - # force-reinstall: latest version of packages are reinstalled - if [[ ${USE_FORCE_REINSTALL} == 'true' ]]; then - INSTALLATION=${INSTALLATION/"pip3 install"/"pip3 install --force-reinstall"} - fi - # extra-index-url: extra dependencies are downloaded from pypi - if [[ ${USE_EXTRA_INDEX_URL} == 'true' ]]; then - INSTALLATION=${INSTALLATION/"--index-url"/"--extra-index-url"} - fi - - # use-meta-cdn: use meta cdn for pypi download - if [[ ${USE_META_CDN} == 'true' ]]; then - INSTALLATION=${INSTALLATION/"download.pytorch.org"/"d3kup0pazkvub8.cloudfront.net"} - fi - - - if [[ ${TORCH_ONLY} == 'true' ]]; then - INSTALLATION=${INSTALLATION/"torchvision torchaudio"/""} - TEST_SUFFIX=" --package torchonly" - fi - - # if RELESE version is passed as parameter - install speific version - if [[ ! -z ${RELEASE_VERSION} ]]; then - INSTALLATION=${INSTALLATION/"torch "/"torch==${RELEASE_VERSION} "} - INSTALLATION=${INSTALLATION/"-y pytorch "/"-y pytorch==${RELEASE_VERSION} "} - INSTALLATION=${INSTALLATION/"::pytorch "/"::pytorch==${RELEASE_VERSION} "} - - if [[ ${USE_VERSION_SET} == 'true' ]]; then - INSTALLATION=${INSTALLATION/"torchvision "/"torchvision==${VISION_RELEASE_VERSION} "} - INSTALLATION=${INSTALLATION/"torchaudio "/"torchaudio==${AUDIO_RELEASE_VERSION} "} - fi - fi - - export OLD_PATH=${PATH} - # Workaround macos-arm64 runners. Issue: https://github.com/pytorch/test-infra/issues/4342 - if [[ ${TARGET_OS} == 'macos-arm64' ]]; then - export PATH="${CONDA_PREFIX}/bin:${PATH}" - fi - - # Make sure we remove previous installation if it exists - if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then - pip3 uninstall -y torch torchaudio torchvision - fi - eval $INSTALLATION - - if [[ ${TARGET_OS} == 'linux' ]]; then - export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib" - export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH - source ${PWD}/check_binary.sh - fi - - # We are only interested in CUDA tests and Python 3.8-3.11. Not all requirement libraries are available for 3.12 yet. - if [[ ${INCLUDE_TEST_OPS:-} == 'true' && ${MATRIX_GPU_ARCH_TYPE} == 'cuda' && ${MATRIX_PYTHON_VERSION} != "3.12" ]]; then - source ./.github/scripts/validate_test_ops.sh - fi - - # Regular smoke test - ${PYTHON_RUN} ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} - # For pip install also test with numpy 2.0.0 for python 3.8 or above - if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' && ${MATRIX_PYTHON_VERSION} != "3.8" ]]; then - pip3 install numpy==2.0.0 --force-reinstall - ${PYTHON_RUN} ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} - fi - - - if [[ ${TARGET_OS} == 'macos-arm64' ]]; then - export PATH=${OLD_PATH} - fi - - # Use case CUDA_VISIBLE_DEVICES: https://github.com/pytorch/pytorch/issues/128819 - if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda' ]]; then - python -c "import torch;import os;print(torch.cuda.device_count(), torch.__version__);os.environ['CUDA_VISIBLE_DEVICES']='0';print(torch.empty(2, device='cuda'))" - fi - - # this is optional step - if [[ ${TARGET_OS} != linux* ]]; then - conda deactivate - conda env remove -n ${ENV_NAME} - fi - -fi diff --git a/.github/scripts/validate_pipy.sh b/.github/scripts/validate_pipy.sh deleted file mode 100644 index 5858e4c28..000000000 --- a/.github/scripts/validate_pipy.sh +++ /dev/null @@ -1,20 +0,0 @@ -conda create -yn ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg -conda activate ${ENV_NAME}_pypi - -TEST_SUFFIX="" -RELEASE_SUFFIX="" -# if RELESE version is passed as parameter - install speific version -if [[ ! -z ${RELEASE_VERSION} ]]; then - RELEASE_SUFFIX="==${RELEASE_VERSION}" -fi - -if [[ ${TORCH_ONLY} == 'true' ]]; then - TEST_SUFFIX=" --package torchonly" - pip3 install torch${RELEASE_SUFFIX} -else - pip3 install torch${RELEASE_SUFFIX} torchvision torchaudio -fi - -python ./test/smoke_test/smoke_test.py ${TEST_SUFFIX} --runtime-error-check disabled -conda deactivate -conda env remove -p ${ENV_NAME}_pypi diff --git a/.github/scripts/validate_poetry.sh b/.github/scripts/validate_poetry.sh deleted file mode 100644 index 6b7fe2412..000000000 --- a/.github/scripts/validate_poetry.sh +++ /dev/null @@ -1,31 +0,0 @@ - -conda create -y -n ${ENV_NAME}_poetry python=${MATRIX_PYTHON_VERSION} numpy ffmpeg -conda activate ${ENV_NAME}_poetry -curl -sSL https://install.python-poetry.org | python3 - --git https://github.com/python-poetry/poetry.git@master -export PATH="/root/.local/bin:$PATH" - -poetry --version -poetry new test_poetry -cd test_poetry - -TEST_SUFFIX="" -if [[ ${TORCH_ONLY} == 'true' ]]; then - TEST_SUFFIX=" --package torchonly" -fi - -RELEASE_SUFFIX="" -# if RELESE version is passed as parameter - install speific version -if [[ ! -z ${RELEASE_VERSION} ]]; then - RELEASE_SUFFIX="@${RELEASE_VERSION}" -fi - -if [[ ${TORCH_ONLY} == 'true' ]]; then - poetry --quiet add torch${RELEASE_SUFFIX} -else - poetry --quiet add torch${RELEASE_SUFFIX} torchaudio torchvision -fi - -python ../test/smoke_test/smoke_test.py ${TEST_SUFFIX} --runtime-error-check disabled -conda deactivate -conda env remove -p ${ENV_NAME}_poetry -cd .. diff --git a/.github/scripts/validate_test_ops.sh b/.github/scripts/validate_test_ops.sh deleted file mode 100644 index 5df646705..000000000 --- a/.github/scripts/validate_test_ops.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -set -eux -o pipefail - -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -BRANCH="" -if [[ ${MATRIX_CHANNEL} == "test" || ${MATRIX_CHANNEL} == "release" ]]; then - SHORT_VERSION=${MATRIX_STABLE_VERSION%.*} - BRANCH="--branch release/${SHORT_VERSION}" -fi - - -# Clone the Pytorch branch -retry git clone ${BRANCH} --depth 1 https://github.com/pytorch/pytorch.git -retry git submodule update --init --recursive -pushd pytorch - -pip install expecttest numpy pyyaml jinja2 packaging hypothesis unittest-xml-reporting scipy - -# Run pytorch cuda wheels validation -# Detect ReduceLogicKernel (ReduceOp and kernel) IMA -python test/test_ops.py -k test_dtypes_all_cuda -# Detect BinaryMulKernel (elementwise binary functor internal mul) IMA -python test/test_torch.py -k test_index_reduce_reduce_prod_cuda_int32 -# Detect BinaryBitwiseOpsKernels (at::native::BitwiseAndFunctor) IMA -python test/test_binary_ufuncs.py -k test_contig_vs_every_other___rand___cuda_int32 -# Detect MaxMinElementwiseKernel (maximum) IMA -python test/test_schema_check.py -k test_schema_correctness_clamp_cuda_int8 - -pushd /tmp -# Detect StepKernel (nextafter) IMA -python -c "import torch; print(torch.nextafter(torch.tensor([-4.5149, -5.9053, -0.9516, -2.3615, 1.5591], device='cuda:0'), torch.tensor(3.8075, device='cuda:0')))" -# Detect BinaryGeometricKernels (atan2) IMA -python -c "import torch; x = (torch.randn((2,1,1), dtype=torch.float, device='cuda')*5).to(torch.float32); y=(torch.randn((), dtype=torch.float, device='cuda')*5).to(torch.float32); print(torch.atan2(x,y))" -popd diff --git a/.github/scripts/xpu_env_helper.bat b/.github/scripts/xpu_env_helper.bat deleted file mode 100644 index 7f331ebc2..000000000 --- a/.github/scripts/xpu_env_helper.bat +++ /dev/null @@ -1,17 +0,0 @@ -call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" - -set args=%1 -shift -:start -if [%1] == [] goto done -set args=%args% %1 -shift -goto start - -:done -if "%args%" == "" ( - echo Usage: xpu_env_helper.bat [command] [args] - echo e.g. xpu_env_helper.bat icpx --version -) - -%args% || exit /b 1 diff --git a/.github/workflows/test-check-binary.yml b/.github/workflows/test-check-binary.yml deleted file mode 100644 index e29ee7ba9..000000000 --- a/.github/workflows/test-check-binary.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Test check_binary - -on: - push: - branches: - - main - pull_request: - paths: - - .github/workflows/test-check-binary.yml - - check_binary.sh - - test/smoke_test/smoke_test.py - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} - cancel-in-progress: true - -jobs: - check_binary_linux_cpu: - name: Test check_binary.sh for Linux CPU - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: "pytorch/builder" - docker-image: python:3.11 - script: | - pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cpu - DESIRED_PYTHON=3.11 DESIRED_CUDA=cpu PACKAGE_TYPE=manywheel ./check_binary.sh - - check_binary_linux_cuda: - name: Test check_binary.sh for Linux CUDA - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - repository: "pytorch/builder" - runner: linux.4xlarge.nvidia.gpu - docker-image: python:3.11 - script: | - pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu121 - DESIRED_PYTHON=3.11 DESIRED_CUDA=cu121 PACKAGE_TYPE=manywheel ./check_binary.sh diff --git a/.github/workflows/test-validate-domain-library.yml b/.github/workflows/test-validate-domain-library.yml deleted file mode 100644 index 6c651e709..000000000 --- a/.github/workflows/test-validate-domain-library.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Test validate domain library - -on: - pull_request: - paths: - - .github/workflows/validate-domain-library.yml - - .github/workflows/test-validate-domain-library.yml - workflow_dispatch: - -jobs: - test-validate-domain-library: - uses: ./.github/workflows/validate-domain-library.yml - with: - package_type: "conda,wheel" - os: "all" - channel: "release" - repository: "pytorch/builder" - ref: main - smoke_test: "echo test" diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml deleted file mode 100644 index bae4f9221..000000000 --- a/.github/workflows/validate-aarch64-linux-binaries.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: Validate Aarch64 linux binaries - -on: - workflow_call: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: string - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - use-version-set: - description: 'Applies when version is used, use version for each domain' - default: false - required: false - type: boolean - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - workflow_dispatch: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: choice - options: - - release - - nightly - - test - - all - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - -jobs: - generate-aarch64-linux-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: linux-aarch64 - channel: ${{ inputs.channel }} - with-cuda: disable - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - - linux-aarch64: - needs: generate-aarch64-linux-matrix - strategy: - matrix: ${{ fromJson(needs.generate-aarch64-linux-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - name: ${{ matrix.build_name }} - with: - runner: ${{ matrix.validation_runner }} - repository: "pytorch/builder" - ref: ${{ github.ref }} - job-name: ${{ matrix.build_name }} - docker-image: ${{ matrix.container_image }} - binary-matrix: ${{ toJSON(matrix) }} - no-sudo: true - script: | - set -ex - export DESIRED_PYTHON=${{ matrix.python_version }} - echo "/opt/conda/bin" >> $GITHUB_PATH - ############################################################################### - # Install conda - # disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path" - # when using Python version, less than the conda latest - ############################################################################### - echo 'Installing conda-forge' - curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh - chmod +x /mambaforge.sh - /mambaforge.sh -b -p /opt/conda - rm /mambaforge.sh - source /opt/conda/etc/profile.d/conda.sh - conda config --set ssl_verify False - - export ENV_NAME="conda-env-${{ github.run_id }}" - export TARGET_OS="linux-aarch64" - export TORCH_ONLY=${{ inputs.torchonly }} - export RELEASE_VERSION=${{ inputs.version }} - export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }} - export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }} - export USE_META_CDN=${{ inputs.use-meta-cdn }} - export USE_VERSION_SET=${{ inputs.use-version-set }} - if [[ ${USE_VERSION_SET} == 'true' ]]; then - export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }} - export AUDIO_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchaudio }} - fi - - printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json - eval "$(conda shell.bash hook)" - - # NB: The latest conda 23.11.0 pulls in some dependencies of conda-libmamba-solver that - # require GLIBC_2.25, which is not available in the current aarch64 image causing the - # subsequence git command to fail. Basically, they don't work with CentOS 7 which AML 2 - # is based on https://github.com/ContinuumIO/anaconda-issues/issues/12822 - unset LD_LIBRARY_PATH - - # Standard case: Validate binaries - source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml deleted file mode 100644 index b13b9168f..000000000 --- a/.github/workflows/validate-binaries.yml +++ /dev/null @@ -1,189 +0,0 @@ -name: Validate binaries - -# A reusable workflow that triggers a set of jobs that perform a smoke test / validation of pytorch binaries. -# Optionally restricts validation to the specified OS and channel. -# For the details about parameter values, see: -# pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main -# For an example of the `workflow_call` usage see: -# https://github.com/pytorch/builder/pull/1144 -on: - workflow_call: - inputs: - os: - description: "Operating system to generate for (linux, windows, macos, macos-arm64)" - required: true - type: string - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: string - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - include-test-ops: - description: 'Include Test Ops tests (only Linux)' - default: false - required: false - type: boolean - use-version-set: - description: 'Use version for each domain' - default: false - required: false - type: boolean - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use_split_build: - description: | - [Experimental] Use Split Build - required: false - type: boolean - default: false - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - workflow_dispatch: - inputs: - os: - description: "Operating system to generate for (linux, windows, macos, macos-arm64)" - required: true - type: choice - default: all - options: - - windows - - linux - - linux-aarch64 - - macos - - all - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: choice - default: test - options: - - release - - nightly - - test - - all - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate' - default: "" - required: false - type: string - include-test-ops: - description: 'Include Test Ops tests (only Linux)' - default: false - required: false - type: boolean - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use_split_build: - description: | - [Experimental] Use Split Build - required: false - type: boolean - default: false - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - - -jobs: - generate-release-matrix: - uses: pytorch/test-infra/.github/workflows/generate_release_matrix.yml@main - with: - version: ${{ inputs.version }} - - win: - if: inputs.os == 'windows' || inputs.os == 'all' - needs: generate-release-matrix - uses: ./.github/workflows/validate-windows-binaries.yml - with: - channel: ${{ inputs.channel }} - torchonly: ${{ inputs.torchonly }} - version: ${{ inputs.version }} - release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - use-force-reinstall: ${{ inputs.use-force-reinstall }} - use-extra-index-url: ${{ inputs.use-extra-index-url }} - use-meta-cdn: ${{ inputs.use-meta-cdn }} - - linux: - if: inputs.os == 'linux' || inputs.os == 'all' - needs: generate-release-matrix - uses: ./.github/workflows/validate-linux-binaries.yml - with: - channel: ${{ inputs.channel }} - torchonly: ${{ inputs.torchonly }} - version: ${{ inputs.version }} - release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} - include-test-ops: ${{ inputs.include-test-ops }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - use-force-reinstall: ${{ inputs.use-force-reinstall }} - use_split_build: ${{ inputs.use_split_build }} - use-extra-index-url: ${{ inputs.use-extra-index-url }} - use-meta-cdn: ${{ inputs.use-meta-cdn }} - - linux-aarch64: - if: inputs.os == 'linux-aarch64' || inputs.os == 'all' - needs: generate-release-matrix - uses: ./.github/workflows/validate-aarch64-linux-binaries.yml - with: - channel: ${{ inputs.channel }} - torchonly: ${{ inputs.torchonly }} - version: ${{ inputs.version }} - release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - use-force-reinstall: ${{ inputs.use-force-reinstall }} - use-extra-index-url: ${{ inputs.use-extra-index-url }} - use-meta-cdn: ${{ inputs.use-meta-cdn }} - - mac-arm64: - if: inputs.os == 'macos' || inputs.os == 'all' - needs: generate-release-matrix - uses: ./.github/workflows/validate-macos-arm64-binaries.yml - with: - channel: ${{ inputs.channel }} - torchonly: ${{ inputs.torchonly }} - version: ${{ inputs.version }} - release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - use-force-reinstall: ${{ inputs.use-force-reinstall }} - use-extra-index-url: ${{ inputs.use-extra-index-url }} - use-meta-cdn: ${{ inputs.use-meta-cdn }} diff --git a/.github/workflows/validate-docker-images.yml b/.github/workflows/validate-docker-images.yml deleted file mode 100644 index 46ad0aa2a..000000000 --- a/.github/workflows/validate-docker-images.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Validate Nightly Docker Images -on: - pull_request: - paths: - - .github/workflows/validate-docker-images.yml - workflow_call: - inputs: - channel: - description: 'PyTorch channel to use (nightly, test, release, all)' - required: true - type: string - default: 'nightly' - generate_dockerhub_images: - description: 'Generate Docker Hub images (strip ghcr.io/ prefix for release)' - default: false - required: false - type: boolean - ref: - description: 'Reference to checkout, defaults to empty' - default: "" - required: false - type: string - workflow_dispatch: - inputs: - channel: - description: 'PyTorch channel to use (nightly, test, release, all)' - required: true - type: choice - default: 'nightly' - options: - - 'nightly' - - 'test' - - 'release' - generate_dockerhub_images: - description: 'Generate Docker Hub images (strip ghcr.io/ prefix for release)' - default: false - required: false - type: boolean - ref: - description: 'Reference to checkout, defaults to empty' - default: "" - required: false - type: string - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_docker_release_matrix.yml@main - with: - channel: ${{ inputs.channel }} - generate_dockerhub_images : ${{ inputs.generate_dockerhub_images }} - secrets: inherit - run-gpu-tests: - needs: generate-matrix - name: cuda${{ matrix.cuda }}-cudnn${{ matrix.cudnn_version }}-${{ matrix.image_type }} - strategy: - matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: ${{ matrix.validation_runner }} - repository: "pytorch/builder" - ref: ${{ inputs.ref || github.ref }} - job-name: cuda${{ matrix.cuda }}-cudnn${{ matrix.cudnn_version }}-${{ matrix.image_type }} - binary-matrix: ${{ toJSON(matrix) }} - docker-image: ${{matrix.docker}} - timeout: 180 - script: | - set -ex - - export MATRIX_GPU_ARCH_VERSION="${{ matrix.cuda }}" - export MATRIX_IMAGE_TYPE="${{ matrix.image_type }}" - export TARGET_OS="linux" - TORCH_COMPILE_CHECK="--torch-compile-check enabled" - if [[ ${MATRIX_IMAGE_TYPE} == "runtime" ]]; then - TORCH_COMPILE_CHECK="--torch-compile-check disabled" - fi - export MATRIX_GPU_ARCH_TYPE="cuda" - if [[ ${MATRIX_GPU_ARCH_VERSION} == "cpu" ]]; then - export MATRIX_GPU_ARCH_TYPE="cpu" - fi - python test/smoke_test/smoke_test.py --package torchonly --runtime-error-check disabled ${TORCH_COMPILE_CHECK} diff --git a/.github/workflows/validate-domain-library.yml b/.github/workflows/validate-domain-library.yml deleted file mode 100644 index 6d117109a..000000000 --- a/.github/workflows/validate-domain-library.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: Validate domain libary - -# A reusable workflow that triggers a set of jobs that perform a smoke test / validation of pytorch binaries. -# Optionally restricts validation to the specified OS and channel. -# For the details about parameter values, see: -# pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main -on: - workflow_call: - inputs: - os: - description: "Operating system to generate for (linux, windows, macos, macos-arm64)" - required: false - type: string - default: "all" - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: string - ref: - description: 'Reference to checkout, defaults to empty' - default: "" - required: false - type: string - package_type: - description: "Package type (conda, wheel, all)" - required: false - type: string - default: "all" - repository: - description: "Path to repository to checkout" - required: true - type: string - smoke_test: - description: "Path to a smoke test script" - required: true - type: string - with_cuda: - description: "With cuda enable/disable" - required: false - type: string - default: disable - -jobs: - generate-linux-matrix: - if: (inputs.os == 'linux' || inputs.os == 'all') - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: ${{ inputs.package_type }} - os: linux - channel: ${{ inputs.channel }} - with-cuda: ${{ inputs.with_cuda }} - generate-windows-matrix: - if: (inputs.os == 'windows' || inputs.os == 'all') - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: ${{ inputs.package_type }} - os: windows - channel: ${{ inputs.channel }} - with-cuda: ${{ inputs.with_cuda }} - generate-macos-arm64-matrix: - if: (inputs.os == 'macos-arm64' || inputs.os == 'all') - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: ${{ inputs.package_type }} - os: macos-arm64 - channel: ${{ inputs.channel }} - with-cuda: ${{ inputs.with_cuda }} - validate-linux: - if: (inputs.os == 'linux' || inputs.os == 'all') - needs: generate-linux-matrix - strategy: - matrix: ${{ fromJson(needs.generate-linux-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - name: "linux-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - with: - runner: ${{ matrix.validation_runner }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref || github.ref }} - job-name: "linux-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - binary-matrix: ${{ toJSON(matrix) }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export SMOKE_TEST="${{ inputs.smoke_test }}" - eval $SMOKE_TEST - validate-windows: - if: (inputs.os == 'windows' || inputs.os == 'all') - needs: generate-windows-matrix - strategy: - matrix: ${{ fromJson(needs.generate-windows-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - name: "windows-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - with: - runner: ${{ matrix.validation_runner }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref || github.ref }} - job-name: "windows-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - binary-matrix: ${{ toJSON(matrix) }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export SMOKE_TEST="${{ inputs.smoke_test }}" - export TARGET_OS="windows" - eval $SMOKE_TEST - validate-macos: - if: (inputs.os == 'macos' || inputs.os == 'all') - needs: generate-macos-matrix - strategy: - matrix: ${{ fromJson(needs.generate-macos-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - name: "macos-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - with: - runner: ${{ matrix.validation_runner }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref || github.ref }} - job-name: "macos-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - binary-matrix: ${{ toJSON(matrix) }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export TARGET_OS="macos" - export SMOKE_TEST="${{ inputs.smoke_test }}" - eval $SMOKE_TEST - validate-macos-arm64: - if: (inputs.os == 'macos-arm64' || inputs.os == 'all') - needs: generate-macos-matrix - strategy: - matrix: ${{ fromJson(needs.generate-macos-arm64-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - name: "macos-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - with: - runner: ${{ matrix.validation_runner }} - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref || github.ref }} - job-name: "macos-arm64-${{ matrix.package_type }}-${{ matrix.python_version }}-${{ matrix.desired_cuda }}" - binary-matrix: ${{ toJSON(matrix) }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export TARGET_OS="macos-arm64" - eval $SMOKE_TEST diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml deleted file mode 100644 index dd40a9dad..000000000 --- a/.github/workflows/validate-linux-binaries.yml +++ /dev/null @@ -1,179 +0,0 @@ -name: Validate linux binaries - -on: - workflow_call: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: string - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - use-version-set: - description: 'Applies when version is used, use version for each domain' - default: false - required: false - type: boolean - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - include-test-ops: - description: 'Include Test Ops tests (only Linux)' - default: false - required: false - type: boolean - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use_split_build: - description: | - [Experimental] Use split build - required: false - type: boolean - default: false - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - workflow_dispatch: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: choice - options: - - release - - nightly - - test - - all - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - include-test-ops: - description: 'Include Test Ops tests (only Linux)' - default: false - required: false - type: boolean - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use_split_build: - description: | - [Experimental] Use split build - required: false - type: boolean - default: false - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - -jobs: - generate-linux-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel,libtorch # We stopped producing conda nightlies - os: linux - channel: ${{ inputs.channel }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - use_split_build: ${{ inputs.use_split_build }} - with-xpu: enable - - linux: - needs: generate-linux-matrix - strategy: - matrix: ${{ fromJson(needs.generate-linux-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - name: ${{ matrix.build_name }} - with: - runner: ${{ matrix.validation_runner }} - repository: "pytorch/builder" - ref: ${{ github.ref }} - job-name: ${{ matrix.build_name }} - docker-image: ${{ (matrix.gpu_arch_type == 'xpu' && matrix.container_image) || (matrix.gpu_arch_version == '12.6' && 'pytorch/almalinux-builder:cpu-main') || 'pytorch/conda-builder' }} - binary-matrix: ${{ toJSON(matrix) }} - timeout: 180 - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export TORCH_ONLY=${{ inputs.torchonly }} - export INCLUDE_TEST_OPS=${{ inputs.include-test-ops }} - export USE_ONLY_DL_PYTORCH_ORG=${{ inputs.use-only-dl-pytorch-org }} - export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }} - export USE_META_CDN=${{ inputs.use-meta-cdn }} - export RELEASE_VERSION=${{ inputs.version }} - export USE_VERSION_SET=${{ inputs.use-version-set }} - if [[ ${USE_VERSION_SET} == 'true' ]]; then - export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }} - export AUDIO_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchaudio }} - fi - - export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }} - export TARGET_OS="linux" - # Due to xpu doesn't use pytorch/conda-builder image, need to install conda firstly - if [[ ${{ matrix.gpu_arch_type }} == 'xpu' ]]; then - source ./common/install_conda.sh - fi - eval "$(conda shell.bash hook)" - printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json - - # Special case PyPi installation package. And Install of PyPi package via poetry - if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" && \ - ${MATRIX_GPU_ARCH_VERSION} == "12.4" && \ - ${MATRIX_CHANNEL} == "release" && \ - ${USE_ONLY_DL_PYTORCH_ORG} == "false" ]]; then - source ./.github/scripts/validate_pipy.sh - source ./.github/scripts/validate_poetry.sh - fi - - # Standart case: Validate binaries - source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml deleted file mode 100644 index be9a4c0c3..000000000 --- a/.github/workflows/validate-macos-arm64-binaries.yml +++ /dev/null @@ -1,140 +0,0 @@ -name: Validate MacOS ARM64 Binaries - -on: - workflow_call: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: string - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - use-version-set: - description: 'Applies when version is used, use version for each domain' - default: false - required: false - type: boolean - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - workflow_dispatch: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: choice - options: - - release - - nightly - - test - - all - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - -jobs: - generate-macos-arm64-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel,libtorch # We stopped producing conda nightlies - os: macos-arm64 - channel: ${{ inputs.channel }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - - macos-arm64: - needs: generate-macos-arm64-matrix - strategy: - matrix: ${{ fromJson(needs.generate-macos-arm64-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - name: ${{ matrix.build_name }} - with: - runner: ${{ matrix.validation_runner }} - repository: "pytorch/builder" - ref: ${{ github.ref }} - job-name: ${{ matrix.build_name }} - binary-matrix: ${{ toJSON(matrix) }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export TARGET_OS="macos-arm64" - export TORCH_ONLY=${{ inputs.torchonly }} - export RELEASE_VERSION=${{ inputs.version }} - export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }} - export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }} - export USE_META_CDN=${{ inputs.use-meta-cdn }} - export USE_VERSION_SET=${{ inputs.use-version-set }} - - if [[ ${{ matrix.package_type }} == "conda" ]]; then - export IS_M1_CONDA_BUILD_JOB=1 - fi - - if [[ ${USE_VERSION_SET} == 'true' ]]; then - export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }} - export AUDIO_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchaudio }} - fi - - printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json - source ./.github/scripts/validate_binaries.sh diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml deleted file mode 100644 index 13ad52b17..000000000 --- a/.github/workflows/validate-nightly-binaries.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Scheduled validation of the nightly binaries -name: cron - -on: - schedule: - # At 3:30 pm UTC (8:30 am PDT) - - cron: "30 15 * * *" - # Have the ability to trigger this job manually through the API - workflow_dispatch: - push: - branches: - - main - paths: - - .github/workflows/validate-nightly-binaries.yml - - .github/workflows/validate-linux-binaries.yml - - .github/workflows/validate-windows-binaries.yml - - .github/workflows/validate-macos-arm64-binaries.yml - - test/smoke_test/* - -jobs: - nightly: - uses: ./.github/workflows/validate-binaries.yml - with: - channel: nightly - os: all - use_split_build: true - use-meta-cdn: true diff --git a/.github/workflows/validate-nightly-pypi-wheel-binary-size.yml b/.github/workflows/validate-nightly-pypi-wheel-binary-size.yml deleted file mode 100644 index 24fffc16e..000000000 --- a/.github/workflows/validate-nightly-pypi-wheel-binary-size.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Validate Nightly PyPI Wheel Binary Size -on: - pull_request: - paths: - - .github/workflows/validate-nightly-pypi-wheel-binary-size.yml - workflow_dispatch: - schedule: - # At 2:30 pm UTC (7:30 am PDT) - - cron: "30 14 * * *" - -jobs: - nightly-pypi-binary-size-validation: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - repository: pytorch/test-infra - - name: Install requirements - run: | - pip3 install -r tools/binary_size_validation/requirements.txt - - name: Run validation - run: | - python tools/binary_size_validation/binary_size_validation.py \ - --url https://download.pytorch.org/whl/nightly/cu121/torch/ \ - --include "linux" --only-latest-version --threshold 750 diff --git a/.github/workflows/validate-release-binaries.yml b/.github/workflows/validate-release-binaries.yml deleted file mode 100644 index 1a8c51571..000000000 --- a/.github/workflows/validate-release-binaries.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Scheduled validation of the release binaries -name: cron - -on: - schedule: - # At 3 am UTC (7 am PDT) - - cron: "0 3 * * *" - # Have the ability to trigger this job manually through the API - workflow_dispatch: - push: - branches: - - main - paths: - - .github/workflows/validate-release-binaries.yml - - .github/workflows/validate-linux-binaries.yml - - .github/workflows/validate-windows-binaries.yml - - .github/workflows/validate-macos-arm64-binaries.yml - - test/smoke_test/* - pull_request: - paths: - - .github/workflows/validate-nightly-binaries.yml - - .github/workflows/validate-linux-binaries.yml - - .github/workflows/validate-windows-binaries.yml - - .github/workflows/validate-macos-arm64-binaries.yml - - .github/scripts/validate_binaries.sh - - test/smoke_test/* - -jobs: - release: - uses: ./.github/workflows/validate-binaries.yml - with: - channel: release - os: all diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml deleted file mode 100644 index cb1a6a73e..000000000 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Validate manywheel binaries - -# This workflow validates the size of the manywheel binaries after repackaging for PyPi -# Specify the direct URLs to the binaries (from https://download.pytorch.org/whl/test/torch/) in the matrix -# along with the python version. -# -# The workflow will: -# * download the binaries, -# * run release/pypi/prep_binary_for_pypi.sh -# * run smoke tests on the repackaged binaries -# * display the size before and after repackaging as the workflow annotation -# * optionally upload the repackaged binaries as artifacts (for debug or promotion) - -on: - pull_request: - paths: - - .github/workflows/validate-repackaged-binary-sizes.yml - - release/pypi/prep_binary_for_pypi.sh - -jobs: - validate-binary-size: - strategy: - fail-fast: false - matrix: - whl: - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp312-cp312-linux_x86_64.whl - python: "3.12" - artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp311-cp311-linux_x86_64.whl - python: "3.11" # python version to use for smoke tests - upload_artifact: false # upload the repackaged binary as an artifact - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp310-cp310-linux_x86_64.whl - python: "3.10" - artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp39-cp39-linux_x86_64.whl - python: "3.9" - artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp38-cp38-linux_x86_64.whl - python: "3.8" - artifact: false - - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - with: - runner: linux.g5.4xlarge.nvidia.gpu - job-name: "Validate binary size" - upload-artifact: ${{ matrix.whl.upload_artifact == 'true' && 'repackaged-binary' || '' }} - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export MATRIX_GPU_ARCH_VERSION="12.1" - export MATRIX_GPU_ARCH_TYPE="cuda" - export MATRIX_CUDA_VER="12.1" - export DESIRED_PYTHON="${{ matrix.whl.python }}" - export MATRIX_PACKAGE_TYPE="wheel" - export TARGET_OS="linux" - - # install zip - sudo yum install zip -y - - # install patchelf - chmod a+x common/install_patchelf.sh - sudo common/install_patchelf.sh - - # download torch whl - wget ${{ matrix.whl.url }} - FILENAME=$(ls -1 *.whl | head -n 1) - SIZE_BEFORE=$(du -h $FILENAME | cut -f1) - - # repackage into manywheel - release/pypi/prep_binary_for_pypi.sh $FILENAME - - NEW_FILENAME=$(ls -1 *.whl | head -n 1) - echo "::notice:: $FILENAME before: $SIZE_BEFORE after: $(du -h $NEW_FILENAME | cut -f1)" - - # cp to ${RUNNER_ARTIFACT_DIR} - cp $NEW_FILENAME ${RUNNER_ARTIFACT_DIR}/ - - # create conda env - conda create -y -n $ENV_NAME python=$DESIRED_PYTHON - conda activate $ENV_NAME - - # install torch - pip install numpy pillow $NEW_FILENAME - - # run smoke test - python ./test/smoke_test/smoke_test.py --package=torchonly diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml deleted file mode 100644 index 5b709b198..000000000 --- a/.github/workflows/validate-windows-binaries.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: Validate Windows binary images - -on: - workflow_call: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: string - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - use-version-set: - description: 'Applies when version is used, use version for each domain' - default: false - required: false - type: boolean - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - workflow_dispatch: - inputs: - channel: - description: "Channel to use (nightly, test, release, all)" - required: true - type: choice - options: - - release - - nightly - - test - - all - torchonly: - description: 'Validate torchonly' - default: false - required: false - type: boolean - version: - description: 'Version to validate - optional' - default: "" - required: false - type: string - release-matrix: - description: 'Release matrix - optional' - default: "" - required: false - type: string - use-only-dl-pytorch-org: - description: 'Use only download.pytorch.org when generating wheel install command' - default: false - required: false - type: boolean - use-force-reinstall: - description: 'Use force-reinstall for pip binaries' - default: false - required: false - type: boolean - use-meta-cdn: - description: 'Use meta cdn for installing pip binaries' - default: false - required: false - type: boolean - use-extra-index-url: - description: 'Use extra-index url for pip tests' - default: false - required: false - type: boolean - -jobs: - generate-windows-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel,libtorch # We stopped producing conda nightlies - os: windows - channel: ${{ inputs.channel }} - use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }} - with-xpu: enable - - win: - needs: generate-windows-matrix - strategy: - matrix: ${{ fromJson(needs.generate-windows-matrix.outputs.matrix) }} - fail-fast: false - uses: pytorch/test-infra/.github/workflows/windows_job.yml@main - name: ${{ matrix.build_name }} - with: - runner: ${{ matrix.package_type == 'libtorch' && 'windows.4xlarge' || matrix.validation_runner }} - repository: "pytorch/builder" - ref: ${{ github.ref }} - job-name: ${{ matrix.build_name }} - binary-matrix: ${{ toJSON(matrix) }} - timeout: 60 - script: | - set -ex - export ENV_NAME="conda-env-${{ github.run_id }}" - export TARGET_OS="windows" - export TORCH_ONLY=${{ inputs.torchonly }} - export RELEASE_VERSION=${{ inputs.version }} - export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }} - export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }} - export USE_META_CDN=${{ inputs.use-meta-cdn }} - export USE_VERSION_SET=${{ inputs.use-version-set }} - if [[ ${USE_VERSION_SET} == 'true' ]]; then - export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }} - export AUDIO_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchaudio }} - fi - - printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json - source /c/Jenkins/Miniconda3/etc/profile.d/conda.sh - if [[ ${MATRIX_GPU_ARCH_TYPE} == "cuda" ]]; then - ./windows/internal/driver_update.bat - elif [[ ${MATRIX_GPU_ARCH_TYPE} == "xpu" ]]; then - export CUDA_VERSION=xpu - ./windows/internal/xpu_install.bat - fi - source ./.github/scripts/validate_binaries.sh diff --git a/check_binary.sh b/check_binary.sh deleted file mode 100755 index 952ee5459..000000000 --- a/check_binary.sh +++ /dev/null @@ -1,390 +0,0 @@ -#!/bin/bash -set -eux -o pipefail - -# This script checks the following things on binaries -# 1. The gcc abi matches DESIRED_DEVTOOLSET -# 2. MacOS binaries do not link against OpenBLAS -# 3. There are no protobuf symbols of any sort anywhere (turned off, because -# this is currently not true) -# 4. Standard Python imports work -# 5. MKL is available everywhere except for MacOS wheels -# 6. XNNPACK is available everywhere except for MacOS wheels -# 7. CUDA is setup correctly and does not hang -# 8. Magma is available for CUDA builds -# 9. CuDNN is available for CUDA builds -# -# This script needs the env variables DESIRED_PYTHON, DESIRED_CUDA, -# DESIRED_DEVTOOLSET and PACKAGE_TYPE -# -# This script expects PyTorch to be installed into the active Python (the -# Python returned by `which python`). Or, if this is testing a libtorch -# Pythonless binary, then it expects to be in the root folder of the unzipped -# libtorch package. - - -if [[ -z ${DESIRED_PYTHON:-} ]]; then - export DESIRED_PYTHON=${MATRIX_PYTHON_VERSION:-} -fi -if [[ -z ${DESIRED_CUDA:-} ]]; then - export DESIRED_CUDA=${MATRIX_DESIRED_CUDA:-} -fi -if [[ -z ${DESIRED_DEVTOOLSET:-} ]]; then - export DESIRED_DEVTOOLSET=${MATRIX_DESIRED_DEVTOOLSET:-} -fi -if [[ -z ${PACKAGE_TYPE:-} ]]; then - export PACKAGE_TYPE=${MATRIX_PACKAGE_TYPE:-} -fi - -# The install root depends on both the package type and the os -# All MacOS packages use conda, even for the wheel packages. -if [[ "$PACKAGE_TYPE" == libtorch ]]; then - # NOTE: Only $PWD works on both CentOS and Ubuntu - export install_root="$PWD" -else - - if [[ $DESIRED_PYTHON =~ ([0-9].[0-9]+)t ]]; then - # For python that is maj.mint keep original version - py_dot="$DESIRED_PYTHON" - elif [[ $DESIRED_PYTHON =~ ([0-9].[0-9]+) ]]; then - # Strip everything but major.minor from DESIRED_PYTHON version - py_dot="${BASH_REMATCH[0]}" - else - echo "Unexpected ${DESIRED_PYTHON} format" - exit 1 - fi - export install_root="$(dirname $(which python))/../lib/python${py_dot}/site-packages/torch/" -fi - -############################################################################### -# Setup XPU ENV -############################################################################### -if [[ "$DESIRED_CUDA" == 'xpu' ]]; then - set +u - # Refer https://www.intel.com/content/www/us/en/developer/articles/tool/pytorch-prerequisites-for-intel-gpus.html - source /opt/intel/oneapi/compiler/latest/env/vars.sh - source /opt/intel/oneapi/pti/latest/env/vars.sh -fi - -############################################################################### -# Check GCC ABI -############################################################################### - -# NOTE [ Building libtorch with old vs. new gcc ABI ] -# -# Packages built with one version of ABI could not be linked against by client -# C++ libraries that were compiled using the other version of ABI. Since both -# gcc ABIs are still common in the wild, we need to support both ABIs. Currently: -# -# - All the nightlies built on CentOS 7 + devtoolset7 use the old gcc ABI. -# - All the nightlies built on Ubuntu 16.04 + gcc 5.4 use the new gcc ABI. - -echo "Checking that the gcc ABI is what we expect" -if [[ "$(uname)" != 'Darwin' ]]; then - function is_expected() { - if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* || "$DESIRED_CUDA" == *"rocm"* ]]; then - if [[ "$1" -gt 0 || "$1" == "ON " ]]; then - echo 1 - fi - else - if [[ -z "$1" || "$1" == 0 || "$1" == "OFF" ]]; then - echo 1 - fi - fi - } - - # First we check that the env var in TorchConfig.cmake is correct - - # We search for D_GLIBCXX_USE_CXX11_ABI=1 in torch/TorchConfig.cmake - torch_config="${install_root}/share/cmake/Torch/TorchConfig.cmake" - if [[ ! -f "$torch_config" ]]; then - echo "No TorchConfig.cmake found!" - ls -lah "$install_root/share/cmake/Torch" - exit 1 - fi - echo "Checking the TorchConfig.cmake" - cat "$torch_config" - - # The sed call below is - # don't print lines by default (only print the line we want) - # -n - # execute the following expression - # e - # replace lines that match with the first capture group and print - # s/.*D_GLIBCXX_USE_CXX11_ABI=\(.\)".*/\1/p - # any characters, D_GLIBCXX_USE_CXX11_ABI=, exactly one any character, a - # quote, any characters - # Note the exactly one single character after the '='. In the case that the - # variable is not set the '=' will be followed by a '"' immediately and the - # line will fail the match and nothing will be printed; this is what we - # want. Otherwise it will capture the 0 or 1 after the '='. - # /.*D_GLIBCXX_USE_CXX11_ABI=\(.\)".*/ - # replace the matched line with the capture group and print - # /\1/p - actual_gcc_abi="$(sed -ne 's/.*D_GLIBCXX_USE_CXX11_ABI=\(.\)".*/\1/p' < "$torch_config")" - if [[ "$(is_expected "$actual_gcc_abi")" != 1 ]]; then - echo "gcc ABI $actual_gcc_abi not as expected." - exit 1 - fi - - # We also check that there are [not] cxx11 symbols in libtorch - # - echo "Checking that symbols in libtorch.so have the right gcc abi" - python3 "$(dirname ${BASH_SOURCE[0]})/test/check_binary_symbols.py" - - echo "cxx11 symbols seem to be in order" -fi # if on Darwin - -############################################################################### -# Check for no OpenBLAS -# TODO Check for no Protobuf symbols (not finished) -# Print *all* runtime dependencies -############################################################################### -# We have to loop through all shared libraries for this -if [[ "$(uname)" == 'Darwin' ]]; then - all_dylibs=($(find "$install_root" -name '*.dylib')) - for dylib in "${all_dylibs[@]}"; do - echo "All dependencies of $dylib are $(otool -L $dylib) with rpath $(otool -l $dylib | grep LC_RPATH -A2)" - - # Check that OpenBlas is not linked to on Macs - echo "Checking the OpenBLAS is not linked to" - if [[ -n "$(otool -L $dylib | grep -i openblas)" ]]; then - echo "ERROR: Found openblas as a dependency of $dylib" - echo "Full dependencies is: $(otool -L $dylib)" - exit 1 - fi - - # Check for protobuf symbols - #proto_symbols="$(nm $dylib | grep protobuf)" || true - #if [[ -n "$proto_symbols" ]]; then - # echo "ERROR: Detected protobuf symbols in $dylib" - # echo "Symbols are $proto_symbols" - # exit 1 - #fi - done -else - all_libs=($(find "$install_root" -name '*.so')) - for lib in "${all_libs[@]}"; do - echo "All dependencies of $lib are $(ldd $lib) with runpath $(objdump -p $lib | grep RUNPATH)" - - # Check for protobuf symbols - #proto_symbols=$(nm $lib | grep protobuf) || true - #if [[ -n "$proto_symbols" ]]; then - # echo "ERROR: Detected protobuf symbols in $lib" - # echo "Symbols are $proto_symbols" - # exit 1 - #fi - done -fi - -setup_link_flags () { - REF_LIB="-Wl,-R${install_root}/lib" - if [[ "$(uname)" == 'Darwin' ]]; then - REF_LIB="-Wl,-rpath ${install_root}/lib" - fi - ADDITIONAL_LINKER_FLAGS="" - if [[ "$(uname)" == 'Linux' ]]; then - ADDITIONAL_LINKER_FLAGS="-Wl,--no-as-needed" - fi - C10_LINK_FLAGS="" - if [ -f "${install_root}/lib/libc10.so" ] || [ -f "${install_root}/lib/libc10.dylib" ]; then - C10_LINK_FLAGS="-lc10" - fi - TORCH_CPU_LINK_FLAGS="" - if [ -f "${install_root}/lib/libtorch_cpu.so" ] || [ -f "${install_root}/lib/libtorch_cpu.dylib" ]; then - TORCH_CPU_LINK_FLAGS="-ltorch_cpu" - fi - TORCH_CUDA_LINK_FLAGS="" - if [ -f "${install_root}/lib/libtorch_cuda.so" ] || [ -f "${install_root}/lib/libtorch_cuda.dylib" ]; then - TORCH_CUDA_LINK_FLAGS="-ltorch_cuda" - elif [ -f "${install_root}/lib/libtorch_cuda_cpp.so" ] && [ -f "${install_root}/lib/libtorch_cuda_cpp.so" ] || \ - [ -f "${install_root}/lib/libtorch_cuda_cu.dylib" ] && [ -f "${install_root}/lib/libtorch_cuda_cu.dylib" ]; then - TORCH_CUDA_LINK_FLAGS="-ltorch_cuda_cpp -ltorch_cuda_cu" - fi -} - -TEST_CODE_DIR="$(dirname $(realpath ${BASH_SOURCE[0]}))/test_example_code" -build_and_run_example_cpp () { - if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - GLIBCXX_USE_CXX11_ABI=1 - else - GLIBCXX_USE_CXX11_ABI=0 - fi - setup_link_flags - g++ ${TEST_CODE_DIR}/$1.cpp -I${install_root}/include -I${install_root}/include/torch/csrc/api/include -D_GLIBCXX_USE_CXX11_ABI=$GLIBCXX_USE_CXX11_ABI -std=gnu++17 -L${install_root}/lib ${REF_LIB} ${ADDITIONAL_LINKER_FLAGS} -ltorch $TORCH_CPU_LINK_FLAGS $TORCH_CUDA_LINK_FLAGS $C10_LINK_FLAGS -o $1 - ./$1 -} - -build_example_cpp_with_incorrect_abi () { - if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - GLIBCXX_USE_CXX11_ABI=0 - else - GLIBCXX_USE_CXX11_ABI=1 - fi - set +e - setup_link_flags - g++ ${TEST_CODE_DIR}/$1.cpp -I${install_root}/include -I${install_root}/include/torch/csrc/api/include -D_GLIBCXX_USE_CXX11_ABI=$GLIBCXX_USE_CXX11_ABI -std=gnu++17 -L${install_root}/lib ${REF_LIB} ${ADDITIONAL_LINKER_FLAGS} -ltorch $TORCH_CPU_LINK_FLAGS $TORCH_CUDA_LINK_FLAGS $C10_LINK_FLAGS -o $1 - ERRCODE=$? - set -e - if [ "$ERRCODE" -eq "0" ]; then - echo "Building example with incorrect ABI didn't throw error. Aborting." - exit 1 - else - echo "Building example with incorrect ABI throws expected error. Proceeding." - fi -} - -############################################################################### -# Check simple Python/C++ calls -############################################################################### -if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then - # NS: Set LD_LIBRARY_PATH for CUDA builds, but perhaps it should be removed - if [[ "$DESIRED_CUDA" == "cu"* ]]; then - export LD_LIBRARY_PATH=/usr/local/cuda/lib64 - fi - build_and_run_example_cpp simple-torch-test - # `_GLIBCXX_USE_CXX11_ABI` is always ignored by gcc in devtoolset7, so we test - # the expected failure case for Ubuntu 16.04 + gcc 5.4 only. - if [[ "$DESIRED_DEVTOOLSET" == *"cxx11-abi"* ]]; then - build_example_cpp_with_incorrect_abi simple-torch-test - fi -else - pushd /tmp - python -c 'import torch' - popd -fi - -############################################################################### -# Check torch.git_version -############################################################################### -if [[ "$PACKAGE_TYPE" != 'libtorch' ]]; then - pushd /tmp - python -c 'import torch; assert torch.version.git_version != "Unknown"' - python -c 'import torch; assert torch.version.git_version != None' - popd -fi - - -############################################################################### -# Check for MKL -############################################################################### - -if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then - echo "Checking that MKL is available" - build_and_run_example_cpp check-torch-mkl -elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" ]]; then - if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]]; then - if [[ "$(uname -m)" == "aarch64" ]]; then - echo "Checking that MKLDNN is available on aarch64" - pushd /tmp - python -c 'import torch; exit(0 if torch.backends.mkldnn.is_available() else 1)' - popd - else - echo "Checking that MKL is available" - pushd /tmp - python -c 'import torch; exit(0 if torch.backends.mkl.is_available() else 1)' - popd - fi - fi -fi - -############################################################################### -# Check for XNNPACK -############################################################################### - -if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then - echo "Checking that XNNPACK is available" - build_and_run_example_cpp check-torch-xnnpack -else - if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" ]]; then - echo "Checking that XNNPACK is available" - pushd /tmp - python -c 'import torch.backends.xnnpack; exit(0 if torch.backends.xnnpack.enabled else 1)' - popd - fi -fi - -############################################################################### -# Check CUDA configured correctly -############################################################################### -# Skip these for Windows machines without GPUs -if [[ "$OSTYPE" == "msys" ]]; then - GPUS=$(wmic path win32_VideoController get name) - if [[ ! "$GPUS" == *NVIDIA* ]]; then - echo "Skip CUDA tests for machines without a Nvidia GPU card" - exit 0 - fi -fi - -# Test that CUDA builds are setup correctly -if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" ]]; then - if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then - build_and_run_example_cpp check-torch-cuda - else - pushd /tmp - echo "Checking that CUDA archs are setup correctly" - timeout 20 python -c 'import torch; torch.randn([3,5]).cuda()' - - # These have to run after CUDA is initialized - - echo "Checking that magma is available" - python -c 'import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)' - - echo "Checking that CuDNN is available" - python -c 'import torch; exit(0 if torch.backends.cudnn.is_available() else 1)' - - # Validates builds is free of linker regressions reported in https://github.com/pytorch/pytorch/issues/57744 - echo "Checking that exception handling works" - python -c "import torch; from unittest import TestCase;TestCase().assertRaises(RuntimeError, lambda:torch.eye(7, 7, device='cuda:7'))" - - echo "Checking that basic RNN works" - python ${TEST_CODE_DIR}/rnn_smoke.py - - echo "Checking that basic CNN works" - python ${TEST_CODE_DIR}/cnn_smoke.py - - echo "Test that linalg works" - python -c "import torch;x=torch.rand(3,3,device='cuda');print(torch.linalg.svd(torch.mm(x.t(), x)))" - - popd - fi # if libtorch -fi # if cuda - -########################## -# Run parts of smoke tests -########################## -if [[ "$PACKAGE_TYPE" != 'libtorch' ]]; then - pushd "$(dirname ${BASH_SOURCE[0]})/test/smoke_test" - python -c "from smoke_test import test_linalg; test_linalg()" - if [[ "$DESIRED_CUDA" == *cuda* ]]; then - python -c "from smoke_test import test_linalg; test_linalg('cuda')" - fi - popd -fi - -############################################################################### -# Check PyTorch supports TCP_TLS gloo transport -############################################################################### - -if [[ "$(uname)" == 'Linux' && "$PACKAGE_TYPE" != 'libtorch' ]]; then - GLOO_CHECK="import torch.distributed as dist -try: - dist.init_process_group('gloo', rank=0, world_size=1) -except RuntimeError as e: - print(e) -" - RESULT=`GLOO_DEVICE_TRANSPORT=TCP_TLS MASTER_ADDR=localhost MASTER_PORT=63945 python -c "$GLOO_CHECK"` - GLOO_TRANSPORT_IS_NOT_SUPPORTED='gloo transport is not supported' - if [[ "$RESULT" =~ "$GLOO_TRANSPORT_IS_NOT_SUPPORTED" ]]; then - echo "PyTorch doesn't support TLS_TCP transport, please build with USE_GLOO_WITH_OPENSSL=1" - exit 1 - fi -fi - -############################################################################### -# Check for C++ ABI compatibility between gcc7 and gcc9 compiled binaries -############################################################################### -if [[ "$(uname)" == 'Linux' && ("$PACKAGE_TYPE" == 'conda' || "$PACKAGE_TYPE" == 'manywheel')]]; then - pushd /tmp - python -c "import torch; exit(0 if torch.compiled_with_cxx11_abi() else (0 if torch._C._PYBIND11_BUILD_ABI == '_cxxabi1011' else 1))" - popd -fi diff --git a/conda/.DS_Store b/conda/.DS_Store deleted file mode 100644 index 5008ddfcf..000000000 Binary files a/conda/.DS_Store and /dev/null differ diff --git a/conda/.gitignore b/conda/.gitignore deleted file mode 100644 index e4e5f6c8b..000000000 --- a/conda/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*~ \ No newline at end of file diff --git a/release/README.md b/release/README.md deleted file mode 100644 index a80144b17..000000000 --- a/release/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# PyTorch Release Scripts - -These are a collection of scripts that are to be used for release activities. - -> NOTE: All scripts should do no actual work unless the `DRY_RUN` environment variable is set -> to `disabled`. -> The basic idea being that there should be no potential to do anything dangerous unless -> `DRY_RUN` is explicitly set to `disabled`. - -## Requirements to actually run these scripts -* AWS access to pytorch account (META Employees: `bunnylol cloud pytorch`) -* Access to upload conda packages to the [`pytorch`](https://anaconda.org/pytorch) conda channel -* Access to the PyPI repositories (like [torch](https://pypi.org/project/torch)) - -## Promote pypi to staging - -Following steps needed in order to promote pypi to staging: -1. Edit `release_versions.sh` and set correct version -2. Run promote script : `./pypi/promote_pypi_to_staging.sh` -3. Edit and run `../analytics/validate_pypi_staging.py` to perform initial prevalidation of binaries for pypi promotion -4. Manually inspect and spot check binaries staged for pypi promotion by logging into s3 and downloading packages - -## Promote - -These are scripts related to promotion of release candidates to GA channels, these -can actually be used to promote pytorch, libtorch, and related domain libraries. - -> NOTE: Currently the script requires some knowledge on when to comment things out / comment things in - -> TODO: Make the script not rely on commenting things out / commenting this in - -### Usage - -```bash -./promote.sh -``` - -## Restoring backups - -All release candidates from `pytorch/pytorch` are currently backed up -to `s3://pytorch-backup/${TAG_NAME}` and can be restored to the test channels with the -`restore-backup.sh` script. - -Which backup to restore from is dictated by the `RESTORE_FROM` environment variable. - -### Usage -```bash -RESTORE_FROM=v1.5.0-rc5 ./restore-backup.sh -``` diff --git a/release/cut-release-branch.sh b/release/cut-release-branch.sh deleted file mode 100644 index 468dbfb18..000000000 --- a/release/cut-release-branch.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -: ' -So you are looking to cut a release branch? Well you came -to the right script. - -This script can be used to cut any branch on any repository - -For `pytorch/pytorch` usage would be like: -> DRY_RUN=disabled cut-release-branch.sh - -For `pytorch/builder` or domains usage would be like: -> DRY_RUN=disabled GIT_BRANCH_TO_CUT_FROM=main RELEASE_VERSION=1.11 cut-release-branch.sh -' - -set -eou pipefail - -GIT_TOP_DIR=$(git rev-parse --show-toplevel) -GIT_REMOTE=${GIT_REMOTE:-origin} -GIT_BRANCH_TO_CUT_FROM=${GIT_BRANCH_TO_CUT_FROM:-viable/strict} - -# should output something like 1.11 -RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} - -DRY_RUN_FLAG="--dry-run" -if [[ ${DRY_RUN:-enabled} == "disabled" ]]; then - DRY_RUN_FLAG="" -fi - - -( - set -x - git fetch --all - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" -) - -for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do - if git rev-parse --verify "${branch}" >/dev/null 2>/dev/null; then - echo "+ Branch ${branch} already exists, skipping..." - continue - else - ( - set -x - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" - git checkout -b "${branch}" - git push "${GIT_REMOTE}" "${branch}" - ) - fi -done diff --git a/release/promote.sh b/release/promote.sh deleted file mode 100644 index 8b7939783..000000000 --- a/release/promote.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/release_versions.sh" - -# Make sure to update these versions when doing a release first -PYTORCH_VERSION=${PYTORCH_VERSION:-2.3.0} -TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.18.0} -TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.3.0} -TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.18.0} -TORCHREC_VERSION=${TORCHREC_VERSION:-0.8.0} -TENSORRT_VERSION=${TENSORRT_VERSION:-2.2.0} -FBGEMMGPU_VERSION=${FBGEMMGPU_VERSION:-1.0.0} - -DRY_RUN=${DRY_RUN:-enabled} - -promote_s3() { - local package_name - package_name=$1 - local package_type - package_type=$2 - local promote_version - promote_version=$3 - - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} ${package_type} packages' =-=-=-=" - ( - set -x - TEST_PYTORCH_PROMOTE_VERSION="${promote_version}" \ - PACKAGE_NAME="${package_name}" \ - PACKAGE_TYPE="${package_type}" \ - TEST_WITHOUT_GIT_TAG=1 \ - DRY_RUN="${DRY_RUN}" ${DIR}/promote/s3_to_s3.sh - ) - echo -} - -promote_conda() { - local package_name - package_name=$1 - local package_type - package_type=$2 - local promote_version - promote_version=$3 - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} ${package_type} packages' =-=-=-=" - ( - ANACONDA="echo + anaconda" - if [[ "${DRY_RUN:-enabled}" = "disabled" ]]; then - ANACONDA="anaconda" - set -x - else - echo "DRY_RUN enabled not actually doing work" - fi - ${ANACONDA} copy --to-owner ${PYTORCH_CONDA_TO:-pytorch} ${PYTORCH_CONDA_FROM:-pytorch-test}/${package_name}/${promote_version} - ) - echo -} - -promote_pypi() { - local package_name - package_name=$1 - local promote_version - promote_version=$2 - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi' =-=-=-=" - ( - set -x - TEST_PYTORCH_PROMOTE_VERSION="${promote_version}" \ - PACKAGE_NAME="${package_name}" \ - TEST_WITHOUT_GIT_TAG=1 \ - DRY_RUN="${DRY_RUN}" ${DIR}/promote/wheel_to_pypi.sh - ) - echo -} - -# Promote s3 dependencies -# promote_s3 "certifi" whl "2022.12.7" -# promote_s3 "charset_normalizer" whl "2.1.1" -# promote_s3 "cmake" whl "3.25" -# promote_s3 "colorama" whl "0.4.6" -# promote_s3 "triton" whl "2.0.0" -# promote_s3 "pytorch_triton_rocm" whl "2.0.1" -# promote_s3 "tqdm" whl "4.64.1" -# promote_s3 "Pillow" whl "9.3.0" -# for python 3.8-3.11 -# promote_s3 "numpy" whl "1.24.1" -# for python 3.7 older pytorch versions -# promote_s3 "numpy" whl "1.21.6" -# promote_s3 "urllib3" whl "1.26.13" -# promote_s3 "lit" whl "15.0.7" -# promote_s3 "sympy" whl "1.11.1" -# promote_s3 "typing_extensions" whl "4.4.0" -# promote_s3 "filelock" whl "3.9.0" -# promote_s3 "mpmath" whl "1.2.1" -# promote_s3 "MarkupSafe" whl "2.1.2" -# promote_s3 "Jinja2" whl "3.1.2" -# promote_s3 "idna" whl "3.4" -# promote_s3 "networkx" whl "3.0" -# promote_s3 "packaging" whl "22.0" -# promote_s3 "requests" whl "2.28.1" - -# promote_s3 torch whl "${PYTORCH_VERSION}" -# promote_s3 torchvision whl "${TORCHVISION_VERSION}" -# promote_s3 torchaudio whl "${TORCHAUDIO_VERSION}" -# promote_s3 torchtext whl "${TORCHTEXT_VERSION}" -# promote_s3 torchrec whl "${TORCHREC_VERSION}" -# promote_s3 fbgemm-gpu whl "${FBGEMMGPU_VERSION}" -# promote_s3 "libtorch-*" libtorch "${PYTORCH_VERSION}" -# promote_s3 "torch_tensorrt" whl "${TENSORRT_VERSION}" - -# promote_conda torchtriton conda "2.1.0" -# promote_conda pytorch-cuda conda "11.8" -# promote_conda pytorch-cuda conda "12.1" - -# promote_conda pytorch conda "${PYTORCH_VERSION}" -# promote_conda torchvision conda "${TORCHVISION_VERSION}" -# promote_conda torchaudio conda "${TORCHAUDIO_VERSION}" -# promote_conda torchtext conda "${TORCHTEXT_VERSION}" - -# Uncomment these to promote to pypi -LINUX_VERSION_SUFFIX="%2Bcu102" -WIN_VERSION_SUFFIX="%2Bcpu" -# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torch "${PYTORCH_VERSION}" -# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torch "${PYTORCH_VERSION}" -# PLATFORM="win_amd64" VERSION_SUFFIX="${WIN_VERSION_SUFFIX}" promote_pypi torch "${PYTORCH_VERSION}" -# PLATFORM="macosx_11_0" VERSION_SUFFIX="" promote_pypi torch "${PYTORCH_VERSION}" # m1 mac - -# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torchvision "${TORCHVISION_VERSION}" -# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torchvision "${TORCHVISION_VERSION}" -# PLATFORM="win_amd64" VERSION_SUFFIX="${WIN_VERSION_SUFFIX}" promote_pypi torchvision "${TORCHVISION_VERSION}" -# PLATFORM="macosx_11_0" VERSION_SUFFIX="" promote_pypi torchvision "${TORCHVISION_VERSION}" - -# PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" -# PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" -# PLATFORM="win_amd64" VERSION_SUFFIX="${WIN_VERSION_SUFFIX}" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" -# PLATFORM="macosx_11_0" VERSION_SUFFIX="" promote_pypi torchaudio "${TORCHAUDIO_VERSION}" diff --git a/release/promote/common_utils.sh b/release/promote/common_utils.sh deleted file mode 100644 index e50ccb110..000000000 --- a/release/promote/common_utils.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -exit_if_not_on_git_tag() { - # Have an override for debugging purposes - if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]] ;then - >&2 echo "+ WARN: Continuing without being on a git tag" - exit 0 - fi - # Exit if we're not currently on a git tag - if ! git describe --tags --exact >/dev/null 2>/dev/null; then - >&2 echo "- ERROR: Attempting to promote on a non-git tag, must have tagged current commit locally first" - exit 1 - fi - # Exit if we're currently on an RC - if git describe --tags | grep "-rc" >/dev/null 2>/dev/null; then - >&2 echo "- ERROR: Attempting to promote on a non GA git tag, current tag must be a GA tag" - >&2 echo " Example: v1.5.0" - exit 1 - fi -} - -get_pytorch_version() { - if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]];then - if [[ -z "${TEST_PYTORCH_PROMOTE_VERSION-}" ]]; then - >&2 echo "- ERROR: Specified TEST_WITHOUT_GIT_TAG without specifying TEST_PYTORCH_PROMOTE_VERSION" - >&2 echo "- TEST_PYTORCH_PROMOTE_VERSION must be specified" - exit 1 - else - echo "${TEST_PYTORCH_PROMOTE_VERSION}" - exit 0 - fi - fi - exit_if_not_on_git_tag - # Echo git tag, strip leading v - git describe --tags | sed -e 's/^v//' -} - -aws_promote() { - package_name=$1 - pytorch_version=$(get_pytorch_version) - # Dry run by default - DRY_RUN=${DRY_RUN:-enabled} - DRY_RUN_FLAG="--dryrun" - if [[ $DRY_RUN = "disabled" ]]; then - DRY_RUN_FLAG="" - fi - AWS=${AWS:-aws} - ( - set -x - ${AWS} s3 cp ${DRY_RUN_FLAG} \ - --only-show-errors \ - --acl public-read \ - --recursive \ - --exclude '*' \ - --include "*${package_name}-${pytorch_version}*" \ - "${PYTORCH_S3_FROM/\/$//}" \ - "${PYTORCH_S3_TO/\/$//}" - ) - # ^ We grep for package_name-.*pytorch_version to avoid any situations where domain libraries have - # the same version on our S3 buckets -} diff --git a/release/promote/conda_to_conda.sh b/release/promote/conda_to_conda.sh deleted file mode 100755 index c890f9c18..000000000 --- a/release/promote/conda_to_conda.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-pytorch} -PYTORCH_CONDA_FROM=${PYTORCH_CONDA_FROM:-pytorch-test} -PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch} -CONDA_PLATFORMS="linux-64 osx-64 win-64 noarch" - -pytorch_version="$(get_pytorch_version)" - -tmp_dir="$(mktemp -d)" -pushd "${tmp_dir}" -trap 'rm -rf ${tmp_dir}' EXIT - -conda_search() { - conda search -q "${PYTORCH_CONDA_FROM}::${PACKAGE_NAME}==${pytorch_version}" -c "${PYTORCH_CONDA_FROM}" --platform "${platform}" \ - | grep -e "^${PACKAGE_NAME}" \ - | awk -F ' *' '{print $3}' \ - | xargs -I % echo "https://anaconda.org/${PYTORCH_CONDA_FROM}/${PACKAGE_NAME}/${pytorch_version}/download/${platform}/${PACKAGE_NAME}-${pytorch_version}-%.tar.bz2" -} - -pkgs_to_download=() -for platform in ${CONDA_PLATFORMS}; do - pkgs_to_download+=($(\ - conda_search 2>/dev/null || true - )) - # Create directory where packages will eventually be downloaded - mkdir -p "${platform}" -done - -my_curl() { - local dl_url=$1 - local start=$(date +%s) - # downloads should be distinguished by platform which should be the second - # to last field in the url, this is to avoid clobbering same named files - # for different platforms - dl_dir=$(echo "${dl_url}" | rev | cut -d'/' -f 2 | rev) - dl_name=$(echo "${dl_url}" | rev | cut -d'/' -f 1 | rev) - curl -fsSL -o "${dl_dir}/${dl_name}" "${dl_url}" - local end=$(date +%s) - local diff=$(( end - start )) - echo "+ ${dl_url} took ${diff}s" -} -export -f my_curl - -# Download all packages in parallel -printf '%s\n' "${pkgs_to_download[@]}" \ - | xargs -P 10 -I % bash -c '(declare -t my_curl); my_curl %' - -# dry run by default -DRY_RUN=${DRY_RUN:-enabled} -ANACONDA="true anaconda" -if [[ $DRY_RUN = "disabled" ]]; then - ANACONDA="anaconda" -fi -( - # We use --skip here to avoid re-uploading files we've already uploaded - set -x - ${ANACONDA} upload --skip -u ${PYTORCH_CONDA_TO} $(find . -name '*.bz2') -) - -popd diff --git a/release/promote/prep_binary_for_pypi.sh b/release/promote/prep_binary_for_pypi.sh deleted file mode 100755 index bb088f554..000000000 --- a/release/promote/prep_binary_for_pypi.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Preps binaries for publishing to pypi by removing the -# version suffix we normally add for all binaries -# (outside of default ones, CUDA 10.2 currently) - -# Usage is: -# $ prep_binary_for_pypy.sh - -# Will output a whl in your current directory - -set -eou pipefail -shopt -s globstar - -OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)} - -tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir}' EXIT - -for whl_file in "$@"; do - whl_file=$(realpath "${whl_file}") - whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped" - mkdir -pv "${whl_dir}" - ( - set -x - unzip -q "${whl_file}" -d "${whl_dir}" - ) - version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2) - version_with_suffix_escaped=${version_with_suffix/+/%2B} - # Remove all suffixed +bleh versions - version_no_suffix=${version_with_suffix/+*/} - new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}") - dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1) - basename_dist_info_folder=$(basename "${dist_info_folder}") - dirname_dist_info_folder=$(dirname "${dist_info_folder}") - ( - set -x - find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \; - # Moves distinfo from one with a version suffix to one without - # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info - mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}" - cd "${whl_dir}" - zip -qr "${new_whl_file}" . - ) -done diff --git a/release/promote/s3_to_s3.sh b/release/promote/s3_to_s3.sh deleted file mode 100755 index 21495ac3b..000000000 --- a/release/promote/s3_to_s3.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} -PACKAGE_TYPE=${PACKAGE_TYPE:-whl} - -PYTORCH_S3_BUCKET=${PYTORCH_S3_BUCKET:-s3://pytorch} -FROM=${FROM:-test} -PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${FROM}} -TO=${TO:-} -PYTORCH_S3_TO=${PYTORCH_S3_TO:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${TO}} - -aws_promote "${PACKAGE_NAME}" diff --git a/release/promote/wheel_to_pypi.sh b/release/promote/wheel_to_pypi.sh deleted file mode 100755 index 1e9b7411f..000000000 --- a/release/promote/wheel_to_pypi.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/common_utils.sh" - -# Allow for users to pass PACKAGE_NAME -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} - -pytorch_version="$(get_pytorch_version)" -# Refers to the specific package we'd like to promote -# i.e. VERSION_SUFFIX='%2Bcu102' -# torch-1.8.0+cu102 -> torch-1.8.0 -VERSION_SUFFIX=${VERSION_SUFFIX:-} -# Refers to the specific platofmr we'd like to promote -# i.e. PLATFORM=linux_x86_64 -# For domains like torchaudio / torchtext this is to be left blank -PLATFORM=${PLATFORM:-} - -pkgs_to_promote=$(\ - curl -fsSL https://download.pytorch.org/whl/torch_stable.html \ - | grep "${PACKAGE_NAME}-${pytorch_version}${VERSION_SUFFIX}-" \ - | grep "${PLATFORM}" \ - | cut -d '"' -f2 -) - -tmp_dir="$(mktemp -d)" -output_tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT -pushd "${output_tmp_dir}" - -# Dry run by default -DRY_RUN=${DRY_RUN:-enabled} -# On dry run just echo the commands that are meant to be run -TWINE_UPLOAD="echo twine upload" -if [[ $DRY_RUN = "disabled" ]]; then - TWINE_UPLOAD="twine upload" -fi - -for pkg in ${pkgs_to_promote}; do - pkg_basename="$(basename "${pkg}")" - # Don't attempt to change if manylinux2014 - if [[ "${pkg}" != *manylinux2014* ]]; then - pkg_basename="$(basename "${pkg//linux/manylinux1}")" - fi - orig_pkg="${tmp_dir}/${pkg_basename}" - ( - set -x - # Download package, sub out linux for manylinux1 - curl -fsSL -o "${orig_pkg}" "https://download.pytorch.org/whl/${pkg}" - ) - - if [[ -n "${VERSION_SUFFIX}" ]]; then - OUTPUT_DIR="${output_tmp_dir}" ${DIR}/prep_binary_for_pypi.sh "${orig_pkg}" - else - mv "${orig_pkg}" "${output_tmp_dir}/" - fi - - ( - set -x - ${TWINE_UPLOAD} \ - --disable-progress-bar \ - --non-interactive \ - ./*.whl - rm -rf ./*.whl - ) -done diff --git a/release/pypi/prep_binary_for_pypi.sh b/release/pypi/prep_binary_for_pypi.sh deleted file mode 100755 index 154b22852..000000000 --- a/release/pypi/prep_binary_for_pypi.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -# Preps binaries for publishing to pypi by removing the -# version suffix we normally add for all binaries -# (outside of default ones, CUDA 10.2 currently) - -# Usage is: -# $ prep_binary_for_pypy.sh - -# Will output a whl in your current directory - -set -eou pipefail -shopt -s globstar - -# Function copied from manywheel/build_common.sh -make_wheel_record() { - FPATH=$1 - if echo $FPATH | grep RECORD >/dev/null 2>&1; then - # if the RECORD file, then - echo "$FPATH,," - else - HASH=$(openssl dgst -sha256 -binary $FPATH | openssl base64 | sed -e 's/+/-/g' | sed -e 's/\//_/g' | sed -e 's/=//g') - FSIZE=$(ls -nl $FPATH | awk '{print $5}') - echo "$FPATH,sha256=$HASH,$FSIZE" - fi -} - -OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)} - -tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir}' EXIT - -DEBUG=${DEBUG:-} - -for whl_file in "$@"; do - whl_file=$(realpath "${whl_file}") - whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped" - mkdir -pv "${whl_dir}" - ( - set -x - unzip -q "${whl_file}" -d "${whl_dir}" - ) - version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2 | tr -d "[:space:]") - version_with_suffix_escaped=${version_with_suffix/+/%2B} - - # Remove all suffixed +bleh versions - version_no_suffix=${version_with_suffix/+*/} - new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}") - dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1) - basename_dist_info_folder=$(basename "${dist_info_folder}") - dirname_dist_info_folder=$(dirname "${dist_info_folder}") - ( - set -x - - find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \; - # Moves distinfo from one with a version suffix to one without - # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info - mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}" - cd "${whl_dir}" - - ( - set +x - # copied from manywheel/build_common.sh - # regenerate the RECORD file with new hashes - record_file="${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}/RECORD" - if [[ -e $record_file ]]; then - echo "Generating new record file $record_file" - : > "$record_file" - # generate records for folders in wheel - find * -type f | while read fname; do - make_wheel_record "$fname" >>"$record_file" - done - fi - ) - - rm -rf "${new_whl_file}" - zip -qr9 "${new_whl_file}" . - ) -done diff --git a/release/pypi/promote_pypi_to_production.sh b/release/pypi/promote_pypi_to_production.sh deleted file mode 100644 index 4da59ac84..000000000 --- a/release/pypi/promote_pypi_to_production.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/../release_versions.sh" - -PYTORCH_DIR=${PYTORCH_DIR:-~/pytorch} - -DRY_RUN=${DRY_RUN:-enabled} - -TWINE_UPLOAD="true twine upload" -if [[ ${DRY_RUN:-enabled} = "disabled" ]]; then - TWINE_UPLOAD="twine upload" -fi - -promote_staging_binaries() { - local package_name - package_name=$1 - local promote_version - promote_version=$2 - - ( - TMP_DIR=$(mktemp -d) - trap 'rm -rf ${TMP_DIR}' EXIT - pushd "${TMP_DIR}" - set -x - aws s3 sync "s3://pytorch-backup/${package_name}-${promote_version}-pypi-staging/" "${TMP_DIR}/" - ${TWINE_UPLOAD} --skip-existing "${TMP_DIR}/*.whl" - ) -} - -promote_staging_binaries torch "${PYTORCH_VERSION}" -promote_staging_binaries torchvision "${TORCHVISION_VERSION}" -promote_staging_binaries torchaudio "${TORCHAUDIO_VERSION}" - -promote_staging_binaries executorch "${EXECUTORCH_VERSION}" -#promote_staging_binaries torchtext "${TORCHTEXT_VERSION}" -#promote_staging_binaries torchdata "${TORCHDATA_VERSION}" - -#promote_staging_binaries torchao "${TORCHAO_VERSION}" -#promote_staging_binaries torchtune "${TORCHTUNE_VERSION}" diff --git a/release/pypi/promote_pypi_to_staging.sh b/release/pypi/promote_pypi_to_staging.sh deleted file mode 100644 index 72f04e3da..000000000 --- a/release/pypi/promote_pypi_to_staging.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/../release_versions.sh" - -DRY_RUN=${DRY_RUN:-enabled} - -upload_pypi_to_staging() { - local package_name - package_name=$1 - local promote_version - promote_version=$2 - echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi staging' =-=-=-=" - ( - set -x - PACKAGE_VERSION="${promote_version}" PACKAGE_NAME="${package_name}" DRY_RUN="${DRY_RUN}" bash ${DIR}/upload_pypi_to_staging.sh - ) - echo -} - -# Uncomment these to promote to pypi -LINUX_VERSION_SUFFIX="%2Bcu124" -CPU_VERSION_SUFFIX="%2Bcpu" -MACOS_X86_64="macosx_.*_x86_64" -MACOS_ARM64="macosx_.*_arm64" - -PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu124" upload_pypi_to_staging torch "${PYTORCH_VERSION}" -PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torch "${PYTORCH_VERSION}" -PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torch "${PYTORCH_VERSION}" -PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torch "${PYTORCH_VERSION}" - -PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu124" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" -PLATFORM="linux_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" -PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" -PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchvision "${TORCHVISION_VERSION}" - -PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu124" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" -PLATFORM="linux_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" -PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" -PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchaudio "${TORCHAUDIO_VERSION}" - -#PLATFORM="manylinux2014_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torchao "${TORCHAO_VERSION}" -#PLATFORM="" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtune "${TORCHTUNE_VERSION}" - -#PLATFORM="linux_x86" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging executorch "${EXECUTORCH_VERSION}" -#PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging executorch "${EXECUTORCH_VERSION}" - -#PLATFORM="linux_x86" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}" -#PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}" -#PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchtext "${TORCHTEXT_VERSION}" - -#PLATFORM="linux_x86_64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}" -#PLATFORM="win_amd64" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}" -#PLATFORM="${MACOS_ARM64}" VERSION_SUFFIX="" upload_pypi_to_staging torchdata "${TORCHDATA_VERSION}" diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh deleted file mode 100644 index 732c049ba..000000000 --- a/release/pypi/upload_pypi_to_staging.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -# Allow for users to pass PACKAGE_NAME - -# For use with other packages, i.e. torchvision, etc. -PACKAGE_NAME=${PACKAGE_NAME:-torch} - -PACKAGE_VERSION=${PACKAGE_VERSION:-1.0.0} -# Refers to the specific package we'd like to promote -# i.e. VERSION_SUFFIX='%2Bcu102' -# torch-1.8.0+cu102 -> torch-1.8.0 -VERSION_SUFFIX=${VERSION_SUFFIX:-} -# Refers to the specific platofmr we'd like to promote -# i.e. PLATFORM=linux_x86_64 -# For domains like torchaudio / torchtext this is to be left blank -PLATFORM=${PLATFORM:-} -# Refers to the specific architecture we'd like to promote -# i.e. cpu, cu121, cu124 -ARCH==${ARCH:-cpu} - -pkgs_to_promote=$(\ - curl -fsSL "https://download.pytorch.org/whl/test/${ARCH}/${PACKAGE_NAME}/index.html" \ - | grep "${PACKAGE_NAME}-${PACKAGE_VERSION}${VERSION_SUFFIX}-" \ - | grep "${PLATFORM}" \ - | cut -d '"' -f2 -) - -tmp_dir="$(mktemp -d)" -output_tmp_dir="$(mktemp -d)" -trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT -pushd "${output_tmp_dir}" - -# Dry run by default -DRY_RUN=${DRY_RUN:-enabled} -# On dry run just echo the commands that are meant to be run -DRY_RUN_FLAG="--dryrun" -if [[ $DRY_RUN = "disabled" ]]; then - DRY_RUN_FLAG="" -fi - -for pkg in ${pkgs_to_promote}; do - pkg_basename="$(basename "${pkg}")" - - if [[ "${pkg}" != *aarch64* && "${pkg}" != *torchao* ]]; then - # sub out linux for manylinux1 - pkg_basename="$(basename "${pkg//linux/manylinux1}")" - elif [[ "${pkg}" == *manylinux_2_17_aarch64* ]]; then - # strip manylinux_2_17 from core filename - pkg_basename="$(basename "${pkg//manylinux_2_17_aarch64./}")" - elif [[ "${pkg}" == *linux_aarch64* ]]; then - # domains change linux_aarch64 to manylinux2014_aarch64 - pkg_basename="$(basename "${pkg//linux_aarch64/manylinux2014_aarch64}")" - fi - - orig_pkg="${tmp_dir}/${pkg_basename}" - ( - set -x - curl -fSL -o "${orig_pkg}" "https://download.pytorch.org${pkg}" - ) - - if [[ -n "${VERSION_SUFFIX}" ]]; then - OUTPUT_DIR="${output_tmp_dir}" bash "${DIR}/prep_binary_for_pypi.sh" "${orig_pkg}" - else - mv "${orig_pkg}" "${output_tmp_dir}/" - fi - - ( - set -x - aws s3 cp ${DRY_RUN_FLAG} *.whl "s3://pytorch-backup/${PACKAGE_NAME}-${PACKAGE_VERSION}-pypi-staging/" - rm -rf ./*.whl - ) -done diff --git a/release/release_versions.sh b/release/release_versions.sh deleted file mode 100644 index 32e8c47d8..000000000 --- a/release/release_versions.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# Make sure to update these versions when doing a release first -PYTORCH_VERSION=${PYTORCH_VERSION:-2.3.1} -TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.18.1} -TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.3.1} -TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.18.0} -TORCHREC_VERSION=${TORCHREC_VERSION:-0.8.0} -TENSORRT_VERSION=${TENSORRT_VERSION:-2.4.0} -EXECUTORCH_VERSION=${EXECUTORCH_VERSION:-0.3.0} -TORCHAO_VERSION=${TORCHAO_VERSION:-0.4.0} -TORCHTUNE_VERSION=${TORCHTUNE_VERSION:-0.2.1} -FBGEMMGPU_VERSION=${FBGEMMGPU_VERSION:-1.0.0} diff --git a/release/restore-backup.sh b/release/restore-backup.sh deleted file mode 100755 index 6cf79549d..000000000 --- a/release/restore-backup.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -set -eou pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -source "${DIR}/promote/common_utils.sh" - -if [[ -z "${RESTORE_FROM:-}" ]]; then - echo "ERROR: RESTORE_FROM environment variable must be specified" - echo " example: RESTORE_FROM=v1.6.0-rc3 ${0}" - exit 1 -fi - -DRY_RUN=${DRY_RUN:-enabled} - -PYTORCH_S3_BACKUP_BUCKET=${PYTORCH_S3_BACKUP_BUCKET:-s3://pytorch-backup/${RESTORE_FROM}} -PYTORCH_S3_TEST_BUCKET=${PYTORCH_S3_TEST_BUCKET:-s3://pytorch/} -PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BACKUP_BUCKET}} -PYTORCH_S3_TO=${PYTORCH_S3_TO:-s3://pytorch/} - -restore_wheels() { - aws_promote torch whl -} - -restore_libtorch() { - aws_promote libtorch-* libtorch -} - -ANACONDA="true anaconda" -if [[ ${DRY_RUN} = "disabled" ]]; then - ANACONDA="anaconda" -fi -PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch-test} - -upload_conda() { - local pkg - pkg=${1} - ( - set -x - ${ANACONDA} upload --skip -u "${PYTORCH_CONDA_TO}" "${pkg}" - ) -} - -export -f upload_conda - -restore_conda() { - TMP_DIR="$(mktemp -d)" - trap 'rm -rf ${TMP_DIR}' EXIT - ( - set -x - aws s3 cp --recursive "${PYTORCH_S3_BACKUP_BUCKET}/conda" "${TMP_DIR}/" - ) - export ANACONDA - export PYTORCH_CONDA_TO - # Should upload all bz2 packages in parallel for quick restoration - find "${TMP_DIR}" -name '*.bz2' -type f \ - | xargs -P 10 -I % bash -c "(declare -t upload_conda); upload_conda %" -} - - -restore_wheels -restore_libtorch -restore_conda diff --git a/run_tests.sh b/run_tests.sh deleted file mode 100755 index 2046501ca..000000000 --- a/run_tests.sh +++ /dev/null @@ -1,731 +0,0 @@ -#!/bin/bash -set -eux -o pipefail - -# Essentially runs pytorch/test/run_test.py, but keeps track of which tests to -# skip in a centralized place. -# -# TODO Except for a few tests, this entire file is a giant TODO. Why are these -# tests # failing? -# TODO deal with Windows - -# This script expects to be in the pytorch root folder -if [[ ! -d 'test' || ! -f 'test/run_test.py' ]]; then - echo "builder/test.sh expects to be run from the Pytorch root directory " \ - "but I'm actually in $(pwd)" - exit 2 -fi - -# Allow master skip of all tests -if [[ -n "${SKIP_ALL_TESTS:-}" ]]; then - exit 0 -fi - -# If given specific test params then just run those -if [[ -n "${RUN_TEST_PARAMS:-}" ]]; then - echo "$(date) :: Calling user-command $(pwd)/test/run_test.py ${RUN_TEST_PARAMS[@]}" - python test/run_test.py ${RUN_TEST_PARAMS[@]} - exit 0 -fi - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -# Parameters -############################################################################## -if [[ "$#" != 3 ]]; then - if [[ -z "${DESIRED_PYTHON:-}" || -z "${DESIRED_CUDA:-}" || -z "${PACKAGE_TYPE:-}" ]]; then - echo "USAGE: run_tests.sh PACKAGE_TYPE DESIRED_PYTHON DESIRED_CUDA" - echo "The env variable PACKAGE_TYPE must be set to 'conda' or 'manywheel' or 'libtorch'" - echo "The env variable DESIRED_PYTHON must be set like '2.7mu' or '3.6m' etc" - echo "The env variable DESIRED_CUDA must be set like 'cpu' or 'cu80' etc" - exit 1 - fi - package_type="$PACKAGE_TYPE" - py_ver="$DESIRED_PYTHON" - cuda_ver="$DESIRED_CUDA" -else - package_type="$1" - py_ver="$2" - cuda_ver="$3" -fi - -if [[ "$cuda_ver" == 'cpu-cxx11-abi' ]]; then - cuda_ver="cpu" -fi - -# cu80, cu90, cu100, cpu -if [[ ${#cuda_ver} -eq 4 ]]; then - cuda_ver_majmin="${cuda_ver:2:1}.${cuda_ver:3:1}" -elif [[ ${#cuda_ver} -eq 5 ]]; then - cuda_ver_majmin="${cuda_ver:2:2}.${cuda_ver:4:1}" -fi - -NUMPY_PACKAGE="" -if [[ ${py_ver} == "3.10" ]]; then - PROTOBUF_PACKAGE="protobuf>=3.17.2" - NUMPY_PACKAGE="numpy>=1.21.2" -else - PROTOBUF_PACKAGE="protobuf=3.14.0" -fi - -# Environment initialization -if [[ "$package_type" == conda || "$(uname)" == Darwin ]]; then - # Install the testing dependencies - retry conda install -yq future hypothesis ${NUMPY_PACKAGE} ${PROTOBUF_PACKAGE} pytest setuptools six typing_extensions pyyaml -else - retry pip install -qr requirements.txt || true - retry pip install -q hypothesis protobuf pytest setuptools || true - numpy_ver=1.15 - case "$(python --version 2>&1)" in - *2* | *3.5* | *3.6*) - numpy_ver=1.11 - ;; - esac - retry pip install -q "numpy==${numpy_ver}" || true -fi - -echo "Testing with:" -pip freeze -conda list || true - -############################################################################## -# Smoke tests -############################################################################## -# TODO use check_binary.sh, which requires making sure it runs on Windows -pushd / -echo "Smoke testing imports" -python -c 'import torch' - -# Test that MKL is there -if [[ "$(uname)" == 'Darwin' && "$package_type" == *wheel ]]; then - echo 'Not checking for MKL on Darwin wheel packages' -else - echo "Checking that MKL is available" - python -c 'import torch; exit(0 if torch.backends.mkl.is_available() else 1)' -fi - -if [[ "$OSTYPE" == "msys" ]]; then - GPUS=$(wmic path win32_VideoController get name) - if [[ ! "$GPUS" == *NVIDIA* ]]; then - echo "Skip CUDA tests for machines without a Nvidia GPU card" - exit 0 - fi -fi - -# Test that the version number is consistent during building and testing -if [[ "$PYTORCH_BUILD_NUMBER" -gt 1 ]]; then - expected_version="${PYTORCH_BUILD_VERSION}.post${PYTORCH_BUILD_NUMBER}" -else - expected_version="${PYTORCH_BUILD_VERSION}" -fi -echo "Checking that we are testing the package that is just built" -python -c "import torch; exit(0 if torch.__version__ == '$expected_version' else 1)" - -# Test that CUDA builds are setup correctly -if [[ "$cuda_ver" != 'cpu' ]]; then - cuda_installed=1 - nvidia-smi || cuda_installed=0 - if [[ "$cuda_installed" == 0 ]]; then - echo "Skip CUDA tests for machines without a Nvidia GPU card" - else - # Test CUDA archs - echo "Checking that CUDA archs are setup correctly" - timeout 20 python -c 'import torch; torch.randn([3,5]).cuda()' - - # These have to run after CUDA is initialized - echo "Checking that magma is available" - python -c 'import torch; torch.rand(1).cuda(); exit(0 if torch.cuda.has_magma else 1)' - echo "Checking that CuDNN is available" - python -c 'import torch; exit(0 if torch.backends.cudnn.is_available() else 1)' - fi -fi - -# Check that OpenBlas is not linked to on MacOS -if [[ "$(uname)" == 'Darwin' ]]; then - echo "Checking the OpenBLAS is not linked to" - all_dylibs=($(find "$(python -c "import site; print(site.getsitepackages()[0])")"/torch -name '*.dylib')) - for dylib in "${all_dylibs[@]}"; do - if [[ -n "$(otool -L $dylib | grep -i openblas)" ]]; then - echo "Found openblas as a dependency of $dylib" - echo "Full dependencies is: $(otool -L $dylib)" - exit 1 - fi - done - - echo "Checking that OpenMP is available" - python -c "import torch; exit(0 if torch.backends.openmp.is_available() else 1)" -fi - -popd - -# TODO re-enable the other tests after the nightlies are moved to CI. This is -# because the binaries keep breaking, often from additional tests, that aren't -# real problems. Once these are on circleci and a smoke-binary-build is added -# to PRs then this should stop happening and these can be re-enabled. -echo "Not running unit tests. Hopefully these problems are caught by CI" -exit 0 - - -############################################################################## -# Running unit tests (except not right now) -############################################################################## -echo "$(date) :: Starting tests for $package_type package for python$py_ver and $cuda_ver" - -# We keep track of exact tests to skip, as otherwise we would be hardly running -# any tests. But b/c of issues working with pytest/normal-python-test/ and b/c -# of special snowflake tests in test/run_test.py we also take special care of -# those -tests_to_skip=() - -# -# Entire file exclusions -############################################################################## -entire_file_exclusions=("-x") - -# cpp_extensions doesn't work with pytest, so we exclude it from the pytest run -# here and then manually run it later. Note that this is only because this -# entire_fil_exclusions flag is only passed to the pytest run -entire_file_exclusions+=("cpp_extensions") - -# TODO temporary line to fix next days nightlies, but should be removed when -# issue is fixed -entire_file_exclusions+=('type_info') - -if [[ "$cuda_ver" == 'cpu' ]]; then - # test/test_cuda.py exits early if the installed torch is not built with - # CUDA, but the exit doesn't work when running with pytest, so pytest will - # still try to run all the CUDA tests and then fail - entire_file_exclusions+=("cuda") - entire_file_exclusions+=("nccl") -fi - -if [[ "$(uname)" == 'Darwin' || "$OSTYPE" == "msys" ]]; then - # pytest on Mac doesn't like the exits in these files - entire_file_exclusions+=('c10d') - entire_file_exclusions+=('distributed') - - # pytest doesn't mind the exit but fails the tests. On Mac we run this - # later without pytest - entire_file_exclusions+=('thd_distributed') -fi - - -# -# Universal flaky tests -############################################################################## - -# RendezvousEnvTest sometimes hangs forever -# Otherwise it will fail on CUDA with -# Traceback (most recent call last): -# File "test_c10d.py", line 179, in test_common_errors -# next(gen) -# AssertionError: ValueError not raised -tests_to_skip+=('RendezvousEnvTest and test_common_errors') - -# This hung forever once on conda_3.5_cu92 -tests_to_skip+=('TestTorch and test_sum_dim') - -# test_trace_warn isn't actually flaky, but it doesn't work with pytest so we -# just skip it -tests_to_skip+=('TestJit and test_trace_warn') -# -# Python specific flaky tests -############################################################################## - -# test_dataloader.py:721: AssertionError -# looks like a timeout, but interestingly only appears on python 3 -if [[ "$py_ver" == 3* ]]; then - tests_to_skip+=('TestDataLoader and test_proper_exit') -fi - -# -# CUDA flaky tests, all package types -############################################################################## -if [[ "$cuda_ver" != 'cpu' ]]; then - - # - # DistributedDataParallelTest - # All of these seem to fail - tests_to_skip+=('DistributedDataParallelTest') - - # - # RendezvousEnvTest - # Traceback (most recent call last): - # File "test_c10d.py", line 201, in test_nominal - # store0, rank0, size0 = next(gen0) - # File "/opt/python/cp36-cp36m/lib/python3.6/site-packages/torch/distributed/rendezvous.py", line 131, in _env_rendezvous_handler - # store = TCPStore(master_addr, master_port, start_daemon) - # RuntimeError: Address already in use - tests_to_skip+=('RendezvousEnvTest and test_nominal') - - # - # TestCppExtension - # - # Traceback (most recent call last): - # File "test_cpp_extensions.py", line 134, in test_jit_cudnn_extension - # with_cuda=True) - # File "/opt/python/cp35-cp35m/lib/python3.5/site-packages/torch/utils/cpp_extension.py", line 552, in load - # with_cuda) - # File "/opt/python/cp35-cp35m/lib/python3.5/site-packages/torch/utils/cpp_extension.py", line 729, in _jit_compile - # return _import_module_from_library(name, build_directory) - # File "/opt/python/cp35-cp35m/lib/python3.5/site-packages/torch/utils/cpp_extension.py", line 867, in _import_module_from_library - # return imp.load_module(module_name, file, path, description) - # File "/opt/python/cp35-cp35m/lib/python3.5/imp.py", line 243, in load_module - # return load_dynamic(name, filename, file) - # File "/opt/python/cp35-cp35m/lib/python3.5/imp.py", line 343, in load_dynamic - # return _load(spec) - # File "", line 693, in _load - # File "", line 666, in _load_unlocked - # File "", line 577, in module_from_spec - # File "", line 938, in create_module - # File "", line 222, in _call_with_frames_removed - # ImportError: libcudnn.so.7: cannot open shared object file: No such file or directory - tests_to_skip+=('TestCppExtension and test_jit_cudnn_extension') - - # - # TestCuda - # - - # 3.7_cu80 - # RuntimeError: CUDA error: out of memory - tests_to_skip+=('TestCuda and test_arithmetic_large_tensor') - - # 3.7_cu80 - # RuntimeError: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch-nightly_1538097262541/work/aten/src/THC/THCTensorCopy.cu:205 - tests_to_skip+=('TestCuda and test_autogpu') - - # - # TestDistBackend - # - - # Traceback (most recent call last): - # File "test_thd_distributed.py", line 1046, in wrapper - # self._join_and_reduce(fn) - # File "test_thd_distributed.py", line 1108, in _join_and_reduce - # self.assertEqual(p.exitcode, first_process.exitcode) - # File "/pytorch/test/common.py", line 399, in assertEqual - # super(TestCase, self).assertEqual(x, y, message) - # AssertionError: None != 77 : - tests_to_skip+=('TestDistBackend and test_all_gather_group') - tests_to_skip+=('TestDistBackend and test_all_reduce_group_max') - tests_to_skip+=('TestDistBackend and test_all_reduce_group_min') - tests_to_skip+=('TestDistBackend and test_all_reduce_group_sum') - tests_to_skip+=('TestDistBackend and test_all_reduce_group_product') - tests_to_skip+=('TestDistBackend and test_barrier_group') - tests_to_skip+=('TestDistBackend and test_broadcast_group') - - # Traceback (most recent call last): - # File "test_thd_distributed.py", line 1046, in wrapper - # self._join_and_reduce(fn) - # File "test_thd_distributed.py", line 1108, in _join_and_reduce - # self.assertEqual(p.exitcode, first_process.exitcode) - # File "/pytorch/test/common.py", line 397, in assertEqual - # super(TestCase, self).assertLessEqual(abs(x - y), prec, message) - # AssertionError: 12 not less than or equal to 1e-05 - tests_to_skip+=('TestDistBackend and test_barrier') - - # Traceback (most recent call last): - # File "test_distributed.py", line 1267, in wrapper - # self._join_and_reduce(fn) - # File "test_distributed.py", line 1350, in _join_and_reduce - # self.assertEqual(p.exitcode, first_process.exitcode) - # File "/pytorch/test/common.py", line 399, in assertEqual - # super(TestCase, self).assertEqual(x, y, message) - # AssertionError: None != 1 - tests_to_skip+=('TestDistBackend and test_broadcast') - - # Memory leak very similar to all the conda ones below, but appears on manywheel - # 3.6m_cu80 - # AssertionError: 1605632 not less than or equal to 1e-05 : __main__.TestEndToEndHybridFrontendModels.test_vae_cuda leaked 1605632 bytes CUDA memory on device 0 - tests_to_skip+=('TestEndToEndHybridFrontendModels and test_vae_cuda') - - # ________________________ TestNN.test_embedding_bag_cuda ________________________ - # - # self = - # dtype = torch.float32 - # - # @unittest.skipIf(not TEST_CUDA, "CUDA unavailable") - # @repeat_test_for_types(ALL_TENSORTYPES) - # @skipIfRocm - # def test_embedding_bag_cuda(self, dtype=torch.float): - # self._test_EmbeddingBag(True, 'sum', False, dtype) - # self._test_EmbeddingBag(True, 'mean', False, dtype) - # self._test_EmbeddingBag(True, 'max', False, dtype) - # if dtype != torch.half: - # # torch.cuda.sparse.HalfTensor is not enabled. - # self._test_EmbeddingBag(True, 'sum', True, dtype) - # > self._test_EmbeddingBag(True, 'mean', True, dtype) - # - # test_nn.py:2144: - # _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ - # test_nn.py:2062: in _test_EmbeddingBag - # _test_vs_Embedding(N, D, B, L) - # test_nn.py:2059: in _test_vs_Embedding - # self.assertEqual(es_weight_grad, e.weight.grad, needed_prec) - # common.py:373: in assertEqual - # assertTensorsEqual(x, y) - # common.py:365: in assertTensorsEqual - # self.assertLessEqual(max_err, prec, message) - # E AssertionError: tensor(0.0000, device='cuda:0', dtype=torch.float32) not less than or equal to 2e-05 : - # 1 failed, 1202 passed, 19 skipped, 2 xfailed, 796 warnings in 1166.73 seconds = - # Traceback (most recent call last): - # File "test/run_test.py", line 391, in - # main() - # File "test/run_test.py", line 383, in main - # raise RuntimeError(message) - tests_to_skip+=('TestNN and test_embedding_bag_cuda') -fi - - -########################################################################## -# Conda specific flaky tests -########################################################################## - -# Only on Anaconda's python 2.7 -# So, this doesn't really make sense. All the mac jobs are run on the same -# machine, so the wheel jobs still use conda to silo their python -# installations. The wheel job for Python 2.7 should use the exact same Python -# from conda as the conda job for Python 2.7. Yet, this only appears on the -# conda jobs. -if [[ "$package_type" == 'conda' && "$py_ver" == '2.7' ]]; then - # Traceback (most recent call last): - # File "test_jit.py", line 6281, in test_wrong_return_type - # @torch.jit.script - # File "/Users/administrator/nightlies/2018_09_30/wheel_build_dirs/conda_2.7/conda/envs/env_py2.7_0_20180930/lib/python2.7/site-packages/torch/jit/__init__.py", line 639, in script - # graph = _jit_script_compile(ast, rcb) - # File "/Users/administrator/nightlies/2018_09_30/wheel_build_dirs/conda_2.7/conda/envs/env_py2.7_0_20180930/lib/python2.7/site-packages/torch/jit/annotations.py", line 80, in get_signature - # return parse_type_line(type_line) - # File "/Users/administrator/nightlies/2018_09_30/wheel_build_dirs/conda_2.7/conda/envs/env_py2.7_0_20180930/lib/python2.7/site-packages/torch/jit/annotations.py", line 131, in parse_type_line - # return arg_types, ann_to_type(ret_ann) - # File "/Users/administrator/nightlies/2018_09_30/wheel_build_dirs/conda_2.7/conda/envs/env_py2.7_0_20180930/lib/python2.7/site-packages/torch/jit/annotations.py", line 192, in ann_to_type - # return TupleType([ann_to_type(a) for a in ann.__args__]) - # TypeError: 'TupleInstance' object is not iterable - tests_to_skip+=('TestScript and test_wrong_return_type') -fi - -# Lots of memory leaks on CUDA -if [[ "$package_type" == 'conda' && "$cuda_ver" != 'cpu' ]]; then - - # 3.7_cu92 - # AssertionError: 63488 not less than or equal to 1e-05 : __main__.TestEndToEndHybridFrontendModels.test_mnist_cuda leaked 63488 bytes CUDA memory on device 0 - tests_to_skip+=('TestEndToEndHybridFrontendModels and test_mnist_cuda') - - # 2.7_cu92 - # AssertionError: __main__.TestNN.test_BatchNorm3d_momentum_eval_cuda leaked -1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestNN and test_BatchNorm3d_momentum_eval_cuda') - - # - # All of test_BCE is flaky - tests_to_skip+=('TestNN and test_BCE') - - # 3.5_cu80 - # AssertionError: 3584 not less than or equal to 1e-05 : test_nn.TestNN.test_BCEWithLogitsLoss_cuda_double leaked 3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_BCEWithLogitsLoss_cuda_double') - - # 2.7_cu92 - # AssertionError: __main__.TestNN.test_ConvTranspose2d_cuda leaked -1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestNN and test_ConvTranspose2d_cuda') - - # 3.7_cu90 - # AssertionError: 1024 not less than or equal to 1e-05 : __main__.TestNN.test_ConvTranspose3d_cuda leaked -1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestNN and test_ConvTranspose3d_cuda') - - # - # - # CTCLoss - # These are all flaky - tests_to_skip+=('TestNN and test_CTCLoss') - - # 2.7_cu90 - # 2.7_cu92 - # 3.5_cu90 x2 - # 3.6_cu90 - # 3.7_cu80 x3 - # 3.7_cu90 - # AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_1d_target_cuda_double leaked 37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_1d_target_cuda_double') - - # 2.7_cu80 --18944 - # 2.7_cu92 - # 3.5_cu90 --18944 x2 - # 3.5_cu92 --18944 x2 - # 3.6_cu90 --18944 - # 3.6_cu92 --18944 - # 3.7_cu80 - # AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_1d_target_cuda_float leaked -37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_1d_target_cuda_float') - - # 3.5_cu90 - # 3.7_cu92 - # AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_1d_target_sum_reduction_cuda_double leaked 37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_1d_target_sum_reduction_cuda_double') - - # 3.7_cu92 - # AssertionError: 18432 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_1d_target_sum_reduction_cuda_float leaked -18432 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_1d_target_sum_reduction_cuda_float') - - # 3.5_cu92 x2 - # 3.6_cu80 - # 3.7_cu90 - # AssertionError: AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_2d_int_target_cuda_double leaked 37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_2d_int_target_cuda_double') - - # 3.5_cu92 - # 3.6_cu80 --37376 - # 3.6_cu92 - # AssertionError: 18944 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_2d_int_target_cuda_float leaked 18944 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_2d_int_target_cuda_float') - - # 2.7_cu90 - # 3.5_cu80 - # 3.7_cu80 x2 - # AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_2d_int_target_sum_reduction_cuda_double leaked 37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_2d_int_target_sum_reduction_cuda_double') - - # 2.7_cu90 - # 2.7_cu92 --18944 - # AssertionError: __main__.TestNN.test_CTCLoss_2d_int_target_sum_reduction_cuda_float leaked -37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_2d_int_target_sum_reduction_cuda_float') - - # 2.7_cu92 - # AssertionError: __main__.TestNN.test_CTCLoss_cuda_double leaked 37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_cuda_double') - - # 2.7_cu92 - # AssertionError: __main__.TestNN.test_CTCLoss_cuda_float leaked 18944 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_cuda_float') - - # 2.7_cu92 - # 3.5_cu90 x2 - # 3.5_cu92 - # 3.5_cu92 - # 3.6_cu80 x2 - # AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_sum_reduction_cuda_double leaked 37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_sum_reduction_cuda_double') - - # 2.7_cu92 --18944 - # 3.6_cu80 - # AssertionError: 37376 not less than or equal to 1e-05 : __main__.TestNN.test_CTCLoss_sum_reduction_cuda_float leaked -37376 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_CTCLoss_sum_reduction_cuda_float') - - # - # - # NLLLoss - # These are all flaky - tests_to_skip+=('TestNN and NLLLoss') - - # 3.5_cu90 x2 - # AssertionError: 3584 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_cuda_double leaked 3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_cuda_double') - - # 2.7_cu80 - # AssertionError: __main__.TestNN.test_NLLLoss_2d_cuda_float leaked 2560 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_cuda_float') - - # 2.7_cu80 - # 2.7_cu92 - # 3.6_cu80 x2 - # AssertionError: 1536 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_cuda_half leaked 1536 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_cuda_half') - - # 2.7_cu90 - # 3.6_cu80 x2 - # 3.6_cu90 - # 3.6_cu92 - # AssertionError: 3584 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_ignore_index_cuda_double leaked 3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_ignore_index_cuda_double') - - # 3.6_cu80 x2 - # 3.6_cu90 - # AssertionError: 3584 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_ignore_index_cuda_float leaked -3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_ignore_index_cuda_float') - - # 3.6_cu90 - # AssertionError: 3584 not less than or equal to 1e-05 : test_nn.TestNN.test_NLLLoss_2d_weights_cuda_double leaked 3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_ignore_index_cuda_half') - - # 3.6_cu80 - # AssertionError: 3584 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_sum_reduction_cuda_double leaked 3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_sum_reduction_cuda_double') - - # 3.6_cu80 - # AssertionError: 2560 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_sum_reduction_cuda_float leaked 2560 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_sum_reduction_cuda_float') - - # 3.7_cu92 - # AssertionError: 1536 not less than or equal to 1e-05 : test_nn.TestNN.test_NLLLoss_2d_weights_cuda_half leaked 1536 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_weights_cuda_half') - - # 3.6_cu80 - # AssertionError: 1536 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_2d_sum_reduction_cuda_half leaked 1536 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_sum_reduction_cuda_half') - - # 2.7_cu92 - # AssertionError: __main__.TestNN.test_NLLLoss_2d_weights_cuda_float leaked 2560 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_2d_weights_cuda_float') - - # 3.5_cu80 x2 - # 3.6_cu90 - # AssertionError: 1536 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_dim_is_3_cuda_double leaked 1536 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_dim_is_3_cuda_double') - - # 3.6_cu80 - # AssertionError: 1536 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_dim_is_3_sum_reduction_cuda_float leaked 1536 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_dim_is_3_sum_reduction_cuda_float') - - # 3.6_cu80 - # 3.7_cu80 x2 - # AssertionError: 1536 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_dim_is_3_sum_reduction_cuda_half leaked 1536 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_dim_is_3_sum_reduction_cuda_half') - - # 3.5_cu80 - # 3.7_cu80 x2 - # AssertionError: 10752 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_higher_dim_cuda_double leaked 10752 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_higher_dim_cuda_double') - - # 3.5_cu80 - # 3.7_cu80 --10752 x2 - # AssertionError: 5120 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_higher_dim_cuda_float leaked -5120 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_higher_dim_cuda_float') - - # 3.5_cu80 - # 3.5 cu90 - # AssertionError: 3584 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_higher_dim_cuda_half leaked 3584 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_higher_dim_cuda_half') - - # 3.5_cu90 - # AssertionError: 10752 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_higher_dim_sum_reduction_cuda_double leaked 10752 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_higher_dim_sum_reduction_cuda_double') - - # 3.5_cu90 - # AssertionError: 5120 not less than or equal to 1e-05 : __main__.TestNN.test_NLLLoss_higher_dim_sum_reduction_cuda_float leaked -5120 bytes CUDA memory on device 0 - #tests_to_skip+=('TestNN and test_NLLLoss_higher_dim_sum_reduction_cuda_float') - - # ______________________ TestNN.test_variable_sequence_cuda ______________________ - # common_utils.py:277: in wrapper - # method(*args, **kwargs) - # common_utils.py:241: in __exit__ - # self.name, after - before, i)) - # common_utils.py:399: in assertEqual - # super(TestCase, self).assertLessEqual(abs(x - y), prec, message) - # E AssertionError: 1024 not less than or equal to 1e-05 : test_nn.TestNN.test_variable_sequence_cuda leaked 1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestNN and test_variable_sequence_cuda') - - # 3.7_cu90 - # AssertionError: 1024 not less than or equal to 1e-05 : __main__.TestJit.test_fuse_last_device_cuda leaked 1024 bytes CUDA memory on device 1 - tests_to_skip+=('TestJit and test_fuse_last_device_cuda') - - # 3.7_cu92 x2 - # AssertionError: 1024 not less than or equal to 1e-05 : __main__.TestJit.test_ge_cuda leaked 1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestJit and test_ge_cuda') - - # 3.5_cu90 - # AssertionError: 1024 not less than or equal to 1e-05 : test_jit.TestJit.test_comparison_ge_le_cuda leaked -1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestJit and test_comparison_ge_le_cuda') - - # 3.6_cu92 - # 3.7_cu92 - # AssertionError: 1024 not less than or equal to 1e-05 : __main__.TestJit.test_relu_cuda leaked 1024 bytes CUDA memory on device 0 - tests_to_skip+=('TestJit and test_relu_cuda') - - # 3.7_cu92 x3 - # AssertionError: 1024 not less than or equal to 1e-05 : __main__.TestScript.test_milstm_fusion_cuda leaked 1024 bytes CUDA memory on device 1 - tests_to_skip+=('TestScript and test_milstm_fusion_cuda') -fi - - -############################################################################## -# MacOS specific flaky tests -############################################################################## - -if [[ "$(uname)" == 'Darwin' ]]; then - # TestCppExtensions by default uses a temp folder in /tmp. This doesn't - # work for this Mac machine cause there is only one machine and /tmp is - # shared. (All the linux builds are on docker so have their own /tmp). - tests_to_skip+=('TestCppExtension') -fi - -if [[ "$(uname)" == 'Darwin' && "$package_type" == 'conda' ]]; then - - # - # TestDistBackend - # Seems like either most of the Mac builds get this error or none of them - # do - # - - # Traceback (most recent call last): - # File "test_thd_distributed.py", line 1046, in wrapper - # self._join_and_reduce(fn) - # File "test_thd_distributed.py", line 1120, in _join_and_reduce - # first_process.exitcode == SKIP_IF_SMALL_WORLDSIZE_EXIT_CODE - # AssertionError - tests_to_skip+=('TestDistBackend and test_reduce_group_max') - - # Traceback (most recent call last): - # File "test_thd_distributed.py", line 1046, in wrapper - # self._join_and_reduce(fn) - # File "test_thd_distributed.py", line 1132, in _join_and_reduce - # self.assertEqual(first_process.exitcode, 0) - # File "/Users/administrator/nightlies/2018_10_01/wheel_build_dirs/conda_2.7/pytorch/test/common.py", line 397, in assertEqual - # super(TestCase, self).assertLessEqual(abs(x - y), prec, message) - # AssertionError: 1 not less than or equal to 1e-05 - tests_to_skip+=('TestDistBackend and test_isend') - tests_to_skip+=('TestDistBackend and test_reduce_group_min') - tests_to_skip+=('TestDistBackend and test_reduce_max') - tests_to_skip+=('TestDistBackend and test_reduce_min') - tests_to_skip+=('TestDistBackend and test_reduce_group_max') - tests_to_skip+=('TestDistBackend and test_reduce_group_min') - tests_to_skip+=('TestDistBackend and test_reduce_max') - tests_to_skip+=('TestDistBackend and test_reduce_min') - tests_to_skip+=('TestDistBackend and test_reduce_product') - tests_to_skip+=('TestDistBackend and test_reduce_sum') - tests_to_skip+=('TestDistBackend and test_scatter') - tests_to_skip+=('TestDistBackend and test_send_recv') - tests_to_skip+=('TestDistBackend and test_send_recv_any_source') -fi - - -# Turn the set of tests to skip into an invocation that pytest understands -excluded_tests_logic='' -for exclusion in "${tests_to_skip[@]}"; do - if [[ -z "$excluded_tests_logic" ]]; then - # Only true for i==0 - excluded_tests_logic="not ($exclusion)" - else - excluded_tests_logic="$excluded_tests_logic and not ($exclusion)" - fi -done - - -############################################################################## -# Run the tests -############################################################################## -echo -echo "$(date) :: Calling 'python test/run_test.py -v -p pytest ${entire_file_exclusions[@]} -- --disable-pytest-warnings -k '$excluded_tests_logic'" - -python test/run_test.py -v -p pytest ${entire_file_exclusions[@]} -- --disable-pytest-warnings -k "'" "$excluded_tests_logic" "'" - -echo -echo "$(date) :: Finished 'python test/run_test.py -v -p pytest ${entire_file_exclusions[@]} -- --disable-pytest-warnings -k '$excluded_tests_logic'" - -# cpp_extensions don't work with pytest, so we run them without pytest here, -# except there's a failure on CUDA builds (documented above), and -# cpp_extensions doesn't work on a shared mac machine (also documented above) -if [[ "$cuda_ver" == 'cpu' && "$(uname)" != 'Darwin' ]]; then - echo - echo "$(date) :: Calling 'python test/run_test.py -v -i cpp_extensions'" - python test/run_test.py -v -i cpp_extensions - echo - echo "$(date) :: Finished 'python test/run_test.py -v -i cpp_extensions'" -fi - -# thd_distributed can run on Mac but not in pytest -if [[ "$(uname)" == 'Darwin' ]]; then - echo - echo "$(date) :: Calling 'python test/run_test.py -v -i thd_distributed'" - python test/run_test.py -v -i thd_distributed - echo - echo "$(date) :: Finished 'python test/run_test.py -v -i thd_distributed'" -fi diff --git a/test/check_binary_symbols.py b/test/check_binary_symbols.py deleted file mode 100755 index dba11fbe9..000000000 --- a/test/check_binary_symbols.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -import concurrent.futures -import distutils.sysconfig -import itertools -import functools -import os -import re -from pathlib import Path -from typing import Any, List, Tuple - -# We also check that there are [not] cxx11 symbols in libtorch -# -# To check whether it is using cxx11 ABI, check non-existence of symbol: -PRE_CXX11_SYMBOLS=( - "std::basic_string<", - "std::list", -) -# To check whether it is using pre-cxx11 ABI, check non-existence of symbol: -CXX11_SYMBOLS=( - "std::__cxx11::basic_string", - "std::__cxx11::list", -) -# NOTE: Checking the above symbols in all namespaces doesn't work, because -# devtoolset7 always produces some cxx11 symbols even if we build with old ABI, -# and CuDNN always has pre-cxx11 symbols even if we build with new ABI using gcc 5.4. -# Instead, we *only* check the above symbols in the following namespaces: -LIBTORCH_NAMESPACE_LIST=( - "c10::", - "at::", - "caffe2::", - "torch::", -) - - -def _apply_libtorch_symbols(symbols): - return [re.compile(f"{x}.*{y}") for (x,y) in itertools.product(LIBTORCH_NAMESPACE_LIST, symbols)] - - -LIBTORCH_CXX11_PATTERNS = _apply_libtorch_symbols(CXX11_SYMBOLS) - -LIBTORCH_PRE_CXX11_PATTERNS = _apply_libtorch_symbols(PRE_CXX11_SYMBOLS) - -@functools.lru_cache(100) -def get_symbols(lib :str ) -> List[Tuple[str, str, str]]: - from subprocess import check_output - lines = check_output(f'nm "{lib}"|c++filt', shell=True) - return [x.split(' ', 2) for x in lines.decode('latin1').split('\n')[:-1]] - - -def grep_symbols(lib: str, patterns: List[Any]) -> List[str]: - def _grep_symbols(symbols: List[Tuple[str, str, str]], patterns: List[Any]) -> List[str]: - rc = [] - for _s_addr, _s_type, s_name in symbols: - for pattern in patterns: - if pattern.match(s_name): - rc.append(s_name) - continue - return rc - all_symbols = get_symbols(lib) - num_workers= 32 - chunk_size = (len(all_symbols) + num_workers - 1 ) // num_workers - def _get_symbols_chunk(i): - return all_symbols[i * chunk_size : (i + 1) * chunk_size] - - with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: - tasks = [executor.submit(_grep_symbols, _get_symbols_chunk(i), patterns) for i in range(num_workers)] - return functools.reduce(list.__add__, (x.result() for x in tasks), []) - - -def check_lib_symbols_for_abi_correctness(lib: str, pre_cxx11_abi: bool = True) -> None: - print(f"lib: {lib}") - cxx11_symbols = grep_symbols(lib, LIBTORCH_CXX11_PATTERNS) - pre_cxx11_symbols = grep_symbols(lib, LIBTORCH_PRE_CXX11_PATTERNS) - num_cxx11_symbols = len(cxx11_symbols) - num_pre_cxx11_symbols = len(pre_cxx11_symbols) - print(f"num_cxx11_symbols: {num_cxx11_symbols}") - print(f"num_pre_cxx11_symbols: {num_pre_cxx11_symbols}") - if pre_cxx11_abi: - if num_cxx11_symbols > 0: - raise RuntimeError(f"Found cxx11 symbols, but there shouldn't be any, see: {cxx11_symbols[:100]}") - if num_pre_cxx11_symbols < 1000: - raise RuntimeError("Didn't find enough pre-cxx11 symbols.") - # Check for no recursive iterators, regression test for https://github.com/pytorch/pytorch/issues/133437 - rec_iter_symbols = grep_symbols(lib, [re.compile("std::filesystem::recursive_directory_iterator.*")]) - if len(rec_iter_symbols) > 0: - raise RuntimeError(f"recursive_directory_iterator in used pre-CXX11 binaries, see; {rec_iter_symbols}") - else: - if num_pre_cxx11_symbols > 0: - raise RuntimeError(f"Found pre-cxx11 symbols, but there shouldn't be any, see: {pre_cxx11_symbols[:100]}") - if num_cxx11_symbols < 100: - raise RuntimeError("Didn't find enought cxx11 symbols") - - -def main() -> None: - if "install_root" in os.environ: - install_root = Path(os.getenv("install_root")) # noqa: SIM112 - else: - if os.getenv("PACKAGE_TYPE") == "libtorch": - install_root = Path(os.getcwd()) - else: - install_root = Path(distutils.sysconfig.get_python_lib()) / "torch" - - libtorch_cpu_path = install_root / "lib" / "libtorch_cpu.so" - pre_cxx11_abi = "cxx11-abi" not in os.getenv("DESIRED_DEVTOOLSET", "") - check_lib_symbols_for_abi_correctness(libtorch_cpu_path, pre_cxx11_abi) - - -if __name__ == "__main__": - main() diff --git a/test/smoke_test/assets/dog2.jpg b/test/smoke_test/assets/dog2.jpg deleted file mode 100644 index 528dfec72..000000000 Binary files a/test/smoke_test/assets/dog2.jpg and /dev/null differ diff --git a/test/smoke_test/assets/rgb_pytorch.jpg b/test/smoke_test/assets/rgb_pytorch.jpg deleted file mode 100644 index d49e658b9..000000000 Binary files a/test/smoke_test/assets/rgb_pytorch.jpg and /dev/null differ diff --git a/test/smoke_test/assets/rgb_pytorch.png b/test/smoke_test/assets/rgb_pytorch.png deleted file mode 100644 index c9d08e6c7..000000000 Binary files a/test/smoke_test/assets/rgb_pytorch.png and /dev/null differ diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py deleted file mode 100644 index d8ba7f8d4..000000000 --- a/test/smoke_test/smoke_test.py +++ /dev/null @@ -1,352 +0,0 @@ -import os -import re -import sys -import argparse -import torch -import json -import importlib -import subprocess -import torch._dynamo -import torch.nn as nn -import torch.nn.functional as F -from pathlib import Path - -if "MATRIX_GPU_ARCH_VERSION" in os.environ: - gpu_arch_ver = os.getenv("MATRIX_GPU_ARCH_VERSION") -else: - gpu_arch_ver = os.getenv("GPU_ARCH_VERSION") # Use fallback if available -gpu_arch_type = os.getenv("MATRIX_GPU_ARCH_TYPE") -channel = os.getenv("MATRIX_CHANNEL") -package_type = os.getenv("MATRIX_PACKAGE_TYPE") -target_os = os.getenv("TARGET_OS", sys.platform) -BASE_DIR = Path(__file__).parent.parent.parent - -is_cuda_system = gpu_arch_type == "cuda" -NIGHTLY_ALLOWED_DELTA = 3 - -MODULES = [ - { - "name": "torchvision", - "repo": "https://github.com/pytorch/vision.git", - "smoke_test": "./vision/test/smoke_test.py", - "extension": "extension", - "repo_name": "vision", - }, - { - "name": "torchaudio", - "repo": "https://github.com/pytorch/audio.git", - "smoke_test": "./audio/test/smoke_test/smoke_test.py --no-ffmpeg", - "extension": "_extension", - "repo_name": "audio", - }, -] - - -class Net(nn.Module): - def __init__(self): - super().__init__() - self.conv1 = nn.Conv2d(1, 32, 3, 1) - self.conv2 = nn.Conv2d(32, 64, 3, 1) - self.fc1 = nn.Linear(9216, 1) - - def forward(self, x): - x = self.conv1(x) - x = self.conv2(x) - x = F.max_pool2d(x, 2) - x = torch.flatten(x, 1) - output = self.fc1(x) - return output - -def load_json_from_basedir(filename: str): - try: - with open(BASE_DIR / filename) as fptr: - return json.load(fptr) - except FileNotFoundError as exc: - raise ImportError(f"File {filename} not found error: {exc.strerror}") from exc - except json.JSONDecodeError as exc: - raise ImportError(f"Invalid JSON {filename}") from exc - -def read_release_matrix(): - return load_json_from_basedir("release_matrix.json") - -def test_numpy(): - import numpy as np - x = np.arange(5) - torch.tensor(x) - -def check_version(package: str) -> None: - release_version = os.getenv("RELEASE_VERSION") - # if release_version is specified, use it to validate the packages - if(release_version): - release_matrix = read_release_matrix() - stable_version = release_matrix["torch"] - else: - stable_version = os.getenv("MATRIX_STABLE_VERSION") - - # only makes sense to check nightly package where dates are known - if channel == "nightly": - check_nightly_binaries_date(package) - elif stable_version is not None: - if not torch.__version__.startswith(stable_version): - raise RuntimeError( - f"Torch version mismatch, expected {stable_version} for channel {channel}. But its {torch.__version__}" - ) - - if release_version and package == "all": - for module in MODULES: - imported_module = importlib.import_module(module["name"]) - module_version = imported_module.__version__ - if not module_version.startswith(release_matrix[module["name"]]): - raise RuntimeError( - f"{module['name']} version mismatch, expected: \ - {release_matrix[module['name']]} for channel {channel}. But its {module_version}" - ) - else: - print(f"{module['name']} version actual: {module_version} expected: \ - {release_matrix[module['name']]} for channel {channel}.") - - else: - print(f"Skip version check for channel {channel} as stable version is None") - - -def check_nightly_binaries_date(package: str) -> None: - from datetime import datetime - format_dt = '%Y%m%d' - - date_t_str = re.findall("dev\\d+", torch.__version__) - date_t_delta = datetime.now() - datetime.strptime(date_t_str[0][3:], format_dt) - if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA: - raise RuntimeError( - f"the binaries are from {date_t_str} and are more than {NIGHTLY_ALLOWED_DELTA} days old!" - ) - - if package == "all": - for module in MODULES: - imported_module = importlib.import_module(module["name"]) - module_version = imported_module.__version__ - date_m_str = re.findall("dev\\d+", module_version) - date_m_delta = datetime.now() - datetime.strptime(date_m_str[0][3:], format_dt) - print(f"Nightly date check for {module['name']} version {module_version}") - if date_m_delta.days > NIGHTLY_ALLOWED_DELTA: - raise RuntimeError( - f"Expected {module['name']} to be less then {NIGHTLY_ALLOWED_DELTA} days. But its {date_m_delta}" - ) - - -def test_cuda_runtime_errors_captured() -> None: - cuda_exception_missed = True - try: - print("Testing test_cuda_runtime_errors_captured") - torch._assert_async(torch.tensor(0, device="cuda")) - torch._assert_async(torch.tensor(0 + 0j, device="cuda")) - except RuntimeError as e: - if re.search("CUDA", f"{e}"): - print(f"Caught CUDA exception with success: {e}") - cuda_exception_missed = False - else: - raise e - if cuda_exception_missed: - raise RuntimeError("Expected CUDA RuntimeError but have not received!") - - -def smoke_test_cuda(package: str, runtime_error_check: str, torch_compile_check: str) -> None: - if not torch.cuda.is_available() and is_cuda_system: - raise RuntimeError(f"Expected CUDA {gpu_arch_ver}. However CUDA is not loaded.") - - if package == 'all' and is_cuda_system: - for module in MODULES: - imported_module = importlib.import_module(module["name"]) - # TBD for vision move extension module to private so it will - # be _extention. - version = "N/A" - if module["extension"] == "extension": - version = imported_module.extension._check_cuda_version() - else: - version = imported_module._extension._check_cuda_version() - print(f"{module['name']} CUDA: {version}") - - # torch.compile is available on macos-arm64 and Linux for python 3.8-3.13 - if (torch_compile_check == "enabled" and sys.version_info < (3, 13, 0) - and target_os in ["linux", "linux-aarch64", "macos-arm64", "darwin"]): - smoke_test_compile("cuda" if torch.cuda.is_available() else "cpu") - - if torch.cuda.is_available(): - if torch.version.cuda != gpu_arch_ver: - raise RuntimeError( - f"Wrong CUDA version. Loaded: {torch.version.cuda} Expected: {gpu_arch_ver}" - ) - print(f"torch cuda: {torch.version.cuda}") - # todo add cudnn version validation - print(f"torch cudnn: {torch.backends.cudnn.version()}") - print(f"cuDNN enabled? {torch.backends.cudnn.enabled}") - - torch.cuda.init() - print("CUDA initialized successfully") - print(f"Number of CUDA devices: {torch.cuda.device_count()}") - for i in range(torch.cuda.device_count()): - print(f"Device {i}: {torch.cuda.get_device_name(i)}") - - # nccl is availbale only on Linux - if (sys.platform in ["linux", "linux2"]): - print(f"torch nccl version: {torch.cuda.nccl.version()}") - - if runtime_error_check == "enabled": - test_cuda_runtime_errors_captured() - - -def smoke_test_conv2d() -> None: - import torch.nn as nn - - print("Testing smoke_test_conv2d") - # With square kernels and equal stride - m = nn.Conv2d(16, 33, 3, stride=2) - # non-square kernels and unequal stride and with padding - m = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2)) - assert m is not None - # non-square kernels and unequal stride and with padding and dilation - basic_conv = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2), dilation=(3, 1)) - input = torch.randn(20, 16, 50, 100) - output = basic_conv(input) - - if is_cuda_system: - print("Testing smoke_test_conv2d with cuda") - conv = nn.Conv2d(3, 3, 3).cuda() - x = torch.randn(1, 3, 24, 24, device="cuda") - with torch.cuda.amp.autocast(): - out = conv(x) - assert out is not None - - supported_dtypes = [torch.float16, torch.float32, torch.float64] - for dtype in supported_dtypes: - print(f"Testing smoke_test_conv2d with cuda for {dtype}") - conv = basic_conv.to(dtype).cuda() - input = torch.randn(20, 16, 50, 100, device="cuda").type(dtype) - output = conv(input) - assert output is not None - - -def test_linalg(device="cpu") -> None: - print(f"Testing smoke_test_linalg on {device}") - A = torch.randn(5, 3, device=device) - U, S, Vh = torch.linalg.svd(A, full_matrices=False) - assert U.shape == A.shape and S.shape == torch.Size([3]) and Vh.shape == torch.Size([3, 3]) - torch.dist(A, U @ torch.diag(S) @ Vh) - - U, S, Vh = torch.linalg.svd(A) - assert U.shape == torch.Size([5, 5]) and S.shape == torch.Size([3]) and Vh.shape == torch.Size([3, 3]) - torch.dist(A, U[:, :3] @ torch.diag(S) @ Vh) - - A = torch.randn(7, 5, 3, device=device) - U, S, Vh = torch.linalg.svd(A, full_matrices=False) - torch.dist(A, U @ torch.diag_embed(S) @ Vh) - - if device == "cuda": - supported_dtypes = [torch.float32, torch.float64] - for dtype in supported_dtypes: - print(f"Testing smoke_test_linalg with cuda for {dtype}") - A = torch.randn(20, 16, 50, 100, device=device, dtype=dtype) - torch.linalg.svd(A) - - -def smoke_test_compile(device: str = "cpu") -> None: - supported_dtypes = [torch.float16, torch.float32, torch.float64] - - def foo(x: torch.Tensor) -> torch.Tensor: - return torch.sin(x) + torch.cos(x) - - for dtype in supported_dtypes: - print(f"Testing smoke_test_compile for {device} and {dtype}") - x = torch.rand(3, 3, device=device).type(dtype) - x_eager = foo(x) - x_pt2 = torch.compile(foo)(x) - torch.testing.assert_close(x_eager, x_pt2) - - # Check that SIMD were detected for the architecture - if device == "cpu": - from torch._inductor.codecache import pick_vec_isa - isa = pick_vec_isa() - if not isa: - raise RuntimeError("Can't detect vectorized ISA for CPU") - print(f"Picked CPU ISA {type(isa).__name__} bit width {isa.bit_width()}") - - # Reset torch dynamo since we are changing mode - torch._dynamo.reset() - dtype = torch.float32 - torch.set_float32_matmul_precision('high') - print(f"Testing smoke_test_compile with mode 'max-autotune' for {dtype}") - x = torch.rand(64, 1, 28, 28, device=device).type(torch.float32) - model = Net().to(device=device) - x_pt2 = torch.compile(model, mode="max-autotune")(x) - - -def smoke_test_modules(): - cwd = os.getcwd() - for module in MODULES: - if module["repo"]: - if not os.path.exists(f"{cwd}/{module['repo_name']}"): - print(f"Path does not exist: {cwd}/{module['repo_name']}") - try: - subprocess.check_output( - f"git clone --depth 1 {module['repo']}", - stderr=subprocess.STDOUT, - shell=True, - ) - except subprocess.CalledProcessError as exc: - raise RuntimeError( - f"Cloning {module['repo']} FAIL: {exc.returncode} Output: {exc.output}" - ) from exc - try: - smoke_test_command = f"python3 {module['smoke_test']}" - if target_os == 'windows': - smoke_test_command = f"python {module['smoke_test']}" - output = subprocess.check_output( - smoke_test_command, stderr=subprocess.STDOUT, shell=True, - universal_newlines=True) - except subprocess.CalledProcessError as exc: - raise RuntimeError(f"Module {module['name']} FAIL: {exc.returncode} Output: {exc.output}") from exc - else: - print(f"Output: \n{output}\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "--package", - help="Package to include in smoke testing", - type=str, - choices=["all", "torchonly"], - default="all", - ) - parser.add_argument( - "--runtime-error-check", - help="No Runtime Error check", - type=str, - choices=["enabled", "disabled"], - default="enabled", - ) - parser.add_argument( - "--torch-compile-check", - help="Check torch compile", - type=str, - choices=["enabled", "disabled"], - default="enabled", - ) - options = parser.parse_args() - print(f"torch: {torch.__version__}") - print(torch.__config__.parallel_info()) - - check_version(options.package) - smoke_test_conv2d() - test_linalg() - test_numpy() - if is_cuda_system: - test_linalg("cuda") - - if options.package == "all": - smoke_test_modules() - - smoke_test_cuda(options.package, options.runtime_error_check, options.torch_compile_check) - - -if __name__ == "__main__": - main() diff --git a/test_example_code/CMakeLists.txt b/test_example_code/CMakeLists.txt deleted file mode 100644 index 1724a6ed0..000000000 --- a/test_example_code/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(simple-torch-test) - -find_package(Torch REQUIRED) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}") - - -add_executable(simple-torch-test simple-torch-test.cpp) -target_include_directories(simple-torch-test PRIVATE ${TORCH_INCLUDE_DIRS}) -target_link_libraries(simple-torch-test "${TORCH_LIBRARIES}") -set_property(TARGET simple-torch-test PROPERTY CXX_STANDARD 17) - -find_package(CUDAToolkit 11.8) - -target_link_libraries(simple-torch-test CUDA::cudart CUDA::cufft CUDA::cusparse CUDA::cublas CUDA::cusolver) -find_library(CUDNN_LIBRARY NAMES cudnn) -target_link_libraries(simple-torch-test ${CUDNN_LIBRARY} ) -if (MSVC) - file(GLOB TORCH_DLLS "$ENV{CUDA_PATH}/bin/cudnn64_8.dll" "$ENV{NVTOOLSEXT_PATH}/bin/x64/*.dll") - message("dlls to copy " ${TORCH_DLLS}) - add_custom_command(TARGET simple-torch-test - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${TORCH_DLLS} - $) -endif (MSVC) diff --git a/test_example_code/check-torch-cuda.cpp b/test_example_code/check-torch-cuda.cpp deleted file mode 100644 index 35e9f8ebc..000000000 --- a/test_example_code/check-torch-cuda.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include - -int main(int argc, const char* argv[]) { - std::cout << "Checking that CUDA archs are setup correctly" << std::endl; - TORCH_CHECK(torch::rand({ 3, 5 }, torch::Device(torch::kCUDA)).defined(), "CUDA archs are not setup correctly"); - - // These have to run after CUDA is initialized - - std::cout << "Checking that magma is available" << std::endl; - TORCH_CHECK(torch::hasMAGMA(), "MAGMA is not available"); - - std::cout << "Checking that CuDNN is available" << std::endl; - TORCH_CHECK(torch::cuda::cudnn_is_available(), "CuDNN is not available"); - return 0; -} diff --git a/test_example_code/check-torch-mkl.cpp b/test_example_code/check-torch-mkl.cpp deleted file mode 100644 index 419898d79..000000000 --- a/test_example_code/check-torch-mkl.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(int argc, const char* argv[]) { - TORCH_CHECK(torch::hasMKL(), "MKL is not available"); - return 0; -} diff --git a/test_example_code/check-torch-xnnpack.cpp b/test_example_code/check-torch-xnnpack.cpp deleted file mode 100644 index b0e901954..000000000 --- a/test_example_code/check-torch-xnnpack.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include - -int main(int argc, const char* argv[]) { - TORCH_CHECK(at::globalContext().isXNNPACKAvailable(), "XNNPACK is not available"); - return 0; -} diff --git a/test_example_code/cnn_smoke.py b/test_example_code/cnn_smoke.py deleted file mode 100644 index 3269a2b74..000000000 --- a/test_example_code/cnn_smoke.py +++ /dev/null @@ -1,36 +0,0 @@ -r""" -It's used to check basic rnn features with cuda. -For example, it would throw exception if some components are missing -""" - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim - -class SimpleCNN(nn.Module): - def __init__(self): - super().__init__() - self.conv = nn.Conv2d(1, 1, 3) - self.pool = nn.MaxPool2d(2, 2) - - def forward(self, inputs): - output = self.pool(F.relu(self.conv(inputs))) - output = output.view(1) - return output - -# Mock one infer -device = torch.device("cuda:0") -net = SimpleCNN().to(device) -net_inputs = torch.rand((1, 1, 5, 5), device=device) -outputs = net(net_inputs) -print(outputs) - -criterion = nn.MSELoss() -optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.1) - -# Mock one step training -label = torch.full((1,), 1.0, dtype=torch.float, device=device) -loss = criterion(outputs, label) -loss.backward() -optimizer.step() diff --git a/test_example_code/rnn_smoke.py b/test_example_code/rnn_smoke.py deleted file mode 100644 index 5965b30ea..000000000 --- a/test_example_code/rnn_smoke.py +++ /dev/null @@ -1,13 +0,0 @@ -r""" -It's used to check basic rnn features with cuda. -For example, it would throw exception if missing some components are missing -""" - -import torch -import torch.nn as nn - -device = torch.device("cuda:0") -rnn = nn.RNN(10, 20, 2).to(device) -inputs = torch.randn(5, 3, 10).to(device) -h0 = torch.randn(2, 3, 20).to(device) -output, hn = rnn(inputs, h0) diff --git a/test_example_code/simple-torch-test.cpp b/test_example_code/simple-torch-test.cpp deleted file mode 100644 index 1b9453fe6..000000000 --- a/test_example_code/simple-torch-test.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main(int argc, const char* argv[]) { - TORCH_WARN("Simple test passed!"); - return 0; -} diff --git a/wheel/build_wheel.sh b/wheel/build_wheel.sh deleted file mode 100755 index d91b29be0..000000000 --- a/wheel/build_wheel.sh +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env bash -set -ex -SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -# Env variables that should be set: -# DESIRED_PYTHON -# Which Python version to build for in format 'Maj.min' e.g. '2.7' or '3.6' -# -# PYTORCH_FINAL_PACKAGE_DIR -# **absolute** path to folder where final whl packages will be stored. The -# default should not be used when calling this from a script. The default -# is 'whl', and corresponds to the default in the wheel/upload.sh script. -# -# MAC_PACKAGE_WORK_DIR -# absolute path to a workdir in which to clone an isolated conda -# installation and pytorch checkout. If the pytorch checkout already exists -# then it will not be overwritten. - -# Function to retry functions that sometimes timeout or have flaky failures -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - -# Parameters -if [[ -n "$DESIRED_PYTHON" && -n "$PYTORCH_BUILD_VERSION" && -n "$PYTORCH_BUILD_NUMBER" ]]; then - desired_python="$DESIRED_PYTHON" - build_version="$PYTORCH_BUILD_VERSION" - build_number="$PYTORCH_BUILD_NUMBER" -else - if [ "$#" -ne 3 ]; then - echo "illegal number of parameters. Need PY_VERSION BUILD_VERSION BUILD_NUMBER" - echo "for example: build_wheel.sh 2.7 0.1.6 20" - echo "Python version should be in format 'M.m'" - exit 1 - fi - desired_python=$1 - build_version=$2 - build_number=$3 -fi - -echo "Building for Python: $desired_python Version: $build_version Build: $build_number" -python_nodot="$(echo $desired_python | tr -d m.u)" - -# Version: setup.py uses $PYTORCH_BUILD_VERSION.post$PYTORCH_BUILD_NUMBER if -# PYTORCH_BUILD_NUMBER > 1 -if [[ -n "$OVERRIDE_PACKAGE_VERSION" ]]; then - # This will be the *exact* version, since build_number<1 - build_version="$OVERRIDE_PACKAGE_VERSION" - build_number=0 - build_number_prefix='' -else - if [[ $build_number -eq 1 ]]; then - build_number_prefix="" - else - build_number_prefix=".post$build_number" - fi -fi -export PYTORCH_BUILD_VERSION=$build_version -export PYTORCH_BUILD_NUMBER=$build_number - -package_type="${PACKAGE_TYPE:-wheel}" -# Fill in empty parameters with defaults -if [[ -z "$TORCH_PACKAGE_NAME" ]]; then - TORCH_PACKAGE_NAME='torch' -fi -TORCH_PACKAGE_NAME="$(echo $TORCH_PACKAGE_NAME | tr '-' '_')" -if [[ -z "$PYTORCH_REPO" ]]; then - PYTORCH_REPO='pytorch' -fi -if [[ -z "$PYTORCH_BRANCH" ]]; then - PYTORCH_BRANCH="v${build_version}" -fi -if [[ -z "$RUN_TEST_PARAMS" ]]; then - RUN_TEST_PARAMS=() -fi -if [[ -z "$PYTORCH_FINAL_PACKAGE_DIR" ]]; then - if [[ -n "$BUILD_PYTHONLESS" ]]; then - PYTORCH_FINAL_PACKAGE_DIR='libtorch' - else - PYTORCH_FINAL_PACKAGE_DIR='whl' - fi -fi -mkdir -p "$PYTORCH_FINAL_PACKAGE_DIR" || true - -# Create an isolated directory to store this builds pytorch checkout and conda -# installation -if [[ -z "$MAC_PACKAGE_WORK_DIR" ]]; then - MAC_PACKAGE_WORK_DIR="$(pwd)/tmp_wheel_conda_${DESIRED_PYTHON}_$(date +%H%M%S)" -fi -mkdir -p "$MAC_PACKAGE_WORK_DIR" || true -if [[ -n ${GITHUB_ACTIONS} ]]; then - pytorch_rootdir="${PYTORCH_ROOT:-${MAC_PACKAGE_WORK_DIR}/pytorch}" -else - pytorch_rootdir="${MAC_PACKAGE_WORK_DIR}/pytorch" -fi -whl_tmp_dir="${MAC_PACKAGE_WORK_DIR}/dist" -mkdir -p "$whl_tmp_dir" - -mac_version='macosx_11_0_arm64' -libtorch_arch='arm64' - -# Create a consistent wheel package name to rename the wheel to -wheel_filename_new="${TORCH_PACKAGE_NAME}-${build_version}${build_number_prefix}-cp${python_nodot}-none-${mac_version}.whl" - -########################################################### - -# Have a separate Pytorch repo clone -if [[ ! -d "$pytorch_rootdir" ]]; then - git clone "https://github.com/${PYTORCH_REPO}/pytorch" "$pytorch_rootdir" - pushd "$pytorch_rootdir" - if ! git checkout "$PYTORCH_BRANCH" ; then - echo "Could not checkout $PYTORCH_BRANCH, so trying tags/v${build_version}" - git checkout tags/v${build_version} - fi - popd -fi -pushd "$pytorch_rootdir" -git submodule update --init --recursive -popd - -########################## -# now build the binary - - -export TH_BINARY_BUILD=1 -export INSTALL_TEST=0 # dont install test binaries into site-packages -export MACOSX_DEPLOYMENT_TARGET=10.15 -export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} - -SETUPTOOLS_PINNED_VERSION="=46.0.0" -PYYAML_PINNED_VERSION="=5.3" -EXTRA_CONDA_INSTALL_FLAGS="" -case $desired_python in - 3.13) - echo "Using 3.13 deps" - SETUPTOOLS_PINNED_VERSION=">=68.0.0" - PYYAML_PINNED_VERSION=">=6.0.1" - NUMPY_PINNED_VERSION="=2.1.0" - ;; - 3.12) - echo "Using 3.12 deps" - SETUPTOOLS_PINNED_VERSION=">=68.0.0" - PYYAML_PINNED_VERSION=">=6.0.1" - NUMPY_PINNED_VERSION="=2.0.2" - ;; - 3.11) - echo "Using 3.11 deps" - SETUPTOOLS_PINNED_VERSION=">=46.0.0" - PYYAML_PINNED_VERSION=">=5.3" - NUMPY_PINNED_VERSION="=2.0.2" - ;; - 3.10) - echo "Using 3.10 deps" - SETUPTOOLS_PINNED_VERSION=">=46.0.0" - PYYAML_PINNED_VERSION=">=5.3" - NUMPY_PINNED_VERSION="=2.0.2" - ;; - 3.9) - echo "Using 3.9 deps" - SETUPTOOLS_PINNED_VERSION=">=46.0.0" - PYYAML_PINNED_VERSION=">=5.3" - NUMPY_PINNED_VERSION="=2.0.2" - ;; - *) - echo "Using default deps" - NUMPY_PINNED_VERSION="=1.11.3" - ;; -esac - -# Install into a fresh env -tmp_env_name="wheel_py$python_nodot" -conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "$tmp_env_name" python="$desired_python" -source activate "$tmp_env_name" - -pip install -q "numpy=${NUMPY_PINNED_VERSION}" "pyyaml${PYYAML_PINNED_VERSION}" requests -retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq llvm-openmp=14.0.6 cmake ninja "setuptools${SETUPTOOLS_PINNED_VERSION}" typing_extensions -retry pip install -qr "${pytorch_rootdir}/requirements.txt" || true - -# For USE_DISTRIBUTED=1 on macOS, need libuv and pkg-config to find libuv. -export USE_DISTRIBUTED=1 -retry conda install ${EXTRA_CONDA_INSTALL_FLAGS} -yq libuv pkg-config - -if [[ -n "$CROSS_COMPILE_ARM64" ]]; then - export CMAKE_OSX_ARCHITECTURES=arm64 -fi -export USE_MKLDNN=OFF -export USE_QNNPACK=OFF -export BUILD_TEST=OFF - -pushd "$pytorch_rootdir" -echo "Calling setup.py bdist_wheel at $(date)" - -if [[ "$USE_SPLIT_BUILD" == "true" ]]; then - echo "Calling setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)" - BUILD_LIBTORCH_WHL=1 BUILD_PYTHON_ONLY=0 python setup.py bdist_wheel -d "$whl_tmp_dir" - echo "Finished setup.py bdist_wheel for split build (BUILD_LIBTORCH_WHL)" - echo "Calling setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)" - BUILD_PYTHON_ONLY=1 BUILD_LIBTORCH_WHL=0 python setup.py bdist_wheel -d "$whl_tmp_dir" --cmake - echo "Finished setup.py bdist_wheel for split build (BUILD_PYTHON_ONLY)" -else - python setup.py bdist_wheel -d "$whl_tmp_dir" -fi - -echo "Finished setup.py bdist_wheel at $(date)" - -if [[ $package_type != 'libtorch' ]]; then - echo "delocating wheel dependencies" - retry pip install https://github.com/matthew-brett/delocate/archive/refs/tags/0.10.4.zip - echo "found the following wheels:" - find $whl_tmp_dir -name "*.whl" - echo "running delocate" - find $whl_tmp_dir -name "*.whl" | xargs -I {} delocate-wheel -v {} - find $whl_tmp_dir -name "*.whl" - find $whl_tmp_dir -name "*.whl" | xargs -I {} delocate-listdeps {} - echo "Finished delocating wheels at $(date)" -fi - -echo "The wheel is in $(find $whl_tmp_dir -name '*.whl')" - -wheel_filename_gen=$(find $whl_tmp_dir -name '*.whl' | head -n1 | xargs -I {} basename {}) -popd - -if [[ -z "$BUILD_PYTHONLESS" ]]; then - # Copy the whl to a final destination before tests are run - echo "Renaming Wheel file: $wheel_filename_gen to $wheel_filename_new" - cp "$whl_tmp_dir/$wheel_filename_gen" "$PYTORCH_FINAL_PACKAGE_DIR/$wheel_filename_new" - - ########################## - # now test the binary, unless it's cross compiled arm64 - if [[ -z "$CROSS_COMPILE_ARM64" ]]; then - pip uninstall -y "$TORCH_PACKAGE_NAME" || true - pip uninstall -y "$TORCH_PACKAGE_NAME" || true - - # Create new "clean" conda environment for testing - conda create ${EXTRA_CONDA_INSTALL_FLAGS} -yn "test_conda_env" python="$desired_python" - conda activate test_conda_env - - pip install "$PYTORCH_FINAL_PACKAGE_DIR/$wheel_filename_new" -v - - echo "$(date) :: Running tests" - pushd "$pytorch_rootdir" - "${SOURCE_DIR}/../run_tests.sh" 'wheel' "$desired_python" 'cpu' - popd - echo "$(date) :: Finished tests" - fi -else - pushd "$pytorch_rootdir" - - mkdir -p libtorch/{lib,bin,include,share} - cp -r "$(pwd)/build/lib" "$(pwd)/libtorch/" - - # for now, the headers for the libtorch package will just be - # copied in from the wheel - unzip -d any_wheel "$whl_tmp_dir/$wheel_filename_gen" - if [[ -d $(pwd)/any_wheel/torch/include ]]; then - cp -r "$(pwd)/any_wheel/torch/include" "$(pwd)/libtorch/" - else - cp -r "$(pwd)/any_wheel/torch/lib/include" "$(pwd)/libtorch/" - fi - cp -r "$(pwd)/any_wheel/torch/share/cmake" "$(pwd)/libtorch/share/" - if [[ "${libtorch_arch}" == "x86_64" ]]; then - if [[ -x "$(pwd)/any_wheel/torch/.dylibs/libiomp5.dylib" ]]; then - cp -r "$(pwd)/any_wheel/torch/.dylibs/libiomp5.dylib" "$(pwd)/libtorch/lib/" - else - cp -r "$(pwd)/any_wheel/torch/lib/libiomp5.dylib" "$(pwd)/libtorch/lib/" - fi - else - cp -r "$(pwd)/any_wheel/torch/lib/libomp.dylib" "$(pwd)/libtorch/lib/" - fi - rm -rf "$(pwd)/any_wheel" - - echo $PYTORCH_BUILD_VERSION > libtorch/build-version - echo "$(pushd $pytorch_rootdir && git rev-parse HEAD)" > libtorch/build-hash - - zip -rq "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip" libtorch - cp "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-$PYTORCH_BUILD_VERSION.zip" \ - "$PYTORCH_FINAL_PACKAGE_DIR/libtorch-macos-${libtorch_arch}-latest.zip" -fi