From d1fe9bc94f70a8d68f54587ecd0d353c1144a920 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 13:55:05 -0800 Subject: [PATCH 1/7] Release 2.2.0 prep script modifications --- release/pypi/prep_binary_for_pypi.sh | 8 -------- release/pypi/promote_pypi_to_staging.sh | 3 +-- release/pypi/upload_pypi_to_staging.sh | 3 +++ release/release_versions.sh | 8 ++++---- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/release/pypi/prep_binary_for_pypi.sh b/release/pypi/prep_binary_for_pypi.sh index fdd9bf4a0..154b22852 100755 --- a/release/pypi/prep_binary_for_pypi.sh +++ b/release/pypi/prep_binary_for_pypi.sh @@ -52,14 +52,6 @@ for whl_file in "$@"; do ( set -x - # Special build with pypi cudnn remove it from version - if [[ $whl_file == *"with.pypi.cudnn"* ]]; then - rm -rf "${whl_dir}/caffe2" - rm -rf "${whl_dir}"/torch/lib/libnvrtc* - - sed -i -e "s/-with-pypi-cudnn//g" "${whl_dir}/torch/version.py" - fi - find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \; # Moves distinfo from one with a version suffix to one without # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info diff --git a/release/pypi/promote_pypi_to_staging.sh b/release/pypi/promote_pypi_to_staging.sh index 46cd958cd..dbc00e24a 100644 --- a/release/pypi/promote_pypi_to_staging.sh +++ b/release/pypi/promote_pypi_to_staging.sh @@ -21,13 +21,12 @@ upload_pypi_to_staging() { } # Uncomment these to promote to pypi -PYTORCH_LINUX_VERSION_SUFFIX="%2Bcu121.with.pypi.cudnn" LINUX_VERSION_SUFFIX="%2Bcu121" CPU_VERSION_SUFFIX="%2Bcpu" MACOS_X86_64="macosx_.*_x86_64" MACOS_ARM64="macosx_.*_arm64" -PLATFORM="linux_x86_64" VERSION_SUFFIX="${PYTORCH_LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torch "${PYTORCH_VERSION}" +PLATFORM="linux_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" upload_pypi_to_staging torch "${PYTORCH_VERSION}" PLATFORM="manylinux2014_aarch64" VERSION_SUFFIX="" upload_pypi_to_staging torch "${PYTORCH_VERSION}" PLATFORM="win_amd64" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torch "${PYTORCH_VERSION}" PLATFORM="${MACOS_X86_64}" VERSION_SUFFIX="" upload_pypi_to_staging torch "${PYTORCH_VERSION}" # intel mac diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh index b1a7ddf6d..c25a98320 100644 --- a/release/pypi/upload_pypi_to_staging.sh +++ b/release/pypi/upload_pypi_to_staging.sh @@ -5,6 +5,9 @@ set -eou pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Allow for users to pass PACKAGE_NAME +# Set TMPDIR +#export TMPDIR="/home/ec2-user/github/builder/release/pypi/work" + # For use with other packages, i.e. torchvision, etc. PACKAGE_NAME=${PACKAGE_NAME:-torch} diff --git a/release/release_versions.sh b/release/release_versions.sh index 981a18ea0..88b6cbcd3 100644 --- a/release/release_versions.sh +++ b/release/release_versions.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Make sure to update these versions when doing a release first -PYTORCH_VERSION=${PYTORCH_VERSION:-2.1.1} -TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.16.1} -TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.1.1} -TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.16.1} +PYTORCH_VERSION=${PYTORCH_VERSION:-2.2.0} +TORCHVISION_VERSION=${TORCHVISION_VERSION:-0.17.0} +TORCHAUDIO_VERSION=${TORCHAUDIO_VERSION:-2.2.0} +TORCHTEXT_VERSION=${TORCHTEXT_VERSION:-0.17.0} TORCHDATA_VERSION=${TORCHDATA_VERSION:-0.7.1} TORCHREC_VERSION=${TORCHREC_VERSION:-0.6.0} FBGEMMGPU_VERSION=${FBGEMMGPU_VERSION:-0.6.0} From 89745cabaa083768329810ff5061b7b87da09d49 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 13:57:08 -0800 Subject: [PATCH 2/7] typo --- release/pypi/upload_pypi_to_staging.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh index c25a98320..f00271715 100644 --- a/release/pypi/upload_pypi_to_staging.sh +++ b/release/pypi/upload_pypi_to_staging.sh @@ -5,9 +5,6 @@ set -eou pipefail DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Allow for users to pass PACKAGE_NAME -# Set TMPDIR -#export TMPDIR="/home/ec2-user/github/builder/release/pypi/work" - # For use with other packages, i.e. torchvision, etc. PACKAGE_NAME=${PACKAGE_NAME:-torch} @@ -36,10 +33,8 @@ pushd "${output_tmp_dir}" # Dry run by default DRY_RUN=${DRY_RUN:-enabled} # On dry run just echo the commands that are meant to be run -TWINE_UPLOAD="echo twine upload" DRY_RUN_FLAG="--dryrun" if [[ $DRY_RUN = "disabled" ]]; then - TWINE_UPLOAD="twine upload" DRY_RUN_FLAG="" fi From a906f1289bed6ea56e7a3461a37a07190501a34b Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 14:34:43 -0800 Subject: [PATCH 3/7] test --- .../validate-repackaged-binary-sizes.yml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml index 695c68d3a..d99307b5e 100644 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ b/.github/workflows/validate-repackaged-binary-sizes.yml @@ -23,21 +23,21 @@ jobs: fail-fast: false matrix: whl: - - url: https://download.pytorch.org/whl/test/cu117_pypi_cudnn/torch-1.13.1%2Bcu117.with.pypi.cudnn-cp310-cp310-linux_x86_64.whl - python: "3.10" # python version to use for smoke tests - upload_artifact: false # upload the repackaged binary as an artifact - - url: https://download.pytorch.org/whl/test/cu117_pypi_cudnn/torch-1.13.1%2Bcu117.with.pypi.cudnn-cp37-cp37m-linux_x86_64.whl - python: "3.7" + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp312-cp312-linux_x86_64.whl + python: "3.12" artifact: false - - url: https://download.pytorch.org/whl/test/cu117_pypi_cudnn/torch-1.13.1%2Bcu117.with.pypi.cudnn-cp38-cp38-linux_x86_64.whl - python: "3.8" + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp311-cp311-linux_x86_64.whl + python: "3.11" # python version to use for smoke tests + upload_artifact: false # upload the repackaged binary as an artifact + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp310-cp310-linux_x86_64.whl + python: "3.10" artifact: false - - url: https://download.pytorch.org/whl/test/cu117_pypi_cudnn/torch-1.13.1%2Bcu117.with.pypi.cudnn-cp39-cp39-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp319-cp319-linux_x86_64.whl python: "3.9" artifact: false - # - url: https://download.pytorch.org/whl/test/cu117_pypi_cudnn/torch-1.13.1%2Bcu117.with.pypi.cudnn-cp311-cp311-linux_x86_64.whl - # python: "3.11" - # artifact: false + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp318-cp318-linux_x86_64.whl + python: "3.8" + artifact: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: @@ -55,34 +55,34 @@ jobs: export PACKAGE_TYPE="wheel" export TARGET_OS="linux" export INSTALLATION="" - + # install zip sudo yum install zip -y - + # install patchelf chmod a+x common/install_patchelf.sh sudo common/install_patchelf.sh - + # download torch whl wget ${{ matrix.whl.url }} FILENAME=$(ls -1 *.whl | head -n 1) SIZE_BEFORE=$(du -h $FILENAME | cut -f1) - + # repackage into manywheel release/pypi/prep_binary_for_pypi.sh $FILENAME - + NEW_FILENAME=$(ls -1 *.whl | head -n 1) echo "::notice:: $FILENAME before: $SIZE_BEFORE after: $(du -h $NEW_FILENAME | cut -f1)" - + # cp to ${RUNNER_ARTIFACT_DIR} cp $NEW_FILENAME ${RUNNER_ARTIFACT_DIR}/ - + # create conda env conda create -y -n $ENV_NAME python=$DESIRED_PYTHON conda activate $ENV_NAME - + # install torch pip install numpy pillow $NEW_FILENAME - + # run smoke test - python ./test/smoke_test/smoke_test.py --package=torchonly \ No newline at end of file + python ./test/smoke_test/smoke_test.py --package=torchonly From a917649185f816fed15566010bef64c3281c5f05 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 14:43:04 -0800 Subject: [PATCH 4/7] fix --- .../validate-repackaged-binary-sizes.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml index d99307b5e..9507c877a 100644 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ b/.github/workflows/validate-repackaged-binary-sizes.yml @@ -23,19 +23,19 @@ jobs: fail-fast: false matrix: whl: - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp312-cp312-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp312-cp312-linux_x86_64.whl python: "3.12" artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp311-cp311-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp311-cp311-linux_x86_64.whl python: "3.11" # python version to use for smoke tests upload_artifact: false # upload the repackaged binary as an artifact - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp310-cp310-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp310-cp310-linux_x86_64.whl python: "3.10" artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp319-cp319-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp319-cp319-linux_x86_64.whl python: "3.9" artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0-cu121-cp318-cp318-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp318-cp318-linux_x86_64.whl python: "3.8" artifact: false @@ -47,11 +47,11 @@ jobs: script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export GPU_ARCH_VER="11.7" + export GPU_ARCH_VER="12.1" export GPU_ARCH_TYPE="cuda" - export CUDA_VER="11.7" + export CUDA_VER="12.1" export DESIRED_PYTHON="${{ matrix.whl.python }}" - export DESIRED_CUDA="cu117" + export DESIRED_CUDA="cu121" export PACKAGE_TYPE="wheel" export TARGET_OS="linux" export INSTALLATION="" From e5de8792e40f37de67b0bc715bb77e7c3c0ac328 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 14:53:05 -0800 Subject: [PATCH 5/7] test --- .github/workflows/validate-repackaged-binary-sizes.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml index 9507c877a..0542a2d7b 100644 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ b/.github/workflows/validate-repackaged-binary-sizes.yml @@ -23,19 +23,19 @@ jobs: fail-fast: false matrix: whl: - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp312-cp312-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp312-cp312-linux_x86_64.whl python: "3.12" artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp311-cp311-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp311-cp311-linux_x86_64.whl python: "3.11" # python version to use for smoke tests upload_artifact: false # upload the repackaged binary as an artifact - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp310-cp310-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp310-cp310-linux_x86_64.whl python: "3.10" artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp319-cp319-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp39-cp39-linux_x86_64.whl python: "3.9" artifact: false - - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0+cu121-cp318-cp318-linux_x86_64.whl + - url: https://download.pytorch.org/whl/test/cu121/torch-2.2.0%2Bcu121-cp38-cp38-linux_x86_64.whl python: "3.8" artifact: false From ae28417e21f213a8d81928f80808f92c6133ed36 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 16:02:33 -0800 Subject: [PATCH 6/7] test --- .github/workflows/validate-repackaged-binary-sizes.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml index 0542a2d7b..3f1ac307b 100644 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ b/.github/workflows/validate-repackaged-binary-sizes.yml @@ -47,14 +47,12 @@ jobs: script: | set -ex export ENV_NAME="conda-env-${{ github.run_id }}" - export GPU_ARCH_VER="12.1" - export GPU_ARCH_TYPE="cuda" - export CUDA_VER="12.1" + export MATRIX_GPU_ARCH_VERSION="12.1" + export MATRIX_GPU_ARCH_TYPE="cuda" + export MATRIX_CUDA_VER="12.1" export DESIRED_PYTHON="${{ matrix.whl.python }}" - export DESIRED_CUDA="cu121" - export PACKAGE_TYPE="wheel" + export MATRIX_PACKAGE_TYPE="wheel" export TARGET_OS="linux" - export INSTALLATION="" # install zip sudo yum install zip -y From a7a58b761e733307c5b95cf4d14891e93bb86723 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 24 Jan 2024 16:56:19 -0800 Subject: [PATCH 7/7] test --- .github/workflows/validate-repackaged-binary-sizes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-repackaged-binary-sizes.yml b/.github/workflows/validate-repackaged-binary-sizes.yml index 3f1ac307b..cb1a6a73e 100644 --- a/.github/workflows/validate-repackaged-binary-sizes.yml +++ b/.github/workflows/validate-repackaged-binary-sizes.yml @@ -41,7 +41,7 @@ jobs: uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: - runner: linux.4xlarge.nvidia.gpu + runner: linux.g5.4xlarge.nvidia.gpu job-name: "Validate binary size" upload-artifact: ${{ matrix.whl.upload_artifact == 'true' && 'repackaged-binary' || '' }} script: |