Skip to content

Commit db71e37

Browse files
atalmanatalman
committed
Use linux_job for linux workers
Test Testing Test testing Tetsing testing Change linux binary action test Simplify version check
1 parent f504a35 commit db71e37

File tree

2 files changed

+56
-131
lines changed

2 files changed

+56
-131
lines changed

‎.github/workflows/validate-linux-binaries.yml

Lines changed: 48 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ on:
77
description: "Channel to use (nightly, test, release, all)"
88
required: true
99
type: string
10+
ref:
11+
description: 'Reference to checkout, defaults to empty'
12+
default: ""
13+
required: false
14+
type: string
1015
workflow_dispatch:
1116
inputs:
1217
channel:
@@ -18,108 +23,57 @@ on:
1823
- nightly
1924
- test
2025
- all
26+
ref:
27+
description: 'Reference to checkout, defaults to empty'
28+
default: ""
29+
required: false
30+
type: string
2131

2232
jobs:
23-
generate-linux-conda-matrix:
24-
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
25-
with:
26-
package-type: conda
27-
os: linux
28-
channel: ${{ inputs.channel }}
29-
generate-linux-wheel-matrix:
30-
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
31-
with:
32-
package-type: wheel
33-
os: linux
34-
channel: ${{ inputs.channel }}
35-
with-py311: enable
36-
generate-linux-libtorch-matrix:
33+
generate-linux-matrix:
3734
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
3835
with:
39-
package-type: libtorch
36+
package-type: all
4037
os: linux
4138
channel: ${{ inputs.channel }}
4239

43-
linux-conda:
44-
needs: generate-linux-conda-matrix
40+
linux:
41+
needs: generate-linux-matrix
4542
strategy:
46-
matrix:
47-
${{ fromJson(needs.generate-linux-conda-matrix.outputs.matrix) }}
43+
matrix: ${{ fromJson(needs.generate-linux-matrix.outputs.matrix) }}
4844
fail-fast: false
49-
runs-on: ${{ matrix.validation_runner }}
50-
steps:
51-
- name: Validate binary conda
52-
uses: pytorch/builder/.github/actions/validate-binary@main
53-
with:
54-
gpu_arch_type: ${{ matrix.gpu_arch_type }}
55-
gpu_arch_ver: ${{ matrix.gpu_arch_version }}
56-
installation: ${{ matrix.installation }}
57-
python_version: ${{ matrix.python_version }}
58-
desired_cuda: ${{ matrix.desired_cuda }}
59-
dev_toolset: ''
60-
package_type: conda
61-
target_os: linux
62-
63-
linux-wheel:
64-
needs: generate-linux-wheel-matrix
65-
strategy:
66-
matrix:
67-
${{ fromJson(needs.generate-linux-wheel-matrix.outputs.matrix) }}
68-
fail-fast: false
69-
runs-on: ${{ matrix.validation_runner }}
70-
steps:
71-
- name: Checkout PyTorch builder
72-
uses: actions/checkout@v2
73-
- name: Validate binary wheel
74-
uses: ./.github/actions/validate-binary
75-
with:
76-
gpu_arch_type: ${{ matrix.gpu_arch_type }}
77-
gpu_arch_ver: ${{ matrix.gpu_arch_version }}
78-
installation: ${{ matrix.installation }}
79-
installation_pypi: ${{ matrix.installation_pypi }}
80-
python_version: ${{ matrix.python_version }}
81-
desired_cuda: ${{ matrix.desired_cuda }}
82-
dev_toolset: ''
83-
package_type: wheel
84-
target_os: linux
45+
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
46+
name: ${{ matrix.build_name }}
47+
with:
48+
runner: ${{ matrix.validation_runner }}
49+
repository: "pytorch/builder"
50+
ref: ${{ inputs.ref || github.ref }}
51+
job-name: ${{ matrix.build_name }}
52+
script: |
53+
set -ex
54+
export ENV_NAME="conda-env-${{ github.run_id }}"
55+
export GPU_ARCH_VER="${{ matrix.gpu_arch_version }}"
56+
export GPU_ARCH_TYPE="${{ matrix.gpu_arch_type }}"
57+
export INSTALLATION="${{ matrix.installation }}"
58+
export CUDA_VER="${{ matrix.desired_cuda }}"
59+
export DESIRED_PYTHON="${{ matrix.python_version }}"
60+
export DESIRED_CUDA="${{ matrix.desired_cuda }}"
61+
export DESIRED_DEVTOOLSET="${{ matrix.dev_toolset }}"
62+
export PACKAGE_TYPE="${{ matrix.package_type }}"
63+
export TARGET_OS="linux"
8564
86-
linux-libt:
87-
needs: generate-linux-libtorch-matrix
88-
strategy:
89-
matrix:
90-
${{ fromJson(needs.generate-linux-libtorch-matrix.outputs.matrix) }}
91-
fail-fast: false
92-
runs-on: ${{ matrix.validation_runner }}
93-
env:
94-
PYTHON_VERSION: ${{ matrix.python_version }}
95-
steps:
96-
- name: Checkout PyTorch builder
97-
uses: actions/checkout@v2
98-
- name: Install Conda
99-
uses: conda-incubator/setup-miniconda@v2
100-
with:
101-
python-version: ${{ matrix.python_version }}
102-
auto-update-conda: true
103-
miniconda-version: "latest"
104-
activate-environment: testenv
105-
- name: Install pytorch and smoke test
106-
shell: bash
107-
env:
108-
INSTALLATION: ${{ matrix.installation }}
109-
ENV_NAME: conda-env-${{ github.run_id }}
110-
DESIRED_PYTHON: ${{ matrix.python_version }}
111-
DESIRED_CUDA: ${{ matrix.desired_cuda }}
112-
DESIRED_DEVTOOLSET: ${{ matrix.devtoolset }}
113-
PACKAGE_TYPE: libtorch
114-
run: |
115-
sudo apt-get install unzip -y
116-
set -ex
117-
curl ${INSTALLATION} -o libtorch.zip
65+
if [[ ${{ matrix.package_type }} == "libtorch" ]]; then
66+
curl ${{ matrix.installation }} -o libtorch.zip
11867
unzip libtorch.zip
119-
conda create -yp ${ENV_NAME} python=${{ matrix.python_version }} numpy
120-
export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
121-
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
122-
if [[ ${{ matrix.libtorch_variant }} == "static-with-deps" ]]; then
123-
conda run -p ${ENV_NAME} --cwd libtorch env LD_LIBRARY_PATH=${LD_LIBRARY_PATH} bash ./../check_binary.sh
124-
fi
125-
conda env remove -p ${ENV_NAME}
68+
else
69+
conda create -y -n ${ENV_NAME} python=${{ matrix.python_version }} numpy pillow
70+
conda activate ${ENV_NAME}
71+
eval $INSTALLATION
72+
export CONDA_LIBRARY_PATH="$(dirname $(which python))/../lib"
73+
export LD_LIBRARY_PATH=$CONDA_LIBRARY_PATH:$LD_LIBRARY_PATH
74+
75+
python --version
76+
python ./test/smoke_test/smoke_test.py
77+
78+
${PWD}/check_binary.sh
79+
fi

‎test/smoke_test/smoke_test.py

Lines changed: 8 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -14,26 +14,6 @@
1414
SCRIPT_DIR = Path(__file__).parent
1515
NIGHTLY_ALLOWED_DELTA = 3
1616

17-
# helper function to return the conda installed packages
18-
# and return package we are insterseted in
19-
def get_anaconda_output_for_package(pkg_name_str):
20-
import subprocess as sp
21-
22-
# If we are installing using conda just list package name
23-
if installation_str.find("conda install") != -1:
24-
cmd = "conda list --explicit"
25-
output = sp.getoutput(cmd)
26-
for item in output.split("\n"):
27-
if pkg_name_str in item:
28-
return item
29-
return f"{pkg_name_str} can't be found"
30-
else:
31-
cmd = "conda list -f " + pkg_name_str
32-
output = sp.getoutput(cmd)
33-
# Get the last line only
34-
return output.strip().split('\n')[-1]
35-
36-
3717
def check_nightly_binaries_date(package: str) -> None:
3818
from datetime import datetime, timedelta
3919
format_dt = '%Y%m%d'
@@ -79,23 +59,14 @@ def smoke_test_cuda(package: str) -> None:
7959
if(package == 'all'):
8060
import torchaudio
8161
import torchvision
82-
# There is an issue with current windows runners calling conda from python
83-
# https://github.com/pytorch/test-infra/issues/1054
84-
if installation_str.find("nightly") != -1 or platform.system() == "Windows" :
85-
# just print out cuda version, as version check were already performed during import
86-
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
87-
print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")
88-
else:
89-
# torchaudio runtime added the cuda verison check on 09/23/2022 via
90-
# https://github.com/pytorch/audio/pull/2707
91-
# so relying on anaconda output for pytorch-test and pytorch channel
92-
torchaudio_allstr = get_anaconda_output_for_package(torchaudio.__name__)
93-
if (
94-
is_cuda_system
95-
and "cu" + str(gpu_arch_ver).replace(".", "") not in torchaudio_allstr
96-
):
97-
raise RuntimeError(
98-
f"CUDA version issue. Loaded: {torchaudio_allstr} Expected: {gpu_arch_ver}"
62+
63+
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
64+
print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")
65+
if gpu_arch_ver != torch.ops.torchvision._cuda_version() or
66+
gpu_arch_ver != torch.ops.torchaudio.cuda_version():
67+
raise RuntimeError(
68+
f"Wrong CUDA version. Vision: {torch.ops.torchvision._cuda_version()} \
69+
Audio: {ttorch.ops.torchaudio.cuda_version()} Expected: {gpu_arch_ver}"
9970
)
10071

10172
def smoke_test_conv2d() -> None:

0 commit comments

Comments
 (0)