Skip to content

Commit 2067561

Browse files
atalmanatalman
committed
Validate linux binaries change
Add options Import torchvision Adding python 3.11 install pass package to check nightly binaries date Test test Add python 3.11 code testing Adding python 3.11 test Add python 3.11 validation Adding zlib develop install Install zlib etc.. Adding zlib1g as well testing testing Adding validate windows binary Trying to workaround testing Refacor smoke test Add import statement fix datetime call
1 parent f3b676a commit 2067561

File tree

3 files changed

+45
-32
lines changed

3 files changed

+45
-32
lines changed

‎.github/actions/validate-binary/action.yml

Lines changed: 24 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ runs:
4444
run: |
4545
nvidia-smi
4646
- name: Install Conda Linux
47-
if: ${{ inputs.target_os == 'linux' }}
47+
if: ${{ inputs.target_os == 'linux' && inputs.python_version != '3.11' }}
4848
uses: conda-incubator/setup-miniconda@v2
4949
with:
5050
python-version: ${{ inputs.python_version }}
@@ -66,16 +66,29 @@ runs:
6666
DESIRED_DEVTOOLSET: ${{ inputs.dev_toolset }}
6767
PACKAGE_TYPE: ${{ inputs.package_type }}
6868
run: |
69-
set -ex
70-
conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy
71-
conda run -p ${ENV_NAME} $INSTALLATION
72-
if [ $DESIRED_PYTHON == "3.11" ]; then
73-
conda run -p ${ENV_NAME} python3 --package torchonly ./test/smoke_test/smoke_test.py
69+
70+
if [ $DESIRED_PYTHON == '3.11' ]; then
71+
set -ex
72+
export CPYTHON_VERSIONS=3.11.0
73+
sudo apt-get install build-essential gdb lcov libbz2-dev libffi-dev \
74+
libgdbm-dev liblzma-dev libncurses5-dev libreadline6-dev \
75+
libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g zlib1g-dev -y
76+
77+
export PYTHON_PATH="/opt/_internal/cpython-3.11.0rc2/bin/"
78+
export PIP_PATH="${PYTHON_PATH}/pip"
79+
export PIP_INSTALLATION="${INSTALLATION/pip3/"$PIP_PATH"}"
80+
./common/install_cpython.sh
81+
82+
eval ${PYTHON_PATH}/python --version
83+
eval ${PIP_INSTALLATION}
84+
eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly
7485
else
86+
set -ex
87+
conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy
88+
conda run -p ${ENV_NAME} $INSTALLATION
7589
conda run -p ${ENV_NAME} python3 ./test/smoke_test/smoke_test.py
90+
export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
91+
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
92+
conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh
93+
conda env remove -p ${ENV_NAME}
7694
fi
77-
78-
export LD_LIBRARY_PATH="$(dirname $(which python))/lib"
79-
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib
80-
conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh
81-
conda env remove -p ${ENV_NAME}

‎.github/workflows/validate-linux-binaries.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
package-type: wheel
3333
os: linux
3434
channel: ${{ inputs.channel }}
35+
with-py311: enable
3536
generate-linux-libtorch-matrix:
3637
uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main
3738
with:
@@ -70,7 +71,7 @@ jobs:
7071
- name: Checkout PyTorch builder
7172
uses: actions/checkout@v2
7273
- name: Validate binary wheel
73-
uses: .github/actions/validate-binary
74+
uses: ./.github/actions/validate-binary
7475
with:
7576
gpu_arch_type: ${{ matrix.gpu_arch_type }}
7677
gpu_arch_ver: ${{ matrix.gpu_arch_version }}

‎test/smoke_test/smoke_test.py

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,13 @@
55
import argparse
66
import torch
77

8-
9-
108
gpu_arch_ver = os.getenv("GPU_ARCH_VER")
119
gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
1210
# use installation env variable to tell if it is nightly channel
1311
installation_str = os.getenv("INSTALLATION")
1412
is_cuda_system = gpu_arch_type == "cuda"
1513
SCRIPT_DIR = Path(__file__).parent
14+
NIGHTLY_ALLOWED_DELTA = 3
1615

1716
# helper function to return the conda installed packages
1817
# and return package we are insterseted in
@@ -35,32 +34,31 @@ def get_anaconda_output_for_package(pkg_name_str):
3534

3635

3736
def check_nightly_binaries_date(package: str) -> None:
37+
from datetime import datetime, timedelta
38+
format_dt = '%Y%m%d'
39+
3840
torch_str = torch.__version__
3941
date_t_str = re.findall("dev\d+", torch.__version__)
42+
date_t_delta = datetime.now() - datetime.strptime(date_t_str.lstrip("dev"), format_dt)
43+
if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA:
44+
raise RuntimeError(
45+
f"the binaries are from {date_t_str} and are more than {NIGHTLY_ALLOWED_DELTA} days old!"
46+
)
4047

4148
if(package == "all"):
4249
ta_str = torchaudio.__version__
4350
tv_str = torchvision.__version__
4451
date_ta_str = re.findall("dev\d+", torchaudio.__version__)
4552
date_tv_str = re.findall("dev\d+", torchvision.__version__)
53+
date_ta_delta = datetime.now() - datetime.strptime(date_ta_str.lstrip("dev"), format_dt)
54+
date_tv_delta = datetime.now() - datetime.strptime(date_tv_str.lstrip("dev"), format_dt)
4655

4756
# check that the above three lists are equal and none of them is empty
48-
if not date_t_str or not date_t_str == date_ta_str == date_tv_str:
57+
if date_ta_delta.days > NIGHTLY_ALLOWED_DELTA or date_tv_delta.days > NIGHTLY_ALLOWED_DELTA:
4958
raise RuntimeError(
50-
f"Expected torch, torchaudio, torchvision to be the same date. But they are from {date_t_str}, {date_ta_str}, {date_tv_str} respectively"
59+
f"Expected torchaudio, torchvision to be less then {NIGHTLY_ALLOWED_DELTA} days. But they are from {date_ta_str}, {date_tv_str} respectively"
5160
)
5261

53-
# check that the date is recent, at this point, date_torch_str is not empty
54-
binary_date_str = date_t_str[0][3:]
55-
from datetime import datetime
56-
57-
binary_date_obj = datetime.strptime(binary_date_str, "%Y%m%d").date()
58-
today_obj = datetime.today().date()
59-
delta = today_obj - binary_date_obj
60-
if delta.days >= 2:
61-
raise RuntimeError(
62-
f"the binaries are from {binary_date_obj} and are more than 2 days old!"
63-
)
6462

6563
def smoke_test_cuda(package: str) -> None:
6664
if not torch.cuda.is_available() and is_cuda_system:
@@ -76,6 +74,8 @@ def smoke_test_cuda(package: str) -> None:
7674
print(f"cuDNN enabled? {torch.backends.cudnn.enabled}")
7775

7876
if(package == 'all'):
77+
import torchaudio
78+
import torchvision
7979
if installation_str.find("nightly") != -1:
8080
# just print out cuda version, as version check were already performed during import
8181
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
@@ -165,6 +165,7 @@ def smoke_test_torchvision_resnet50_classify(device: str = "cpu") -> None:
165165

166166

167167
def smoke_test_torchaudio() -> None:
168+
import torchaudio
168169
import torchaudio.compliance.kaldi # noqa: F401
169170
import torchaudio.datasets # noqa: F401
170171
import torchaudio.functional # noqa: F401
@@ -184,20 +185,18 @@ def main() -> None:
184185
choices=["all", "torchonly"],
185186
default="all",
186187
)
187-
188+
options = parser.parse_args()
188189
print(f"torch: {torch.__version__}")
190+
189191
smoke_test_cuda(options.package)
190192
smoke_test_conv2d()
191193

192194
# only makes sense to check nightly package where dates are known
193195
if installation_str.find("nightly") != -1:
194-
check_nightly_binaries_date()
196+
check_nightly_binaries_date(options.package)
195197

196198
if options.package == "all":
197199
import torchaudio
198-
# the following import would invoke
199-
# _check_cuda_version()
200-
# via torchvision.extension._check_cuda_version()
201200
import torchvision
202201
print(f"torchvision: {torchvision.__version__}")
203202
print(f"torchaudio: {torchaudio.__version__}")

0 commit comments

Comments
 (0)