Skip to content

Add cuda and date check to smoke test #1145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Oct 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/validate-windows-binary/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ runs:
env:
GPU_ARCH_VER: ${{ inputs.gpu_arch_ver }}
GPU_ARCH_TYPE: ${{ inputs.gpu_arch_type }}
INSTALLATION: ${{ inputs.installation }}
CUDA_VER: ${{ inputs.desired_cuda }}
run: |
conda install numpy pillow python=${{ inputs.python_version }}
Expand Down
60 changes: 60 additions & 0 deletions test/smoke_test/smoke_test.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,57 @@
import os
import re
import sys
import torch
# the following import would invoke
# _check_cuda_version()
# via torchvision.extension._check_cuda_version()
import torchvision
import torchaudio
from pathlib import Path

gpu_arch_ver = os.getenv("GPU_ARCH_VER")
gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
# use installation env variable to tell if it is nightly channel
installation_str = os.getenv("INSTALLATION")
is_cuda_system = gpu_arch_type == "cuda"
SCRIPT_DIR = Path(__file__).parent

# helper function to return the conda list output, e.g.
# torchaudio 0.13.0.dev20220922 py39_cu102 pytorch-nightly
def get_anaconda_output_for_package(pkg_name_str):
import subprocess as sp

# ignore the header row:
# Name Version Build Channel
cmd = 'conda list -f ' + pkg_name_str
output = sp.getoutput(cmd)
# Get the last line only
return output.strip().split('\n')[-1]

def check_nightly_binaries_date() -> None:
torch_str = torch.__version__
ta_str = torchaudio.__version__
tv_str = torchvision.__version__

date_t_str = re.findall('dev\d+', torch.__version__ )
date_ta_str = re.findall('dev\d+', torchaudio.__version__ )
date_tv_str = re.findall('dev\d+', torchvision.__version__ )

# check that the above three lists are equal and none of them is empty
if not date_t_str or not date_t_str == date_ta_str == date_tv_str:
raise RuntimeError(f"Expected torch, torchaudio, torchvision to be the same date. But they are from {date_t_str}, {date_ta_str}, {date_tv_str} respectively")

# check that the date is recent, at this point, date_torch_str is not empty
binary_date_str = date_t_str[0][3:]
from datetime import datetime

binary_date_obj = datetime.strptime(binary_date_str, '%Y%m%d').date()
today_obj = datetime.today().date()
delta = today_obj - binary_date_obj
if delta.days >= 2:
raise RuntimeError(f"the binaries are from {binary_date_obj} and are more than 2 days old!")


def smoke_test_cuda() -> None:
if(not torch.cuda.is_available() and is_cuda_system):
raise RuntimeError(f"Expected CUDA {gpu_arch_ver}. However CUDA is not loaded.")
Expand All @@ -19,6 +61,19 @@ def smoke_test_cuda() -> None:
print(f"torch cuda: {torch.version.cuda}")
# todo add cudnn version validation
print(f"torch cudnn: {torch.backends.cudnn.version()}")
print(f"cuDNN enabled? {torch.backends.cudnn.enabled}")

if installation_str.find('nightly') != -1:
# just print out cuda version, as version check were already performed during import
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")
else:
# torchaudio runtime added the cuda verison check on 09/23/2022 via
# https://github.com/pytorch/audio/pull/2707
# so relying on anaconda output for pytorch-test and pytorch channel
torchaudio_allstr = get_anaconda_output_for_package(torchaudio.__name__)
if is_cuda_system and 'cu'+str(gpu_arch_ver).replace(".", "") not in torchaudio_allstr:
raise RuntimeError(f"CUDA version issue. Loaded: {torchaudio_allstr} Expected: {gpu_arch_ver}")

def smoke_test_conv2d() -> None:
import torch.nn as nn
Expand Down Expand Up @@ -95,6 +150,11 @@ def main() -> None:
print(f"torchvision: {torchvision.__version__}")
print(f"torchaudio: {torchaudio.__version__}")
smoke_test_cuda()

# only makes sense to check nightly package where dates are known
if installation_str.find('nightly') != -1:
check_nightly_binaries_date()

smoke_test_conv2d()
smoke_test_torchaudio()
smoke_test_torchvision()
Expand Down