Skip to content

Commit d034c3c

Browse files
atalmanatalman
authored
Refactor smoke tests to configure module included in the release (#1223)
1 parent b970528 commit d034c3c

File tree

3 files changed

+55
-107
lines changed

3 files changed

+55
-107
lines changed

‎.github/workflows/validate-nightly-binaries.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ on:
2525
- .github/workflows/validate-macos-binaries.yml
2626
- .github/workflows/validate-macos-arm64-binaries.yml
2727
- test/smoke_test/*
28-
2928
jobs:
3029
nightly:
3130
uses: ./.github/workflows/validate-binaries.yml

‎.github/workflows/validate-windows-binaries.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ jobs:
4949
repository: "pytorch/builder"
5050
ref: ${{ inputs.ref || github.ref }}
5151
job-name: ${{ matrix.build_name }}
52+
timeout: 60
5253
script: |
5354
set -ex
5455
export ENV_NAME="conda-env-${{ github.run_id }}"

‎test/smoke_test/smoke_test.py

Lines changed: 54 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import argparse
66
import torch
77
import platform
8+
import importlib
9+
import subprocess
810

911
gpu_arch_ver = os.getenv("GPU_ARCH_VER")
1012
gpu_arch_type = os.getenv("GPU_ARCH_TYPE")
@@ -14,6 +16,21 @@
1416
SCRIPT_DIR = Path(__file__).parent
1517
NIGHTLY_ALLOWED_DELTA = 3
1618

19+
MODULES = [
20+
{
21+
"name": "torchvision",
22+
"repo": "https://github.com/pytorch/vision.git",
23+
"smoke_test": "python ./vision/test/smoke_test.py",
24+
"extension": "extension",
25+
},
26+
{
27+
"name": "torchaudio",
28+
"repo": "https://github.com/pytorch/audio.git",
29+
"smoke_test": "python ./audio/test/smoke_test/smoke_test.py --no-ffmpeg",
30+
"extension": "_extension",
31+
},
32+
]
33+
1734
def check_nightly_binaries_date(package: str) -> None:
1835
from datetime import datetime, timedelta
1936
format_dt = '%Y%m%d'
@@ -27,33 +44,16 @@ def check_nightly_binaries_date(package: str) -> None:
2744
)
2845

2946
if(package == "all"):
30-
import torchaudio
31-
import torchvision
32-
ta_str = torchaudio.__version__
33-
tv_str = torchvision.__version__
34-
date_ta_str = re.findall("dev\d+", torchaudio.__version__)
35-
date_tv_str = re.findall("dev\d+", torchvision.__version__)
36-
date_ta_delta = datetime.now() - datetime.strptime(date_ta_str[0][3:], format_dt)
37-
date_tv_delta = datetime.now() - datetime.strptime(date_tv_str[0][3:], format_dt)
38-
39-
# check that the above three lists are equal and none of them is empty
40-
if date_ta_delta.days > NIGHTLY_ALLOWED_DELTA or date_tv_delta.days > NIGHTLY_ALLOWED_DELTA:
41-
raise RuntimeError(
42-
f"Expected torchaudio, torchvision to be less then {NIGHTLY_ALLOWED_DELTA} days. But they are from {date_ta_str}, {date_tv_str} respectively"
43-
)
44-
45-
def check_cuda_version(version: str, dlibary: str):
46-
version = torch.ops.torchaudio.cuda_version()
47-
if version is not None and torch.version.cuda is not None:
48-
version_str = str(version)
49-
ta_version = f"{version_str[:-3]}.{version_str[-2]}"
50-
t_version = torch.version.cuda.split(".")
51-
t_version = f"{t_version[0]}.{t_version[1]}"
52-
if ta_version != t_version:
53-
raise RuntimeError(
54-
"Detected that PyTorch and {dlibary} were compiled with different CUDA versions. "
55-
f"PyTorch has CUDA version {t_version} whereas {dlibary} has CUDA version {ta_version}. "
56-
)
47+
for module in MODULES:
48+
imported_module = importlib.import_module(module["name"])
49+
module_version = imported_module.__version__
50+
date_m_str = re.findall("dev\d+", module_version)
51+
date_m_delta = datetime.now() - datetime.strptime(date_m_str[0][3:], format_dt)
52+
print(f"Nightly date check for {module['name']} version {module_version}")
53+
if date_m_delta.days > NIGHTLY_ALLOWED_DELTA:
54+
raise RuntimeError(
55+
f"Expected {module['name']} to be less then {NIGHTLY_ALLOWED_DELTA} days. But its {date_m_delta}"
56+
)
5757

5858
def smoke_test_cuda(package: str) -> None:
5959
if not torch.cuda.is_available() and is_cuda_system:
@@ -69,12 +69,16 @@ def smoke_test_cuda(package: str) -> None:
6969
print(f"cuDNN enabled? {torch.backends.cudnn.enabled}")
7070

7171
if(package == 'all' and is_cuda_system):
72-
import torchaudio
73-
import torchvision
74-
print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}")
75-
print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}")
76-
check_cuda_version(torch.ops.torchvision._cuda_version(), "TorchVision")
77-
check_cuda_version(torch.ops.torchaudio.cuda_version(), "TorchAudio")
72+
for module in MODULES:
73+
imported_module = importlib.import_module(module["name"])
74+
# TBD for vision move extension module to private so it will
75+
# be _extention.
76+
version = "N/A"
77+
if module["extension"] == "extension":
78+
version = imported_module.extension._check_cuda_version()
79+
else:
80+
version = imported_module._extension._check_cuda_version()
81+
print(f"{module['name']} CUDA: {version}")
7882

7983

8084
def smoke_test_conv2d() -> None:
@@ -97,67 +101,20 @@ def smoke_test_conv2d() -> None:
97101
out = conv(x)
98102

99103

100-
def smoke_test_torchvision() -> None:
101-
print(
102-
"Is torchvision useable?",
103-
all(
104-
x is not None
105-
for x in [torch.ops.image.decode_png, torch.ops.torchvision.roi_align]
106-
),
107-
)
108-
109-
110-
def smoke_test_torchvision_read_decode() -> None:
111-
from torchvision.io import read_image
112-
113-
img_jpg = read_image(str(SCRIPT_DIR / "assets" / "rgb_pytorch.jpg"))
114-
if img_jpg.ndim != 3 or img_jpg.numel() < 100:
115-
raise RuntimeError(f"Unexpected shape of img_jpg: {img_jpg.shape}")
116-
img_png = read_image(str(SCRIPT_DIR / "assets" / "rgb_pytorch.png"))
117-
if img_png.ndim != 3 or img_png.numel() < 100:
118-
raise RuntimeError(f"Unexpected shape of img_png: {img_png.shape}")
119-
120-
121-
def smoke_test_torchvision_resnet50_classify(device: str = "cpu") -> None:
122-
from torchvision.io import read_image
123-
from torchvision.models import resnet50, ResNet50_Weights
124-
125-
img = read_image(str(SCRIPT_DIR / "assets" / "dog2.jpg")).to(device)
126-
127-
# Step 1: Initialize model with the best available weights
128-
weights = ResNet50_Weights.DEFAULT
129-
model = resnet50(weights=weights).to(device)
130-
model.eval()
131-
132-
# Step 2: Initialize the inference transforms
133-
preprocess = weights.transforms()
134-
135-
# Step 3: Apply inference preprocessing transforms
136-
batch = preprocess(img).unsqueeze(0)
137-
138-
# Step 4: Use the model and print the predicted category
139-
prediction = model(batch).squeeze(0).softmax(0)
140-
class_id = prediction.argmax().item()
141-
score = prediction[class_id].item()
142-
category_name = weights.meta["categories"][class_id]
143-
expected_category = "German shepherd"
144-
print(f"{category_name}: {100 * score:.1f}%")
145-
if category_name != expected_category:
146-
raise RuntimeError(
147-
f"Failed ResNet50 classify {category_name} Expected: {expected_category}"
148-
)
149-
150-
151-
def smoke_test_torchaudio() -> None:
152-
import torchaudio
153-
import torchaudio.compliance.kaldi # noqa: F401
154-
import torchaudio.datasets # noqa: F401
155-
import torchaudio.functional # noqa: F401
156-
import torchaudio.models # noqa: F401
157-
import torchaudio.pipelines # noqa: F401
158-
import torchaudio.sox_effects # noqa: F401
159-
import torchaudio.transforms # noqa: F401
160-
import torchaudio.utils # noqa: F401
104+
def smoke_test_modules():
105+
for module in MODULES:
106+
if module["repo"]:
107+
subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True)
108+
try:
109+
output = subprocess.check_output(
110+
module["smoke_test"], stderr=subprocess.STDOUT, shell=True,
111+
universal_newlines=True)
112+
except subprocess.CalledProcessError as exc:
113+
raise RuntimeError(
114+
f"Module {module['name']} FAIL: {exc.returncode} Output: {exc.output}"
115+
)
116+
else:
117+
print("Output: \n{}\n".format(output))
161118

162119

163120
def main() -> None:
@@ -171,25 +128,16 @@ def main() -> None:
171128
)
172129
options = parser.parse_args()
173130
print(f"torch: {torch.__version__}")
174-
175131
smoke_test_cuda(options.package)
176132
smoke_test_conv2d()
177133

134+
if options.package == "all":
135+
smoke_test_modules()
136+
178137
# only makes sense to check nightly package where dates are known
179138
if installation_str.find("nightly") != -1:
180139
check_nightly_binaries_date(options.package)
181140

182-
if options.package == "all":
183-
import torchaudio
184-
import torchvision
185-
print(f"torchvision: {torchvision.__version__}")
186-
print(f"torchaudio: {torchaudio.__version__}")
187-
smoke_test_torchaudio()
188-
smoke_test_torchvision()
189-
smoke_test_torchvision_read_decode()
190-
smoke_test_torchvision_resnet50_classify()
191-
if torch.cuda.is_available():
192-
smoke_test_torchvision_resnet50_classify("cuda")
193141

194142
if __name__ == "__main__":
195143
main()

0 commit comments

Comments
 (0)