Skip to content

Commit 28508a3

Browse files
atalmanatalman
authored
Add smoke tests conv,linalg,compile. And better version check. (#1333)
* Add smoke tests conv,linalg,compile * Add version check * Fix typo Fix version check Add not * Add exception for python 3.11 * fix typo * Try to exit after CUDA Runtime exception * Restrict carsh test only to conda * Restrict carsh test only to conda * Fix tests * Turn off cuda runtime issue * tests * more tests * test * remove compile step * test * disable some of the tests * testing * Remove extra index url * test * Fix tests * Additional smoke tests Remove release blocking changes
1 parent 7d4c980 commit 28508a3

File tree

2 files changed

+72
-13
lines changed

2 files changed

+72
-13
lines changed

‎.github/scripts/validate_binaries.sh

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@ else
1717
conda env remove -n ${ENV_NAME}
1818
else
1919

20+
21+
2022
# Special case Pypi installation package, only applicable to linux nightly CUDA 11.7 builds, wheel package
21-
if [[ ${TARGET_OS} == 'linux' && ${MATRIX_CHANNEL} == 'nightly' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' ]]; then
22-
conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy
23+
if [[ ${TARGET_OS} == 'linux' && ${MATRIX_GPU_ARCH_VERSION} == '11.7' && ${MATRIX_PACKAGE_TYPE} == 'manywheel' ]]; then
24+
conda create -yp ${ENV_NAME}_pypi python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
2325
INSTALLATION_PYPI=${MATRIX_INSTALLATION/"cu117"/"cu117_pypi_cudnn"}
2426
INSTALLATION_PYPI=${INSTALLATION_PYPI/"torchvision torchaudio"/""}
2527
INSTALLATION_PYPI=${INSTALLATION_PYPI/"index-url"/"extra-index-url"}
@@ -29,9 +31,11 @@ else
2931
conda env remove -p ${ENV_NAME}_pypi
3032
fi
3133

32-
conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy pillow
34+
# Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
35+
conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
3336
conda activate ${ENV_NAME}
3437
INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"}
38+
INSTALLATION=${INSTALLATION/"extra-index-url"/"index-url"}
3539
eval $INSTALLATION
3640

3741
if [[ ${TARGET_OS} == 'linux' ]]; then

‎test/smoke_test/smoke_test.py

Lines changed: 65 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010

1111
gpu_arch_ver = os.getenv("MATRIX_GPU_ARCH_VERSION")
1212
gpu_arch_type = os.getenv("MATRIX_GPU_ARCH_TYPE")
13-
# use installation env variable to tell if it is nightly channel
14-
installation_str = os.getenv("MATRIX_INSTALLATION")
13+
channel = os.getenv("MATRIX_CHANNEL")
14+
stable_version = os.getenv("MATRIX_STABLE_VERSION")
15+
package_type = os.getenv("MATRIX_PACKAGE_TYPE")
16+
1517
is_cuda_system = gpu_arch_type == "cuda"
1618
SCRIPT_DIR = Path(__file__).parent
1719
NIGHTLY_ALLOWED_DELTA = 3
@@ -31,6 +33,16 @@
3133
},
3234
]
3335

36+
def check_version(package: str) -> None:
37+
# only makes sense to check nightly package where dates are known
38+
if channel == "nightly":
39+
check_nightly_binaries_date(options.package)
40+
else:
41+
if not torch.__version__.startswith(stable_version):
42+
raise RuntimeError(
43+
f"Torch version mismatch, expected {stable_version} for channel {channel}. But its {torch.__version__}"
44+
)
45+
3446
def check_nightly_binaries_date(package: str) -> None:
3547
from datetime import datetime, timedelta
3648
format_dt = '%Y%m%d'
@@ -58,6 +70,7 @@ def check_nightly_binaries_date(package: str) -> None:
5870
def test_cuda_runtime_errors_captured() -> None:
5971
cuda_exception_missed=True
6072
try:
73+
print("Testing test_cuda_runtime_errors_captured")
6174
torch._assert_async(torch.tensor(0, device="cuda"))
6275
torch._assert_async(torch.tensor(0 + 0j, device="cuda"))
6376
except RuntimeError as e:
@@ -95,29 +108,73 @@ def smoke_test_cuda(package: str) -> None:
95108
print(f"torch cudnn: {torch.backends.cudnn.version()}")
96109
print(f"cuDNN enabled? {torch.backends.cudnn.enabled}")
97110

98-
# This check has to be run last, since its messing up CUDA runtime
111+
# torch.compile is available only on Linux and python 3.8-3.10
112+
if (sys.platform == "linux" or sys.platform == "linux2") and sys.version_info < (3, 11, 0):
113+
smoke_test_compile()
114+
99115
test_cuda_runtime_errors_captured()
100116

101117

102118
def smoke_test_conv2d() -> None:
103119
import torch.nn as nn
104120

105-
print("Calling smoke_test_conv2d")
121+
print("Testing smoke_test_conv2d")
106122
# With square kernels and equal stride
107123
m = nn.Conv2d(16, 33, 3, stride=2)
108124
# non-square kernels and unequal stride and with padding
109125
m = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2))
110126
# non-square kernels and unequal stride and with padding and dilation
111-
m = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2), dilation=(3, 1))
127+
basic_conv = nn.Conv2d(16, 33, (3, 5), stride=(2, 1), padding=(4, 2), dilation=(3, 1))
112128
input = torch.randn(20, 16, 50, 100)
113-
output = m(input)
129+
output = basic_conv(input)
130+
114131
if is_cuda_system:
115132
print("Testing smoke_test_conv2d with cuda")
116133
conv = nn.Conv2d(3, 3, 3).cuda()
117134
x = torch.randn(1, 3, 24, 24).cuda()
118135
with torch.cuda.amp.autocast():
119136
out = conv(x)
120137

138+
supported_dtypes = [torch.float16, torch.float32, torch.float64]
139+
for dtype in supported_dtypes:
140+
print(f"Testing smoke_test_conv2d with cuda for {dtype}")
141+
conv = basic_conv.to(dtype).cuda()
142+
input = torch.randn(20, 16, 50, 100, device="cuda").type(dtype)
143+
output = conv(input)
144+
145+
def smoke_test_linalg() -> None:
146+
print("Testing smoke_test_linalg")
147+
A = torch.randn(5, 3)
148+
U, S, Vh = torch.linalg.svd(A, full_matrices=False)
149+
U.shape, S.shape, Vh.shape
150+
torch.dist(A, U @ torch.diag(S) @ Vh)
151+
152+
U, S, Vh = torch.linalg.svd(A)
153+
U.shape, S.shape, Vh.shape
154+
torch.dist(A, U[:, :3] @ torch.diag(S) @ Vh)
155+
156+
A = torch.randn(7, 5, 3)
157+
U, S, Vh = torch.linalg.svd(A, full_matrices=False)
158+
torch.dist(A, U @ torch.diag_embed(S) @ Vh)
159+
160+
if is_cuda_system:
161+
supported_dtypes = [torch.float32, torch.float64]
162+
for dtype in supported_dtypes:
163+
print(f"Testing smoke_test_linalg with cuda for {dtype}")
164+
A = torch.randn(20, 16, 50, 100, device="cuda").type(dtype)
165+
torch.linalg.svd(A)
166+
167+
def smoke_test_compile() -> None:
168+
supported_dtypes = [torch.float16, torch.float32, torch.float64]
169+
def foo(x: torch.Tensor) -> torch.Tensor:
170+
return torch.sin(x) + torch.cos(x)
171+
for dtype in supported_dtypes:
172+
print(f"Testing smoke_test_compile for {dtype}")
173+
x = torch.rand(3, 3, device="cuda").type(dtype)
174+
x_eager = foo(x)
175+
x_pt2 = torch.compile(foo)(x)
176+
print(torch.allclose(x_eager, x_pt2))
177+
121178

122179
def smoke_test_modules():
123180
for module in MODULES:
@@ -146,15 +203,13 @@ def main() -> None:
146203
)
147204
options = parser.parse_args()
148205
print(f"torch: {torch.__version__}")
206+
check_version(options.package)
149207
smoke_test_conv2d()
208+
smoke_test_linalg()
150209

151210
if options.package == "all":
152211
smoke_test_modules()
153212

154-
# only makes sense to check nightly package where dates are known
155-
if installation_str.find("nightly") != -1:
156-
check_nightly_binaries_date(options.package)
157-
158213
smoke_test_cuda(options.package)
159214

160215

0 commit comments

Comments
 (0)