From f2ba0a4aa9949f6b30d60f4f7fef232021fabe4c Mon Sep 17 00:00:00 2001 From: Jeff Yang Date: Sat, 24 Oct 2020 19:10:47 +0630 Subject: [PATCH 1/5] Update setup.cfg --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 00608f0487c0..cae9c5153343 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,3 +28,5 @@ markers = distributed: mark a test with distributed option multinode_distributed: mark a test with multi-node distributed option tpu: mark a test as requiring XLA +addopts = + --color=yes From 741bd68b3a3ebe248034b71966912fa77ad5445a Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Sat, 14 Nov 2020 15:46:36 +0630 Subject: [PATCH 2/5] [metrics] update ssim --- tests/ignite/metrics/test_ssim.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index b158c181e79b..69e857f8717d 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -1,5 +1,6 @@ import os +import numpy as np import pytest import torch @@ -50,21 +51,35 @@ def test_invalid_ssim(): def test_ssim(): device = "cuda" if torch.cuda.is_available() else "cpu" + atol = 7e-5 ssim = SSIM(data_range=1.0, device=device) - y_pred = torch.rand(16, 3, 64, 64, device=device) + y_pred = torch.rand(12, 3, 64, 64, device=device) y = y_pred * 0.65 ssim.update((y_pred, y)) + ignite_ssim = ssim.compute() np_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() np_y = np_pred * 0.65 - np_ssim = ski_ssim(np_pred, np_y, win_size=11, multichannel=True, gaussian_weights=True, data_range=1.0) - - assert isinstance(ssim.compute(), torch.Tensor) - assert torch.allclose(ssim.compute(), torch.tensor(np_ssim, dtype=torch.float64, device=device), atol=1e-4) + np_ssim = ski_ssim( + np_pred, + np_y, + win_size=11, + sigma=1.5, + multichannel=True, + gaussian_weights=True, + data_range=1.0, + use_sample_covariance=False, + ) + + assert isinstance(ignite_ssim, torch.Tensor) + assert ignite_ssim.dtype == torch.float64 + assert ignite_ssim.device == torch.device(device) + assert np.allclose(ignite_ssim.numpy(), np_ssim, atol=atol) + # assert torch.allclose(ssim.compute(), torch.tensor(np_ssim, dtype=torch.float64, device=device), atol=atol) device = "cuda" if torch.cuda.is_available() else "cpu" ssim = SSIM(data_range=1.0, gaussian=False, kernel_size=7, device=device) - y_pred = torch.rand(16, 3, 227, 227, device=device) + y_pred = torch.rand(8, 3, 227, 227, device=device) y = y_pred * 0.65 ssim.update((y_pred, y)) @@ -73,7 +88,7 @@ def test_ssim(): np_ssim = ski_ssim(np_pred, np_y, win_size=7, multichannel=True, gaussian_weights=False, data_range=1.0) assert isinstance(ssim.compute(), torch.Tensor) - assert torch.allclose(ssim.compute(), torch.tensor(np_ssim, dtype=torch.float64, device=device), atol=1e-4) + assert torch.allclose(ssim.compute(), torch.tensor(np_ssim, dtype=torch.float64, device=device), atol=atol) def _test_distrib_integration(device, tol=1e-4): From 7a5f7e11ca6d09b1cf96421b2c698fdba686efaa Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Sun, 15 Nov 2020 17:38:56 +0630 Subject: [PATCH 3/5] use np.allclose instead of torch.allclose --- tests/ignite/metrics/test_ssim.py | 83 +++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index 69e857f8717d..0441fa1164d6 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -21,14 +21,14 @@ def test_zero_div(): def test_invalid_ssim(): - y_pred = torch.rand(16, 1, 32, 32) + y_pred = torch.rand(1, 1, 4, 4) y = y_pred + 0.125 - with pytest.raises(ValueError, match=r"Expected kernel_size to have odd positive number. Got 10."): - ssim = SSIM(data_range=1.0, kernel_size=10) + with pytest.raises(ValueError, match=r"Expected kernel_size to have odd positive number."): + ssim = SSIM(data_range=1.0, kernel_size=2) ssim.update((y_pred, y)) ssim.compute() - with pytest.raises(ValueError, match=r"Expected kernel_size to have odd positive number. Got -1."): + with pytest.raises(ValueError, match=r"Expected kernel_size to have odd positive number."): ssim = SSIM(data_range=1.0, kernel_size=-1) ssim.update((y_pred, y)) ssim.compute() @@ -43,26 +43,49 @@ def test_invalid_ssim(): ssim.update((y_pred, y)) ssim.compute() + with pytest.raises(ValueError, match=r"Expected sigma to have positive number."): + ssim = SSIM(data_range=1.0, sigma=(-1, -1)) + ssim.update((y_pred, y)) + ssim.compute() + with pytest.raises(ValueError, match=r"Argument sigma should be either float or a sequence of float."): ssim = SSIM(data_range=1.0, sigma=1) ssim.update((y_pred, y)) ssim.compute() + with pytest.raises(ValueError, match=r"Expected y_pred and y to have the same shape."): + y = y.squeeze(dim=0) + ssim = SSIM(data_range=1.0) + ssim.update((y_pred, y)) + ssim.compute() + + with pytest.raises(ValueError, match=r"Expected y_pred and y to have BxCxHxW shape."): + y = y.squeeze(dim=0) + ssim = SSIM(data_range=1.0) + ssim.update((y, y)) + ssim.compute() + + with pytest.raises(TypeError, match=r"Expected y_pred and y to have the same data type."): + y = y.double() + ssim = SSIM(data_range=1.0) + ssim.update((y_pred, y)) + ssim.compute() + def test_ssim(): device = "cuda" if torch.cuda.is_available() else "cpu" atol = 7e-5 - ssim = SSIM(data_range=1.0, device=device) - y_pred = torch.rand(12, 3, 64, 64, device=device) - y = y_pred * 0.65 + ssim = SSIM(data_range=1.0, device=device, sigma=1.5) + y_pred = torch.rand(12, 3, 12, 12, device=device) + y = y_pred * 0.8 ssim.update((y_pred, y)) ignite_ssim = ssim.compute() - np_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() - np_y = np_pred * 0.65 - np_ssim = ski_ssim( - np_pred, - np_y, + skimg_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() + skimg_y = skimg_pred * 0.8 + skimg_ssim = ski_ssim( + skimg_pred, + skimg_y, win_size=11, sigma=1.5, multichannel=True, @@ -74,21 +97,22 @@ def test_ssim(): assert isinstance(ignite_ssim, torch.Tensor) assert ignite_ssim.dtype == torch.float64 assert ignite_ssim.device == torch.device(device) - assert np.allclose(ignite_ssim.numpy(), np_ssim, atol=atol) - # assert torch.allclose(ssim.compute(), torch.tensor(np_ssim, dtype=torch.float64, device=device), atol=atol) + assert np.allclose(ignite_ssim.numpy(), skimg_ssim, atol=atol) - device = "cuda" if torch.cuda.is_available() else "cpu" ssim = SSIM(data_range=1.0, gaussian=False, kernel_size=7, device=device) - y_pred = torch.rand(8, 3, 227, 227, device=device) - y = y_pred * 0.65 + y_pred = torch.rand(8, 3, 8, 8, device=device) + y = y_pred * 0.8 ssim.update((y_pred, y)) + ignite_ssim = ssim.compute() - np_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() - np_y = np_pred * 0.65 - np_ssim = ski_ssim(np_pred, np_y, win_size=7, multichannel=True, gaussian_weights=False, data_range=1.0) + skimg_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() + skimg_y = skimg_pred * 0.8 + skimg_ssim = ski_ssim(skimg_pred, skimg_y, win_size=7, multichannel=True, gaussian_weights=False, data_range=1.0) - assert isinstance(ssim.compute(), torch.Tensor) - assert torch.allclose(ssim.compute(), torch.tensor(np_ssim, dtype=torch.float64, device=device), atol=atol) + assert isinstance(ignite_ssim, torch.Tensor) + assert ignite_ssim.dtype == torch.float64 + assert ignite_ssim.device == torch.device(device) + assert np.allclose(ignite_ssim.numpy(), skimg_ssim, atol=atol) def _test_distrib_integration(device, tol=1e-4): @@ -100,7 +124,7 @@ def _test_distrib_integration(device, tol=1e-4): offset = n_iters * s def _test(metric_device): - y_pred = torch.rand(offset * idist.get_world_size(), 3, 28, 28, dtype=torch.float, device=device) + y_pred = torch.rand(offset * idist.get_world_size(), 3, 12, 12, dtype=torch.float, device=device) y = y_pred * 0.65 def update(engine, i): @@ -120,7 +144,16 @@ def update(engine, i): np_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() np_true = np_pred * 0.65 - true_res = ski_ssim(np_pred, np_true, win_size=11, multichannel=True, gaussian_weights=True, data_range=1.0) + true_res = ski_ssim( + np_pred, + np_true, + win_size=11, + sigma=1.5, + multichannel=True, + gaussian_weights=True, + data_range=1.0, + use_sample_covariance=False, + ) assert pytest.approx(res, abs=tol) == true_res @@ -157,7 +190,7 @@ def _test_distrib_accumulator_device(device): type(ssim._kernel.device), ssim._kernel.device, type(metric_device), metric_device ) - y_pred = torch.rand(4, 3, 28, 28, dtype=torch.float, device=device) + y_pred = torch.rand(1, 3, 12, 12, dtype=torch.float, device=device) y = y_pred * 0.65 ssim.update((y_pred, y)) From d36b35060646eaf2c9f6895bef37228b58de0b65 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Tue, 17 Nov 2020 10:36:16 +0630 Subject: [PATCH 4/5] Apply suggestions from code review --- tests/ignite/metrics/test_ssim.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index 0441fa1164d6..d995b12f8baa 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -100,7 +100,7 @@ def test_ssim(): assert np.allclose(ignite_ssim.numpy(), skimg_ssim, atol=atol) ssim = SSIM(data_range=1.0, gaussian=False, kernel_size=7, device=device) - y_pred = torch.rand(8, 3, 8, 8, device=device) + y_pred = torch.rand(8, 3, 28, 28, device=device) y = y_pred * 0.8 ssim.update((y_pred, y)) ignite_ssim = ssim.compute() @@ -124,7 +124,7 @@ def _test_distrib_integration(device, tol=1e-4): offset = n_iters * s def _test(metric_device): - y_pred = torch.rand(offset * idist.get_world_size(), 3, 12, 12, dtype=torch.float, device=device) + y_pred = torch.rand(offset * idist.get_world_size(), 3, 28, 28, dtype=torch.float, device=device) y = y_pred * 0.65 def update(engine, i): @@ -190,7 +190,7 @@ def _test_distrib_accumulator_device(device): type(ssim._kernel.device), ssim._kernel.device, type(metric_device), metric_device ) - y_pred = torch.rand(1, 3, 12, 12, dtype=torch.float, device=device) + y_pred = torch.rand(2, 3, 28, 28, dtype=torch.float, device=device) y = y_pred * 0.65 ssim.update((y_pred, y)) From b410c69b08b8521e6b6298c005c8dd7021c4f5d1 Mon Sep 17 00:00:00 2001 From: ydcjeff Date: Thu, 19 Nov 2020 19:07:42 +0630 Subject: [PATCH 5/5] extract into _test_ssim --- tests/ignite/metrics/test_ssim.py | 41 ++++++++++++++----------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index d995b12f8baa..3c77dda13f56 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -72,12 +72,9 @@ def test_invalid_ssim(): ssim.compute() -def test_ssim(): - device = "cuda" if torch.cuda.is_available() else "cpu" +def _test_ssim(y_pred, y, data_range, kernel_size, sigma, gaussian, use_sample_covariance, device): atol = 7e-5 - ssim = SSIM(data_range=1.0, device=device, sigma=1.5) - y_pred = torch.rand(12, 3, 12, 12, device=device) - y = y_pred * 0.8 + ssim = SSIM(data_range=data_range, sigma=sigma, device=device) ssim.update((y_pred, y)) ignite_ssim = ssim.compute() @@ -86,12 +83,12 @@ def test_ssim(): skimg_ssim = ski_ssim( skimg_pred, skimg_y, - win_size=11, - sigma=1.5, + win_size=kernel_size, + sigma=sigma, multichannel=True, - gaussian_weights=True, - data_range=1.0, - use_sample_covariance=False, + gaussian_weights=gaussian, + data_range=data_range, + use_sample_covariance=use_sample_covariance, ) assert isinstance(ignite_ssim, torch.Tensor) @@ -99,20 +96,20 @@ def test_ssim(): assert ignite_ssim.device == torch.device(device) assert np.allclose(ignite_ssim.numpy(), skimg_ssim, atol=atol) - ssim = SSIM(data_range=1.0, gaussian=False, kernel_size=7, device=device) - y_pred = torch.rand(8, 3, 28, 28, device=device) - y = y_pred * 0.8 - ssim.update((y_pred, y)) - ignite_ssim = ssim.compute() - skimg_pred = y_pred.permute(0, 2, 3, 1).cpu().numpy() - skimg_y = skimg_pred * 0.8 - skimg_ssim = ski_ssim(skimg_pred, skimg_y, win_size=7, multichannel=True, gaussian_weights=False, data_range=1.0) +def test_ssim(): + device = "cuda" if torch.cuda.is_available() else "cpu" + y_pred = torch.rand(8, 3, 224, 224, device=device) + y = y_pred * 0.8 + _test_ssim( + y_pred, y, data_range=1.0, kernel_size=7, sigma=1.5, gaussian=False, use_sample_covariance=True, device=device + ) - assert isinstance(ignite_ssim, torch.Tensor) - assert ignite_ssim.dtype == torch.float64 - assert ignite_ssim.device == torch.device(device) - assert np.allclose(ignite_ssim.numpy(), skimg_ssim, atol=atol) + y_pred = torch.rand(12, 3, 28, 28, device=device) + y = y_pred * 0.8 + _test_ssim( + y_pred, y, data_range=1.0, kernel_size=11, sigma=1.5, gaussian=True, use_sample_covariance=False, device=device + ) def _test_distrib_integration(device, tol=1e-4):