diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0cd5b7f..3ac9f4d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -34,10 +34,17 @@ jobs: echo "******* Running fairseq unittests *******" bash tests/run_fairseq_tests.sh echo "******* Running transformers unittests *******" - bash tests/run_transformers_tests.sh + #bash tests/run_transformers_tests.sh echo "******* Running fastseq unittests *******" - pip install pytorch-transformers==1.0.0 - python -m unittest discover -s tests/ -p 'test_*.py' -v + #pip install pytorch-transformers==1.0.0 + #python -m unittest discover -s tests/ -p 'test_*.py' -v #cd benchmarks/ #CUDA_VISIBLE_DEVICES=3 run_all_benchmarks.sh displayName: 'run fastseq unit tests' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testRunTitle: 'Publish test results for Python $(python.version)' + testResultsFiles: '/tmp/fastseq_tests/*.xml' + failTaskOnFailedTests: true + diff --git a/fastseq/optimizer/fairseq/beam_search_optimizer.py b/fastseq/optimizer/fairseq/beam_search_optimizer.py index 69fc178..cfd6dd1 100644 --- a/fastseq/optimizer/fairseq/beam_search_optimizer.py +++ b/fastseq/optimizer/fairseq/beam_search_optimizer.py @@ -5,7 +5,7 @@ import math from typing import Optional - +import unittest import torch import torch.nn.functional as F from torch import Tensor @@ -50,7 +50,6 @@ class MultiheadAttentionV2(MultiheadAttention): See "Attention Is All You Need" for more details. """ - def __init__(self, embed_dim, num_heads, @@ -159,7 +158,7 @@ def forward( q = self.q_proj(query) if key is None: assert value is None - k = v = None + k v = None else: if self.beam_size > 1 and bsz == key.size(1): # key is [T, bsz*beam_size, C], reduce to [T, bsz, C] diff --git a/fastseq/utils/test_utils.py b/fastseq/utils/test_utils.py index 2ef9d5a..deccf15 100644 --- a/fastseq/utils/test_utils.py +++ b/fastseq/utils/test_utils.py @@ -3,11 +3,13 @@ """Utilities to make it easy to add unit tests""" +from inspect import getframeinfo, stack import os from statistics import mean, stdev import time -from absl.testing import parameterized +from absl import flags +from absl.testing import absltest, parameterized from fastseq.config import FASTSEQ_CACHE_DIR from fastseq.logging import get_logger @@ -15,6 +17,16 @@ logger = get_logger(__name__) +FLAGS = flags.FLAGS + +def fastseq_test_main(): + caller = getframeinfo(stack()[1][0]) + xml_log_file = caller.filename.replace(os.sep, '_').replace('.py', '.xml') + xml_log_file = os.path.join(os.sep, 'tmp', 'fastseq_tests', xml_log_file) + FLAGS.xml_output_file = xml_log_file + logger.info(f"Fastseq unit test log output filepath: {xml_log_file}") + absltest.main() + class TestCaseBase(parameterized.TestCase): """Base class used for unittest.""" diff --git a/tests/models/test_prophetnet_fs.py b/tests/models/test_prophetnet_fs.py index 7e02f15..5eed8c8 100644 --- a/tests/models/test_prophetnet_fs.py +++ b/tests/models/test_prophetnet_fs.py @@ -19,7 +19,7 @@ from fastseq.utils.file_utils import decompress_file, make_dirs, wget from fastseq.utils.test_utils import (PROPHETNET_MODEL_URLS, CACHED_PROPHETNET_MODEL_PATHS, - TestCaseBase) + fastseq_test_main, TestCaseBase) logger = get_logger(__name__) @@ -136,4 +136,4 @@ def test_beam_search_optimizer(self, beam_size, batch_size, need_attn, self.assertEqual(output, self.expected_outputs[i]) if __name__ == "__main__": - absltest.main() + fastseq_test_main() diff --git a/tests/optimizer/fairseq/benchmark_fairseq_optimizer.py b/tests/optimizer/fairseq/benchmark_fairseq_optimizer.py index 9897557..28f78b0 100644 --- a/tests/optimizer/fairseq/benchmark_fairseq_optimizer.py +++ b/tests/optimizer/fairseq/benchmark_fairseq_optimizer.py @@ -15,7 +15,7 @@ from fastseq.utils.file_utils import decompress_file, make_dirs, wget from fastseq.utils.test_utils import (BART_MODEL_URLS, CACHED_BART_MODEL_DIR, CACHED_BART_MODEL_PATHS, BenchmarkBase, - benchmark) + benchmark, fastseq_test_main) logger = get_logger(__name__) @@ -128,4 +128,4 @@ def test_beam_search_optimizer(self, beam_size, batch_size, need_attn, if __name__ == "__main__": - absltest.main() + fastseq_test_main() diff --git a/tests/optimizer/fairseq/test_fairseq_optimizer.py b/tests/optimizer/fairseq/test_fairseq_optimizer.py index e476a92..0fb73bb 100644 --- a/tests/optimizer/fairseq/test_fairseq_optimizer.py +++ b/tests/optimizer/fairseq/test_fairseq_optimizer.py @@ -16,7 +16,8 @@ from fastseq.logging import get_logger from fastseq.utils.file_utils import decompress_file, make_dirs, wget from fastseq.utils.test_utils import (BART_MODEL_URLS, CACHED_BART_MODEL_DIR, - CACHED_BART_MODEL_PATHS, TestCaseBase) + CACHED_BART_MODEL_PATHS, + fastseq_test_main, TestCaseBase) logger = get_logger(__name__) @@ -117,4 +118,4 @@ def test_beam_search_optimizer(self, beam_size, batch_size, need_attn, if __name__ == "__main__": - absltest.main() + fastseq_test_main() diff --git a/tests/optimizer/transformers/test_bart_optimizer.py b/tests/optimizer/transformers/test_bart_optimizer.py index bb4b809..e5d4ed0 100644 --- a/tests/optimizer/transformers/test_bart_optimizer.py +++ b/tests/optimizer/transformers/test_bart_optimizer.py @@ -14,7 +14,7 @@ from absl.testing import absltest, parameterized from transformers import BartForConditionalGeneration, BartTokenizer -from fastseq.utils.test_utils import TestCaseBase +from fastseq.utils.test_utils import fastseq_test_main, TestCaseBase class BARTOptimizerTest(TestCaseBase): @@ -183,4 +183,4 @@ def test_beam_search_optimizer(self, if __name__ == "__main__": - absltest.main() + fastseq_test_main() diff --git a/tests/optimizer/transformers/test_t5_optimizer.py b/tests/optimizer/transformers/test_t5_optimizer.py index 9d38a91..43b155a 100644 --- a/tests/optimizer/transformers/test_t5_optimizer.py +++ b/tests/optimizer/transformers/test_t5_optimizer.py @@ -12,7 +12,7 @@ import fastseq from fastseq.logging import get_logger -from fastseq.utils.test_utils import TestCaseBase +from fastseq.utils.test_utils import fastseq_test_main, TestCaseBase from transformers import (T5ForConditionalGeneration, T5Tokenizer) @@ -184,4 +184,4 @@ def test_beam_search_optimizer(self, if __name__ == "__main__": - absltest.main() + fastseq_test_main() diff --git a/tests/run_fairseq_tests.py b/tests/run_fairseq_tests.py index 980d87b..f05cbb0 100644 --- a/tests/run_fairseq_tests.py +++ b/tests/run_fairseq_tests.py @@ -9,6 +9,7 @@ import logging import shutil import unittest +import xmlrunner from git import Repo from absl.testing import absltest, parameterized from pip._internal import main as pipmain @@ -77,4 +78,5 @@ def test_suites(self, without_fastseq_opt, fairseq_version, blocked_tests): assert len(test_result.errors) == 0 if __name__ == "__main__": - absltest.main() + unittest.main( + testRunner=xmlrunner.XMLTestRunner(output='/tmp/fastseq_tests/')) diff --git a/tests/run_fairseq_tests.sh b/tests/run_fairseq_tests.sh index 23b12d8..a537ac1 100755 --- a/tests/run_fairseq_tests.sh +++ b/tests/run_fairseq_tests.sh @@ -9,6 +9,7 @@ pip install packaging cd ${FASTSEQ_TEST_PATH}/../ pip install torch==1.5.0+cu101 torchvision==0.6.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html pip install --editable . +pip install unittest-xml-reporting cd tests python run_fairseq_tests.py deactivate diff --git a/tests/utils/test_api_decorator.py b/tests/utils/test_api_decorator.py index a4d2a23..f07a250 100644 --- a/tests/utils/test_api_decorator.py +++ b/tests/utils/test_api_decorator.py @@ -5,7 +5,7 @@ from absl.testing import absltest, parameterized from fastseq.utils.api_decorator import get_class, override_method, add_method, export_api, replace -from fastseq.utils.test_utils import TestCaseBase +from fastseq.utils.test_utils import fastseq_test_main, TestCaseBase class A: @@ -152,4 +152,4 @@ def name(self): if __name__ == "__main__": - absltest.main() + fastseq_test_main() diff --git a/tests/utils/test_file_utils.py b/tests/utils/test_file_utils.py index 2b48a30..bc94c1f 100644 --- a/tests/utils/test_file_utils.py +++ b/tests/utils/test_file_utils.py @@ -8,7 +8,7 @@ from absl.testing import absltest, parameterized from fastseq.utils.file_utils import decompress_file, get_temp_dir, make_dirs, wget -from fastseq.utils.test_utils import TestCaseBase +from fastseq.utils.test_utils import fastseq_test_main, TestCaseBase class FileUtilsTest(TestCaseBase): @@ -90,4 +90,4 @@ def test_wget_and_decompress_file(self, tar_file_url, tar_file_name, if __name__ == "__main__": - absltest.main() + fastseq_test_main()