From a09f2fd474784cd0ae744e3a0484c4d5275865c9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 3 Aug 2021 10:00:32 +0100 Subject: [PATCH 01/10] Run tests with pytest, not bash --- scripts/install.sh | 2 ++ scripts/run_pytorch.sh | 34 ++++++++++++---------------------- test_run_python_code.py | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 test_run_python_code.py diff --git a/scripts/install.sh b/scripts/install.sh index 2c4115c5..5d1f4886 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -9,6 +9,8 @@ conda install -y pytorch torchvision -c pytorch # Also install torchaudio conda install -y -c pytorch torchaudio +conda install pytest + # Dependencies required to load models conda install -y regex pillow tqdm boto3 requests numpy\ h5py scipy matplotlib unidecode ipython pyyaml diff --git a/scripts/run_pytorch.sh b/scripts/run_pytorch.sh index e662c757..ce0d97f1 100755 --- a/scripts/run_pytorch.sh +++ b/scripts/run_pytorch.sh @@ -3,30 +3,20 @@ set -e . ~/miniconda3/etc/profile.d/conda.sh conda activate base -ALL_FILE=$(find *.md ! -name README.md) +ALL_FILES=$(find *.md ! -name README.md) TEMP_PY="temp.py" +PYTHON_CODE_DIR="python_code" CUDAS="nvidia" -for f in $ALL_FILE -do - echo "Running pytorch example in $f" - # FIXME: NVIDIA models checkoints are on cuda - if [[ $f = $CUDAS* ]]; then - echo "...skipped due to cuda checkpoints." - elif [[ $f = "pytorch_fairseq_translation"* ]]; then - echo "...temporarily disabled" - # FIXME: torch.nn.modules.module.ModuleAttributeError: 'autoShape' object has no attribute 'fuse' - elif [[ $f = "ultralytics_yolov5"* ]]; then - echo "...temporarily disabled" - elif [[ $f = "huggingface_pytorch-transformers"* ]]; then - echo "...temporarily disabled" - # FIXME: rate limiting - else - sed -n '/^```python/,/^```/ p' < $f | sed '/^```/ d' > $TEMP_PY - python $TEMP_PY +mkdir $PYTHON_CODE_DIR - if [ -f "$TEMP_PY" ]; then - rm $TEMP_PY - fi - fi +for f in $ALL_FILES +do + f_no_ext=${f%.md} # remove .md extension + out_py=$PYTHON_CODE_DIR/$f_no_ext.py + echo "Extracting Python code from $f into $out_py" + sed -n '/^```python/,/^```/ p' < $f | sed '/^```/ d' > $out_py done + +pytest -v -s test_run_python_code.py +rm -r $PYTHON_CODE_DIR diff --git a/test_run_python_code.py b/test_run_python_code.py new file mode 100644 index 00000000..f888a082 --- /dev/null +++ b/test_run_python_code.py @@ -0,0 +1,22 @@ +import pytest +import glob + + +ALL_FILES = glob.glob("python_code/*.py") + + +@pytest.mark.parametrize('file_path', ALL_FILES) +def test_run_file(file_path): + if 'nvidia' in file_path: + # FIXME: NVIDIA models checkoints are on cuda + pytest.skip("temporarily disabled") + if 'pytorch_fairseq_translation' in file_path: + pytest.skip("temporarily disabled") + if 'ultralytics_yolov5' in file_path: + # FIXME torch.nn.modules.module.ModuleAttributeError: 'autoShape' object has no attribute 'fuse + pytest.skip("temporarily disabled") + if 'huggingface_pytorch-transformers' in file_path: + # FIXME torch.nn.modules.module.ModuleAttributeError: 'autoShape' object has no attribute 'fuse + pytest.skip("temporarily disabled") + + exec(open(file_path).read()) From b664c0dcc11a8516c62d56c262b4be9cbbef4970 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 3 Aug 2021 10:03:50 +0100 Subject: [PATCH 02/10] comments --- scripts/run_pytorch.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/run_pytorch.sh b/scripts/run_pytorch.sh index ce0d97f1..19e26548 100755 --- a/scripts/run_pytorch.sh +++ b/scripts/run_pytorch.sh @@ -10,6 +10,9 @@ CUDAS="nvidia" mkdir $PYTHON_CODE_DIR +# Quick rundown: for each file we extract the python code that's within +# the ``` markers and we put that code in a corresponding .py file in $PYTHON_CODE_DIR +# Then we execute each of these python files with pytest in test_run_python_code.py for f in $ALL_FILES do f_no_ext=${f%.md} # remove .md extension From 680e19bd27dc2a62fb80dc760c620a73962c61c6 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 3 Aug 2021 10:10:13 +0100 Subject: [PATCH 03/10] forgot -y --- scripts/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install.sh b/scripts/install.sh index 5d1f4886..6fb5a46f 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -9,7 +9,7 @@ conda install -y pytorch torchvision -c pytorch # Also install torchaudio conda install -y -c pytorch torchaudio -conda install pytest +conda install -y pytest # Dependencies required to load models conda install -y regex pillow tqdm boto3 requests numpy\ From a7fe7145d40d2426f6d108eeee7fe165a1ad8afb Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 3 Aug 2021 10:11:18 +0100 Subject: [PATCH 04/10] remove unneeded stuff --- scripts/run_pytorch.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/run_pytorch.sh b/scripts/run_pytorch.sh index 19e26548..da90b27c 100755 --- a/scripts/run_pytorch.sh +++ b/scripts/run_pytorch.sh @@ -4,9 +4,7 @@ set -e conda activate base ALL_FILES=$(find *.md ! -name README.md) -TEMP_PY="temp.py" PYTHON_CODE_DIR="python_code" -CUDAS="nvidia" mkdir $PYTHON_CODE_DIR From dc0bc3230295ca9ff86d8f68186781c40e9916c5 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 3 Aug 2021 10:22:40 +0100 Subject: [PATCH 05/10] Add store_test_result step --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 922c53db..be4da9f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,9 @@ jobs: name: Run Pytorch scripts command: ./scripts/run_pytorch.sh no_output_timeout: 1h + - store_test_results: + path: test-results + workflows: version: 2 From 23ea83d3a6220916f126ddc00e95e88ac558ebaa Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 31 Aug 2021 09:51:10 +0100 Subject: [PATCH 06/10] try to add report in test-results/junit.xml --- scripts/run_pytorch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_pytorch.sh b/scripts/run_pytorch.sh index 6324e131..2ae9bd7c 100755 --- a/scripts/run_pytorch.sh +++ b/scripts/run_pytorch.sh @@ -18,5 +18,5 @@ do sed -n '/^```python/,/^```/ p' < $f | sed '/^```/ d' > $out_py done -pytest -v -s test_run_python_code.py +pytest --junitxml=test-results/junit.xml -v -s test_run_python_code.py rm -r $PYTHON_CODE_DIR From 456589f0bbc1482930810ca7ece5946d68244de9 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 31 Aug 2021 09:54:57 +0100 Subject: [PATCH 07/10] Let last line be pytest --- scripts/run_pytorch.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/run_pytorch.sh b/scripts/run_pytorch.sh index 2ae9bd7c..e0945e51 100755 --- a/scripts/run_pytorch.sh +++ b/scripts/run_pytorch.sh @@ -19,4 +19,3 @@ do done pytest --junitxml=test-results/junit.xml -v -s test_run_python_code.py -rm -r $PYTHON_CODE_DIR From 67ae8ae72af6c20f8d1ecacfbd34e7038b3d869f Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 31 Aug 2021 11:49:39 +0100 Subject: [PATCH 08/10] use importlib instead of open() because it messes up globals --- test_run_python_code.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_run_python_code.py b/test_run_python_code.py index 8111af68..10dce979 100644 --- a/test_run_python_code.py +++ b/test_run_python_code.py @@ -1,8 +1,10 @@ +import importlib import pytest import glob -ALL_FILES = glob.glob("python_code/*.py") +PYTHON_CODE_DIR = "python_code" +ALL_FILES = glob.glob(PYTHON_CODE_DIR + "/*.py") @pytest.mark.parametrize('file_path', ALL_FILES) @@ -21,4 +23,5 @@ def test_run_file(file_path): if 'pytorch_fairseq_roberta' in file_path: pytest.skip("temporarily disabled") - exec(open(file_path).read()) + module = file_path[:-3].replace('/', '.') # :-3 to remove the .py extension + importlib.import_module(module) From 73a1018fe59b6eb848f1be06806a44e559dfb284 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 31 Aug 2021 15:04:53 +0100 Subject: [PATCH 09/10] Use subprocesses to avoid import conflicts --- test_run_python_code.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test_run_python_code.py b/test_run_python_code.py index 10dce979..f44a2856 100644 --- a/test_run_python_code.py +++ b/test_run_python_code.py @@ -1,4 +1,5 @@ -import importlib +from subprocess import check_output, STDOUT, CalledProcessError +import sys import pytest import glob @@ -23,5 +24,18 @@ def test_run_file(file_path): if 'pytorch_fairseq_roberta' in file_path: pytest.skip("temporarily disabled") - module = file_path[:-3].replace('/', '.') # :-3 to remove the .py extension - importlib.import_module(module) + # We just run the python files in a separate sub-process. We really want a + # subprocess here because otherwise we might run into package versions + # issues: imagine script A that needs torchvivion 0.9 and script B that + # needs torchvision 0.10. If script A is run prior to script B in the same + # process, script B will still be run with torchvision 0.9 because the only + # "import torchvision" statement that counts is the first one, and even + # torchub sys.path shenanigans can do nothing about this. By creating + # subprocesses we're sure that all file executions are fully independent. + try: + # This is inspired (and heavily simplified) from + # https://github.com/cloudpipe/cloudpickle/blob/343da119685f622da2d1658ef7b3e2516a01817f/tests/testutils.py#L177 + out = check_output([sys.executable, file_path], stderr=STDOUT) + print(out.decode()) + except CalledProcessError as e: + raise RuntimeError(f"Script {file_path} errored with output:\n{e.output.decode()}") From 60eb03eaee6c5938e4cd9b6abbbe142111c8ff48 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 31 Aug 2021 15:18:55 +0100 Subject: [PATCH 10/10] remove -s to have cleaner output --- scripts/run_pytorch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_pytorch.sh b/scripts/run_pytorch.sh index e0945e51..a4b9132a 100755 --- a/scripts/run_pytorch.sh +++ b/scripts/run_pytorch.sh @@ -18,4 +18,4 @@ do sed -n '/^```python/,/^```/ p' < $f | sed '/^```/ d' > $out_py done -pytest --junitxml=test-results/junit.xml -v -s test_run_python_code.py +pytest --junitxml=test-results/junit.xml test_run_python_code.py -vv