From c8770052493b015bf78c9d989ee1fe311cba3b7b Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:30:36 -0700 Subject: [PATCH 1/6] Added CodeQL and Bandit security checks as GitHub Actions --- .circleci/regenerate.py | 2 +- .../unittest/linux/scripts/environment.yml | 1 + .../unittest/windows/scripts/environment.yml | 1 + .github/workflows/bandit.yml | 23 ++++++++++ .github/workflows/codeql.yml | 43 +++++++++++++++++++ packaging/torchvision/meta.yaml | 3 ++ torchvision/datasets/voc.py | 2 +- 7 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/bandit.yml create mode 100644 .github/workflows/codeql.yml diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index d70860de86f..f61992ef06a 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -295,7 +295,7 @@ def ios_workflows(indentation=6, nightly=False): env = jinja2.Environment( loader=jinja2.FileSystemLoader(d), lstrip_blocks=True, - autoescape=False, + autoescape=True, keep_trailing_newline=True, ) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index dcad1abfa31..f685c2f85aa 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -16,3 +16,4 @@ dependencies: - pillow>=4.1.1 - scipy - av + - defusedxml diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index b4f32cb3cad..105fadab8af 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -17,3 +17,4 @@ dependencies: - scipy - av - dataclasses + - defusedxml diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml new file mode 100644 index 00000000000..93bae80f9bd --- /dev/null +++ b/.github/workflows/bandit.yml @@ -0,0 +1,23 @@ +# GitHub Actions Bandit Workflow + +name: Bandit + +on: + pull_request: + branches: [ master ] + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + # Task will fail if any high-severity issues are found + # Ignoring submodules + - name: Run Bandit Security Analysis + run: | + python -m pip install bandit + python -m bandit -r . -x ./third_party -lll diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..387d82ec343 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,43 @@ +# GitHub Actions CodeQL Workflow + +name: CodeQL + +on: + pull_request: + branches: [ master ] + + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: python, cpp + + - name: Install Ninja + run: | + sudo apt-get update -y + sudo apt-get install -y ninja-build + + - name: Update submodules + run: git submodule update --init --recursive + + - name: Install Torch + run: | + python -m pip install cmake + python -m pip install torch==1.8.1+cpu -f https://download.pytorch.org/whl/torch_stable.html + sudo ln -s /usr/bin/ninja /usr/bin/ninja-build + + - name: Build TorchVision + run: python setup.py develop --user + + # If any code scanning alerts are found, they will be under Security -> CodeQL + # Link: https://github.com/pytorch/vision/security/code-scanning + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index 8279c276433..959e65b84e6 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -12,6 +12,7 @@ requirements: - jpeg <=9b # NOTE: The only ffmpeg version that we build is actually 4.2 - ffmpeg >=4.2 # [not win] + - defusedxml host: - python @@ -28,6 +29,7 @@ requirements: - jpeg <=9b - pillow >=4.1.1 - defaults::numpy >=1.11 + - defusedxml {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} @@ -55,6 +57,7 @@ test: - av >=8.0.1 - jpeg <=9b - ca-certificates + - defusedxml about: diff --git a/torchvision/datasets/voc.py b/torchvision/datasets/voc.py index 9f6685e865c..c61a7a5c0e8 100644 --- a/torchvision/datasets/voc.py +++ b/torchvision/datasets/voc.py @@ -2,7 +2,7 @@ import tarfile import collections from .vision import VisionDataset -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET from PIL import Image from typing import Any, Callable, Dict, Optional, Tuple, List from .utils import download_and_extract_archive, verify_str_arg From b60154d087bbe70a2a9115d55953cd1160de1cf5 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Wed, 31 Mar 2021 14:35:13 -0700 Subject: [PATCH 2/6] Nit fix on defusedxml.ElementTree --- torchvision/datasets/voc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/torchvision/datasets/voc.py b/torchvision/datasets/voc.py index c61a7a5c0e8..b54e43f3cfa 100644 --- a/torchvision/datasets/voc.py +++ b/torchvision/datasets/voc.py @@ -2,7 +2,8 @@ import tarfile import collections from .vision import VisionDataset -import defusedxml.ElementTree as ET +import xml.etree.ElementTree as ET +import defusedxml.ElementTree as DET from PIL import Image from typing import Any, Callable, Dict, Optional, Tuple, List from .utils import download_and_extract_archive, verify_str_arg @@ -203,7 +204,7 @@ def __getitem__(self, index: int) -> Tuple[Any, Any]: tuple: (image, target) where target is a dictionary of the XML tree. """ img = Image.open(self.images[index]).convert("RGB") - target = self.parse_voc_xml(ET.parse(self.annotations[index]).getroot()) + target = self.parse_voc_xml(DET.parse(self.annotations[index]).getroot()) if self.transforms is not None: img, target = self.transforms(img, target) From 5e5bea9e9674256ad06fb4361c67f7541299f7c5 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 1 Apr 2021 14:54:24 -0700 Subject: [PATCH 3/6] Remove defusedxml as hard requirement --- .circleci/regenerate.py | 3 ++- .circleci/unittest/linux/scripts/environment.yml | 1 - .circleci/unittest/windows/scripts/environment.yml | 1 - packaging/torchvision/meta.yaml | 3 --- torchvision/datasets/voc.py | 9 +++++++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index f61992ef06a..c853ec273e4 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -15,6 +15,7 @@ """ import jinja2 +from jinja2 import select_autoescape import yaml import os.path @@ -295,7 +296,7 @@ def ios_workflows(indentation=6, nightly=False): env = jinja2.Environment( loader=jinja2.FileSystemLoader(d), lstrip_blocks=True, - autoescape=True, + autoescape=select_autoescape(enabled_extensions=('html', 'xml')), keep_trailing_newline=True, ) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index f685c2f85aa..dcad1abfa31 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -16,4 +16,3 @@ dependencies: - pillow>=4.1.1 - scipy - av - - defusedxml diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index 105fadab8af..b4f32cb3cad 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -17,4 +17,3 @@ dependencies: - scipy - av - dataclasses - - defusedxml diff --git a/packaging/torchvision/meta.yaml b/packaging/torchvision/meta.yaml index 959e65b84e6..8279c276433 100644 --- a/packaging/torchvision/meta.yaml +++ b/packaging/torchvision/meta.yaml @@ -12,7 +12,6 @@ requirements: - jpeg <=9b # NOTE: The only ffmpeg version that we build is actually 4.2 - ffmpeg >=4.2 # [not win] - - defusedxml host: - python @@ -29,7 +28,6 @@ requirements: - jpeg <=9b - pillow >=4.1.1 - defaults::numpy >=1.11 - - defusedxml {{ environ.get('CONDA_PYTORCH_CONSTRAINT') }} {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT') }} @@ -57,7 +55,6 @@ test: - av >=8.0.1 - jpeg <=9b - ca-certificates - - defusedxml about: diff --git a/torchvision/datasets/voc.py b/torchvision/datasets/voc.py index b54e43f3cfa..68076848710 100644 --- a/torchvision/datasets/voc.py +++ b/torchvision/datasets/voc.py @@ -3,7 +3,6 @@ import collections from .vision import VisionDataset import xml.etree.ElementTree as ET -import defusedxml.ElementTree as DET from PIL import Image from typing import Any, Callable, Dict, Optional, Tuple, List from .utils import download_and_extract_archive, verify_str_arg @@ -203,8 +202,14 @@ def __getitem__(self, index: int) -> Tuple[Any, Any]: Returns: tuple: (image, target) where target is a dictionary of the XML tree. """ + # If defusedxml exists, use defusedxml.ElementTree instead. + # Else, use xml.ElementTree as normal. + try: + import defusedxml.ElementTree as ET + except ImportError: + pass img = Image.open(self.images[index]).convert("RGB") - target = self.parse_voc_xml(DET.parse(self.annotations[index]).getroot()) + target = self.parse_voc_xml(ET.parse(self.annotations[index]).getroot()) if self.transforms is not None: img, target = self.transforms(img, target) From 9ce0af5aa27395f93be6fb2b393d16bdda567aed Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Thu, 1 Apr 2021 17:04:18 -0700 Subject: [PATCH 4/6] Changed diffusedxml/xml importing --- torchvision/datasets/voc.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/torchvision/datasets/voc.py b/torchvision/datasets/voc.py index 68076848710..80f3f5b42e4 100644 --- a/torchvision/datasets/voc.py +++ b/torchvision/datasets/voc.py @@ -2,7 +2,11 @@ import tarfile import collections from .vision import VisionDataset -import xml.etree.ElementTree as ET +import xml.etree.ElementTree.Element as ET_Element +try: + import defusedxml.ElementTree.parse as ET_parse +except ImportError: + import xml.etree.ElementTree.parse as ET_parse from PIL import Image from typing import Any, Callable, Dict, Optional, Tuple, List from .utils import download_and_extract_archive, verify_str_arg @@ -202,21 +206,15 @@ def __getitem__(self, index: int) -> Tuple[Any, Any]: Returns: tuple: (image, target) where target is a dictionary of the XML tree. """ - # If defusedxml exists, use defusedxml.ElementTree instead. - # Else, use xml.ElementTree as normal. - try: - import defusedxml.ElementTree as ET - except ImportError: - pass img = Image.open(self.images[index]).convert("RGB") - target = self.parse_voc_xml(ET.parse(self.annotations[index]).getroot()) + target = self.parse_voc_xml(ET_parse(self.annotations[index]).getroot()) if self.transforms is not None: img, target = self.transforms(img, target) return img, target - def parse_voc_xml(self, node: ET.Element) -> Dict[str, Any]: + def parse_voc_xml(self, node: ET_Element) -> Dict[str, Any]: voc_dict: Dict[str, Any] = {} children = list(node) if children: From 33acc79a9eee473d2f5feaa691776d4e8b656efc Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Thu, 1 Apr 2021 18:43:57 -0700 Subject: [PATCH 5/6] Fix compilation --- torchvision/datasets/voc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torchvision/datasets/voc.py b/torchvision/datasets/voc.py index 80f3f5b42e4..557392e1bbf 100644 --- a/torchvision/datasets/voc.py +++ b/torchvision/datasets/voc.py @@ -2,11 +2,11 @@ import tarfile import collections from .vision import VisionDataset -import xml.etree.ElementTree.Element as ET_Element +from xml.etree.ElementTree import Element as ET_Element try: - import defusedxml.ElementTree.parse as ET_parse + from defusedxml.ElementTree import parse as ET_parse except ImportError: - import xml.etree.ElementTree.parse as ET_parse + from xml.etree.ElementTree import parse as ET_parse from PIL import Image from typing import Any, Callable, Dict, Optional, Tuple, List from .utils import download_and_extract_archive, verify_str_arg From fc48e1ef26e2be47d6fb996e000926fab4c81f37 Mon Sep 17 00:00:00 2001 From: Mustafa Bal <5262061+mstfbl@users.noreply.github.com> Date: Mon, 5 Apr 2021 15:58:31 -0700 Subject: [PATCH 6/6] Removed Bandit specific changes --- .circleci/regenerate.py | 3 +-- torchvision/datasets/voc.py | 10 +++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index c853ec273e4..d70860de86f 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -15,7 +15,6 @@ """ import jinja2 -from jinja2 import select_autoescape import yaml import os.path @@ -296,7 +295,7 @@ def ios_workflows(indentation=6, nightly=False): env = jinja2.Environment( loader=jinja2.FileSystemLoader(d), lstrip_blocks=True, - autoescape=select_autoescape(enabled_extensions=('html', 'xml')), + autoescape=False, keep_trailing_newline=True, ) diff --git a/torchvision/datasets/voc.py b/torchvision/datasets/voc.py index 557392e1bbf..9f6685e865c 100644 --- a/torchvision/datasets/voc.py +++ b/torchvision/datasets/voc.py @@ -2,11 +2,7 @@ import tarfile import collections from .vision import VisionDataset -from xml.etree.ElementTree import Element as ET_Element -try: - from defusedxml.ElementTree import parse as ET_parse -except ImportError: - from xml.etree.ElementTree import parse as ET_parse +import xml.etree.ElementTree as ET from PIL import Image from typing import Any, Callable, Dict, Optional, Tuple, List from .utils import download_and_extract_archive, verify_str_arg @@ -207,14 +203,14 @@ def __getitem__(self, index: int) -> Tuple[Any, Any]: tuple: (image, target) where target is a dictionary of the XML tree. """ img = Image.open(self.images[index]).convert("RGB") - target = self.parse_voc_xml(ET_parse(self.annotations[index]).getroot()) + target = self.parse_voc_xml(ET.parse(self.annotations[index]).getroot()) if self.transforms is not None: img, target = self.transforms(img, target) return img, target - def parse_voc_xml(self, node: ET_Element) -> Dict[str, Any]: + def parse_voc_xml(self, node: ET.Element) -> Dict[str, Any]: voc_dict: Dict[str, Any] = {} children = list(node) if children: