ci: Bring mypy typechecking back (aws#2503)

aahung · mndeveci · commit a41a05048a98 · 2021-01-18T18:49:49.000-08:00
* * Revert "Revert "chore: Enable basic mypy typechecking"" This brings back commit 9d29984. * chore: Enable basic mypy typechecking, 2nd round after reverting * Remove import from builtins * Loose boto3-stubs version, matching boto3 aws#2493
diff --git a/Makefile b/Makefile
@@ -35,6 +35,8 @@ smoke-test:
 lint:
 	# Linter performs static analysis to catch latent bugs
 	pylint --rcfile .pylintrc samcli
+	# mypy performs type check
+	mypy setup.py samcli tests
 
 # Command to run everytime you make changes to verify everything works
 dev: lint test
diff --git a/appveyor.yml b/appveyor.yml
@@ -79,6 +79,7 @@ for:
       - "venv\\Scripts\\activate"
       - "pytest --cov samcli --cov-report term-missing --cov-fail-under 94 tests/unit"
       - "pylint --rcfile .pylintrc samcli"
+      - "mypy setup.py samcli tests"
       - "pytest -n 4 tests/functional"
 
   - 
@@ -132,6 +133,7 @@ for:
     test_script:
       - "pytest --cov samcli --cov-report term-missing --cov-fail-under 94 tests/unit"
       - "pylint --rcfile .pylintrc samcli"
+      - "mypy setup.py samcli tests"
       - "pytest -n 4 tests/functional"
 
       # Runs only in Linux, logging docker hub when running canary and docker cred is available
diff --git a/mypy.ini b/mypy.ini
@@ -0,0 +1,59 @@
+# https://mypy.readthedocs.io/en/stable/config_file.html#config-file-format
+
+[mypy]
+warn_return_any=True
+warn_unused_configs=True
+no_implicit_optional=True
+warn_redundant_casts=True
+warn_unused_ignores=True
+warn_unreachable=True
+
+#
+# ignore errors in testdata
+#
+
+[mypy-tests.integration.testdata.*]
+ignore_errors=True
+
+#
+# below are packages/modules that do not have stubs available
+#
+
+[mypy-pywintypes]
+ignore_missing_imports=True
+
+[mypy-botocore,botocore.*]
+ignore_missing_imports=True
+
+[mypy-docker,docker.*]
+ignore_missing_imports=True
+
+[mypy-aws_lambda_builders,aws_lambda_builders.*]
+ignore_missing_imports=True
+
+[mypy-cookiecutter,cookiecutter.*]
+ignore_missing_imports=True
+
+[mypy-serverlessrepo,serverlessrepo.*]
+ignore_missing_imports=True
+
+[mypy-tomlkit]
+ignore_missing_imports=True
+
+[mypy-samtranslator,samtranslator.*]
+ignore_missing_imports=True
+
+[mypy-jmespath]
+ignore_missing_imports=True
+
+[mypy-chevron]
+ignore_missing_imports=True
+
+[mypy-parameterized]
+ignore_missing_imports=True
+
+[mypy-setuptools]
+ignore_missing_imports=True
+
+[mypy-watchdog,watchdog.*]
+ignore_missing_imports=True
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -2,6 +2,10 @@ coverage==5.3
 pytest-cov==2.10.1
 pylint~=2.6.0
 
+# type checking and related stubs
+mypy~=0.790
+boto3-stubs[essential]~=1.14
+
 # Test requirements
 pytest==6.1.1
 parameterized==0.7.4
diff --git a/samcli/lib/generated_sample_events/events.py b/samcli/lib/generated_sample_events/events.py
@@ -6,7 +6,7 @@
 import json
 import base64
 import warnings
-from requests.utils import quote as url_quote
+from urllib.parse import quote as url_quote
 
 with warnings.catch_warnings():
     # https://github.com/aws/aws-sam-cli/issues/2381
diff --git a/samcli/lib/logs/formatter.py b/samcli/lib/logs/formatter.py
@@ -5,15 +5,6 @@
 import json
 import functools
 
-try:
-    # Python2
-    from itertools import imap
-except ImportError:
-    # Python3 already has `map` defined, alias to imap
-    # We do this to prevent accidentally using the built-in ``map`` in Python2. In Python2, ``map`` does a full
-    # evaluation of the iterator whereas imap does a lazy evaluation. For performance reasons, we need to use ``imap``.
-    from builtins import map as imap
-
 
 class LogsFormatter:
     """
@@ -106,7 +97,7 @@ def do_format(self, event_iterable):
 
             # Make sure the operation has access to certain basic objects like colored
             partial_op = functools.partial(operation, colored=self.colored)
-            event_iterable = imap(partial_op, event_iterable)
+            event_iterable = map(partial_op, event_iterable)
 
         return event_iterable
 
diff --git a/samcli/lib/package/packageable_resources.py b/samcli/lib/package/packageable_resources.py
@@ -4,6 +4,7 @@
 import logging
 import os
 import shutil
+from typing import Optional
 
 import jmespath
 from botocore.utils import set_value_from_jmespath
@@ -50,14 +51,14 @@
 
 
 class Resource:
-    RESOURCE_TYPE = None
-    PROPERTY_NAME = None
+    RESOURCE_TYPE: Optional[str] = None
+    PROPERTY_NAME: Optional[str] = None
     PACKAGE_NULL_PROPERTY = True
     # Set this property to True in base class if you want the exporter to zip
     # up the file before uploading This is useful for Lambda functions.
     FORCE_ZIP = False
-    EXPORT_DESTINATION = None
-    ARTIFACT_TYPE = None
+    EXPORT_DESTINATION: Optional[str] = None
+    ARTIFACT_TYPE: Optional[str] = None
 
     def __init__(self, uploader, code_signer):
         self.uploader = uploader
@@ -75,8 +76,8 @@ class ResourceZip(Resource):
     Base class representing a CloudFormation resource that can be exported
     """
 
-    RESOURCE_TYPE = None
-    PROPERTY_NAME = None
+    RESOURCE_TYPE: Optional[str] = None
+    PROPERTY_NAME: Optional[str] = None
     PACKAGE_NULL_PROPERTY = True
     # Set this property to True in base class if you want the exporter to zip
     # up the file before uploading This is useful for Lambda functions.
@@ -149,8 +150,8 @@ class ResourceImageDict(Resource):
     Base class representing a CFN Image based resource that can be exported.
     """
 
-    RESOURCE_TYPE = None
-    PROPERTY_NAME = None
+    RESOURCE_TYPE: Optional[str] = None
+    PROPERTY_NAME: Optional[str] = None
     FORCE_ZIP = False
     ARTIFACT_TYPE = IMAGE
     EXPORT_DESTINATION = "ecr"
@@ -196,10 +197,10 @@ class ResourceImage(Resource):
     Base class representing a SAM Image based resource that can be exported.
     """
 
-    RESOURCE_TYPE = None
-    PROPERTY_NAME = None
+    RESOURCE_TYPE: Optional[str] = None
+    PROPERTY_NAME: Optional[str] = None
     FORCE_ZIP = False
-    ARTIFACT_TYPE = IMAGE
+    ARTIFACT_TYPE: Optional[str] = IMAGE
     EXPORT_DESTINATION = "ecr"
 
     def __init__(self, uploader, code_signer):
@@ -242,9 +243,9 @@ class ResourceWithS3UrlDict(ResourceZip):
     an dict like {Bucket: "", Key: "", Version: ""}
     """
 
-    BUCKET_NAME_PROPERTY = None
-    OBJECT_KEY_PROPERTY = None
-    VERSION_PROPERTY = None
+    BUCKET_NAME_PROPERTY: Optional[str] = None
+    OBJECT_KEY_PROPERTY: Optional[str] = None
+    VERSION_PROPERTY: Optional[str] = None
     ARTIFACT_TYPE = ZIP
     EXPORT_DESTINATION = "s3"
 
diff --git a/samcli/lib/providers/provider.py b/samcli/lib/providers/provider.py
@@ -202,6 +202,10 @@ def name(self):
     def codeuri(self):
         return self._codeuri
 
+    @codeuri.setter
+    def codeuri(self, codeuri):
+        self._codeuri = codeuri
+
     @property
     def version(self):
         return self._version
@@ -211,10 +215,6 @@ def layer_arn(self):
         layer_arn, _ = self.arn.rsplit(":", 1)
         return layer_arn
 
-    @codeuri.setter
-    def codeuri(self, codeuri):
-        self._codeuri = codeuri
-
     @property
     def build_method(self):
         return self._build_method
@@ -259,7 +259,7 @@ def binary_media_types(self):
 
 _CorsTuple = namedtuple("Cors", ["allow_origin", "allow_methods", "allow_headers", "max_age"])
 
-_CorsTuple.__new__.__defaults__ = (
+_CorsTuple.__new__.__defaults__ = (  # type: ignore
     None,  # Allow Origin defaults to None
     None,  # Allow Methods is optional and defaults to empty
     None,  # Allow Headers is optional and defaults to empty
diff --git a/samcli/lib/schemas/schemas_aws_config.py b/samcli/lib/schemas/schemas_aws_config.py
@@ -2,7 +2,7 @@
 
 import click
 
-from boto3 import Session
+from boto3.session import Session
 from samcli.commands.local.cli_common.user_exceptions import ResourceNotFound
 
 
diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py
@@ -5,6 +5,7 @@
 import itertools
 import os
 import pathlib
+from typing import Set
 
 _init_path = str(pathlib.Path(os.path.dirname(__file__)).parent.parent)
 _templates = os.path.join(_init_path, "lib", "init", "templates")
@@ -94,14 +95,16 @@ def get_local_lambda_images_location(mapping, runtime):
     "java8.al2": ["maven", "gradle"],
 }
 
-SUPPORTED_DEP_MANAGERS = {
-    c["dependency_manager"]
+SUPPORTED_DEP_MANAGERS: Set[str] = {
+    c["dependency_manager"]  # type: ignore
     for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))
     if c["dependency_manager"]
 }
 
-RUNTIMES = set(
-    itertools.chain(*[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))])
+RUNTIMES: Set[str] = set(
+    itertools.chain(
+        *[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))]  # type: ignore
+    )
 )
 
 # When adding new Lambda runtimes, please update SAM_RUNTIME_TO_SCHEMAS_CODE_LANG_MAPPING
diff --git a/samcli/local/lambdafn/env_vars.py b/samcli/local/lambdafn/env_vars.py
@@ -129,6 +129,10 @@ def memory(self, value):
     def handler(self):
         return self._function["handler"]
 
+    @handler.setter
+    def handler(self, value):
+        self._function["handler"] = value
+
     @property
     def name(self):
         return self._function["name"]
@@ -137,10 +141,6 @@ def name(self):
     def name(self, value):
         self._function["name"] = value
 
-    @handler.setter
-    def handler(self, value):
-        self._function["handler"] = value
-
     def _get_aws_variables(self):
         """
         Returns the AWS specific environment variables that should be available in the Lambda runtime.
diff --git a/samcli/settings/__init__.py b/samcli/settings/__init__.py
@@ -17,7 +17,7 @@
 
 import os
 
-if "__SAM_CLI_TELEMETRY_ENDPOINT_URL" not in os.environ:
-    telemetry_endpoint_url = "https://aws-serverless-tools-telemetry.us-west-2.amazonaws.com/metrics"
-else:
-    telemetry_endpoint_url = os.getenv("__SAM_CLI_TELEMETRY_ENDPOINT_URL")
+
+telemetry_endpoint_url = os.getenv(
+    "__SAM_CLI_TELEMETRY_ENDPOINT_URL", "https://aws-serverless-tools-telemetry.us-west-2.amazonaws.com/metrics"
+)
diff --git a/samcli/yamlhelper.py b/samcli/yamlhelper.py
@@ -21,7 +21,10 @@
 from botocore.compat import OrderedDict
 
 import yaml
-from yaml.resolver import ScalarNode, SequenceNode
+
+# ScalarNode and SequenceNode are not declared in __all__,
+# TODO: we need to double check whether they are public and stable
+from yaml.resolver import ScalarNode, SequenceNode  # type: ignore
 
 TAG_STR = "tag:yaml.org,2002:str"
 
diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py
@@ -149,7 +149,6 @@ def _verify_invoke_built_function(self, template_path, function_logical_id, over
 
 
 class BuildIntegRubyBase(BuildIntegBase):
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"app.rb"}
     EXPECTED_RUBY_GEM = "aws-record"
 
diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py
@@ -33,7 +33,6 @@
 class TestBuildCommand_PythonFunctions_Images(BuildIntegBase):
     template = "template_image.yaml"
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {
         "__init__.py",
         "main.py",
@@ -70,7 +69,6 @@ def test_with_default_requirements(self, runtime, use_container):
     "Skip build tests on windows when running in CI unless overridden",
 )
 class TestBuildCommand_PythonFunctions(BuildIntegBase):
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {
         "__init__.py",
         "main.py",
@@ -186,7 +184,6 @@ def test_unsupported_runtime(self):
     "Skip build tests on windows when running in CI unless overridden",
 )
 class TestBuildCommand_NodeFunctions(BuildIntegBase):
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"node_modules", "main.js"}
     EXPECTED_NODE_MODULES = {"minimal-request-promise"}
 
@@ -665,7 +662,6 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files)
 class TestBuildCommand_SingleFunctionBuilds(BuildIntegBase):
     template = "many-functions-template.yaml"
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {
         "__init__.py",
         "main.py",
@@ -742,7 +738,6 @@ def _get_python_version(self):
 class TestBuildCommand_LayerBuilds(BuildIntegBase):
     template = "layers-functions-template.yaml"
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"__init__.py", "main.py", "requirements.txt"}
     EXPECTED_LAYERS_FILES_PROJECT_MANIFEST = {"__init__.py", "layer.py", "numpy", "requirements.txt"}
 
@@ -906,7 +901,6 @@ class TestBuildCommand_ProvidedFunctions(BuildIntegBase):
     # Test Suite for runtime: provided and where selection of the build workflow is implicitly makefile builder
     # if the makefile is present.
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"__init__.py", "main.py", "requests", "requirements.txt"}
 
     FUNCTION_LOGICAL_ID = "Function"
@@ -980,7 +974,6 @@ class TestBuildWithBuildMethod(BuildIntegBase):
     # Test Suite where `BuildMethod` is explicitly specified.
 
     template = "custom-build-function.yaml"
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"__init__.py", "main.py", "requests", "requirements.txt"}
 
     FUNCTION_LOGICAL_ID = "Function"
diff --git a/tests/integration/init/schemas/schemas_test_data_setup.py b/tests/integration/init/schemas/schemas_test_data_setup.py
@@ -4,7 +4,7 @@
 import tempfile
 from unittest import TestCase
 
-from boto3 import Session
+from boto3.session import Session
 from botocore.exceptions import ClientError
 from click.testing import CliRunner
 from samcli.commands.init import cli as init_cmd
diff --git a/tests/integration/local/invoke/invoke_integ_base.py b/tests/integration/local/invoke/invoke_integ_base.py
diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py
diff --git a/tests/integration/local/start_api/start_api_integ_base.py b/tests/integration/local/start_api/start_api_integ_base.py
diff --git a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py
diff --git a/tests/unit/commands/deploy/test_guided_config.py b/tests/unit/commands/deploy/test_guided_config.py
diff --git a/tests/unit/lib/intrinsic_resolver/test_intrinsic_resolver.py b/tests/unit/lib/intrinsic_resolver/test_intrinsic_resolver.py
diff --git a/tests/unit/lib/schemas/test_schemas_api_caller.py b/tests/unit/lib/schemas/test_schemas_api_caller.py
diff --git a/tests/unit/lib/utils/test_sam_logging.py b/tests/unit/lib/utils/test_sam_logging.py
