Merge 9fcbe13 into bb9acd8

Author: lucywang000

.github/workflows/build.yml

@@ -30,6 +30,9 @@ jobs:
         - python-version: 3.9
           env:
             TOXENV: black
+        - python-version: 3.9
+          env:
+            TOXENV: typing
 
     steps:
     - uses: actions/checkout@v2

.gitignore

@@ -10,3 +10,7 @@ _trial_temp
 .coverage
 coverage.xml
 .cache
+.mypy_cache/
+/index.txt
+.dmypy.json
+.hypothesis/

docs/w3lib.rst

@@ -26,3 +26,6 @@ w3lib Package
 
 .. automodule:: w3lib.url
     :members:
+
+    .. autoclass:: ParseDataURIResult
+       :members:

mypy.ini

@@ -0,0 +1,12 @@
+[mypy]
+exclude = .*flycheck_.*
+show_error_codes = True
+check_untyped_defs = True
+
+[mypy-w3lib.*]
+# All non-tests functions must be typed.
+disallow_untyped_defs = True
+
+[mypy-tests.*]
+# Allow test functions to be untyped
+disallow_untyped_defs = False

tests/test_encoding.py

@@ -38,14 +38,16 @@ def test_bom(self):
         utf32le = b"\xff\xfe\x00\x00\x34\x6c\x00\x00"
         for string in (utf16be, utf16le, utf32be, utf32le):
             bom_encoding, bom = read_bom(string)
+            assert bom_encoding is not None
+            assert bom is not None
             decoded = string[len(bom) :].decode(bom_encoding)
             self.assertEqual(water_unicode, decoded)
         # Body without BOM
-        enc, bom = read_bom("foo")
+        enc, bom = read_bom(b"foo")
         self.assertEqual(enc, None)
         self.assertEqual(bom, None)
         # Empty body
-        enc, bom = read_bom("")
+        enc, bom = read_bom(b"")
         self.assertEqual(enc, None)
         self.assertEqual(bom, None)
 

tests/test_html.py

@@ -69,7 +69,7 @@ def test_illegal_entities(self):
     def test_browser_hack(self):
         # check browser hack for numeric character references in the 80-9F range
         self.assertEqual(replace_entities("x™y", encoding="cp1252"), "x\u2122y")
-        self.assertEqual(replace_entities("x™y", encoding="cp1252"), u"x\u2122y")
+        self.assertEqual(replace_entities("x™y", encoding="cp1252"), "x\u2122y")
 
     def test_missing_semicolon(self):
         for entity, result in (

tests/test_http.py

@@ -1,6 +1,11 @@
 import unittest
 from collections import OrderedDict
-from w3lib.http import basic_auth_header, headers_dict_to_raw, headers_raw_to_dict
+from w3lib.http import (
+    HeadersDictInput,
+    basic_auth_header,
+    headers_dict_to_raw,
+    headers_raw_to_dict,
+)
 
 __doctests__ = ["w3lib.http"]  # for trial support
 
@@ -47,7 +52,9 @@ def test_headers_dict_to_raw(self):
         )
 
     def test_headers_dict_to_raw_listtuple(self):
-        dct = OrderedDict([(b"Content-type", [b"text/html"]), (b"Accept", [b"gzip"])])
+        dct: HeadersDictInput = OrderedDict(
+            [(b"Content-type", [b"text/html"]), (b"Accept", [b"gzip"])]
+        )
         self.assertEqual(
             headers_dict_to_raw(dct), b"Content-type: text/html\r\nAccept: gzip"
         )
@@ -70,12 +77,13 @@ def test_headers_dict_to_raw_listtuple(self):
         )
 
     def test_headers_dict_to_raw_wrong_values(self):
-        dct = OrderedDict(
+        dct: HeadersDictInput = OrderedDict(
             [
                 (b"Content-type", 0),
             ]
         )
         self.assertEqual(headers_dict_to_raw(dct), b"")
+        self.assertEqual(headers_dict_to_raw(dct), b"")
 
         dct = OrderedDict([(b"Content-type", 1), (b"Accept", [b"gzip"])])
         self.assertEqual(headers_dict_to_raw(dct), b"Accept: gzip")

tests/test_url.py

@@ -508,7 +508,7 @@ def test_add_or_replace_parameters(self):
     def test_add_or_replace_parameters_does_not_change_input_param(self):
         url = "http://domain/test?arg=original"
         input_param = {"arg": "value"}
-        new_url = add_or_replace_parameters(url, input_param)  # noqa
+        add_or_replace_parameters(url, input_param)  # noqa
         self.assertEqual(input_param, {"arg": "value"})
 
     def test_url_query_cleaner(self):
@@ -817,15 +817,18 @@ def test_non_ascii_percent_encoding_in_paths(self):
         self.assertEqual(
             canonicalize_url("http://www.example.com/a do?a=1"),
             "http://www.example.com/a%20do?a=1",
-        ),
+        )
+
         self.assertEqual(
             canonicalize_url("http://www.example.com/a %20do?a=1"),
             "http://www.example.com/a%20%20do?a=1",
-        ),
+        )
+
         self.assertEqual(
             canonicalize_url("http://www.example.com/a do£.html?a=1"),
             "http://www.example.com/a%20do%C2%A3.html?a=1",
         )
+
         self.assertEqual(
             canonicalize_url(b"http://www.example.com/a do\xc2\xa3.html?a=1"),
             "http://www.example.com/a%20do%C2%A3.html?a=1",

tests/test_util.py

@@ -20,7 +20,7 @@ def test_deprecation(self):
 class ToBytesTestCase(TestCase):
     def test_type_error(self):
         with raises(TypeError):
-            to_bytes(True)
+            to_bytes(True)  # type: ignore
 
 
 class ToNativeStrTestCase(TestCase):
@@ -32,7 +32,7 @@ def test_deprecation(self):
 class ToUnicodeTestCase(TestCase):
     def test_type_error(self):
         with raises(TypeError):
-            to_unicode(True)
+            to_unicode(True)  # type: ignore
 
 
 class UnicodeToStrTestCase(TestCase):

tox.ini

@@ -22,6 +22,15 @@ deps =
 commands =
     bandit -r -c .bandit.yml {posargs:w3lib}
 
+[testenv:typing]
+basepython = python3
+deps =
+    # mypy would error if pytest (or its sub) not found
+    pytest
+    mypy==0.910
+commands =
+    mypy --show-error-codes {posargs: w3lib tests}
+
 [testenv:flake8]
 basepython = python3
 deps =