Use the ReadableBuffer type in more places (#4245)

This is a follow-up on #4232. memoryview, hashlib, and hmac are updated
to use ReadableBuffer type instead of their own home-spun unions of
bytes, bytearray and whatever else each use case used. mmap is being
handled in #4244, and I'll leave BinaryIO for another day (or possibly
another person) because it's going to require some messy code
duplication because the relevant methods are defined in IO[AnyStr].

There's one corner case I'm not quite sure how best to handle: the
documentation for hmac.digest claim that the parmaeters have the same
meanings as in hmac.new, but in CPython the latter has an explicit check
that `key` is bytes or bytearray while the former works with a
memory-view. For now I've matched the documentation.

Also, the documentation for HMAC.update says that `msg` can be any type
supported by hashlib from Python 3.4; but I can't see anything in the
Python 2.7 implementation that would prevent it also taking bytes-like
objects, so I've not tried to treat Python 2 any different to Python 3.

Author: bmerry

stdlib/2/__builtin__.pyi

@@ -15,7 +15,11 @@ from io import (
     TextIOWrapper, FileIO, BufferedRandom, BufferedReader, BufferedWriter
 )
 from types import TracebackType, CodeType
-from _typeshed import AnyPath, OpenBinaryMode, OpenTextMode, OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenBinaryModeReading, SupportsWrite
+from _typeshed import (
+    AnyPath, OpenBinaryMode, OpenTextMode,
+    OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenBinaryModeReading,
+    SupportsWrite, ReadableBuffer,
+)
 from typing_extensions import Literal
 import sys
 
@@ -821,12 +825,12 @@ class memoryview(Sized, Container[_mv_container_type]):
         f_contiguous: bool
         contiguous: bool
         nbytes: int
-        def __init__(self, obj: Union[bytes, bytearray, memoryview]) -> None: ...
+        def __init__(self, obj: ReadableBuffer) -> None: ...
         def __enter__(self) -> memoryview: ...
         def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: ...
         def cast(self, format: str, shape: Union[List[int], Tuple[int]] = ...) -> memoryview: ...
     else:
-        def __init__(self, obj: Union[bytes, bytearray, buffer, memoryview]) -> None: ...
+        def __init__(self, obj: ReadableBuffer) -> None: ...
 
     @overload
     def __getitem__(self, i: int) -> _mv_container_type: ...

stdlib/2and3/_typeshed/__init__.pyi

@@ -70,5 +70,9 @@ class SupportsReadline(Protocol[_T_co]):
 class SupportsWrite(Protocol[_T_contra]):
     def write(self, __s: _T_contra) -> int: ...
 
-ReadableBuffer = Union[bytes, bytearray, memoryview, array.array, mmap.mmap]
-WriteableBuffer = Union[bytearray, memoryview, array.array, mmap.mmap]
+if sys.version_info >= (3,):
+    ReadableBuffer = Union[bytes, bytearray, memoryview, array.array, mmap.mmap]
+    WriteableBuffer = Union[bytearray, memoryview, array.array, mmap.mmap]
+else:
+    ReadableBuffer = Union[bytes, bytearray, memoryview, array.array, mmap.mmap, buffer]
+    WriteableBuffer = Union[bytearray, memoryview, array.array, mmap.mmap, buffer]

stdlib/2and3/builtins.pyi

@@ -15,7 +15,11 @@ from io import (
     TextIOWrapper, FileIO, BufferedRandom, BufferedReader, BufferedWriter
 )
 from types import TracebackType, CodeType
-from _typeshed import AnyPath, OpenBinaryMode, OpenTextMode, OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenBinaryModeReading, SupportsWrite
+from _typeshed import (
+    AnyPath, OpenBinaryMode, OpenTextMode,
+    OpenBinaryModeUpdating, OpenBinaryModeWriting, OpenBinaryModeReading,
+    SupportsWrite, ReadableBuffer,
+)
 from typing_extensions import Literal
 import sys
 
@@ -821,12 +825,12 @@ class memoryview(Sized, Container[_mv_container_type]):
         f_contiguous: bool
         contiguous: bool
         nbytes: int
-        def __init__(self, obj: Union[bytes, bytearray, memoryview]) -> None: ...
+        def __init__(self, obj: ReadableBuffer) -> None: ...
         def __enter__(self) -> memoryview: ...
         def __exit__(self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: ...
         def cast(self, format: str, shape: Union[List[int], Tuple[int]] = ...) -> memoryview: ...
     else:
-        def __init__(self, obj: Union[bytes, bytearray, buffer, memoryview]) -> None: ...
+        def __init__(self, obj: ReadableBuffer) -> None: ...
 
     @overload
     def __getitem__(self, i: int) -> _mv_container_type: ...

stdlib/2and3/hmac.pyi

@@ -3,44 +3,45 @@
 from typing import Any, AnyStr, Callable, Optional, Union, overload
 from types import ModuleType
 import sys
+from _typeshed import ReadableBuffer
 
 _B = Union[bytes, bytearray]
 
 # TODO more precise type for object of hashlib
 _Hash = Any
+_DigestMod = Union[str, Callable[[], _Hash], ModuleType]
 
 digest_size: None
 
 if sys.version_info >= (3, 8):
-    _DigestMod = Union[str, Callable[[], _Hash], ModuleType]
     # In reality digestmod has a default value, but the function always throws an error
     # if the argument is not given, so we pretend it is a required argument.
     @overload
-    def new(key: _B, msg: Optional[_B], digestmod: _DigestMod) -> HMAC: ...
+    def new(key: _B, msg: Optional[ReadableBuffer], digestmod: _DigestMod) -> HMAC: ...
     @overload
     def new(key: _B, *, digestmod: _DigestMod) -> HMAC: ...
 elif sys.version_info >= (3, 4):
-    def new(key: _B, msg: Optional[_B] = ...,
-            digestmod: Optional[Union[str, Callable[[], _Hash], ModuleType]] = ...) -> HMAC: ...
+    def new(key: _B, msg: Optional[ReadableBuffer] = ...,
+            digestmod: Optional[_DigestMod] = ...) -> HMAC: ...
 else:
-    def new(key: _B, msg: Optional[_B] = ...,
-            digestmod: Optional[Union[Callable[[], _Hash], ModuleType]] = ...) -> HMAC: ...
+    def new(key: _B, msg: Optional[ReadableBuffer] = ...,
+            digestmod: Optional[_DigestMod] = ...) -> HMAC: ...
 
 class HMAC:
     if sys.version_info >= (3,):
         digest_size: int
     if sys.version_info >= (3, 4):
         block_size: int
         name: str
-    def update(self, msg: _B) -> None: ...
+    def update(self, msg: ReadableBuffer) -> None: ...
     def digest(self) -> bytes: ...
     def hexdigest(self) -> str: ...
     def copy(self) -> HMAC: ...
 
 @overload
-def compare_digest(a: bytearray, b: bytearray) -> bool: ...
+def compare_digest(a: ReadableBuffer, b: ReadableBuffer) -> bool: ...
 @overload
 def compare_digest(a: AnyStr, b: AnyStr) -> bool: ...
 
 if sys.version_info >= (3, 7):
-    def digest(key: _B, msg: _B, digest: str) -> bytes: ...
+    def digest(key: _B, msg: ReadableBuffer, digest: str) -> bytes: ...

stdlib/3/hashlib.pyi

@@ -2,8 +2,7 @@
 
 import sys
 from typing import AbstractSet, Optional, Union
-
-_DataType = Union[bytes, bytearray, memoryview]
+from _typeshed import ReadableBuffer
 
 class _Hash(object):
     digest_size: int
@@ -14,54 +13,54 @@ class _Hash(object):
     # formally specified, so may not exist on some platforms
     name: str
 
-    def __init__(self, data: _DataType = ...) -> None: ...
+    def __init__(self, data: ReadableBuffer = ...) -> None: ...
 
     def copy(self) -> _Hash: ...
     def digest(self) -> bytes: ...
     def hexdigest(self) -> str: ...
-    def update(self, __data: _DataType) -> None: ...
+    def update(self, __data: ReadableBuffer) -> None: ...
 
 if sys.version_info >= (3, 9):
-    def md5(string: _DataType = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
-    def sha1(string: _DataType = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
-    def sha224(string: _DataType = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
-    def sha256(string: _DataType = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
-    def sha384(string: _DataType = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
-    def sha512(string: _DataType = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
+    def md5(string: ReadableBuffer = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
+    def sha1(string: ReadableBuffer = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
+    def sha224(string: ReadableBuffer = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
+    def sha256(string: ReadableBuffer = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
+    def sha384(string: ReadableBuffer = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
+    def sha512(string: ReadableBuffer = ..., *, usedforsecurity: bool = ...) -> _Hash: ...
 elif sys.version_info >= (3, 8):
-    def md5(string: _DataType = ...) -> _Hash: ...
-    def sha1(string: _DataType = ...) -> _Hash: ...
-    def sha224(string: _DataType = ...) -> _Hash: ...
-    def sha256(string: _DataType = ...) -> _Hash: ...
-    def sha384(string: _DataType = ...) -> _Hash: ...
-    def sha512(string: _DataType = ...) -> _Hash: ...
+    def md5(string: ReadableBuffer = ...) -> _Hash: ...
+    def sha1(string: ReadableBuffer = ...) -> _Hash: ...
+    def sha224(string: ReadableBuffer = ...) -> _Hash: ...
+    def sha256(string: ReadableBuffer = ...) -> _Hash: ...
+    def sha384(string: ReadableBuffer = ...) -> _Hash: ...
+    def sha512(string: ReadableBuffer = ...) -> _Hash: ...
 else:
-    def md5(__string: _DataType = ...) -> _Hash: ...
-    def sha1(__string: _DataType = ...) -> _Hash: ...
-    def sha224(__string: _DataType = ...) -> _Hash: ...
-    def sha256(__string: _DataType = ...) -> _Hash: ...
-    def sha384(__string: _DataType = ...) -> _Hash: ...
-    def sha512(__string: _DataType = ...) -> _Hash: ...
+    def md5(__string: ReadableBuffer = ...) -> _Hash: ...
+    def sha1(__string: ReadableBuffer = ...) -> _Hash: ...
+    def sha224(__string: ReadableBuffer = ...) -> _Hash: ...
+    def sha256(__string: ReadableBuffer = ...) -> _Hash: ...
+    def sha384(__string: ReadableBuffer = ...) -> _Hash: ...
+    def sha512(__string: ReadableBuffer = ...) -> _Hash: ...
 
-def new(name: str, data: _DataType = ...) -> _Hash: ...
+def new(name: str, data: ReadableBuffer = ...) -> _Hash: ...
 
 algorithms_guaranteed: AbstractSet[str]
 algorithms_available: AbstractSet[str]
 
-def pbkdf2_hmac(hash_name: str, password: _DataType, salt: _DataType, iterations: int, dklen: Optional[int] = ...) -> bytes: ...
+def pbkdf2_hmac(hash_name: str, password: ReadableBuffer, salt: ReadableBuffer, iterations: int, dklen: Optional[int] = ...) -> bytes: ...
 
 if sys.version_info >= (3, 6):
     class _VarLenHash(object):
         digest_size: int
         block_size: int
         name: str
 
-        def __init__(self, data: _DataType = ...) -> None: ...
+        def __init__(self, data: ReadableBuffer = ...) -> None: ...
 
         def copy(self) -> _VarLenHash: ...
         def digest(self, __length: int) -> bytes: ...
         def hexdigest(self, __length: int) -> str: ...
-        def update(self, __data: _DataType) -> None: ...
+        def update(self, __data: ReadableBuffer) -> None: ...
 
     sha3_224 = _Hash
     sha3_256 = _Hash
@@ -70,7 +69,7 @@ if sys.version_info >= (3, 6):
     shake_128 = _VarLenHash
     shake_256 = _VarLenHash
 
-    def scrypt(password: _DataType, *, salt: Optional[_DataType] = ..., n: Optional[int] = ..., r: Optional[int] = ..., p: Optional[int] = ..., maxmem: int = ..., dklen: int = ...) -> bytes: ...
+    def scrypt(password: ReadableBuffer, *, salt: Optional[ReadableBuffer] = ..., n: Optional[int] = ..., r: Optional[int] = ..., p: Optional[int] = ..., maxmem: int = ..., dklen: int = ...) -> bytes: ...
 
     class _BlakeHash(_Hash):
         MAX_DIGEST_SIZE: int
@@ -79,9 +78,9 @@ if sys.version_info >= (3, 6):
         SALT_SIZE: int
 
         if sys.version_info >= (3, 9):
-            def __init__(self, __data: _DataType = ..., *, digest_size: int = ..., key: _DataType = ..., salt: _DataType = ..., person: _DataType = ..., fanout: int = ..., depth: int = ..., leaf_size: int = ..., node_offset: int = ..., node_depth: int = ..., inner_size: int = ..., last_node: bool = ..., usedforsecurity: bool = ...) -> None: ...
+            def __init__(self, __data: ReadableBuffer = ..., *, digest_size: int = ..., key: ReadableBuffer = ..., salt: ReadableBuffer = ..., person: ReadableBuffer = ..., fanout: int = ..., depth: int = ..., leaf_size: int = ..., node_offset: int = ..., node_depth: int = ..., inner_size: int = ..., last_node: bool = ..., usedforsecurity: bool = ...) -> None: ...
         else:
-            def __init__(self, __data: _DataType = ..., *, digest_size: int = ..., key: _DataType = ..., salt: _DataType = ..., person: _DataType = ..., fanout: int = ..., depth: int = ..., leaf_size: int = ..., node_offset: int = ..., node_depth: int = ..., inner_size: int = ..., last_node: bool = ...) -> None: ...
+            def __init__(self, __data: ReadableBuffer = ..., *, digest_size: int = ..., key: ReadableBuffer = ..., salt: ReadableBuffer = ..., person: ReadableBuffer = ..., fanout: int = ..., depth: int = ..., leaf_size: int = ..., node_offset: int = ..., node_depth: int = ..., inner_size: int = ..., last_node: bool = ...) -> None: ...
 
     blake2b = _BlakeHash
     blake2s = _BlakeHash