Refactor the middleware setup [starts v7 breaking changes]

.gitignore

@@ -49,6 +49,7 @@ docs/web3.gas_strategies.rst
 docs/web3.middleware.rst
 docs/web3.providers.eth_tester.rst
 docs/web3.providers.rst
+docs/web3.providers.rpc.rst
 docs/web3.providers.websocket.rst
 docs/web3.rst
 docs/web3.scripts.release.rst

docs/conf.py

@@ -86,6 +86,7 @@
     "web3.gas_strategies.rst",
     "web3.middleware.rst",
     "web3.providers.rst",
+    "web3.providers.rpc.rst",
     "web3.providers.websocket.rst",
     "web3.providers.eth_tester.rst",
     "web3.scripts.*",
@@ -224,7 +225,13 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    ("index", "Populus.tex", "Populus Documentation", "The Ethereum Foundation", "manual"),
+    (
+        "index",
+        "Populus.tex",
+        "Populus Documentation",
+        "The Ethereum Foundation",
+        "manual",
+    ),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of

docs/examples.rst

@@ -688,8 +688,8 @@ Inject the middleware into the middleware onion
 
 .. code-block:: python
 
-    from web3.middleware import geth_poa_middleware
-    w3.middleware_onion.inject(geth_poa_middleware, layer=0)
+    from web3.middleware import extradata_to_poa_middleware
+    w3.middleware_onion.inject(extradata_to_poa_middleware, layer=0)
 
 Just remember that you have to sign all transactions locally, as infura does not handle any keys from your wallet ( refer to `this`_  )
 

docs/middleware.rst

@@ -104,18 +104,6 @@ Buffered Gas Estimate
     ``min(w3.eth.estimate_gas + gas_buffer, gas_limit)``
     where the gas_buffer default is 100,000
 
-HTTPRequestRetry
-~~~~~~~~~~~~~~~~~~
-
-.. py:method:: web3.middleware.http_retry_request_middleware
-               web3.middleware.async_http_retry_request_middleware
-
-    This middleware is a default specifically for HTTPProvider that retries failed
-    requests that return the following errors: ``ConnectionError``, ``HTTPError``, ``Timeout``,
-    ``TooManyRedirects``. Additionally there is a whitelist that only allows certain
-    methods to be retried in order to not resend transactions, excluded methods are:
-    ``eth_sendTransaction``, ``personal_signAndSendTransaction``, ``personal_sendTransaction``.
-
 Validation
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -446,16 +434,15 @@ Time-based Cache Middleware
 Proof of Authority
 ~~~~~~~~~~~~~~~~~~
 
-.. py:method:: web3.middleware.geth_poa_middleware
-               web3.middleware.async_geth_poa_middleware
+.. py:class:: web3.middleware.extradata_to_poa_middleware
 
 .. note::
     It's important to inject the middleware at the 0th layer of the middleware onion:
-    ``w3.middleware_onion.inject(geth_poa_middleware, layer=0)``
+    ``w3.middleware_onion.inject(extradata_to_poa_middleware, layer=0)``
 
-The ``geth_poa_middleware`` is required to connect to ``geth --dev`` or the Goerli 
-public network. It may also be needed for other EVM compatible blockchains like Polygon
-or BNB Chain (Binance Smart Chain).
+The ``extradata_to_poa_middleware`` is required to connect to ``geth --dev`` and may
+also be needed for other EVM compatible blockchains like Polygon or
+BNB Chain (Binance Smart Chain).
 
 If the middleware is not injected at the 0th layer of the middleware onion, you may get
 errors like the example below when interacting with your EVM node.
@@ -468,7 +455,8 @@ errors like the example below when interacting with your EVM node.
     for more details. The full extraData is: HexBytes('...')
 
 
-The easiest way to connect to a default ``geth --dev`` instance which loads the middleware is:
+The easiest way to connect to a default ``geth --dev`` instance which loads the
+middleware is:
 
 
 .. code-block:: python
@@ -489,25 +477,26 @@ unique IPC location and loads the middleware:
     # connect to the IPC location started with 'geth --dev --datadir ~/mynode'
     >>> w3 = Web3(IPCProvider('~/mynode/geth.ipc'))
 
-    >>> from web3.middleware import geth_poa_middleware
+    >>> from web3.middleware import extradata_to_poa_middleware
 
     # inject the poa compatibility middleware to the innermost layer (0th layer)
-    >>> w3.middleware_onion.inject(geth_poa_middleware, layer=0)
+    >>> w3.middleware_onion.inject(extradata_to_poa_middleware, layer=0)
 
     # confirm that the connection succeeded
     >>> w3.client_version
     'Geth/v1.7.3-stable-4bb3c89d/linux-amd64/go1.9'
 
-Why is ``geth_poa_middleware`` necessary?
-'''''''''''''''''''''''''''''''''''''''''
+Why is ``extradata_to_poa_middleware`` necessary?
+'''''''''''''''''''''''''''''''''''''''''''''''''
 
 There is no strong community consensus on a single Proof-of-Authority (PoA) standard yet.
 Some nodes have successful experiments running though. One is go-ethereum (geth),
 which uses a prototype PoA for its development mode and the Goerli test network.
 
 Unfortunately, it does deviate from the yellow paper specification, which constrains the
-``extraData`` field in each block to a maximum of 32-bytes. Geth's PoA uses more than
-32 bytes, so this middleware modifies the block data a bit before returning it.
+``extraData`` field in each block to a maximum of 32-bytes. Geth is one such example
+where PoA uses more than 32 bytes, so this middleware modifies the block data a bit
+before returning it.
 
 .. _local-filter:
 

docs/providers.rst

@@ -467,7 +467,6 @@ AsyncHTTPProvider
         - :meth:`Attribute Dict Middleware <web3.middleware.async_attrdict_middleware>`
         - :meth:`Buffered Gas Estimate Middleware <web3.middleware.async_buffered_gas_estimate_middleware>`
         - :meth:`Gas Price Strategy Middleware <web3.middleware.async_gas_price_strategy_middleware>`
-        - :meth:`Geth POA Middleware <web3.middleware.async_geth_poa_middleware>`
         - :meth:`Local Filter Middleware <web3.middleware.async_local_filter_middleware>`
         - :meth:`Simple Cache Middleware <web3.middleware.async_construct_simple_cache_middleware>`
         - :meth:`Stalecheck Middleware <web3.middleware.async_make_stalecheck_middleware>`

ens/async_ens.py

@@ -71,12 +71,14 @@
         AsyncContractFunction,
     )
     from web3.main import AsyncWeb3  # noqa: F401
+    from web3.middleware.base import (  # noqa: F401
+        Middleware,
+    )
     from web3.providers import (  # noqa: F401
         AsyncBaseProvider,
         BaseProvider,
     )
     from web3.types import (  # noqa: F401
-        AsyncMiddleware,
         TxParams,
     )
 
@@ -98,7 +100,7 @@ def __init__(
         self,
         provider: "AsyncBaseProvider" = cast("AsyncBaseProvider", default),
         addr: ChecksumAddress = None,
-        middlewares: Optional[Sequence[Tuple["AsyncMiddleware", str]]] = None,
+        middlewares: Optional[Sequence[Tuple["Middleware", str]]] = None,
     ) -> None:
         """
         :param provider: a single provider used to connect to Ethereum

ens/ens.py

@@ -71,11 +71,13 @@
         Contract,
         ContractFunction,
     )
+    from web3.middleware.base import (  # noqa: F401
+        Middleware,
+    )
     from web3.providers import (  # noqa: F401
         BaseProvider,
     )
     from web3.types import (  # noqa: F401
-        Middleware,
         TxParams,
     )
 

ens/utils.py

@@ -5,7 +5,6 @@
 from typing import (
     TYPE_CHECKING,
     Any,
-    Callable,
     Collection,
     Dict,
     List,
@@ -60,15 +59,15 @@
         AsyncWeb3,
         Web3 as _Web3,
     )
+    from web3.middleware.base import (
+        Middleware,
+    )
     from web3.providers import (  # noqa: F401
         AsyncBaseProvider,
         BaseProvider,
     )
     from web3.types import (  # noqa: F401
         ABIFunction,
-        AsyncMiddleware,
-        Middleware,
-        RPCEndpoint,
     )
 
 
@@ -104,13 +103,14 @@ def customize_web3(w3: "_Web3") -> "_Web3":
         make_stalecheck_middleware,
     )
 
-    if w3.middleware_onion.get("name_to_address"):
-        w3.middleware_onion.remove("name_to_address")
+    if w3.middleware_onion.get("ens_name_to_address"):
+        w3.middleware_onion.remove("ens_name_to_address")
 
     if not w3.middleware_onion.get("stalecheck"):
-        w3.middleware_onion.add(
-            make_stalecheck_middleware(ACCEPTABLE_STALE_HOURS * 3600), name="stalecheck"
+        stalecheck_middleware = make_stalecheck_middleware(
+            ACCEPTABLE_STALE_HOURS * 3600
         )
+        w3.middleware_onion.add(stalecheck_middleware, name="stalecheck")
     return w3
 
 
@@ -299,22 +299,27 @@ def get_abi_output_types(abi: "ABIFunction") -> List[str]:
 
 def init_async_web3(
     provider: "AsyncBaseProvider" = cast("AsyncBaseProvider", default),
-    middlewares: Optional[Sequence[Tuple["AsyncMiddleware", str]]] = (),
+    middlewares: Optional[Sequence[Tuple["Middleware", str]]] = (),
 ) -> "AsyncWeb3":
     from web3 import (
         AsyncWeb3 as AsyncWeb3Main,
     )
     from web3.eth import (
         AsyncEth as AsyncEthMain,
     )
+    from web3.middleware import (
+        make_stalecheck_middleware,
+    )
 
     middlewares = list(middlewares)
     for i, (middleware, name) in enumerate(middlewares):
-        if name == "name_to_address":
+        if name == "ens_name_to_address":
             middlewares.pop(i)
 
     if "stalecheck" not in (name for mw, name in middlewares):
-        middlewares.append((_async_ens_stalecheck_middleware, "stalecheck"))
+        middlewares.append(
+            (make_stalecheck_middleware(ACCEPTABLE_STALE_HOURS * 3600), "stalecheck")
+        )
 
     if provider is default:
         async_w3 = AsyncWeb3Main(
@@ -329,14 +334,3 @@ def init_async_web3(
         )
 
     return async_w3
-
-
-async def _async_ens_stalecheck_middleware(
-    make_request: Callable[["RPCEndpoint", Any], Any], w3: "AsyncWeb3"
-) -> "Middleware":
-    from web3.middleware import (
-        async_make_stalecheck_middleware,
-    )
-
-    middleware = await async_make_stalecheck_middleware(ACCEPTABLE_STALE_HOURS * 3600)
-    return await middleware(make_request, w3)

newsfragments/3169.breaking.rst

@@ -0,0 +1 @@
+Refactor the middleware setup so that request processors and response processors are separated. This will allow for more flexibility in the future and aid in the implementation of features such as batched requests. This PR also closes out a few outstanding issues and will be the start of the breaking changes for `web3.py` ``v7``. Review PR for a full list of changes.

setup.py

@@ -78,6 +78,7 @@
         "jsonschema>=4.0.0",
         "lru-dict>=1.1.6,<1.3.0",
         "protobuf>=4.21.6",
+        "pydantic>=2.4.0",
         "pywin32>=223;platform_system=='Windows'",
         "requests>=2.16.0",
         "typing-extensions>=4.0.1",

tests/conftest.py

@@ -1,4 +1,7 @@
 import pytest
+from typing import (
+    Type,
+)
 
 from eth_utils import (
     event_signature_to_log_topic,
@@ -7,10 +10,14 @@
 from eth_utils.toolz import (
     identity,
 )
+import pytest_asyncio
 
 from web3._utils.contract_sources.contract_data.emitter_contract import (
     EMITTER_CONTRACT_DATA,
 )
+from web3._utils.module_testing.utils import (
+    RequestMocker,
+)
 
 from .utils import (
     get_open_port,
@@ -107,3 +114,11 @@ class LogTopics:
 @pytest.fixture(scope="session")
 def emitter_contract_log_topics():
     return LogTopics
+
+
+# -- mock requests -- #
+
+
+@pytest_asyncio.fixture(scope="function")
+def request_mocker() -> Type[RequestMocker]:
+    return RequestMocker