From 5d3ca4353bcd8a452eff2ce719ab35935e491e73 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 17:36:39 +0300 Subject: [PATCH 01/19] added list of providers in HTTPProvider --- web3/providers/rpc.py | 81 +++++++++++++++++++++++++------------------ 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 0be744f71d..8fd4ac5fd8 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -1,4 +1,13 @@ +from eth_typing import ( + URI, +) +from eth_utils import ( + to_dict, +) import logging +from requests import ( + RequestException, +) from typing import ( Any, Dict, @@ -8,13 +17,6 @@ Union, ) -from eth_typing import ( - URI, -) -from eth_utils import ( - to_dict, -) - from web3._utils.http import ( construct_user_agent, ) @@ -26,6 +28,9 @@ from web3.datastructures import ( NamedElementOnion, ) +from web3.exceptions import ( + CannotHandleRequest, +) from web3.middleware import ( http_retry_request_middleware, ) @@ -42,22 +47,23 @@ class HTTPProvider(JSONBaseProvider): logger = logging.getLogger("web3.providers.HTTPProvider") - endpoint_uri = None + providers = None _request_args = None _request_kwargs = None # type ignored b/c conflict with _middlewares attr on BaseProvider - _middlewares: Tuple[Middleware, ...] = NamedElementOnion([(http_retry_request_middleware, 'http_retry_request')]) # type: ignore # noqa: E501 + _middlewares: Tuple[Middleware, ...] = NamedElementOnion([(http_retry_request_middleware, "http_retry_request")]) # type: ignore # noqa: E501 def __init__( - self, endpoint_uri: Optional[Union[URI, str]] = None, - request_kwargs: Optional[Any] = None, - session: Optional[Any] = None + self, + providers: List[BaseProvider], + request_kwargs: Optional[Any] = None, + session: Optional[Any] = None, ) -> None: - if endpoint_uri is None: - self.endpoint_uri = get_default_http_endpoint() - else: - self.endpoint_uri = URI(endpoint_uri) - + if isinstance(providers, str): + providers = [ + providers, + ] + self.providers = providers self._request_kwargs = request_kwargs or {} if session: @@ -70,28 +76,37 @@ def __str__(self) -> str: @to_dict def get_request_kwargs(self) -> Iterable[Tuple[str, Any]]: - if 'headers' not in self._request_kwargs: - yield 'headers', self.get_request_headers() + if "headers" not in self._request_kwargs: + yield "headers", self.get_request_headers() for key, value in self._request_kwargs.items(): yield key, value def get_request_headers(self) -> Dict[str, str]: return { - 'Content-Type': 'application/json', - 'User-Agent': construct_user_agent(str(type(self))), + "Content-Type": "application/json", + "User-Agent": construct_user_agent(str(type(self))), } def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: - self.logger.debug("Making request HTTP. URI: %s, Method: %s", - self.endpoint_uri, method) - request_data = self.encode_rpc_request(method, params) - raw_response = make_post_request( - self.endpoint_uri, - request_data, - **self.get_request_kwargs() + self.logger.debug( + "Making request HTTP. URI: %s, Method: %s", self.endpoint_uri, method ) - response = self.decode_rpc_response(raw_response) - self.logger.debug("Getting response HTTP. URI: %s, " - "Method: %s, Response: %s", - self.endpoint_uri, method, response) - return response + request_data = self.encode_rpc_request(method, params) + for provider in self.providers: + provider_uri = URI(provider) + try: + raw_response = make_post_request( + URI(provider_uri), request_data, **self.get_request_kwargs() + ) + response = self.decode_rpc_response(raw_response) + self.logger.debug( + "Getting response HTTP. URI: %s, " "Method: %s, Response: %s", + self.provider_uri, + method, + response, + ) + return response + except RequestException: + pass + else: + raise CannotHandleRequest From abe9a2225955a0303f97214e31cde7d13977cdb4 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 17:50:45 +0300 Subject: [PATCH 02/19] added list of providers in HTTPProvider --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 8fd4ac5fd8..2ea7cc15f6 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -55,7 +55,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, - providers: List[BaseProvider], + providers: Union[List[BaseProvider], str], request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From 74287c28e7c4bd6d000edec0f5b307856d0d853c Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 17:59:49 +0300 Subject: [PATCH 03/19] list fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 2ea7cc15f6..384c472966 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -55,7 +55,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, - providers: Union[List[BaseProvider], str], + providers: Union[list[BaseProvider], str], request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From 000f33dab0d50a9c06428f94bdccb44f3dc18b0e Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:06:51 +0300 Subject: [PATCH 04/19] import fix --- web3/providers/rpc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 384c472966..898d695165 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -8,6 +8,9 @@ from requests import ( RequestException, ) +from web3.providers import ( + BaseProvider, +) from typing import ( Any, Dict, From b4d4109f87a0552e4c30dcefcaee65ad1761ad31 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:13:57 +0300 Subject: [PATCH 05/19] annotation fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 898d695165..ff4fe44734 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -58,7 +58,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, - providers: Union[list[BaseProvider], str], + providers: Union[list(BaseProvider), str], request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From 6971ea98d1a3676d2fd936bc2e74a0758dbedba2 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:17:30 +0300 Subject: [PATCH 06/19] annotation fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index ff4fe44734..02fecaad94 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -58,7 +58,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, - providers: Union[list(BaseProvider), str], + providers: Union[list, str], request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From 93378ab96ec8c85d4f32c4ec7025557505e908e9 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:22:27 +0300 Subject: [PATCH 07/19] session --- web3/providers/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 02fecaad94..2c9901d44e 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -69,8 +69,8 @@ def __init__( self.providers = providers self._request_kwargs = request_kwargs or {} - if session: - cache_session(self.endpoint_uri, session) + #if session: + # cache_session(self.endpoint_uri, session) super().__init__() From b3cb4d3424ddc1597ebf6bea94ebaea5e4bc374d Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:26:26 +0300 Subject: [PATCH 08/19] session --- web3/providers/rpc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 2c9901d44e..fd80796a73 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -69,13 +69,13 @@ def __init__( self.providers = providers self._request_kwargs = request_kwargs or {} - #if session: - # cache_session(self.endpoint_uri, session) + if session: + cache_session(self.providers[0], session) super().__init__() def __str__(self) -> str: - return "RPC connection {0}".format(self.endpoint_uri) + return "RPC connection {0}".format(self.providers) @to_dict def get_request_kwargs(self) -> Iterable[Tuple[str, Any]]: @@ -92,7 +92,7 @@ def get_request_headers(self) -> Dict[str, str]: def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: self.logger.debug( - "Making request HTTP. URI: %s, Method: %s", self.endpoint_uri, method + "Making request HTTP. URI: %s, Method: %s", self.provider_uri, method ) request_data = self.encode_rpc_request(method, params) for provider in self.providers: From e0f21b6209416904472bc9513d6930dd2b5361c7 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:35:20 +0300 Subject: [PATCH 09/19] session --- web3/providers/rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index fd80796a73..bf812c4b73 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -91,12 +91,12 @@ def get_request_headers(self) -> Dict[str, str]: } def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: - self.logger.debug( - "Making request HTTP. URI: %s, Method: %s", self.provider_uri, method - ) request_data = self.encode_rpc_request(method, params) for provider in self.providers: provider_uri = URI(provider) + self.logger.debug( + "Making request HTTP. URI: %s, Method: %s", self.provider_uri, method + ) try: raw_response = make_post_request( URI(provider_uri), request_data, **self.get_request_kwargs() From 48cdd4d760272175f5a2ff65a031e576ba43ea91 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 18:40:17 +0300 Subject: [PATCH 10/19] logger fix --- web3/providers/rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index bf812c4b73..3ddf02668b 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -95,7 +95,7 @@ def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: for provider in self.providers: provider_uri = URI(provider) self.logger.debug( - "Making request HTTP. URI: %s, Method: %s", self.provider_uri, method + "Making request HTTP. URI: %s, Method: %s", provider_uri, method ) try: raw_response = make_post_request( @@ -104,7 +104,7 @@ def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: response = self.decode_rpc_response(raw_response) self.logger.debug( "Getting response HTTP. URI: %s, " "Method: %s, Response: %s", - self.provider_uri, + provider_uri, method, response, ) From e2b0f7adfc5f21166b845a31db5b56ec81238134 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:07:37 +0300 Subject: [PATCH 11/19] type mismatch --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 3ddf02668b..dc4b992ba8 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -99,7 +99,7 @@ def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: ) try: raw_response = make_post_request( - URI(provider_uri), request_data, **self.get_request_kwargs() + provider_uri, request_data, **self.get_request_kwargs() ) response = self.decode_rpc_response(raw_response) self.logger.debug( From 61a88aa9dfde34921796cbb91909ab959421508e Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:18:53 +0300 Subject: [PATCH 12/19] randomizing for node balance --- web3/providers/rpc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index dc4b992ba8..5791817118 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -5,6 +5,7 @@ to_dict, ) import logging +import random from requests import ( RequestException, ) @@ -59,6 +60,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, providers: Union[list, str], + randomize: Boolean = False, request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: @@ -66,6 +68,7 @@ def __init__( providers = [ providers, ] + self.randomize = randomize self.providers = providers self._request_kwargs = request_kwargs or {} @@ -92,6 +95,8 @@ def get_request_headers(self) -> Dict[str, str]: def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: request_data = self.encode_rpc_request(method, params) + if self.randomize(): + random.shuffle(self.providers) for provider in self.providers: provider_uri = URI(provider) self.logger.debug( From eae6025b05c848c96fb0250fa31126f9a722b1be Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:22:42 +0300 Subject: [PATCH 13/19] typing fix --- web3/providers/rpc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 5791817118..025cad2005 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -19,6 +19,7 @@ Optional, Tuple, Union, + Boolean, ) from web3._utils.http import ( @@ -60,7 +61,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, providers: Union[list, str], - randomize: Boolean = False, + randomize: Optional[Boolean] = False, request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From 4023f132356023edb7fa9ad4d99a6833be0c089a Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:27:06 +0300 Subject: [PATCH 14/19] typing fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 025cad2005..8dbc23a3d3 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -61,7 +61,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, providers: Union[list, str], - randomize: Optional[Boolean] = False, + randomize: Optional[boolean] = False, request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From a686bfb7cb209b735de407a000b785b4d5df5fa8 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:29:38 +0300 Subject: [PATCH 15/19] typing fix --- web3/providers/rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 8dbc23a3d3..1f6d3121b5 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -19,7 +19,6 @@ Optional, Tuple, Union, - Boolean, ) from web3._utils.http import ( From 9b544a720afb43b1763e152720c14c190191785a Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:31:50 +0300 Subject: [PATCH 16/19] typing fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 1f6d3121b5..de187ae77b 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -60,7 +60,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, providers: Union[list, str], - randomize: Optional[boolean] = False, + randomize: Optional[boole] = False, request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From 3aecf66b634d648ca5676523dfc6f983f0a71933 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:33:53 +0300 Subject: [PATCH 17/19] typing fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index de187ae77b..7b9153883a 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -60,7 +60,7 @@ class HTTPProvider(JSONBaseProvider): def __init__( self, providers: Union[list, str], - randomize: Optional[boole] = False, + randomize: Optional[bool] = False, request_kwargs: Optional[Any] = None, session: Optional[Any] = None, ) -> None: From b0f5a43855e09029d9a5af059c36a770e4e86fc9 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:36:19 +0300 Subject: [PATCH 18/19] typing fix --- web3/providers/rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index 7b9153883a..b0f21d3677 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -95,7 +95,7 @@ def get_request_headers(self) -> Dict[str, str]: def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse: request_data = self.encode_rpc_request(method, params) - if self.randomize(): + if self.randomize: random.shuffle(self.providers) for provider in self.providers: provider_uri = URI(provider) From 67a289628932a4731dea18188081f2afad0c2782 Mon Sep 17 00:00:00 2001 From: dsahdr Date: Thu, 20 Jan 2022 19:40:48 +0300 Subject: [PATCH 19/19] isort --- web3/providers/rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web3/providers/rpc.py b/web3/providers/rpc.py index b0f21d3677..9cf425dcdd 100644 --- a/web3/providers/rpc.py +++ b/web3/providers/rpc.py @@ -9,9 +9,6 @@ from requests import ( RequestException, ) -from web3.providers import ( - BaseProvider, -) from typing import ( Any, Dict, @@ -38,6 +35,9 @@ from web3.middleware import ( http_retry_request_middleware, ) +from web3.providers import ( + BaseProvider, +) from web3.types import ( Middleware, RPCEndpoint,