pull master into asyncify-contract (#2389)

Author: Paul Robinson

Dockerfile

@@ -1,7 +1,6 @@
-FROM python:3.6
+FROM python:3.9
 
 # Set up code directory
-RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # Install Linux dependencies

docker-compose.yml

@@ -3,8 +3,6 @@ services:
   sandbox:
     build:
       context: .
-    environment:
-      PARITY_VERSION: v2.3.5
     volumes:
       - .:/code
     command: tail -f /dev/null

docs/conf.py

@@ -52,8 +52,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'Web3.py'
-copyright = u'2018, Piper Merriam, Jason Carver'
+project = 'Web3.py'
+copyright = '2018, Piper Merriam, Jason Carver'
 
 __version__ = setup_version
 # The version info for the project you're documenting, acts as replacement for
@@ -222,8 +222,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'Populus.tex', u'Populus Documentation',
-   u'Piper Merriam', 'manual'),
+  ('index', 'Populus.tex', 'Populus Documentation',
+   'Piper Merriam', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -252,8 +252,8 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'web3', u'Web3.py Documentation',
-     [u'Piper Merriam'], 1)
+    ('index', 'web3', 'Web3.py Documentation',
+     ['Piper Merriam'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -266,8 +266,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Web3.py', u'Web3.py Documentation',
-   u'Piper Merriam', 'Web3.py', 'Backend agnostic Ethereum client interactions.',
+  ('index', 'Web3.py', 'Web3.py Documentation',
+   'Piper Merriam', 'Web3.py', 'Backend agnostic Ethereum client interactions.',
    'Miscellaneous'),
 ]
 

docs/examples.rst

@@ -992,7 +992,7 @@ The script can be run with: ``python ./eventscanner.py <your JSON-RPC API URL>``
                     # from our in-memory cache
                     block_when = get_block_when(block_number)
 
-                    logger.debug("Processing event %s, block:%d count:%d", evt["event"], evt["blockNumber"])
+                    logger.debug(f"Processing event {evt["event"]}, block: {evt["blockNumber"]} count: {evt["blockNumber"]}")
                     processed = self.state.process_event(block_when, evt)
                     all_processed.append(processed)
 
@@ -1064,8 +1064,8 @@ The script can be run with: ``python ./eventscanner.py <your JSON-RPC API URL>``
                 # Print some diagnostics to logs to try to fiddle with real world JSON-RPC API performance
                 estimated_end_block = current_block + chunk_size
                 logger.debug(
-                    "Scanning token transfers for blocks: %d - %d, chunk size %d, last chunk scan took %f, last logs found %d",
-                    current_block, estimated_end_block, chunk_size, last_scan_duration, last_logs_found)
+                    f"Scanning token transfers for blocks: {current_block} - {estimated_end_block}, chunk size {chunk_size}, last chunk scan took {last_scan_duration}, last logs found {last_logs_found}"
+                )
 
                 start = time.time()
                 actual_end_block, end_block_timestamp, new_entries = self.scan_chunk(current_block, estimated_end_block)
@@ -1116,12 +1116,7 @@ The script can be run with: ``python ./eventscanner.py <your JSON-RPC API URL>``
                 if i < retries - 1:
                     # Give some more verbose info than the default middleware
                     logger.warning(
-                        "Retrying events for block range %d - %d (%d) failed with %s, retrying in %s seconds",
-                        start_block,
-                        end_block,
-                        end_block-start_block,
-                        e,
-                        delay)
+                        f"Retrying events for block range {start_block} - {end_block} ({end_block-start_block}) failed with {e} , retrying in {delay} seconds")
                     # Decrease the `eth_getBlocks` range
                     end_block = start_block + ((end_block - start_block) // 2)
                     # Let the JSON-RPC to recover e.g. from restart
@@ -1175,7 +1170,7 @@ The script can be run with: ``python ./eventscanner.py <your JSON-RPC API URL>``
             toBlock=to_block
         )
 
-        logger.debug("Querying eth_getLogs with the following parameters: %s", event_filter_params)
+        logger.debug(f"Querying eth_getLogs with the following parameters: {event_filter_params}")
 
         # Call JSON-RPC API on your Ethereum node.
         # get_logs() returns raw AttributedDict entries

docs/v4_migration.rst

@@ -68,7 +68,7 @@ printing out new block hashes as they appear:
 .. code-block:: python
 
     >>> def new_block_callback(block_hash):
-    ...     print "New Block: {0}".format(block_hash)
+    ...     print(f"New Block: {block_hash}")
     ...
     >>> new_block_filter = web3.eth.filter('latest')
     >>> new_block_filter.watch(new_block_callback)
@@ -79,7 +79,7 @@ In v4, that same logic:
 
     >>> new_block_filter = web3.eth.filter('latest')
     >>> for block_hash in new_block_filter.get_new_entries():
-    ...     print("New Block: {}".format(block_hash))
+    ...     print(f"New Block: {block_hash}")
 
 The caller is responsible for polling the results from ``get_new_entries()``.
 See :ref:`asynchronous_filters` for examples of filter-event handling with web3 v4.

docs/web3.eth.account.rst

@@ -39,7 +39,7 @@ Hosted Private Key
   not your Ether" in the wise words of Andreas Antonopoulos.
 
 Some Common Uses for Local Private Keys
--------------------------------------------
+---------------------------------------
 
 A very common reason to work with local private keys is to interact
 with a hosted node.
@@ -55,6 +55,59 @@ Using private keys usually involves ``w3.eth.account`` in one way or another. Re
 or see a full list of things you can do in the docs for
 :class:`eth_account.Account <eth_account.account.Account>`.
 
+Read a private key from an environment variable
+-----------------------------------------------
+
+In this example we pass the private key to our Python application in an 
+`environment variable <https://en.wikipedia.org/wiki/Environment_variable>`_.
+This private key is then added to the transaction signing keychain 
+with ``Signing`` middleware.
+
+If unfamiliar, note that you can `export your private keys from Metamask and other wallets <https://metamask.zendesk.com/hc/en-us/articles/360015289632-How-to-Export-an-Account-Private-Key>`_.
+
+.. warning ::
+
+  - **Never** share your private keys.
+  - **Never** put your private keys in source code.
+  - **Never** commit private keys to a Git repository.
+
+Example ``account_test_script.py``
+
+.. code-block:: python
+
+    import os 
+    from eth_account import Account
+    from eth_account.signers.local import LocalAccount
+    from web3.auto import w3
+    from web3.middleware import construct_sign_and_send_raw_middleware
+    
+    private_key = os.environ.get("PRIVATE_KEY")
+    assert private_key is not None, "You must set PRIVATE_KEY environment variable"
+    assert private_key.startswith("0x"), "Private key must start with 0x hex prefix"
+    
+    account: LocalAccount = Account.from_key(private_key)
+    w3.middleware_onion.add(construct_sign_and_send_raw_middleware(account))
+    
+    print(f"Your hot wallet address is {account.address}")
+    
+Example how to run this in UNIX shell:
+
+.. code-block:: shell
+ 
+    # Generate a new 256-bit random integer using openssl UNIX command that acts as a private key.
+    # You can also do:
+    # python -c "from web3 import Web3; w3 = Web3(); acc = w3.eth.account.create(); print(f'private key={w3.toHex(acc.key)}, account={acc.address}')"
+    # Store this in a safe place, like in your password manager.
+    export PRIVATE_KEY=0x`openssl rand -hex 32` 
+    
+    # Run our script
+    python account_test_script.py
+    
+This will print::
+
+    Your hot wallet address is 0x27C8F899bb69E1501BBB96d09d7477a2a7518918
+
+  
 .. _extract_geth_pk:
 
 Extract private key from geth keyfile

docs/web3.eth.rst

@@ -364,7 +364,7 @@ The following methods are available on the ``web3.eth`` namespace.
 
             assert rlp_account == HexaryTrie.get_from_proof(
                 root, trie_key, format_proof_nodes(proof.accountProof)
-            ), "Failed to verify account proof {}".format(proof.address)
+            ), f"Failed to verify account proof {proof.address}"
 
             for storage_proof in proof.storageProof:
                 trie_key = keccak(pad_bytes(b'\x00', 32, storage_proof.key))
@@ -376,7 +376,7 @@ The following methods are available on the ``web3.eth`` namespace.
 
                 assert rlp_value == HexaryTrie.get_from_proof(
                     root, trie_key, format_proof_nodes(storage_proof.proof)
-                ), "Failed to verify storage proof {}".format(storage_proof.key)
+                ), f"Failed to verify storage proof {storage_proof.key}"
 
             return True
 

ens/main.py

@@ -227,10 +227,10 @@ def setup_name(
                 address = resolved
             elif resolved and address != resolved and resolved != EMPTY_ADDR_HEX:
                 raise AddressMismatch(
-                    "Could not set address %r to point to name, because the name resolves to %r. "
-                    "To change the name for an existing address, call setup_address() first." % (
-                        address, resolved
-                    )
+                    f"Could not set address {address!r} to point to name, "
+                    f"because the name resolves to {resolved!r}. "
+                    "To change the name for an existing address, call "
+                    "setup_address() first."
                 )
             if is_none_or_zero_address(address):
                 address = self.owner(name)
@@ -336,9 +336,8 @@ def _assert_control(self, account: ChecksumAddress, name: str,
                         parent_owned: Optional[str] = None) -> None:
         if not address_in(account, self.w3.eth.accounts):
             raise UnauthorizedError(
-                "in order to modify %r, you must control account %r, which owns %r" % (
-                    name, account, parent_owned or name
-                )
+                f"in order to modify {name!r}, you must control account"
+                f" {account!r}, which owns {parent_owned or name!r}"
             )
 
     def _first_owner(self, name: str) -> Tuple[Optional[ChecksumAddress], Sequence[str], str]:
@@ -375,7 +374,7 @@ def _claim_ownership(
                 label_to_hash(label),
                 owner
             ).transact(transact)
-            owned = "%s.%s" % (label, owned)
+            owned = f"{label}.{owned}"
 
     def _set_resolver(
         self,

ens/utils.py

@@ -140,7 +140,7 @@ def sha3_text(val: Union[str, bytes]) -> HexBytes:
 def label_to_hash(label: str) -> HexBytes:
     label = normalize_name(label)
     if '.' in label:
-        raise ValueError("Cannot generate hash for label %r with a '.'" % label)
+        raise ValueError(f"Cannot generate hash for label {label!r} with a '.'")
     return Web3().keccak(text=label)
 
 

ethpm/contract.py

@@ -56,7 +56,7 @@ def __init__(self, address: bytes, **kwargs: Any) -> None:
             )
         validate_address(address)
         # type ignored to allow for undefined **kwargs on `Contract` base class __init__
-        super(LinkableContract, self).__init__(address=address, **kwargs)  # type: ignore
+        super().__init__(address=address, **kwargs)  # type: ignore
 
     @classmethod
     def factory(
@@ -69,15 +69,15 @@ def factory(
             if not is_prelinked_bytecode(to_bytes(hexstr=bytecode), dep_link_refs):
                 needs_bytecode_linking = True
         kwargs = assoc(kwargs, "needs_bytecode_linking", needs_bytecode_linking)
-        return super(LinkableContract, cls).factory(w3, class_name, **kwargs)
+        return super().factory(w3, class_name, **kwargs)
 
     @classmethod
     def constructor(cls, *args: Any, **kwargs: Any) -> ContractConstructor:
         if cls.needs_bytecode_linking:
             raise BytecodeLinkingError(
                 "Contract cannot be deployed until its bytecode is linked."
             )
-        return super(LinkableContract, cls).constructor(*args, **kwargs)
+        return super().constructor(*args, **kwargs)
 
     @classmethod
     def link_bytecode(cls, attr_dict: Dict[str, str]) -> Type["LinkableContract"]:
@@ -111,7 +111,7 @@ def validate_attr_dict(self, attr_dict: Dict[str, str]) -> None:
         """
         Validates that ContractType keys in attr_dict reference existing manifest ContractTypes.
         """
-        attr_dict_names = list(attr_dict.keys())
+        attr_dict_names = attr_dict.keys()
 
         if not self.unlinked_references and not self.linked_references:
             raise BytecodeLinkingError(
@@ -122,8 +122,8 @@ def validate_attr_dict(self, attr_dict: Dict[str, str]) -> None:
         linked_refs = self.linked_references or ({},)
         all_link_refs = unlinked_refs + linked_refs
 
-        all_link_names = [ref["name"] for ref in all_link_refs if ref]
-        if set(attr_dict_names) != set(all_link_names):
+        all_link_names = {ref["name"] for ref in all_link_refs if ref}
+        if attr_dict_names != all_link_names:
             raise BytecodeLinkingError(
                 "All link references must be defined when calling "
                 "`link_bytecode` on a contract factory."

ethpm/tools/builder.py

@@ -498,7 +498,7 @@ def normalize_compiler_output(compiler_output: Dict[str, Any]) -> Dict[str, Any]
     ]
     paths, names = zip(*paths_and_names)
     if len(names) != len(set(names)):
-        duplicates = set([name for name in names if names.count(name) > 1])
+        duplicates = {name for name in names if names.count(name) > 1}
         raise ManifestBuildingError(
             f"Duplicate contract types: {duplicates} were found in the compiler output."
         )

ethpm/validation/manifest.py

@@ -66,11 +66,11 @@ def _load_schema_data() -> Dict[str, Any]:
 
 
 def extract_contract_types_from_deployments(deployment_data: List[Any]) -> Set[str]:
-    contract_types = set(
+    contract_types = {
         deployment["contractType"]
         for chain_deployments in deployment_data
         for deployment in chain_deployments.values()
-    )
+    }
     return contract_types
 
 
@@ -108,11 +108,11 @@ def validate_manifest_deployments(manifest: Dict[str, Any]) -> None:
     """
     Validate that a manifest's deployments contracts reference existing contractTypes.
     """
-    if set(("contractTypes", "deployments")).issubset(manifest):
-        all_contract_types = list(manifest["contractTypes"].keys())
-        all_deployments = list(manifest["deployments"].values())
+    if {"contractTypes", "deployments"}.issubset(manifest):
+        all_contract_types = manifest["contractTypes"].keys()
+        all_deployments = manifest["deployments"].values()
         all_deployment_names = extract_contract_types_from_deployments(all_deployments)
-        missing_contract_types = set(all_deployment_names).difference(
+        missing_contract_types = all_deployment_names.difference(
             all_contract_types
         )
         if missing_contract_types:

newsfragments/2369.misc.rst

@@ -0,0 +1 @@
+bump docker base image to ``3.9`` and remove parity tests from docker.

newsfragments/2372.misc.rst

@@ -0,0 +1,6 @@
+improve the code base by:
+- using `f-strings`.
+- removing redundant `set()` calls.
+- making error messages more readable in the code.
+- removing python2 `u` prefix.
+- other cleanups.

newsfragments/2376.bugfix.rst

@@ -0,0 +1 @@
+Add async default_chain_id and chain_id setter

newsfragments/2380.doc.rst

@@ -0,0 +1 @@
+Document reading private keys from environment variables

newsfragments/2383.breaking-change.rst

@@ -0,0 +1 @@
+Add ``attach_methods()`` to ``Module`` class to facilitate attaching methods to modules.

tests/core/contracts/conftest.py

@@ -1024,7 +1024,7 @@ def invoke_contract(api_call_desig='call',
                     tx_params={}):
     allowable_call_desig = ['call', 'transact', 'estimateGas', 'buildTransaction']
     if api_call_desig not in allowable_call_desig:
-        raise ValueError("allowable_invoke_method must be one of: %s" % allowable_call_desig)
+        raise ValueError(f"allowable_invoke_method must be one of: {allowable_call_desig}")
 
     function = contract.functions[contract_function]
     result = getattr(function(*func_args, **func_kwargs), api_call_desig)(tx_params)

tests/core/contracts/test_contract_ambiguous_functions.py

@@ -153,7 +153,7 @@ def test_find_or_get_functions_by_type(w3, method, args, repr_func, expected):
         (
             'get_function_by_selector',
             (b'\x00' * (4 + 1), ),
-            r'expected value of size 4 bytes. Got: %s bytes' % (4 + 1),
+            f'expected value of size 4 bytes. Got: {(4 + 1)} bytes',
             ValueError
         ),
         (

tests/core/contracts/test_contract_call_interface.py

@@ -726,7 +726,7 @@ def test_reflect_fixed_value(fixed_reflection_contract, function, value):
         ('reflect_short_u', Decimal('0.01'), "no matching argument types"),
         (
             'reflect_short_u',
-            Decimal('1e-%d' % (DEFAULT_DECIMALS + 1)),
+            Decimal(f'1e-{DEFAULT_DECIMALS + 1}'),
             "no matching argument types",
         ),
         ('reflect_short_u', Decimal('25.4' + '9' * DEFAULT_DECIMALS), "no matching argument types"),