From 6127baa4d7c037735c47ccc6c8837ab8eb1c9fd5 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 26 May 2025 17:26:03 +0300 Subject: [PATCH 1/5] feat(orc-372): add stricter marks checking --- Makefile | 2 +- tests/conftest.py | 22 ++++++++++++++----- tests/modules/csm/test_csm_module.py | 1 + .../submodules/consensus/test_consensus.py | 1 + 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index cf7c031d4..91e7c8e63 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ lint: up # Use ORACLE_TEST_PATH to run specific tests, e.g.: # make test ORACLE_TEST_PATH=tests/providers_clients/test_keys_api_client.py test: up - $(EXEC_CMD) pytest $(ORACLE_TEST_PATH) + $(EXEC_CMD) pytest $${ORACLE_TEST_PATH:-tests/} # Use ORACLE_MODULE to run specific module, e.g.: # make run-module ORACLE_MODULE=accounting diff --git a/tests/conftest.py b/tests/conftest.py index 7c4b03ca5..ae655d7ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,13 @@ INTEGRATION_MARKER = 'integration' MAINNET_MARKER = 'mainnet' TESTNET_MARKER = 'testnet' +FORK_MARKER = 'fork' + +INTEGRATION_TESTS_MODIFICATORS_MARKERS = { + MAINNET_MARKER, + TESTNET_MARKER, + FORK_MARKER, +} DUMMY_ADDRESS = "0x0000000000000000000000000000000000000000" @@ -40,14 +47,19 @@ def check_test_marks_compatibility(request): all_test_markers = {x.name for x in request.node.iter_markers()} - if not all_test_markers: - pytest.fail('Test must be marked.') + if not all_test_markers or not {UNIT_MARKER, INTEGRATION_MARKER} & all_test_markers: + pytest.fail('Test must be marked with at least one marker, e.g. @pytest.mark.unit or @pytest.mark.integration.') - elif UNIT_MARKER in all_test_markers and {MAINNET_MARKER, TESTNET_MARKER, INTEGRATION_MARKER} & all_test_markers: + elif ( + UNIT_MARKER in all_test_markers + and {INTEGRATION_MARKER}.union(INTEGRATION_TESTS_MODIFICATORS_MARKERS) & all_test_markers + ): pytest.fail('Test can not be both unit and integration at the same time.') - elif {MAINNET_MARKER, TESTNET_MARKER} & all_test_markers and INTEGRATION_MARKER not in all_test_markers: - pytest.fail('Test can not be run on mainnet or testnet without integration marker.') + elif INTEGRATION_TESTS_MODIFICATORS_MARKERS & all_test_markers and INTEGRATION_MARKER not in all_test_markers: + pytest.fail( + f'Test can not be run with {INTEGRATION_TESTS_MODIFICATORS_MARKERS} markers without @pytest.mark.integration.' + ) @pytest.fixture(autouse=True) diff --git a/tests/modules/csm/test_csm_module.py b/tests/modules/csm/test_csm_module.py index c35cd8997..2118ee574 100644 --- a/tests/modules/csm/test_csm_module.py +++ b/tests/modules/csm/test_csm_module.py @@ -638,6 +638,7 @@ class RewardsTreeTestParam: expected_tree_values: list | Type[ValueError] +@pytest.mark.unit @pytest.mark.parametrize( "param", [ diff --git a/tests/modules/submodules/consensus/test_consensus.py b/tests/modules/submodules/consensus/test_consensus.py index 80c4fb272..89a88d9f5 100644 --- a/tests/modules/submodules/consensus/test_consensus.py +++ b/tests/modules/submodules/consensus/test_consensus.py @@ -355,6 +355,7 @@ class NoContractVersionConsensusImpl(ConsensusImpl): CONSENSUS_VERSION = 1 +@pytest.mark.unit @pytest.mark.parametrize( "impl", [ From 6b959eb35cd30286bb64010735bdbe4eca415c3a Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 26 May 2025 18:50:46 +0300 Subject: [PATCH 2/5] feat(orc-372): update test dependencies --- poetry.lock | 26 +++++++++++++------------- pyproject.toml | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index b5cb2fd64..713ac7207 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1046,14 +1046,14 @@ test = ["eth-hash[pycryptodome]", "pytest (>=7.0.0)", "pytest-xdist (>=2.4.0)"] [[package]] name = "eth-tester" -version = "0.12.1b1" +version = "0.13.0b1" description = "eth-tester: Tools for testing Ethereum applications." category = "dev" optional = false python-versions = "<4,>=3.8" files = [ - {file = "eth_tester-0.12.1b1-py3-none-any.whl", hash = "sha256:aa3f91960e5ce9fe74eac4a0dcb22ffada84b8e28dc11d0f0a69085a5879be60"}, - {file = "eth_tester-0.12.1b1.tar.gz", hash = "sha256:7aeb3b5839fb1bc20e7f15c5e289ba95809fa41117a5ac194e8d270467982832"}, + {file = "eth_tester-0.13.0b1-py3-none-any.whl", hash = "sha256:872108cea7df1340f56bab25b9ed5cf0f835aa467c1a8195d0891238c0e73ab3"}, + {file = "eth_tester-0.13.0b1.tar.gz", hash = "sha256:87fb561d450cd3639ce82eed52e566c902a0ac241f3e9581c6b7545690fcece5"}, ] [package.dependencies] @@ -1065,10 +1065,10 @@ rlp = ">=3.0.0" semantic_version = ">=2.6.0" [package.extras] -dev = ["build (>=0.9.0)", "bump_my_version (>=0.19.0)", "eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pysha3] (>=0.1.4,<1.0.0)", "ipython", "pre-commit (>=3.4.0)", "py-evm (>=0.10.0b0,<0.11.0b0)", "pytest (>=7.0.0)", "pytest-xdist (>=2.0.0,<3)", "towncrier (>=24,<25)", "tox (>=4.0.0)", "twine", "wheel"] +dev = ["build (>=0.9.0)", "bump_my_version (>=0.19.0)", "eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pysha3] (>=0.1.4,<1.0.0)", "ipython", "pre-commit (>=3.4.0)", "py-evm (>=0.12.0b2,<0.13.0b1)", "pytest (>=7.0.0)", "pytest-xdist (>=2.0.0,<3)", "towncrier (>=24,<25)", "tox (>=4.0.0)", "twine", "wheel"] docs = ["towncrier (>=24,<25)"] -py-evm = ["eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pysha3] (>=0.1.4,<1.0.0)", "py-evm (>=0.10.0b0,<0.11.0b0)"] -pyevm = ["eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pysha3] (>=0.1.4,<1.0.0)", "py-evm (>=0.10.0b0,<0.11.0b0)"] +py-evm = ["eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pysha3] (>=0.1.4,<1.0.0)", "py-evm (>=0.12.0b2,<0.13.0b1)"] +pyevm = ["eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "eth-hash[pysha3] (>=0.1.4,<1.0.0)", "py-evm (>=0.12.0b2,<0.13.0b1)"] test = ["eth-hash[pycryptodome] (>=0.1.4,<1.0.0)", "pytest (>=7.0.0)", "pytest-xdist (>=2.0.0,<3)"] [[package]] @@ -2503,24 +2503,24 @@ testutils = ["gitpython (>3)"] [[package]] name = "pytest" -version = "7.4.4" +version = "8.3.5" description = "pytest: simple powerful testing with Python" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, - {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, + {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, + {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, ] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<2.0" +pluggy = ">=1.5,<2" [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-cov" @@ -3346,4 +3346,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "dc8fea94792310447b9fc1ac5ac40c2597998d609bba4b0f438631e2ff8ec389" +content-hash = "3a3b35a518fd17914270a778e29c17fbfe1d8f25dccf6d4135dcb3fd2022fa51" diff --git a/pyproject.toml b/pyproject.toml index b172b0f5e..d27b64b0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,8 @@ license = "GPL 3.0" python = "^3.12" prometheus-client = "0.21.1" timeout-decorator = "^0.5.0" -pytest = "^7.2.1" -pytest-xdist = "^3.2.1" +pytest = "^8.3.5" +pytest-xdist = "^3.6.1" more-itertools = "^10.1.0" web3 = "^7.8.0" web3-multi-provider = { version = "^2.2.0", extras = ["metrics"] } @@ -46,7 +46,7 @@ black = "^24.8" pylint = "^3.2.3" mypy = "^1.10.0" responses = "^0.25.7" -eth-tester = "^0.12.1b1" +eth-tester = "^0.13.0b1" pre-commit = "3.8" ipython = "^9.0" isort = "^6.0.1" From edddd34f28fa828a64a4f1d2cd46410702b519f3 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 26 May 2025 19:06:45 +0300 Subject: [PATCH 3/5] feat(orc-372): add isort to makefile --- Makefile | 3 +++ tests/conftest.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 91e7c8e63..b335cd268 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,9 @@ lint: up $(EXEC_CMD) pylint src tests --jobs=2 $(EXEC_CMD) mypy src +isort: up + $(EXEC_CMD) isort $(ORACLE_ISORT_PATH) + # Use ORACLE_TEST_PATH to run specific tests, e.g.: # make test ORACLE_TEST_PATH=tests/providers_clients/test_keys_api_client.py test: up diff --git a/tests/conftest.py b/tests/conftest.py index ae655d7ae..181e1c0f7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,13 +15,13 @@ from src.web3py.extensions import ( CSM, ConsensusClientModule, + FallbackProviderModule, KeysAPIClientModule, LidoContracts, LidoValidatorsProvider, TransactionUtils, ) from src.web3py.types import Web3 -from src.web3py.extensions import FallbackProviderModule UNIT_MARKER = 'unit' INTEGRATION_MARKER = 'integration' From 6b99fdbf435c1d1d5d50e22e88c685560917b39a Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Mon, 26 May 2025 20:02:05 +0300 Subject: [PATCH 4/5] feat(orc-372): make forktest integration tests --- tests/fork/test_csm_oracle_cycle.py | 1 + tests/fork/test_lido_oracle_cycle.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/fork/test_csm_oracle_cycle.py b/tests/fork/test_csm_oracle_cycle.py index 634d84145..628f65747 100644 --- a/tests/fork/test_csm_oracle_cycle.py +++ b/tests/fork/test_csm_oracle_cycle.py @@ -125,6 +125,7 @@ def missed_initial_frame(frame_config: FrameConfig): @pytest.mark.fork +@pytest.mark.integration @pytest.mark.parametrize( 'module', [csm_module], diff --git a/tests/fork/test_lido_oracle_cycle.py b/tests/fork/test_lido_oracle_cycle.py index 30fad386e..24eb37d96 100644 --- a/tests/fork/test_lido_oracle_cycle.py +++ b/tests/fork/test_lido_oracle_cycle.py @@ -45,6 +45,7 @@ def missed_initial_frame(frame_config: FrameConfig): @pytest.mark.fork +@pytest.mark.integration @pytest.mark.parametrize( 'module', [accounting_module, ejector_module], From 4c03c4d3598f293fc5efdbc91e566d406dde3944 Mon Sep 17 00:00:00 2001 From: chasingrainbows Date: Tue, 27 May 2025 22:49:03 +0300 Subject: [PATCH 5/5] feat(orc-372): replace e2e with integration test --- pyproject.toml | 11 +++--- tests/e2e/conftest.py | 60 ----------------------------- tests/e2e/test_accounting.py | 16 -------- tests/main/__init__.py | 0 tests/main/test_main_cycle_smoke.py | 53 +++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 82 deletions(-) delete mode 100644 tests/e2e/conftest.py delete mode 100644 tests/e2e/test_accounting.py create mode 100644 tests/main/__init__.py create mode 100644 tests/main/test_main_cycle_smoke.py diff --git a/pyproject.toml b/pyproject.toml index d27b64b0a..7568cc362 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,12 +57,11 @@ build-backend = "poetry.core.masonry.api" [tool.pytest.ini_options] markers = [ - "unit: tests with using mocks and don't make external requests", - "integration: tests with using providers", - "e2e: complex tests with using providers and real Ethereum network", - "fork: tests with using forked Ethereum network", - "mainnet: tests that depends on mainnet", - "testnet: tests that depends on testnet", + "unit: tests using mocks without external requests", + "integration: tests involving external providers", + "mainnet: integration tests reliant on the mainnet", + "testnet: integration tests reliant on the testnet", + "fork: integration tests using a forked Ethereum network", ] addopts = "-s -vv --strict-markers" diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py deleted file mode 100644 index 8ee6e21d7..000000000 --- a/tests/e2e/conftest.py +++ /dev/null @@ -1,60 +0,0 @@ -import logging -import time -from logging.handlers import QueueHandler -import pytest - -from pytest import Item -from src.main import main -from multiprocessing import Process, Queue -from src.variables import EXECUTION_CLIENT_URI - - -@pytest.hookimpl(hookwrapper=True) -def pytest_collection_modifyitems(items: list[Item]): - yield - if any(not item.get_closest_marker("e2e") for item in items): - for item in items: - if item.get_closest_marker("e2e"): - item.add_marker( - pytest.mark.skip( - reason="e2e tests are take a lot of time " "and skipped if any other tests are selected" - ) - ) - - -def worker_process(queue, module_name, execution_client_uri): - import src.variables - - src.variables.EXECUTION_CLIENT_URI = [execution_client_uri] - qh = QueueHandler(queue) - root = logging.getLogger() - root.addHandler(qh) - main(module_name) - - -@pytest.fixture(scope="session", params=EXECUTION_CLIENT_URI) -def execution_client_uri(request): - return request.param - - -@pytest.fixture -def start_accounting(caplog, execution_client_uri): - queue = Queue() - listener = logging.handlers.QueueListener(queue, caplog.handler) - listener.start() - - worker = Process(target=worker_process, args=(queue, "accounting", execution_client_uri)) - worker.start() - yield - worker.terminate() - - -def wait_for_message_appeared(caplog, message, timeout=600): - start_time = time.time() - while True: - if message in caplog.messages: - return - if time.time() - start_time > timeout: - break - time.sleep(1) - raise AssertionError(f"Message {message} not found in logs") diff --git a/tests/e2e/test_accounting.py b/tests/e2e/test_accounting.py deleted file mode 100644 index 8816a391e..000000000 --- a/tests/e2e/test_accounting.py +++ /dev/null @@ -1,16 +0,0 @@ -import pytest - -from src.modules.accounting.accounting import Accounting -from tests.e2e.conftest import wait_for_message_appeared - - -@pytest.mark.e2e -def test_app(start_accounting, caplog): - wait_for_message_appeared(caplog, "{'msg': 'Run module as daemon.'}", timeout=10) - wait_for_message_appeared(caplog, "{'msg': 'Check if main data was submitted.', 'value': False}") - wait_for_message_appeared(caplog, "{'msg': 'Check if contract could accept report.', 'value': True}") - wait_for_message_appeared(caplog, "{'msg': 'Execute module.'}") - wait_for_message_appeared(caplog, "{'msg': 'Checking bunker mode'}", timeout=1800) - wait_for_message_appeared( - caplog, f"{'msg': 'Send report hash. Consensus version: [{Accounting.COMPATIBLE_CONSENSUS_VERSION}]'}" - ) diff --git a/tests/main/__init__.py b/tests/main/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/main/test_main_cycle_smoke.py b/tests/main/test_main_cycle_smoke.py new file mode 100644 index 000000000..eeb214e04 --- /dev/null +++ b/tests/main/test_main_cycle_smoke.py @@ -0,0 +1,53 @@ +import logging +import logging.handlers +import multiprocessing +from concurrent.futures import ProcessPoolExecutor + +import pytest + +from src import variables +from src.main import main +from src.types import OracleModule + + +@pytest.mark.mainnet +@pytest.mark.integration +class TestIntegrationMainCycleSmoke: + + def run_main_with_logging(self, module_name, log_queue): + queue_handler = logging.handlers.QueueHandler(log_queue) + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + logger.addHandler(queue_handler) + + main(module_name) + + @pytest.mark.parametrize( + "module_name", + [ + "accounting", + "ejector", + "csm", + ], + ) + def test_main_cycle_smoke__oracle_module__cycle_runs_successfully( + self, monkeypatch, caplog, module_name: OracleModule + ): + monkeypatch.setattr(variables, 'DAEMON', False) + monkeypatch.setattr(variables, 'CYCLE_SLEEP_IN_SECONDS', 0) + monkeypatch.setattr("src.web3py.extensions.CSM.CONTRACT_LOAD_MAX_RETRIES", 3) + monkeypatch.setattr("src.web3py.extensions.CSM.CONTRACT_LOAD_RETRY_DELAY", 0) + + manager = multiprocessing.Manager() + log_queue = manager.Queue() + listener = logging.handlers.QueueListener(log_queue, caplog.handler) + listener.start() + + with ProcessPoolExecutor(max_workers=1) as executor: + future = executor.submit(self.run_main_with_logging, module_name, log_queue) + future.result() + + listener.stop() + + error_logs = [record for record in caplog.records if record.levelno >= logging.ERROR] + assert not error_logs, f"Found error logs: {[record.message for record in error_logs]}"