From 20832eace2fb3892f93d3aa8aa72867c6eb75805 Mon Sep 17 00:00:00 2001 From: James Carr Date: Wed, 28 Jun 2023 17:11:16 -0500 Subject: [PATCH 1/4] test: add implementation of evaluation e2e tests Signed-off-by: Federico Bond Signed-off-by: James Carr --- .gitignore | 3 +- .gitmodules | 3 + Makefile | 7 + open_feature/provider/in_memory_provider.py | 13 +- requirements-dev.in | 2 +- requirements-dev.txt | 83 ++--- test-harness | 1 + tests/features/__init__.py | 0 tests/features/data.py | 80 +++++ tests/features/steps/__init__.py | 0 tests/features/steps/steps.py | 344 ++++++++++++++++++++ 11 files changed, 479 insertions(+), 57 deletions(-) create mode 100644 .gitmodules create mode 160000 test-harness create mode 100644 tests/features/__init__.py create mode 100644 tests/features/data.py create mode 100644 tests/features/steps/__init__.py create mode 100644 tests/features/steps/steps.py diff --git a/.gitignore b/.gitignore index 6e19874f..6cd8ba12 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,5 @@ coverage.xml docs/_build/ # Virtual env directories -.venv \ No newline at end of file +.venv +tests/features/*.feature diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..61d2eb45 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test-harness"] + path = test-harness + url = https://github.com/open-feature/test-harness.git diff --git a/Makefile b/Makefile index dfbe3c7b..c925bb80 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,10 @@ lint: .venv clean: @rm -rf .venv @find -iname "*.pyc" -delete + +.PHONY: e2e +e2e: .venv + # NOTE: only the evaluation feature is run for now + cp test-harness/features/evaluation.feature tests/features/ + behave tests/features/ + rm tests/features/*.feature diff --git a/open_feature/provider/in_memory_provider.py b/open_feature/provider/in_memory_provider.py index 0a375282..68e79fa1 100644 --- a/open_feature/provider/in_memory_provider.py +++ b/open_feature/provider/in_memory_provider.py @@ -1,6 +1,7 @@ from dataclasses import dataclass import typing +from open_feature._backports.strenum import StrEnum from open_feature.evaluation_context.evaluation_context import EvaluationContext from open_feature.exception.error_code import ErrorCode from open_feature.flag_evaluation.reason import Reason @@ -22,12 +23,14 @@ class InMemoryMetadata(Metadata): @dataclass(frozen=True) class InMemoryFlag(typing.Generic[T]): + class State(StrEnum): + ENABLED = "ENABLED" + DISABLED = "DISABLED" + flag_key: str default_variant: str variants: typing.Dict[str, T] - reason: typing.Optional[Reason] = Reason.STATIC - error_code: typing.Optional[ErrorCode] = None - error_message: typing.Optional[str] = None + state: State = State.ENABLED context_evaluator: typing.Optional[ typing.Callable[["InMemoryFlag", EvaluationContext], FlagResolutionDetails[T]] ] = None @@ -42,10 +45,8 @@ def resolve( return FlagResolutionDetails( value=self.variants[self.default_variant], - reason=self.reason, + reason=Reason.STATIC, variant=self.default_variant, - error_code=self.error_code, - error_message=self.error_message, ) diff --git a/requirements-dev.in b/requirements-dev.in index 917bafc9..3b2327e2 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -6,4 +6,4 @@ pip-tools pre-commit flake8 pytest-mock -coverage \ No newline at end of file +coverage diff --git a/requirements-dev.txt b/requirements-dev.txt index cd7725ac..8db4464d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,100 +1,86 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile requirements-dev.in # -astroid==2.11.5 +astroid==2.15.6 # via pylint -attrs==21.4.0 - # via pytest -black==22.12.0 +black==23.7.0 # via -r requirements-dev.in +build==0.10.0 + # via pip-tools cfgv==3.3.1 # via pre-commit -click==8.1.5 +click==8.1.6 # via # black # pip-tools -coverage==6.5.0 +coverage==7.2.7 # via -r requirements-dev.in -dill==0.3.6 +dill==0.3.7 # via pylint -distlib==0.3.6 +distlib==0.3.7 # via virtualenv filelock==3.12.2 # via virtualenv -flake8==4.0.1 +flake8==6.0.0 # via -r requirements-dev.in -identify==2.5.24 +identify==2.5.26 # via pre-commit -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest isort==5.12.0 # via pylint lazy-object-proxy==1.9.0 # via astroid -mccabe==0.6.1 +mccabe==0.7.0 # via # flake8 # pylint -mypy-extensions==0.4.3 +mypy-extensions==1.0.0 # via black nodeenv==1.8.0 # via pre-commit -packaging==21.3 - # via pytest -pathspec==0.9.0 +packaging==23.1 + # via + # black + # build + # pytest +pathspec==0.11.1 # via black -pep517==0.13.0 - # via pip-tools -pip-tools==6.14.0 +pip-tools==7.1.0 # via -r requirements-dev.in -platformdirs==2.6.2 +platformdirs==3.9.1 # via # black # pylint # virtualenv pluggy==1.2.0 # via pytest -pre-commit==2.21.0 +pre-commit==3.3.3 # via -r requirements-dev.in -py==1.11.0 - # via pytest -pycodestyle==2.8.0 +pycodestyle==2.10.0 # via flake8 -pyflakes==2.4.0 +pyflakes==3.0.1 # via flake8 -pylint==2.13.8 +pylint==2.17.5 # via -r requirements-dev.in -pyparsing==3.1.0 - # via packaging +pyproject-hooks==1.0.0 + # via build pytest==7.4.0 # via # -r requirements-dev.in # pytest-mock pytest-mock==3.11.1 # via -r requirements-dev.in -pyyaml==6.0 - # via pre-commit -six==1.16.0 - # via virtualenv -toml==0.10.2 +pyyaml==6.0.1 # via pre-commit -tomli==2.0.1 - # via - # black - # pep517 - # pylint - # pytest -typing-extensions==4.7.1 - # via - # astroid - # black - # pylint -virtualenv==20.14.1 +tomlkit==0.12.1 + # via pylint +virtualenv==20.24.2 # via pre-commit -wheel==0.40.0 +wheel==0.41.0 # via pip-tools wrapt==1.15.0 # via astroid @@ -102,4 +88,3 @@ wrapt==1.15.0 # The following packages are considered to be unsafe in a requirements file: # pip # setuptools -setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/test-harness b/test-harness new file mode 160000 index 00000000..bd13458f --- /dev/null +++ b/test-harness @@ -0,0 +1 @@ +Subproject commit bd13458f7e3587ab2ed98b8017bea3c2eb472cc9 diff --git a/tests/features/__init__.py b/tests/features/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/features/data.py b/tests/features/data.py new file mode 100644 index 00000000..f65b86c9 --- /dev/null +++ b/tests/features/data.py @@ -0,0 +1,80 @@ +from open_feature.evaluation_context.evaluation_context import EvaluationContext +from open_feature.flag_evaluation.reason import Reason +from open_feature.flag_evaluation.resolution_details import FlagResolutionDetails +from open_feature.provider.in_memory_provider import InMemoryFlag + + +def context_func(flag: InMemoryFlag, evaluation_context: EvaluationContext): + expects = {"fn": "Sulisław", "ln": "Świętopełk", "age": 29, "customer": False} + + if expects != evaluation_context.attributes: + return FlagResolutionDetails( + value=flag.variants[flag.default_variant], + reason=Reason.DEFAULT, + variant=flag.default_variant, + ) + + return FlagResolutionDetails( + value=flag.variants["internal"], + reason=Reason.TARGETING_MATCH, + variant="internal", + ) + + +IN_MEMORY_FLAGS = { + "boolean-flag": InMemoryFlag( + flag_key="boolean-flag", + state=InMemoryFlag.State.ENABLED, + default_variant="on", + variants={"on": True, "off": False}, + context_evaluator=None, + ), + "string-flag": InMemoryFlag( + flag_key="string-flag", + state=InMemoryFlag.State.ENABLED, + default_variant="greeting", + variants={"greeting": "hi", "parting": "bye"}, + context_evaluator=None, + ), + "integer-flag": InMemoryFlag( + flag_key="integer-flag", + state=InMemoryFlag.State.ENABLED, + default_variant="ten", + variants={"one": 1, "ten": 10}, + context_evaluator=None, + ), + "float-flag": InMemoryFlag( + flag_key="float-flag", + state=InMemoryFlag.State.ENABLED, + default_variant="half", + variants={"tenth": 0.1, "half": 0.5}, + context_evaluator=None, + ), + "object-flag": InMemoryFlag( + flag_key="object-flag", + state=InMemoryFlag.State.ENABLED, + default_variant="template", + variants={ + "empty": {}, + "template": { + "showImages": True, + "title": "Check out these pics!", + "imagesPerPage": 100, + }, + }, + context_evaluator=None, + ), + "context-aware": InMemoryFlag( + flag_key="context-aware", + state=InMemoryFlag.State.ENABLED, + variants={"internal": "INTERNAL", "external": "EXTERNAL"}, + default_variant="external", + context_evaluator=context_func, + ), + "wrong-flag": InMemoryFlag( + flag_key="wrong-flag", + state="ENABLED", + variants={"one": "uno", "two": "dos"}, + default_variant="one", + ), +} diff --git a/tests/features/steps/__init__.py b/tests/features/steps/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/features/steps/steps.py b/tests/features/steps/steps.py new file mode 100644 index 00000000..fec472f0 --- /dev/null +++ b/tests/features/steps/steps.py @@ -0,0 +1,344 @@ +# flake8: noqa: F811 + +from behave import given, then, when + +from open_feature.evaluation_context.evaluation_context import EvaluationContext +from open_feature.exception.error_code import ErrorCode +from open_feature.flag_evaluation.flag_evaluation_details import FlagEvaluationDetails +from open_feature.flag_evaluation.reason import Reason +from open_feature.open_feature_api import get_client, set_provider +from open_feature.open_feature_client import OpenFeatureClient +from open_feature.provider.in_memory_provider import InMemoryProvider +from tests.features.data import IN_MEMORY_FLAGS + +# Common step definitions + + +@then( + 'the resolved {flag_type} details reason of flag with key "{key}" should be ' + '"{reason}"' +) +def step_impl(context, flag_type, key, expected_reason): + details: FlagEvaluationDetails = None + if flag_type == "boolean": + details = context.boolean_flag_details + assert expected_reason == details.reason.value + + +@given("a provider is registered with cache disabled") +def step_impl(context): + set_provider(InMemoryProvider(IN_MEMORY_FLAGS)) + context.client = get_client(name="Default Provider", version="1.0") + + +@when( + 'a {flag_type} flag with key "{key}" is evaluated with details and default value ' + '"{default_value}"' +) +def step_impl(context, flag_type, key, default_value): + context.client = get_client(name="Default Provider", version="1.0") + if flag_type == "boolean": + context.boolean_flag_details = context.client.get_boolean_details( + key, default_value + ) + elif flag_type == "string": + context.string_flag_details = context.client.get_string_details( + key, default_value + ) + + +@when( + 'a boolean flag with key "{key}" is evaluated with {eval_details} and default ' + 'value "{default_value}"' +) +def step_impl(context, key, eval_details, default_value): + client: OpenFeatureClient = context.client + + context.boolean_flag_details = client.get_boolean_details(key, default_value) + + +@when( + 'a {flag_type} flag with key "{key}" is evaluated with default value ' + '"{default_value}"' +) +def step_impl(context, flag_type, key, default_value): + client: OpenFeatureClient = context.client + + if flag_type == "boolean": + context.boolean_flag_details = client.get_boolean_details(key, default_value) + elif flag_type == "string": + context.string_flag_details = client.get_string_details(key, default_value) + + +@then('the resolved string value should be "{expected_value}"') +def step_impl(context, expected_value): + assert expected_value == context.string_flag_details.value + + +@then('the resolved boolean value should be "{expected_value}"') +def step_impl(context, expected_value): + assert parse_boolean(expected_value) == context.boolean_flag_details.value + + +@when( + 'an integer flag with key "{key}" is evaluated with details and default value ' + "{default_value:d}" +) +def step_impl(context, key, default_value): + context.flag_key = key + context.default_value = default_value + context.integer_flag_details = context.client.get_integer_details( + key, default_value + ) + + +@when( + 'an integer flag with key "{key}" is evaluated with default value {default_value:d}' +) +def step_impl(context, key, default_value): + context.flag_key = key + context.default_value = default_value + context.integer_flag_details = context.client.get_integer_details( + key, default_value + ) + + +@when('a float flag with key "{key}" is evaluated with default value {default_value:f}') +def step_impl(context, key, default_value): + context.flag_key = key + context.default_value = default_value + context.float_flag_details = context.client.get_float_details(key, default_value) + + +@when('an object flag with key "{key}" is evaluated with a null default value') +def step_impl(context, key): + context.flag_key = key + context.default_value = None + context.object_flag_details = context.client.get_object_details(key, None) + + +@then("the resolved integer value should be {expected_value:d}") +def step_impl(context, expected_value): + assert expected_value == context.integer_flag_details.value + + +@then("the resolved float value should be {expected_value:f}") +def step_impl(context, expected_value): + assert expected_value == context.float_flag_details.value + + +# Flag evaluation step definitions +@then( + 'the resolved boolean details value should be "{expected_value}", the variant ' + 'should be "{variant}", and the reason should be "{reason}"' +) +def step_impl(context, expected_value, variant, reason): + assert parse_boolean(expected_value) == context.boolean_flag_details.value + assert variant == context.boolean_flag_details.variant + assert reason == context.boolean_flag_details.reason + + +@then( + 'the resolved string details value should be "{expected_value}", the variant ' + 'should be "{variant}", and the reason should be "{reason}"' +) +def step_impl(context, expected_value, variant, reason): + assert expected_value == context.string_flag_details.value + assert variant == context.string_flag_details.variant + assert reason == context.string_flag_details.reason + + +@then( + 'the resolved object value should be contain fields "{field1}", "{field2}", and ' + '"{field3}", with values "{val1}", "{val2}" and {val3}, respectively' +) +def step_impl(context, field1, field2, field3, val1, val2, val3): + value = context.object_flag_details.value + assert field1 in value + assert field2 in value + assert field3 in value + assert value[field1] == parse_any(val1) + assert value[field2] == parse_any(val2) + assert value[field3] == parse_any(val3) + + +@then('the resolved flag value is "{flag_value}" when the context is empty') +def step_impl(context, flag_value): + context.string_flag_details = context.client.get_boolean_details( + context.flag_key, context.default_value + ) + assert flag_value == context.string_flag_details.value + + +@then( + "the reason should indicate an error and the error code should indicate a missing " + 'flag with "{error_code}"' +) +def step_impl(context, error_code): + assert context.string_flag_details.reason == Reason.ERROR + assert context.string_flag_details.error_code == ErrorCode[error_code] + + +@then("the default {flag_type} value should be returned") +def step_impl(context, flag_type): + flag_details = getattr(context, f"{flag_type}_flag_details") + assert context.default_value == flag_details.value + + +@when( + 'a float flag with key "{key}" is evaluated with details and default value ' + "{default_value:f}" +) +def step_impl(context, key, default_value): + context.float_flag_details = context.client.get_float_details(key, default_value) + + +@then( + "the resolved float details value should be {expected_value:f}, the variant should " + 'be "{variant}", and the reason should be "{reason}"' +) +def step_impl(context, expected_value, variant, reason): + assert expected_value == context.float_flag_details.value + assert variant == context.float_flag_details.variant + assert reason == context.float_flag_details.reason + + +@when( + 'an object flag with key "{key}" is evaluated with details and a null default value' +) +def step_impl(context, key): + context.object_flag_details = context.client.get_object_details(key, None) + + +@then( + 'the resolved object details value should be contain fields "{field1}", "{field2}",' + ' and "{field3}", with values "{val1}", "{val2}" and {val3}, respectively' +) +def step_impl(context, field1, field2, field3, val1, val2, val3): + value = context.object_flag_details.value + assert field1 in value + assert field2 in value + assert field3 in value + assert value[field1] == parse_any(val1) + assert value[field2] == parse_any(val2) + assert value[field3] == parse_any(val3) + + +@then('the variant should be "{variant}", and the reason should be "{reason}"') +def step_impl(context, variant, reason): + assert variant == context.object_flag_details.variant + assert reason == context.object_flag_details.reason + + +@when( + 'context contains keys "{key1}", "{key2}", "{key3}", "{key4}" with values "{val1}",' + ' "{val2}", {val3:d}, "{val4}"' +) +def step_impl(context, key1, key2, key3, key4, val1, val2, val3, val4): + context.evaluation_context = EvaluationContext( + None, + { + key1: val1, + key2: val2, + key3: val3, + key4: parse_boolean(val4), + }, + ) + + +@when('a flag with key "{key}" is evaluated with default value "{default_value}"') +def step_impl(context, key, default_value): + context.flag_key = key + context.default_value = default_value + context.string_flag_details = context.client.get_string_details( + key, default_value, context.evaluation_context + ) + + +@then('the resolved string response should be "{expected_value}"') +def step_impl(context, expected_value): + assert expected_value == context.string_flag_details.value + + +@when( + 'a non-existent string flag with key "{flag_key}" is evaluated with details and a ' + 'default value "{default_value}"' +) +def step_impl(context, flag_key, default_value): + context.flag_key = flag_key + context.default_value = default_value + context.string_flag_details = context.client.get_string_details( + flag_key, default_value + ) + + +@when( + 'a string flag with key "{flag_key}" is evaluated as an integer, with details and a' + " default value {default_value:d}" +) +def step_impl(context, flag_key, default_value): + context.flag_key = flag_key + context.default_value = default_value + context.integer_flag_details = context.client.get_integer_details( + flag_key, default_value + ) + + +@then( + "the reason should indicate an error and the error code should indicate a type " + 'mismatch with "{error_code}"' +) +def step_impl(context, error_code): + assert context.integer_flag_details.reason == Reason.ERROR + assert context.integer_flag_details.error_code == ErrorCode[error_code] + + +# Flag caching step definitions + + +@given( + 'the flag\'s configuration with key "{key}" is updated to defaultVariant ' + '"{variant}"' +) +def step_impl(context, key, variant): + raise NotImplementedError("Step definition not implemented yet") + + +@given("sleep for {duration} milliseconds") +def step_impl(context, duration): + raise NotImplementedError("Step definition not implemented yet") + + +@then('the resolved string details reason should be "{reason}"') +def step_impl(context, reason): + raise NotImplementedError("Step definition not implemented yet") + + +@then( + "the resolved integer details value should be {expected_value:d}, the variant " + 'should be "{variant}", and the reason should be "{reason}"' +) +def step_impl(context, expected_value, variant, reason): + assert expected_value == context.integer_flag_details.value + assert variant == context.integer_flag_details.variant + assert reason == context.integer_flag_details.reason + + +def parse_boolean(value): + if value == "true": + return True + if value == "false": + return False + raise ValueError(f"Invalid boolean value: {value}") + + +def parse_any(value): + if value == "true": + return True + if value == "false": + return False + if value == "null": + return None + if value.isdigit(): + return int(value) + return value From cfed0b9709f546233bab7abac97a34652f05b6d5 Mon Sep 17 00:00:00 2001 From: Federico Bond Date: Tue, 1 Aug 2023 14:09:23 -0300 Subject: [PATCH 2/4] chore: run behave tests in GH workflow Signed-off-by: Federico Bond --- .github/workflows/merge.yml | 9 ++++++++- .github/workflows/pullrequest.yml | 9 ++++++++- requirements-dev.in | 1 + requirements-dev.txt | 12 ++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index b085ac77..3862f049 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -20,6 +20,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + submodules: recursive - name: Cache virtualenvironment uses: actions/cache@v3 @@ -64,5 +66,10 @@ jobs: fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) + - name: Run E2E tests with behave + run: | + cp test-harness/features/evaluation.feature tests/features/ + behave tests/features/ + - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml index 539d34af..94ba2ac4 100644 --- a/.github/workflows/pullrequest.yml +++ b/.github/workflows/pullrequest.yml @@ -23,6 +23,8 @@ jobs: steps: - uses: actions/checkout@v3 + with: + submodules: recursive - name: Cache virtualenvironment uses: actions/cache@v3 @@ -67,5 +69,10 @@ jobs: fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) + - name: Run E2E tests with behave + run: | + cp test-harness/features/evaluation.feature tests/features/ + behave tests/features/ + - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v2 diff --git a/requirements-dev.in b/requirements-dev.in index 3b2327e2..84be89d8 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -7,3 +7,4 @@ pre-commit flake8 pytest-mock coverage +behave diff --git a/requirements-dev.txt b/requirements-dev.txt index 8db4464d..5ccbfe44 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,8 @@ # astroid==2.15.6 # via pylint +behave==1.2.6 + # via -r requirements-dev.in black==23.7.0 # via -r requirements-dev.in build==0.10.0 @@ -47,6 +49,12 @@ packaging==23.1 # black # build # pytest +parse==1.19.1 + # via + # behave + # parse-type +parse-type==0.6.2 + # via behave pathspec==0.11.1 # via black pip-tools==7.1.0 @@ -76,6 +84,10 @@ pytest-mock==3.11.1 # via -r requirements-dev.in pyyaml==6.0.1 # via pre-commit +six==1.16.0 + # via + # behave + # parse-type tomlkit==0.12.1 # via pylint virtualenv==20.24.2 From 9014c07076b5336dc884ff06c9b5ce1eac32a9a4 Mon Sep 17 00:00:00 2001 From: Federico Bond Date: Tue, 1 Aug 2023 14:34:29 -0300 Subject: [PATCH 3/4] fix: replace typing.Union value used in isinstance for compatibility with python<3.10 Signed-off-by: Federico Bond --- open_feature/open_feature_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_feature/open_feature_client.py b/open_feature/open_feature_client.py index 52fda03a..d4f5fb06 100644 --- a/open_feature/open_feature_client.py +++ b/open_feature/open_feature_client.py @@ -385,7 +385,7 @@ def _typecheck_flag_value(value, flag_type): type_map = { FlagType.BOOLEAN: bool, FlagType.STRING: str, - FlagType.OBJECT: typing.Union[dict, list], + FlagType.OBJECT: (dict, list), FlagType.FLOAT: float, FlagType.INTEGER: int, } From 097eafeec164304379d8fcb60f36cd4385d48974 Mon Sep 17 00:00:00 2001 From: Federico Bond Date: Mon, 14 Aug 2023 14:38:01 -0300 Subject: [PATCH 4/4] chore: automatically pull submodule for e2e make target Signed-off-by: Federico Bond --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c925bb80..a68df2b7 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ else $(VENV); pytest endif +test-harness: + git submodule update --init + .PHONY: lint lint: .venv $(VENV); black . @@ -30,8 +33,8 @@ clean: @find -iname "*.pyc" -delete .PHONY: e2e -e2e: .venv +e2e: .venv test-harness # NOTE: only the evaluation feature is run for now cp test-harness/features/evaluation.feature tests/features/ - behave tests/features/ + $(VENV); behave tests/features/ rm tests/features/*.feature