chore: E2E Test Support

.github/workflows/merge.yml

@@ -20,6 +20,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Cache virtualenvironment
         uses: actions/cache@v3
@@ -64,5 +66,10 @@ jobs:
           fail_ci_if_error: true # optional (default = false)
           verbose: true # optional (default = false)
 
+      - name: Run E2E tests with behave
+        run: |
+          cp test-harness/features/evaluation.feature tests/features/
+          behave tests/features/
+
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v2

.github/workflows/pullrequest.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Cache virtualenvironment
         uses: actions/cache@v3
@@ -67,5 +69,10 @@ jobs:
           fail_ci_if_error: true # optional (default = false)
           verbose: true # optional (default = false)
 
+      - name: Run E2E tests with behave
+        run: |
+          cp test-harness/features/evaluation.feature tests/features/
+          behave tests/features/
+
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v2

.gitignore

@@ -50,4 +50,5 @@ coverage.xml
 docs/_build/
 
 # Virtual env directories
-.venv
+.venv
+tests/features/*.feature

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test-harness"]
+	path = test-harness
+	url = https://github.com/open-feature/test-harness.git

Makefile

@@ -18,6 +18,9 @@ else
 	$(VENV); pytest
 endif
 
+test-harness:
+	git submodule update --init
+
 .PHONY: lint
 lint: .venv
 	$(VENV); black .
@@ -28,3 +31,10 @@ lint: .venv
 clean:
 	@rm -rf .venv
 	@find -iname "*.pyc" -delete
+
+.PHONY: e2e
+e2e: .venv test-harness
+	# NOTE: only the evaluation feature is run for now
+	cp test-harness/features/evaluation.feature tests/features/
+	$(VENV); behave tests/features/
+	rm tests/features/*.feature

open_feature/open_feature_client.py

@@ -385,7 +385,7 @@ def _typecheck_flag_value(value, flag_type):
     type_map = {
         FlagType.BOOLEAN: bool,
         FlagType.STRING: str,
-        FlagType.OBJECT: typing.Union[dict, list],
+        FlagType.OBJECT: (dict, list),
         FlagType.FLOAT: float,
         FlagType.INTEGER: int,
     }

open_feature/provider/in_memory_provider.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 import typing
 
+from open_feature._backports.strenum import StrEnum
 from open_feature.evaluation_context.evaluation_context import EvaluationContext
 from open_feature.exception.error_code import ErrorCode
 from open_feature.flag_evaluation.reason import Reason
@@ -22,12 +23,14 @@ class InMemoryMetadata(Metadata):
 
 @dataclass(frozen=True)
 class InMemoryFlag(typing.Generic[T]):
+    class State(StrEnum):
+        ENABLED = "ENABLED"
+        DISABLED = "DISABLED"
+
     flag_key: str
     default_variant: str
     variants: typing.Dict[str, T]
-    reason: typing.Optional[Reason] = Reason.STATIC
-    error_code: typing.Optional[ErrorCode] = None
-    error_message: typing.Optional[str] = None
+    state: State = State.ENABLED
     context_evaluator: typing.Optional[
         typing.Callable[["InMemoryFlag", EvaluationContext], FlagResolutionDetails[T]]
     ] = None
@@ -42,10 +45,8 @@ def resolve(
 
         return FlagResolutionDetails(
             value=self.variants[self.default_variant],
-            reason=self.reason,
+            reason=Reason.STATIC,
             variant=self.default_variant,
-            error_code=self.error_code,
-            error_message=self.error_message,
         )
 
 

requirements-dev.in

@@ -6,4 +6,5 @@ pip-tools
 pre-commit
 flake8
 pytest-mock
-coverage
+coverage
+behave

requirements-dev.txt

@@ -1,105 +1,102 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
 #
 #    pip-compile requirements-dev.in
 #
-astroid==2.11.5
+astroid==2.15.6
     # via pylint
-attrs==21.4.0
-    # via pytest
-black==22.12.0
+behave==1.2.6
+    # via -r requirements-dev.in
+black==23.7.0
     # via -r requirements-dev.in
+build==0.10.0
+    # via pip-tools
 cfgv==3.3.1
     # via pre-commit
-click==8.1.5
+click==8.1.6
     # via
     #   black
     #   pip-tools
-coverage==6.5.0
+coverage==7.2.7
     # via -r requirements-dev.in
-dill==0.3.6
+dill==0.3.7
     # via pylint
-distlib==0.3.6
+distlib==0.3.7
     # via virtualenv
 filelock==3.12.2
     # via virtualenv
-flake8==4.0.1
+flake8==6.0.0
     # via -r requirements-dev.in
-identify==2.5.24
+identify==2.5.26
     # via pre-commit
-iniconfig==1.1.1
+iniconfig==2.0.0
     # via pytest
 isort==5.12.0
     # via pylint
 lazy-object-proxy==1.9.0
     # via astroid
-mccabe==0.6.1
+mccabe==0.7.0
     # via
     #   flake8
     #   pylint
-mypy-extensions==0.4.3
+mypy-extensions==1.0.0
     # via black
 nodeenv==1.8.0
     # via pre-commit
-packaging==21.3
-    # via pytest
-pathspec==0.9.0
+packaging==23.1
+    # via
+    #   black
+    #   build
+    #   pytest
+parse==1.19.1
+    # via
+    #   behave
+    #   parse-type
+parse-type==0.6.2
+    # via behave
+pathspec==0.11.1
     # via black
-pep517==0.13.0
-    # via pip-tools
-pip-tools==6.14.0
+pip-tools==7.1.0
     # via -r requirements-dev.in
-platformdirs==2.6.2
+platformdirs==3.9.1
     # via
     #   black
     #   pylint
     #   virtualenv
 pluggy==1.2.0
     # via pytest
-pre-commit==2.21.0
+pre-commit==3.3.3
     # via -r requirements-dev.in
-py==1.11.0
-    # via pytest
-pycodestyle==2.8.0
+pycodestyle==2.10.0
     # via flake8
-pyflakes==2.4.0
+pyflakes==3.0.1
     # via flake8
-pylint==2.13.8
+pylint==2.17.5
     # via -r requirements-dev.in
-pyparsing==3.1.0
-    # via packaging
+pyproject-hooks==1.0.0
+    # via build
 pytest==7.4.0
     # via
     #   -r requirements-dev.in
     #   pytest-mock
 pytest-mock==3.11.1
     # via -r requirements-dev.in
-pyyaml==6.0
+pyyaml==6.0.1
     # via pre-commit
 six==1.16.0
-    # via virtualenv
-toml==0.10.2
-    # via pre-commit
-tomli==2.0.1
     # via
-    #   black
-    #   pep517
-    #   pylint
-    #   pytest
-typing-extensions==4.7.1
-    # via
-    #   astroid
-    #   black
-    #   pylint
-virtualenv==20.14.1
+    #   behave
+    #   parse-type
+tomlkit==0.12.1
+    # via pylint
+virtualenv==20.24.2
     # via pre-commit
-wheel==0.40.0
+wheel==0.41.0
     # via pip-tools
 wrapt==1.15.0
     # via astroid
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip
 # setuptools
-setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

tests/features/data.py

@@ -0,0 +1,80 @@
+from open_feature.evaluation_context.evaluation_context import EvaluationContext
+from open_feature.flag_evaluation.reason import Reason
+from open_feature.flag_evaluation.resolution_details import FlagResolutionDetails
+from open_feature.provider.in_memory_provider import InMemoryFlag
+
+
+def context_func(flag: InMemoryFlag, evaluation_context: EvaluationContext):
+    expects = {"fn": "Sulisław", "ln": "Świętopełk", "age": 29, "customer": False}
+
+    if expects != evaluation_context.attributes:
+        return FlagResolutionDetails(
+            value=flag.variants[flag.default_variant],
+            reason=Reason.DEFAULT,
+            variant=flag.default_variant,
+        )
+
+    return FlagResolutionDetails(
+        value=flag.variants["internal"],
+        reason=Reason.TARGETING_MATCH,
+        variant="internal",
+    )
+
+
+IN_MEMORY_FLAGS = {
+    "boolean-flag": InMemoryFlag(
+        flag_key="boolean-flag",
+        state=InMemoryFlag.State.ENABLED,
+        default_variant="on",
+        variants={"on": True, "off": False},
+        context_evaluator=None,
+    ),
+    "string-flag": InMemoryFlag(
+        flag_key="string-flag",
+        state=InMemoryFlag.State.ENABLED,
+        default_variant="greeting",
+        variants={"greeting": "hi", "parting": "bye"},
+        context_evaluator=None,
+    ),
+    "integer-flag": InMemoryFlag(
+        flag_key="integer-flag",
+        state=InMemoryFlag.State.ENABLED,
+        default_variant="ten",
+        variants={"one": 1, "ten": 10},
+        context_evaluator=None,
+    ),
+    "float-flag": InMemoryFlag(
+        flag_key="float-flag",
+        state=InMemoryFlag.State.ENABLED,
+        default_variant="half",
+        variants={"tenth": 0.1, "half": 0.5},
+        context_evaluator=None,
+    ),
+    "object-flag": InMemoryFlag(
+        flag_key="object-flag",
+        state=InMemoryFlag.State.ENABLED,
+        default_variant="template",
+        variants={
+            "empty": {},
+            "template": {
+                "showImages": True,
+                "title": "Check out these pics!",
+                "imagesPerPage": 100,
+            },
+        },
+        context_evaluator=None,
+    ),
+    "context-aware": InMemoryFlag(
+        flag_key="context-aware",
+        state=InMemoryFlag.State.ENABLED,
+        variants={"internal": "INTERNAL", "external": "EXTERNAL"},
+        default_variant="external",
+        context_evaluator=context_func,
+    ),
+    "wrong-flag": InMemoryFlag(
+        flag_key="wrong-flag",
+        state="ENABLED",
+        variants={"one": "uno", "two": "dos"},
+        default_variant="one",
+    ),
+}