From b7db844ab1cfc118135d4dd685b4d4d8a9f44abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 18 Sep 2021 00:20:34 +0100 Subject: [PATCH 1/2] Support for environment files in set_env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- docs/changelog/1938.feature.rst | 1 + docs/config.rst | 12 +++++++++- src/tox/config/set_env.py | 42 ++++++++++++++++++++++++--------- src/tox/tox_env/errors.py | 6 ++--- tests/config/test_set_env.py | 38 +++++++++++++++++++++++++---- 5 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 docs/changelog/1938.feature.rst diff --git a/docs/changelog/1938.feature.rst b/docs/changelog/1938.feature.rst new file mode 100644 index 000000000..76a4c4e64 --- /dev/null +++ b/docs/changelog/1938.feature.rst @@ -0,0 +1 @@ +Support for environment files within the :ref:`set_env` configuration via the ``file|`` prefix - by :user:`gaborbernat`. diff --git a/docs/config.rst b/docs/config.rst index fcb758c4c..c7e86f5dd 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -270,7 +270,17 @@ Base options .. conf:: :keys: set_env, setenv - A dictionary of environment variables to set when running commands in the tox environment. + A dictionary of environment variables to set when running commands in the tox environment. Lines starting with a + ``file|`` prefix are considered environment files to load with their location being the content minus this prefix. + + .. note:: + + Environment files are processed using the following rules: + + - blank lines ignored, + - lines starting with the ``#`` character are ignored, + - each line is in KEY=VALUE format; both the key and the value is stripped, + - there is no special handling of quotation marks, they are part of the key or value. .. conf:: :keys: parallel_show_output diff --git a/src/tox/config/set_env.py b/src/tox/config/set_env.py index 33cfae454..030116de4 100644 --- a/src/tox/config/set_env.py +++ b/src/tox/config/set_env.py @@ -1,6 +1,8 @@ +from pathlib import Path from typing import Callable, Dict, Iterator, List, Mapping, Optional, Tuple from tox.config.loader.api import ConfigLoadArgs +from tox.tox_env.errors import Fail Replacer = Callable[[str, ConfigLoadArgs], str] @@ -15,21 +17,39 @@ def __init__(self, raw: str, name: str, env_name: Optional[str]) -> None: for line in raw.splitlines(): if line.strip(): - try: - key, value = self._extract_key_value(line) - if "{" in key: - raise ValueError(f"invalid line {line!r} in set_env") - except ValueError: - _, __, match = find_replace_part(line, 0) - if match: - self._later.append(line) - else: - raise + if line.startswith("file|"): + for key, value in self._read_env_file(line): + self._raw[key] = value else: - self._raw[key] = value + try: + key, value = self._extract_key_value(line) + if "{" in key: + raise ValueError(f"invalid line {line!r} in set_env") + except ValueError: + _, __, match = find_replace_part(line, 0) + if match: + self._later.append(line) + else: + raise + else: + self._raw[key] = value self._materialized: Dict[str, str] = {} self.changed = False + def _read_env_file(self, line: str) -> Iterator[Tuple[str, str]]: + # Our rules in the documentation, some upstream environment file rules (we follow mostly the docker one): + # - https://www.npmjs.com/package/dotenv#rules + # - https://docs.docker.com/compose/env-file/ + env_file = Path(line[len("file|") :]) + if not env_file.exists(): + raise Fail(f"{env_file} does not exist for set_env") + for env_line in env_file.read_text().splitlines(): + env_line = env_line.strip() + if not env_line or env_line.startswith("#"): + continue + key, value = self._extract_key_value(env_line) + yield key, value + @staticmethod def _extract_key_value(line: str) -> Tuple[str, str]: key, sep, value = line.partition("=") diff --git a/src/tox/tox_env/errors.py b/src/tox/tox_env/errors.py index fd3d62170..49194b44f 100644 --- a/src/tox/tox_env/errors.py +++ b/src/tox/tox_env/errors.py @@ -1,13 +1,13 @@ """Defines tox error types""" -class Recreate(RuntimeError): +class Recreate(Exception): # noqa: N818 """Recreate the tox environment""" -class Skip(RuntimeError): +class Skip(Exception): # noqa: N818 """Skip this tox environment""" -class Fail(RuntimeError): +class Fail(Exception): # noqa: N818 """Failed creating env""" diff --git a/tests/config/test_set_env.py b/tests/config/test_set_env.py index 0694a71af..f1e2ce31f 100644 --- a/tests/config/test_set_env.py +++ b/tests/config/test_set_env.py @@ -1,4 +1,4 @@ -from typing import Callable +from typing import Any, Dict, Optional, Protocol import pytest from pytest_mock import MockerFixture @@ -26,13 +26,15 @@ def test_set_env_bad_line() -> None: SetEnv("A", "py", "py") -EvalSetEnv = Callable[[str], SetEnv] +class EvalSetEnv(Protocol): + def __call__(self, tox_ini: str, extra_files: Optional[Dict[str, Any]] = None) -> SetEnv: # noqa: U100 + ... @pytest.fixture() def eval_set_env(tox_project: ToxProjectCreator) -> EvalSetEnv: - def func(tox_ini: str) -> SetEnv: - prj = tox_project({"tox.ini": tox_ini}) + def func(tox_ini: str, extra_files: Optional[Dict[str, Any]] = None) -> SetEnv: + prj = tox_project({"tox.ini": tox_ini, **(extra_files or {})}) result = prj.run("c", "-k", "set_env", "-e", "py") result.assert_success() set_env: SetEnv = result.env_conf("py")["set_env"] @@ -110,3 +112,31 @@ def test_set_env_replacer(eval_set_env: EvalSetEnv, monkeypatch: MonkeyPatch) -> def test_set_env_honor_override(eval_set_env: EvalSetEnv) -> None: set_env = eval_set_env("[testenv]\npackage=skip\nset_env=PIP_DISABLE_PIP_VERSION_CHECK=0") assert set_env.load("PIP_DISABLE_PIP_VERSION_CHECK") == "0" + + +def test_set_env_environment_file(eval_set_env: EvalSetEnv) -> None: + env_file = """ + A=1 + B= 2 + C = 1 + # D = comment # noqa: E800 + E = "1" + F = + """ + set_env = eval_set_env("[testenv]\npackage=skip\nset_env=file|a.txt", extra_files={"a.txt": env_file}) + content = {k: set_env.load(k) for k in set_env} + assert content == { + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "A": "1", + "B": "2", + "C": "1", + "E": '"1"', + "F": "", + } + + +def test_set_env_environment_file_missing(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.ini": "[testenv]\npackage=skip\nset_env=file|magic.txt"}) + result = project.run("r") + result.assert_failed() + assert "py: failed with magic.txt does not exist for set_env" in result.out From 8cf1ea7f9ec23fa4afad643833197bc939a8d627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 18 Sep 2021 08:06:46 +0100 Subject: [PATCH 2/2] PR Feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- docs/config.rst | 6 ++-- src/tox/config/loader/ini/__init__.py | 3 +- src/tox/config/set_env.py | 44 +++++++++++++++------------ src/tox/config/sets.py | 5 +-- src/tox/pytest.py | 2 ++ tests/config/test_set_env.py | 27 +++++++++++----- 6 files changed, 53 insertions(+), 34 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index c7e86f5dd..eb8cbca75 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -271,15 +271,15 @@ Base options :keys: set_env, setenv A dictionary of environment variables to set when running commands in the tox environment. Lines starting with a - ``file|`` prefix are considered environment files to load with their location being the content minus this prefix. + ``file|`` prefix define the location of environment file. .. note:: Environment files are processed using the following rules: - - blank lines ignored, + - blank lines are ignored, - lines starting with the ``#`` character are ignored, - - each line is in KEY=VALUE format; both the key and the value is stripped, + - each line is in KEY=VALUE format; both the key and the value are stripped, - there is no special handling of quotation marks, they are part of the key or value. .. conf:: diff --git a/src/tox/config/loader/ini/__init__.py b/src/tox/config/loader/ini/__init__.py index 34aa18a18..c991a8be9 100644 --- a/src/tox/config/loader/ini/__init__.py +++ b/src/tox/config/loader/ini/__init__.py @@ -81,8 +81,7 @@ def replacer(raw_: str, args_: ConfigLoadArgs) -> str: yield raw if delay_replace: converted = future.result() - if hasattr(converted, "replacer"): # pragma: no branch - converted.replacer = replacer # type: ignore[attr-defined] + converted.use_replacer(replacer, args) # type: ignore[attr-defined] # this can be only set_env that has it def found_keys(self) -> Set[str]: return set(self._section_proxy.keys()) diff --git a/src/tox/config/set_env.py b/src/tox/config/set_env.py index 030116de4..ef4d53628 100644 --- a/src/tox/config/set_env.py +++ b/src/tox/config/set_env.py @@ -8,18 +8,20 @@ class SetEnv: - def __init__(self, raw: str, name: str, env_name: Optional[str]) -> None: - self.replacer: Replacer = lambda s, c: s - self._later: List[str] = [] - self._raw: Dict[str, str] = {} - self._name, self._env_name = name, env_name + def __init__(self, raw: str, name: str, env_name: Optional[str], root: Path) -> None: + self.changed = False + self._materialized: Dict[str, str] = {} # env vars we already loaded + self._raw: Dict[str, str] = {} # could still need replacement + self._needs_replacement: List[str] = [] # env vars that need replacement + self._env_files: List[str] = [] + self._replacer: Replacer = lambda s, c: s + self._name, self._env_name, self._root = name, env_name, root from .loader.ini.replace import find_replace_part for line in raw.splitlines(): if line.strip(): if line.startswith("file|"): - for key, value in self._read_env_file(line): - self._raw[key] = value + self._env_files.append(line[len("file|") :]) else: try: key, value = self._extract_key_value(line) @@ -28,19 +30,23 @@ def __init__(self, raw: str, name: str, env_name: Optional[str]) -> None: except ValueError: _, __, match = find_replace_part(line, 0) if match: - self._later.append(line) + self._needs_replacement.append(line) else: raise else: self._raw[key] = value - self._materialized: Dict[str, str] = {} - self.changed = False - def _read_env_file(self, line: str) -> Iterator[Tuple[str, str]]: + def use_replacer(self, value: Replacer, args: ConfigLoadArgs) -> None: + self._replacer = value + for filename in self._env_files: + self._read_env_file(filename, args) + + def _read_env_file(self, filename: str, args: ConfigLoadArgs) -> None: # Our rules in the documentation, some upstream environment file rules (we follow mostly the docker one): # - https://www.npmjs.com/package/dotenv#rules # - https://docs.docker.com/compose/env-file/ - env_file = Path(line[len("file|") :]) + env_file = Path(self._replacer(filename, args.copy())) # apply any replace options + env_file = env_file if env_file.is_absolute() else self._root / env_file if not env_file.exists(): raise Fail(f"{env_file} does not exist for set_env") for env_line in env_file.read_text().splitlines(): @@ -48,7 +54,7 @@ def _read_env_file(self, line: str) -> Iterator[Tuple[str, str]]: if not env_line or env_line.startswith("#"): continue key, value = self._extract_key_value(env_line) - yield key, value + self._raw[key] = value @staticmethod def _extract_key_value(line: str) -> Tuple[str, str]: @@ -59,12 +65,12 @@ def _extract_key_value(line: str) -> Tuple[str, str]: raise ValueError(f"invalid line {line!r} in set_env") def load(self, item: str, args: Optional[ConfigLoadArgs] = None) -> str: - args = ConfigLoadArgs([], self._name, self._env_name) if args is None else args - args.chain.append(f"env:{item}") if item in self._materialized: return self._materialized[item] raw = self._raw[item] - result = self.replacer(raw, args) # apply any replace options + args = ConfigLoadArgs([], self._name, self._env_name) if args is None else args + args.chain.append(f"env:{item}") + result = self._replacer(raw, args) # apply any replace options result = result.replace(r"\#", "#") # unroll escaped comment with replacement self._materialized[item] = result self._raw.pop(item, None) # if the replace requires the env we may be called again, so allow pop to fail @@ -77,9 +83,9 @@ def __iter__(self) -> Iterator[str]: # start with the materialized ones, maybe we don't need to materialize the raw ones yield from self._materialized.keys() yield from list(self._raw.keys()) # iterating over this may trigger materialization and change the dict - while self._later: - line = self._later.pop(0) - expanded_line = self.replacer(line, ConfigLoadArgs([], self._name, self._env_name)) + while self._needs_replacement: + line = self._needs_replacement.pop(0) + expanded_line = self._replacer(line, ConfigLoadArgs([], self._name, self._env_name)) sub_raw = dict(self._extract_key_value(sub_line) for sub_line in expanded_line.splitlines() if sub_line) self._raw.update(sub_raw) yield from sub_raw.keys() diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index d4aeedc5f..bed770200 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -231,13 +231,14 @@ def set_env_post_process(values: SetEnv) -> SetEnv: def set_env_factory(raw: object) -> SetEnv: if not isinstance(raw, str): raise TypeError(raw) - return SetEnv(raw, self.name, self.env_name) + return SetEnv(raw, self.name, self.env_name, root) + root = self._conf.core["tox_root"] self.add_config( keys=["set_env", "setenv"], of_type=SetEnv, factory=set_env_factory, - default=SetEnv("", self.name, self.env_name), + default=SetEnv("", self.name, self.env_name, root), desc="environment variables to set when running commands in the tox environment", post_process=set_env_post_process, ) diff --git a/src/tox/pytest.py b/src/tox/pytest.py index e16677a12..a075d02dd 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -166,6 +166,8 @@ def _setup_files(dest: Path, base: Optional[Path], content: Dict[str, Any]) -> N ToxProject._setup_files(at_path, None, value) elif isinstance(value, str): at_path.write_text(textwrap.dedent(value)) + elif value is None: + at_path.mkdir() else: msg = f"could not handle {at_path / key} with content {value!r}" # pragma: no cover raise TypeError(msg) # pragma: no cover diff --git a/tests/config/test_set_env.py b/tests/config/test_set_env.py index f1e2ce31f..001877d4d 100644 --- a/tests/config/test_set_env.py +++ b/tests/config/test_set_env.py @@ -1,4 +1,6 @@ -from typing import Any, Dict, Optional, Protocol +import sys +from pathlib import Path +from typing import Any, Dict, Optional import pytest from pytest_mock import MockerFixture @@ -6,9 +8,14 @@ from tox.config.set_env import SetEnv from tox.pytest import MonkeyPatch, ToxProjectCreator +if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from typing import Protocol +else: # pragma: no cover ( None: - set_env = SetEnv("\nA=1\nB = 2\nC= 3\nD= 4", "py", "py") + set_env = SetEnv("\nA=1\nB = 2\nC= 3\nD= 4", "py", "py", Path()) set_env.update({"E": "5 ", "F": "6"}, override=False) keys = list(set_env) @@ -23,19 +30,21 @@ def test_set_env_explicit() -> None: def test_set_env_bad_line() -> None: with pytest.raises(ValueError, match="A"): - SetEnv("A", "py", "py") + SetEnv("A", "py", "py", Path()) class EvalSetEnv(Protocol): - def __call__(self, tox_ini: str, extra_files: Optional[Dict[str, Any]] = None) -> SetEnv: # noqa: U100 + def __call__( + self, tox_ini: str, extra_files: Optional[Dict[str, Any]] = ..., from_cwd: Optional[Path] = ... # noqa: U100 + ) -> SetEnv: ... @pytest.fixture() def eval_set_env(tox_project: ToxProjectCreator) -> EvalSetEnv: - def func(tox_ini: str, extra_files: Optional[Dict[str, Any]] = None) -> SetEnv: + def func(tox_ini: str, extra_files: Optional[Dict[str, Any]] = None, from_cwd: Optional[Path] = None) -> SetEnv: prj = tox_project({"tox.ini": tox_ini, **(extra_files or {})}) - result = prj.run("c", "-k", "set_env", "-e", "py") + result = prj.run("c", "-k", "set_env", "-e", "py", from_cwd=None if from_cwd is None else prj.path / from_cwd) result.assert_success() set_env: SetEnv = result.env_conf("py")["set_env"] return set_env @@ -123,7 +132,9 @@ def test_set_env_environment_file(eval_set_env: EvalSetEnv) -> None: E = "1" F = """ - set_env = eval_set_env("[testenv]\npackage=skip\nset_env=file|a.txt", extra_files={"a.txt": env_file}) + extra = {"A": {"a.txt": env_file}, "B": None, "C": None} + ini = "[testenv]\npackage=skip\nset_env=file|A{/}a.txt\nchange_dir=C" + set_env = eval_set_env(ini, extra_files=extra, from_cwd=Path("B")) content = {k: set_env.load(k) for k in set_env} assert content == { "PIP_DISABLE_PIP_VERSION_CHECK": "1", @@ -139,4 +150,4 @@ def test_set_env_environment_file_missing(tox_project: ToxProjectCreator) -> Non project = tox_project({"tox.ini": "[testenv]\npackage=skip\nset_env=file|magic.txt"}) result = project.run("r") result.assert_failed() - assert "py: failed with magic.txt does not exist for set_env" in result.out + assert f"py: failed with {project.path / 'magic.txt'} does not exist for set_env" in result.out