diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 1d9a45f..a77ebdb 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -20,11 +20,12 @@ requirements: run: - python >=3.6 - - pytask >=0.0.7 + - pytask >=0.0.9 test: requires: - pytest + - pytask-parallel >=0.0.4 source_files: - tox.ini - tests @@ -33,6 +34,7 @@ test: - pytask --help - pytask clean - pytask markers + - pytask collect - pytest tests diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index f749a60..189e945 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -23,6 +23,7 @@ jobs: steps: - uses: actions/checkout@v2 - uses: r-lib/actions/setup-tinytex@v1 + if: runner.os != 'Windows' - uses: goanpeca/setup-miniconda@v1 with: auto-update-conda: true diff --git a/CHANGES.rst b/CHANGES.rst index 887fea4..0812310 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -7,6 +7,12 @@ all releases are available on `Anaconda.org `_. +0.0.8 - 2020-10-29 +------------------ + +- :gh:`11` makes pytask-latex work with pytask v0.0.9. + + 0.0.7 - 2020-10-14 ------------------ diff --git a/README.rst b/README.rst index 4705c6d..6ab08bb 100644 --- a/README.rst +++ b/README.rst @@ -60,9 +60,16 @@ Here is an example where you want to compile ``document.tex`` to a PDF. def task_compile_latex_document(): pass +When the task is executed, you find a ``document.pdf`` in the same folder as your +``document.text``, but you could also compile the report into a another folder by +changing the path in ``produces``. -Note that, LaTeX document which will be compiled must be the first dependency. Add other -dependencies like images after the source file. + +Multiple dependencies and products +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +What happens if a task has more dependencies? Using a list, the LaTeX document which +should be compiled must be found in the first position of the list. .. code-block:: python @@ -72,6 +79,38 @@ dependencies like images after the source file. def task_compile_latex_document(): pass +If you use a dictionary to pass dependencies to the task, pytask-latex will, first, look +for a ``"source"`` key in the dictionary and, secondly, under the key ``0``. + +.. code-block:: python + + @pytask.mark.depends_on({"source": "document.tex", "image": "image.png"}) + def task_compile_document(): + pass + + + # or + + + @pytask.mark.depends_on({0: "document.tex", "image": "image.png"}) + def task_compile_document(): + pass + + + # or two decorators for the function, if you do not assign a name to the image. + + + @pytask.mark.depends_on({"source": "document.tex"}) + @pytask.mark.depends_on("image.png") + def task_compile_document(): + pass + +The same applies to the compiled document which is either in the first position, under +the key ``"document"`` or ``0``. + + +Command Line Arguments +~~~~~~~~~~~~~~~~~~~~~~ To customize the compilation, you can pass some command line arguments to ``latexmk`` via the ``@pytask.mark.latex`` marker. The default is the following. @@ -134,14 +173,34 @@ to include the latex decorator in the parametrization just like with @pytask.mark.parametrize( "produces, latex", [ - ("document.pdf", (["--pdf", "interaction=nonstopmode"])), - ("document.dvi", (["--dvi", "interaction=nonstopmode"])), + ( + "document.pdf", + (["--pdf", "--interaction=nonstopmode", "--synctex=1", "--cd"],), + ), + ( + "document.dvi", + (["--dvi", "--interaction=nonstopmode", "--synctex=1", "--cd"],), + ), ], ) def task_compile_latex_document(): pass +Configuration +------------- + +If you want to change the names of the keys which identify the source file and the +compiled document, change the following default configuration in your pytask +configuration file. + +.. code-block:: ini + + latex_source_key = source + latex_document_key = document + + + Changes ------- diff --git a/environment.yml b/environment.yml index 4fc1a3a..0643acd 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,8 @@ dependencies: - conda-verify # Package dependencies - - pytask >= 0.0.7 + - pytask >= 0.0.9 + - pytask-parallel >= 0.0.4 # Misc - bumpversion diff --git a/setup.cfg b/setup.cfg index 048f6b3..e7f0b46 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.7 +current_version = 0.0.8 parse = (?P\d+)\.(?P\d+)(\.(?P\d+))(\-?((dev)?(?P\d+))?) serialize = {major}.{minor}.{patch}dev{dev} diff --git a/setup.py b/setup.py index 554a3f0..1789bf3 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="pytask-latex", - version="0.0.7", + version="0.0.8", packages=find_packages(where="src"), package_dir={"": "src"}, entry_points={"pytask": ["pytask_latex = pytask_latex.plugin"]}, diff --git a/src/pytask_latex/__init__.py b/src/pytask_latex/__init__.py index 6526deb..a73339b 100644 --- a/src/pytask_latex/__init__.py +++ b/src/pytask_latex/__init__.py @@ -1 +1 @@ -__version__ = "0.0.7" +__version__ = "0.0.8" diff --git a/src/pytask_latex/collect.py b/src/pytask_latex/collect.py index 776cea1..2e2836d 100644 --- a/src/pytask_latex/collect.py +++ b/src/pytask_latex/collect.py @@ -14,7 +14,6 @@ from _pytask.nodes import FilePathNode from _pytask.nodes import PythonFunctionTask from _pytask.parametrize import _copy_func -from _pytask.shared import to_list DEFAULT_OPTIONS = ["--pdf", "--interaction=nonstopmode", "--synctex=1", "--cd"] @@ -37,45 +36,9 @@ def latex(options: Optional[Union[str, Iterable[str]]] = None): return options -def compile_latex_document(depends_on, produces, latex): - """Compile a LaTeX document. - - This function replaces the dummy function of an LaTeX task. It is a nice wrapper - around subprocess. - - The output folder needs to be declared as a relative path to the directory where the - latex source lies. - - 1. It must be relative because bibtex / biber, which is necessary for - bibliographies, does not accept full paths as a safety measure. - 2. Due to the ``--cd`` flag, latexmk will change the directory to the one where the - source files are. Thus, relative to the latex sources. - - See this `discussion on Github - `_ - for additional information. - - """ - latex_document = to_list(depends_on)[0] - compiled_document = to_list(produces)[0] - - if latex_document.stem != compiled_document.stem: - latex.append(f"--jobname={compiled_document.stem}") - - # See comment in doc string. - out_relative_to_latex_source = Path( - os.path.relpath(compiled_document.parent, latex_document.parent) - ).as_posix() - - subprocess.run( - [ - "latexmk", - *latex, - f"--output-directory={out_relative_to_latex_source}", - f"{latex_document.as_posix()}", - ], - check=True, - ) +def compile_latex_document(latex): + """Replaces the dummy function provided by the user.""" + subprocess.run(latex, check=True) @hookimpl @@ -88,52 +51,56 @@ def pytask_collect_task(session, path, name, obj): """ if name.startswith("task_") and callable(obj) and has_marker(obj, "latex"): - # Collect the task. task = PythonFunctionTask.from_path_name_function_session( path, name, obj, session ) - latex_function = _copy_func(compile_latex_document) - latex_function.pytaskmark = copy.deepcopy(task.function.pytaskmark) - - merged_mark = _merge_all_markers(task) - args = latex(*merged_mark.args, **merged_mark.kwargs) - latex_function = functools.partial(latex_function, latex=args) - - task.function = latex_function return task @hookimpl -def pytask_collect_task_teardown(task): - """Perform some checks. - - Remove check for task is none with pytask 0.0.9. - - """ - if task is not None and get_specific_markers_from_task(task, "latex"): - if (len(task.depends_on) == 0) or ( - not ( - isinstance(task.depends_on[0], FilePathNode) - and task.depends_on[0].value.suffix == ".tex" - ) - ): +def pytask_collect_task_teardown(session, task): + """Perform some checks.""" + if get_specific_markers_from_task(task, "latex"): + source = _get_node_from_dictionary( + task.depends_on, session.config["latex_source_key"] + ) + if not (isinstance(source, FilePathNode) and source.value.suffix == ".tex"): raise ValueError( "The first or sole dependency of a LaTeX task must be the document " "which will be compiled and has a .tex extension." ) - if (len(task.produces) == 0) or ( - not ( - isinstance(task.produces[0], FilePathNode) - and task.produces[0].value.suffix in [".pdf", ".ps", ".dvi"] - ) + document = _get_node_from_dictionary( + task.produces, session.config["latex_document_key"] + ) + if not ( + isinstance(document, FilePathNode) + and document.value.suffix in [".pdf", ".ps", ".dvi"] ): raise ValueError( "The first or sole product of a LaTeX task must point to a .pdf, .ps " "or .dvi file which is the compiled document." ) + latex_function = _copy_func(compile_latex_document) + latex_function.pytaskmark = copy.deepcopy(task.function.pytaskmark) + + merged_mark = _merge_all_markers(task) + args = latex(*merged_mark.args, **merged_mark.kwargs) + options = _prepare_cmd_options(session, task, args) + latex_function = functools.partial(latex_function, latex=options) + + task.function = latex_function + + +def _get_node_from_dictionary(obj, key, fallback=0): + if isinstance(obj, Path): + pass + elif isinstance(obj, dict): + obj = obj.get(key) or obj.get(fallback) + return obj + def _merge_all_markers(task): """Combine all information from markers for the compile latex function.""" @@ -142,3 +109,51 @@ def _merge_all_markers(task): for mark_ in latex_marks[1:]: mark = mark.combined_with(mark_) return mark + + +def _prepare_cmd_options(session, task, args): + """Prepare the command line arguments to compile the LaTeX document. + + The output folder needs to be declared as a relative path to the directory where the + latex source lies. + + 1. It must be relative because bibtex / biber, which is necessary for + bibliographies, does not accept full paths as a safety measure. + 2. Due to the ``--cd`` flag, latexmk will change the directory to the one where the + source files are. Thus, relative to the latex sources. + + See this `discussion on Github + `_ + for additional information. + + """ + latex_document = _get_node_from_dictionary( + task.depends_on, session.config["latex_source_key"] + ).value + compiled_document = _get_node_from_dictionary( + task.produces, session.config["latex_document_key"] + ).value + + # Jobname controls the name of the compiled document. No suffix! + if latex_document.stem != compiled_document.stem: + jobname = [f"--jobname={compiled_document.stem}"] + else: + jobname = [] + + # The path to the output directory must be relative from the location of the source + # file. See docstring for more information. + out_relative_to_latex_source = Path( + os.path.relpath(compiled_document.parent, latex_document.parent) + ).as_posix() + + return ( + [ + "latexmk", + *args, + ] + + jobname + + [ + f"--output-directory={out_relative_to_latex_source}", + latex_document.as_posix(), + ] + ) diff --git a/src/pytask_latex/config.py b/src/pytask_latex/config.py index a9b0381..3609d79 100644 --- a/src/pytask_latex/config.py +++ b/src/pytask_latex/config.py @@ -3,6 +3,10 @@ @hookimpl -def pytask_parse_config(config): +def pytask_parse_config(config, config_from_file): """Register the latex marker in the configuration.""" config["markers"]["latex"] = "Tasks which compile LaTeX documents." + config["latex_source_key"] = config_from_file.get("latex_source_key", "source") + config["latex_document_key"] = config_from_file.get( + "latex_document_key", "document" + ) diff --git a/tests/test_collect.py b/tests/test_collect.py index 06068de..85bdef9 100644 --- a/tests/test_collect.py +++ b/tests/test_collect.py @@ -4,7 +4,9 @@ import pytest from _pytask.mark import Mark from _pytask.nodes import FilePathNode +from pytask_latex.collect import _get_node_from_dictionary from pytask_latex.collect import _merge_all_markers +from pytask_latex.collect import _prepare_cmd_options from pytask_latex.collect import DEFAULT_OPTIONS from pytask_latex.collect import latex from pytask_latex.collect import pytask_collect_task @@ -87,12 +89,87 @@ def test_pytask_collect_task(name, expected): (["document.tex"], ["document.out", "document.pdf"], pytest.raises(ValueError)), ], ) -def test_pytask_collect_task_teardown(depends_on, produces, expectation): +@pytest.mark.parametrize("latex_source_key", ["source", "script", "main"]) +@pytest.mark.parametrize("latex_document_key", ["document", "compiled_doc"]) +def test_pytask_collect_task_teardown( + depends_on, produces, expectation, latex_source_key, latex_document_key +): + session = DummyClass() + session.config = { + "latex_source_key": latex_source_key, + "latex_document_key": latex_document_key, + } + task = DummyClass() - task.depends_on = [FilePathNode(n.split(".")[0], Path(n)) for n in depends_on] - task.produces = [FilePathNode(n.split(".")[0], Path(n)) for n in produces] + task.depends_on = { + i: FilePathNode(n.split(".")[0], Path(n)) for i, n in enumerate(depends_on) + } + task.produces = { + i: FilePathNode(n.split(".")[0], Path(n)) for i, n in enumerate(produces) + } task.markers = [Mark("latex", (), {})] task.function = task_dummy + task.function.pytaskmark = task.markers with expectation: - pytask_collect_task_teardown(task) + pytask_collect_task_teardown(session, task) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "obj, key, expected", + [ + (1, "asds", 1), + (1, None, 1), + ({"a": 1}, "a", 1), + ({0: 1}, "a", 1), + ], +) +def test_get_node_from_dictionary(obj, key, expected): + result = _get_node_from_dictionary(obj, key) + assert result == expected + + +@pytest.mark.unit +@pytest.mark.parametrize( + "args", + [ + [], + ["a"], + ["a", "b"], + ], +) +@pytest.mark.parametrize("latex_source_key", ["source", "script"]) +@pytest.mark.parametrize("latex_document_key", ["document", "compiled_doc"]) +@pytest.mark.parametrize("input_name", ["source", "main"]) +@pytest.mark.parametrize("output_name", ["source", "doc"]) +def test_prepare_cmd_options( + args, latex_source_key, latex_document_key, input_name, output_name +): + session = DummyClass() + session.config = { + "latex_source_key": latex_source_key, + "latex_document_key": latex_document_key, + } + + dependency = DummyClass() + dependency.value = Path(f"{input_name}.tex") + product = DummyClass() + product.value = Path(f"{output_name}.pdf") + task = DummyClass() + task.depends_on = {latex_source_key: dependency} + task.produces = {latex_document_key: product} + task.name = "task" + + result = _prepare_cmd_options(session, task, args) + + jobname = ( + [] + if dependency.value.stem == product.value.stem + else [f"--jobname={product.value.stem}"] + ) + + assert result == ["latexmk", *args] + jobname + [ + "--output-directory=.", + f"{input_name}.tex", + ] diff --git a/tests/test_execute.py b/tests/test_execute.py index 9ef3a60..eddc531 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1,4 +1,3 @@ -import itertools import textwrap from contextlib import ExitStack as does_not_raise # noqa: N813 from subprocess import CalledProcessError @@ -16,36 +15,6 @@ class DummyTask: pass -@pytest.mark.end_to_end -@pytest.mark.parametrize( - "dependencies, products", - itertools.product( - ([], ["in.txt"], ["in_1.txt", "in_2.txt"]), - (["out.txt"], ["out_1.txt", "out_2.txt"]), - ), -) -def test_normal_flow_w_varying_dependencies_products(tmp_path, dependencies, products): - source = f""" - import pytask - from pathlib import Path - - - @pytask.mark.depends_on({dependencies}) - @pytask.mark.produces({products}) - def task_dummy(depends_on, produces): - if not isinstance(produces, list): - produces = [produces] - for product in produces: - product.touch() - """ - tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) - for dependency in dependencies: - tmp_path.joinpath(dependency).touch() - - session = main({"paths": tmp_path}) - assert session.exit_code == 0 - - @pytest.mark.unit @pytest.mark.parametrize( "found_latexmk, expectation", @@ -69,14 +38,32 @@ def test_pytask_execute_task_setup(monkeypatch, found_latexmk, expectation): @needs_latexmk @skip_on_github_actions_with_win @pytest.mark.end_to_end -def test_compile_latex_document(runner, tmp_path): +@pytest.mark.parametrize( + "depends_on", + [ + "'document.tex'", + {"source": "document.tex"}, + {0: "document.tex"}, + {"script": "document.tex"}, + ], +) +@pytest.mark.parametrize( + "produces", + [ + "'document.pdf'", + {"document": "document.pdf"}, + {0: "document.pdf"}, + {"compiled_doc": "document.pdf"}, + ], +) +def test_compile_latex_document(runner, tmp_path, depends_on, produces): """Test simple compilation.""" - task_source = """ + task_source = f""" import pytask @pytask.mark.latex - @pytask.mark.depends_on("document.tex") - @pytask.mark.produces("document.pdf") + @pytask.mark.depends_on({depends_on}) + @pytask.mark.produces({produces}) def task_compile_document(): pass @@ -91,6 +78,17 @@ def task_compile_document(): """ tmp_path.joinpath("document.tex").write_text(textwrap.dedent(latex_source)) + config = "[pytask]\n" + if ( + isinstance(depends_on, dict) + and "source" not in depends_on + and 0 not in depends_on + ): + config += "latex_source_key = script\n" + if isinstance(produces, dict) and "document" not in produces and 0 not in produces: + config += "latex_document_key = compiled_doc\n" + tmp_path.joinpath("pytask.ini").write_text(config) + result = runner.invoke(cli, [tmp_path.as_posix()]) assert result.exit_code == 0 diff --git a/tests/test_normal_execution_w_plugin.py b/tests/test_normal_execution_w_plugin.py new file mode 100644 index 0000000..7e06201 --- /dev/null +++ b/tests/test_normal_execution_w_plugin.py @@ -0,0 +1,36 @@ +"""Contains tests which do not require the plugin and ensure normal execution.""" +import textwrap + +import pytest +from pytask import cli + + +@pytest.mark.end_to_end +@pytest.mark.parametrize( + "dependencies", + [[], ["in.txt"], ["in_1.txt", "in_2.txt"]], +) +@pytest.mark.parametrize("products", [["out.txt"], ["out_1.txt", "out_2.txt"]]) +def test_execution_w_varying_dependencies_products( + runner, tmp_path, dependencies, products +): + source = f""" + import pytask + from pathlib import Path + + @pytask.mark.depends_on({dependencies}) + @pytask.mark.produces({products}) + def task_dummy(depends_on, produces): + if isinstance(produces, dict): + produces = produces.values() + elif isinstance(produces, Path): + produces = [produces] + for product in produces: + product.touch() + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + for dependency in dependencies: + tmp_path.joinpath(dependency).touch() + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == 0 diff --git a/tests/test_parallel.py b/tests/test_parallel.py new file mode 100644 index 0000000..6e79114 --- /dev/null +++ b/tests/test_parallel.py @@ -0,0 +1,121 @@ +"""Contains test which ensure that the plugin works with pytask-parallel.""" +import textwrap +import time + +import pytest +from conftest import needs_latexmk +from conftest import skip_on_github_actions_with_win +from pytask import cli + +try: + import pytask_parallel # noqa: F401 +except ImportError: + _IS_PYTASK_PARALLEL_INSTALLED = False +else: + _IS_PYTASK_PARALLEL_INSTALLED = True + + +pytestmark = pytest.mark.skipif( + not _IS_PYTASK_PARALLEL_INSTALLED, reason="Tests require pytask-parallel." +) + + +@pytest.mark.xfail(reason="I don't know.") +@needs_latexmk +@skip_on_github_actions_with_win +@pytest.mark.end_to_end +def test_parallel_parametrization_over_source_files(runner, tmp_path): + source = """ + import pytask + + @pytask.mark.latex + @pytask.mark.parametrize( + "depends_on, produces", + [("document_1.tex", "document_1.pdf"), ("document_2.tex", "document_2.pdf")], + ) + def task_compile_latex_document(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + latex_source = r""" + \documentclass{report} + \begin{document} + He said yeah. + \end{document} + """ + tmp_path.joinpath("document_1.tex").write_text(textwrap.dedent(latex_source)) + + latex_source = r""" + \documentclass{report} + \begin{document} + You better come out with your hands up. + \end{document} + """ + tmp_path.joinpath("document_2.tex").write_text(textwrap.dedent(latex_source)) + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == 0 + duration_normal = time.time() - start + + for name in ["document_1.pdf", "document_2.pdf"]: + tmp_path.joinpath(name).unlink() + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) + assert result.exit_code == 0 + duration_parallel = time.time() - start + + assert duration_parallel < duration_normal + + +@pytest.mark.xfail(reason="I don't know.") +@needs_latexmk +@skip_on_github_actions_with_win +@pytest.mark.end_to_end +def test_parallel_parametrization_over_source_file(runner, tmp_path): + source = """ + import pytask + + @pytask.mark.depends_on("document.tex") + @pytask.mark.parametrize( + "produces, latex", + [ + ( + "document.pdf", + (["--pdf", "--interaction=nonstopmode", "--synctex=1", "--cd"],) + ), + ( + "document.dvi", + (["--dvi", "--interaction=nonstopmode", "--synctex=1", "--cd"],) + ), + ], + ) + def task_compile_latex_document(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + latex_source = r""" + \documentclass{report} + \begin{document} + Ma il mio mistero e chiuso in me + \end{document} + """ + tmp_path.joinpath("document.tex").write_text(textwrap.dedent(latex_source)) + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == 0 + duration_normal = time.time() - start + + for name in ["document.pdf", "document.dvi"]: + tmp_path.joinpath(name).unlink() + + start = time.time() + result = runner.invoke(cli, [tmp_path.as_posix(), "-n", 2]) + assert result.exit_code == 0 + duration_parallel = time.time() - start + + assert duration_parallel < duration_normal diff --git a/tox.ini b/tox.ini index 653a3ee..87c3a57 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,8 @@ basepython = python [testenv:pytest] conda_deps = - pytask >=0.0.7 + pytask >=0.0.9 + pytask-parallel >=0.0.4 pytest pytest-cov pytest-xdist