Add a data catalog.

.gitignore

@@ -15,6 +15,7 @@ _generated
 .eggs
 
 .pytask.sqlite3
+.pytask
 
 build
 dist

.pre-commit-config.yaml

@@ -115,6 +115,7 @@ repos:
                 docs/source/tutorials/repeating_tasks_with_different_inputs.md|
                 docs/source/tutorials/selecting_tasks.md|
                 docs/source/tutorials/set_up_a_project.md|
+                docs/source/tutorials/using_a_data_catalog.md|
                 docs/source/tutorials/write_a_task.md
             )$
 -   repo: https://github.com/nbQA-dev/nbQA

docs/source/_static/md/defining-dependencies-products.md

@@ -0,0 +1,26 @@
+<div class="termy">
+
+```console
+
+$ pytask
+──────────────────────────── Start pytask session ────────────────────────────
+Platform: win32 -- Python <span style="color: var(--termynal-blue)">3.10.0</span>, pytask <span style="color: var(--termynal-blue)">0.4.0</span>, pluggy <span style="color: var(--termynal-blue)">1.0.0</span>
+Root: C:\Users\pytask-dev\git\my_project
+Collected <span style="color: var(--termynal-blue)">2</span> task.
+
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓
+┃ Task                                              ┃ Outcome ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━┩
+│ <span class="termynal-dim">task_data_preparation.py::</span>task_create_random_data │ <span class="termynal-success">.</span>       │
+│ <span class="termynal-dim">task_plot_data.py::</span>task_plot_data                 │ <span class="termynal-success">.</span>       │
+└───────────────────────────────────────────────────┴─────────┘
+
+<span class="termynal-dim">──────────────────────────────────────────────────────────────────────────────</span>
+<span class="termynal-success">╭───────────</span> <span style="font-weight: bold;">Summary</span> <span class="termynal-success">────────────╮</span>
+<span class="termynal-success">│</span> <span style="font-weight: bold;"> 2  Collected tasks </span>           <span class="termynal-success">│</span>
+<span class="termynal-success">│</span> <span class="termynal-success-textonly"> 2  Succeeded        (100.0%) </span> <span class="termynal-success">│</span>
+<span class="termynal-success">╰────────────────────────────────╯</span>
+<span class="termynal-success">───────────────────────── Succeeded in 0.06 seconds ──────────────────────────</span>
+```
+
+</div>

docs/source/conf.py

@@ -82,6 +82,7 @@
 
 intersphinx_mapping = {
     "click": ("https://click.palletsprojects.com/en/8.0.x/", None),
+    "deepdiff": ("https://zepworks.com/deepdiff/current/", None),
     "networkx": ("https://networkx.org/documentation/stable", None),
     "pandas": ("https://pandas.pydata.org/docs", None),
     "pluggy": ("https://pluggy.readthedocs.io/en/latest", None),

docs/source/how_to_guides/hashing_inputs_of_tasks.md

@@ -62,10 +62,10 @@ from interpreter session to interpreter session for security reasons (see
 ```
 
 {class}`list` and {class}`dict` are not hashable by default. Luckily, there are
-libraries who provide this functionality like `deepdiff`. We can use them to pass a
+libraries who provide this functionality like {mod}`deepdiff`. We can use them to pass a
 function to the {class}`~pytask.PythonNode` that generates a stable hash.
 
-First, install `deepdiff`.
+First, install {mod}`deepdiff`.
 
 ```console
 $ pip install deepdiff

docs/source/how_to_guides/index.md

@@ -19,6 +19,7 @@ hashing_inputs_of_tasks
 using_task_returns
 writing_custom_nodes
 how_to_write_a_plugin
+the_data_catalog
 ```
 
 ## Best Practice Guides

docs/source/how_to_guides/the_data_catalog.md

@@ -0,0 +1,73 @@
+# The `DataCatalog` - Revisited
+
+An introduction to the data catalog can be found in the
+[tutorial](../tutorials/using_a_data_catalog.md).
+
+This guide explains some details that were left out of the tutorial.
+
+## Changing the default node
+
+The data catalog uses the {class}`~pytask.PickleNode` by default to serialize any kind
+of Python object. You can use any other node that follows the {protocol}`~pytask.PNode`
+protocol and register it when creating the data catalog.
+
+For example, use the {class}`~pytask.PythonNode` as the default.
+
+```python
+from pytask import PythonNode
+
+
+data_catalog = DataCatalog(default_node=PythonNode)
+```
+
+Or, learn to write your own node by reading {doc}`writing_custom_nodes`.
+
+Here, is an example for a `PickleNode` that uses cloudpickle instead of the normal
+`pickle` module.
+
+```{literalinclude} ../../../docs_src/how_to_guides/the_data_catalog.py
+```
+
+## Changing the name and the default path
+
+By default, the data catalogs store their data in a directory `.pytask/data_catalogs`.
+If you use a `pyproject.toml` with a `[tool.pytask.ini_options]` section, then the
+`.pytask` folder is in the same folder as the configuration file.
+
+The default name for a catalog is `"default"` and so you will find its data in
+`.pytask/data_catalogs/default`. If you assign a different name like
+`"data_management"`, you will find the data in `.pytask/data_catalogs/data_management`.
+
+```python
+data_catalog = DataCatalog(name="data_management")
+```
+
+You can also change the path where the data catalogs will be stored by changing the
+`path` attribute. Here, we store the data catalog's data next to the module where the
+data catalog is defined in `.data`.
+
+```python
+from pathlib import Path
+
+
+data_catalog = DataCatalog(path=Path(__file__).parent / ".data")
+```
+
+## Multiple data catalogs
+
+You can use multiple data catalogs when you want to separate your datasets across
+multiple catalogs or when you want to use the same names multiple times (although it is
+not recommended!).
+
+Make sure you assign different names to the data catalogs so that their data is stored
+in different directories.
+
+```python
+# Stored in .pytask/data_catalog/a
+data_catalog_a = DataCatalog(name="a")
+
+# Stored in .pytask/data_catalog/b
+data_catalog_b = DataCatalog(name="b")
+```
+
+Or, use different paths as explained above.

docs/source/reference_guides/api.md

@@ -33,7 +33,7 @@ To write to the terminal, use pytask's console.
 pytask uses marks to attach additional information to task functions which is processed
 by the host or by plugins. The following marks are available by default.
 
-### Marks
+### Built-in marks
 
 ```{eval-rst}
 .. function:: pytask.mark.depends_on(objects: Any | Iterable[Any] | dict[Any, Any])
@@ -236,7 +236,8 @@ The remaining exceptions convey specific errors.
 
 ```{eval-rst}
 .. autoclass:: pytask.Session
-
+.. autoclass:: pytask.DataCatalog
+   :members:
 ```
 
 ## Protocols
@@ -262,7 +263,11 @@ Nodes are the interface for different kinds of dependencies or products.
 
 ```{eval-rst}
 .. autoclass:: pytask.PathNode
+   :members: load, save
+.. autoclass:: pytask.PickleNode
+   :members: load, save
 .. autoclass:: pytask.PythonNode
+   :members: load, save
 ```
 
 To parse dependencies and products from nodes, use the following functions.

docs/source/tutorials/defining_dependencies_products.md

@@ -3,22 +3,47 @@
 To ensure pytask executes all tasks in the correct order, you need to define
 dependencies and products for each task.
 
-This tutorial offers you different interfaces. One important difference between them is
-that if you are comfortable with type annotations or not afraid to try them, take a look
-at the tabs named `Python 3.10+` or `Python 3.8+`.
+This tutorial offers you different interfaces. If you are comfortable with type
+annotations or not afraid to try them, take a look at the tabs named `Python 3.10+` or
+`Python 3.8+`.
 
 If you want to avoid type annotations for now, look at the tab named `produces`.
 
+The deprecated approaches can be found in the tabs named `Decorators`.
+
 ```{seealso}
 An overview on the different interfaces and their strength and weaknesses is given in
 {doc}`../explanations/interfaces_for_dependencies_products`.
 ```
 
-Let's first focus on how to define products which should already be familiar to you.
+First, we focus on how to define products which should already be familiar to you. Then,
+we focus on how task dependencies can be declared.
+
+We use the same project layout as before and add a `task_plot_data.py` module.
+
+```text
+my_project
+├───pyproject.toml
+│
+├───src
+│   └───my_project
+│       ├────config.py
+│       ├────task_data_preparation.py
+│       └────task_plot_data.py
+│
+├───setup.py
+│
+├───.pytask.sqlite3
+│
+└───bld
+    ├────data.pkl
+    └────plot.png
+```
 
 ## Products
 
-Let's revisit the task from the {doc}`previous tutorial <write_a_task>`.
+Let's revisit the task from the {doc}`previous tutorial <write_a_task>` that we defined
+in `task_data_preparation.py`.
 
 ::::{tab-set}
 
@@ -90,7 +115,9 @@ beneficial for handling paths conveniently and across platforms.
 Most tasks have dependencies and it is important to specify. Then, pytask ensures that
 the dependencies are available before executing the task.
 
-In the example you see a task that creates a plot while relying on some data set.
+As an example, we want to extend our project with another task that plots the data that
+we generated with `task_create_random_data`. The task is called `task_plot_data` and we
+will define it in `task_plot_data.py`.
 
 ::::{tab-set}
 
@@ -104,7 +131,7 @@ pytask assumes that all function arguments that do not have the {class}`~pytask.
 annotation are dependencies of the task.
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_dependencies_py310.py
-:emphasize-lines: 9
+:emphasize-lines: 11
 ```
 
 :::
@@ -119,7 +146,7 @@ pytask assumes that all function arguments that do not have the {class}`~pytask.
 annotation are dependencies of the task.
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_dependencies_py38.py
-:emphasize-lines: 9
+:emphasize-lines: 11
 ```
 
 :::
@@ -134,7 +161,7 @@ pytask assumes that all function arguments that are not passed to the argument
 `produces` are dependencies of the task.
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_dependencies_produces.py
-:emphasize-lines: 7
+:emphasize-lines: 9
 ```
 
 :::
@@ -152,12 +179,17 @@ Equivalent to products, you can use the
 access the dependency path inside the function and load the data.
 
 ```{literalinclude} ../../../docs_src/tutorials/defining_dependencies_products_dependencies_decorators.py
-:emphasize-lines: 7, 9
+:emphasize-lines: 9, 11
 ```
 
 :::
 ::::
 
+Now, let us execute the two paths.
+
+```{include} ../_static/md/defining-dependencies-products.md
+```
+
 ## Relative paths
 
 Dependencies and products do not have to be absolute paths. If paths are relative, they

docs/source/tutorials/index.md

@@ -11,6 +11,7 @@ installation
 set_up_a_project
 write_a_task
 defining_dependencies_products
+using_a_data_catalog
 invoking_pytask
 configuration
 plugins