From f963a2a3b7f31a844b31a3fca0fe653f37b500a2 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 21 Jun 2022 22:12:35 -0600 Subject: [PATCH 1/3] Enable nbconvert config as a sphinx option. This enables users to give nbconvert configuration, such as preprocessor settings, as part of their sphinx config. --- src/nbsphinx.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/nbsphinx.py b/src/nbsphinx.py index 77e51999..0ac12f81 100644 --- a/src/nbsphinx.py +++ b/src/nbsphinx.py @@ -760,8 +760,11 @@ class Exporter(nbconvert.RSTExporter): """ + # TODO: define default preprocessors to include our one to write out notebook just after execution + def __init__(self, execute='auto', kernel_name='', execute_arguments=[], - allow_errors=False, timeout=None, codecell_lexer='none'): + allow_errors=False, timeout=None, codecell_lexer='none', + nbconvert_config=None): """Initialize the Exporter.""" # NB: The following stateful Jinja filters are a hack until @@ -795,13 +798,16 @@ def replace_attachments(text): self._timeout = timeout self._codecell_lexer = codecell_lexer loader = jinja2.DictLoader({'nbsphinx-rst.tpl': RST_TEMPLATE}) - super(Exporter, self).__init__( - template_file='nbsphinx-rst.tpl', extra_loaders=[loader], - config=traitlets.config.Config({ + if nbconvert_config is None: + nbconvert_config = { 'HighlightMagicsPreprocessor': {'enabled': True}, # Work around https://github.com/jupyter/nbconvert/issues/720: 'RegexRemovePreprocessor': {'enabled': False}, - }), + } + + super(Exporter, self).__init__( + template_file='nbsphinx-rst.tpl', extra_loaders=[loader], + config=traitlets.config.Config(nbconvert_config), filters={ 'convert_pandoc': convert_pandoc, 'markdown2rst': markdown2rst, @@ -846,6 +852,9 @@ def from_notebook_node(self, nb, resources=None, **kw): allow_errors = nbsphinx_metadata.get( 'allow_errors', self._allow_errors) timeout = nbsphinx_metadata.get('timeout', self._timeout) + # TODO: Here, just add the execute preprocessor to the exporter list of preprocessors + # rather than executing directly. We can still pass appropriate config values in + # and that way the tag remove preprocessor is run *before* execution rather than after. pp = nbconvert.preprocessors.ExecutePreprocessor( kernel_name=self._kernel_name, extra_arguments=self._execute_arguments, @@ -853,6 +862,7 @@ def from_notebook_node(self, nb, resources=None, **kw): nb, resources = pp.preprocess(nb, resources) if 'nbsphinx_save_notebook' in resources: + # TODO: maybe we write our *own* preprocessor to hook into this stage, right after the execute preprocessor, to save the notebook # Save *executed* notebook *before* the Exporter can change it: nbformat.write(nb, resources['nbsphinx_save_notebook']) @@ -1037,6 +1047,7 @@ def parse(self, inputstring, document): allow_errors=env.config.nbsphinx_allow_errors, timeout=env.config.nbsphinx_timeout, codecell_lexer=env.config.nbsphinx_codecell_lexer, + nbconvert_config=env.config.nbsphinx_nbconvert_config ) try: @@ -2316,6 +2327,7 @@ def setup(app): app.add_config_value('nbsphinx_execute', 'auto', rebuild='env') app.add_config_value('nbsphinx_kernel_name', '', rebuild='env') app.add_config_value('nbsphinx_execute_arguments', [], rebuild='env') + app.add_config_value('nbsphinx_nbconvert_config', None, rebuild='env') app.add_config_value('nbsphinx_allow_errors', False, rebuild='') app.add_config_value('nbsphinx_timeout', None, rebuild='') app.add_config_value('nbsphinx_codecell_lexer', 'none', rebuild='env') From 369a2c0abdb0ae9ac6d4f8068c06f5387f20029e Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 21 Jun 2022 22:17:04 -0600 Subject: [PATCH 2/3] Use nbconvert's preprocessor infrastructure to execute the notebook. This allows the cell removal plugins to remove cells before execution, for example. There is a problem using this nbsphinx with ipywidets - it complains about missing files in the html-collect-pages step. --- src/nbsphinx.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/nbsphinx.py b/src/nbsphinx.py index 0ac12f81..35a66dfd 100644 --- a/src/nbsphinx.py +++ b/src/nbsphinx.py @@ -749,6 +749,13 @@ """ +class Writer(nbconvert.preprocessors.Preprocessor): + def preprocess(self, nb, resources): + if 'nbsphinx_save_notebook' in resources: + # Save *executed* notebook *before* the Exporter can change it: + nbformat.write(nb, resources['nbsphinx_save_notebook']) + return nb, resources + class Exporter(nbconvert.RSTExporter): """Convert Jupyter notebooks to reStructuredText. @@ -760,8 +767,6 @@ class Exporter(nbconvert.RSTExporter): """ - # TODO: define default preprocessors to include our one to write out notebook just after execution - def __init__(self, execute='auto', kernel_name='', execute_arguments=[], allow_errors=False, timeout=None, codecell_lexer='none', nbconvert_config=None): @@ -804,7 +809,6 @@ def replace_attachments(text): # Work around https://github.com/jupyter/nbconvert/issues/720: 'RegexRemovePreprocessor': {'enabled': False}, } - super(Exporter, self).__init__( template_file='nbsphinx-rst.tpl', extra_loaders=[loader], config=traitlets.config.Config(nbconvert_config), @@ -848,23 +852,29 @@ def from_notebook_node(self, nb, resources=None, **kw): not any(c.get('outputs') or c.get('execution_count') for c in nb.cells if c.cell_type == 'code') ) + preprocessors = [] if auto_execute or execute == 'always': allow_errors = nbsphinx_metadata.get( 'allow_errors', self._allow_errors) timeout = nbsphinx_metadata.get('timeout', self._timeout) - # TODO: Here, just add the execute preprocessor to the exporter list of preprocessors - # rather than executing directly. We can still pass appropriate config values in - # and that way the tag remove preprocessor is run *before* execution rather than after. - pp = nbconvert.preprocessors.ExecutePreprocessor( + preprocessors.append(nbconvert.preprocessors.ExecutePreprocessor( kernel_name=self._kernel_name, extra_arguments=self._execute_arguments, - allow_errors=allow_errors, timeout=timeout) - nb, resources = pp.preprocess(nb, resources) - - if 'nbsphinx_save_notebook' in resources: - # TODO: maybe we write our *own* preprocessor to hook into this stage, right after the execute preprocessor, to save the notebook - # Save *executed* notebook *before* the Exporter can change it: - nbformat.write(nb, resources['nbsphinx_save_notebook']) + allow_errors=allow_errors, timeout=timeout)) + # pp = nbconvert.preprocessors.ExecutePreprocessor( + # kernel_name=self._kernel_name, + # extra_arguments=self._execute_arguments, + # allow_errors=allow_errors, timeout=timeout) + # nb, resources = pp.preprocess(nb, resources) + + # if 'nbsphinx_save_notebook' in resources: + # # Save *executed* notebook *before* the Exporter can change it: + # nbformat.write(nb, resources['nbsphinx_save_notebook']) + + preprocessors.append(Writer()) + # Find the existing execute preprocessor and replace it + i = next((i for i,p in enumerate(self._preprocessors) if isinstance(p, nbconvert.preprocessors.ExecutePreprocessor)), len(self._preprocessors)) + self._preprocessors[i:i+1] = preprocessors # Call into RSTExporter rststr, resources = super(Exporter, self).from_notebook_node( From 9e6eb8c3faded502c835647b67cb7a4b115f0ed3 Mon Sep 17 00:00:00 2001 From: Jason Grout Date: Tue, 21 Jun 2022 22:30:34 -0600 Subject: [PATCH 3/3] Fix pep8 warnings --- src/nbsphinx.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nbsphinx.py b/src/nbsphinx.py index 35a66dfd..d88b63bf 100644 --- a/src/nbsphinx.py +++ b/src/nbsphinx.py @@ -873,7 +873,9 @@ def from_notebook_node(self, nb, resources=None, **kw): preprocessors.append(Writer()) # Find the existing execute preprocessor and replace it - i = next((i for i,p in enumerate(self._preprocessors) if isinstance(p, nbconvert.preprocessors.ExecutePreprocessor)), len(self._preprocessors)) + i = next((i for i, p in enumerate(self._preprocessors) + if isinstance(p, nbconvert.preprocessors.ExecutePreprocessor)), + len(self._preprocessors)) self._preprocessors[i:i+1] = preprocessors # Call into RSTExporter