From 344bb57abc08db9a2dbe01d4fb3a53ca2ef7056a Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 7 Jul 2019 00:00:23 -0700 Subject: [PATCH 01/49] Add initial eventlog hook --- MANIFEST.in | 3 ++ jupyter_server/base/handlers.py | 4 ++ .../contentsmanager-actions.json | 30 +++++++++++++++ jupyter_server/serverapp.py | 20 +++++++++- jupyter_server/services/contents/handlers.py | 37 ++++++++++++++++++- 5 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 jupyter_server/event-schemas/contentsmanager-actions.json diff --git a/MANIFEST.in b/MANIFEST.in index 9d4060fc69..b81a6d5536 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,9 @@ include setupbase.py include Dockerfile graft tools +# Event Schemas +graft jupyter_server/event-schemas + # Documentation graft docs exclude docs/\#* diff --git a/jupyter_server/base/handlers.py b/jupyter_server/base/handlers.py index 75467718c8..5185365c4d 100755 --- a/jupyter_server/base/handlers.py +++ b/jupyter_server/base/handlers.py @@ -204,6 +204,10 @@ def jinja_template_vars(self): """User-supplied values to supply to jinja templates.""" return self.settings.get('jinja_template_vars', {}) + @property + def eventlog(self): + return self.settings.get('eventlog') + #--------------------------------------------------------------- # URLs #--------------------------------------------------------------- diff --git a/jupyter_server/event-schemas/contentsmanager-actions.json b/jupyter_server/event-schemas/contentsmanager-actions.json new file mode 100644 index 0000000000..242111722e --- /dev/null +++ b/jupyter_server/event-schemas/contentsmanager-actions.json @@ -0,0 +1,30 @@ +{ + "$id": "jupyter.org/contentsmanager-actions", + "version": 1, + "title": "Contents Manager activities", + "description": "Notebook Server emits this event whenever a contentsmanager action happens", + "type": "object", + "required": ["action", "path"], + "properties": { + "action": { + "enum": [ + "get", + "create", + "save", + "upload", + "rename", + "create", + "copy" + ], + "description": "Action performed by contents manager" + }, + "path": { + "type": "string", + "description": "Logical path the action was performed in" + }, + "source_path": { + "type": "string", + "description": "If action is 'copy', this specifies the source path" + } + } +} \ No newline at end of file diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 21a4e68ca9..e10bc80967 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -33,6 +33,7 @@ import warnings import webbrowser import urllib +from glob import glob from types import ModuleType from base64 import encodebytes @@ -99,6 +100,8 @@ ) from ipython_genutils import py3compat from jupyter_core.paths import jupyter_runtime_dir, jupyter_path +from jupyter_telemetry.eventlog import EventLog + from jupyter_server._sysinfo import get_sys_info from ._tz import utcnow, utcfromtimestamp @@ -279,7 +282,8 @@ def init_settings(self, jupyter_app, kernel_manager, contents_manager, server_root_dir=root_dir, jinja2_env=env, terminals_available=False, # Set later if terminals are available - serverapp=self + serverapp=self, + eventlog=jupyter_app.eventlog ) # allow custom overrides for the tornado web app. @@ -1758,6 +1762,18 @@ def _init_asyncio_patch(): # WindowsProactorEventLoopPolicy is not compatible with tornado 6 # fallback to the pre-3.8 default of Selector asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + def init_eventlog(self): + self.eventlog = EventLog(parent=self) + + schemas_glob = os.path.join( + os.path.dirname(__file__), + 'event-schemas', + '*.json' + ) + + for schema_file in glob(schemas_glob): + with open(schema_file) as f: + self.eventlog.register_schema(json.load(f)) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): @@ -1788,10 +1804,12 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True): self.init_server_extensions() # Initialize all components of the ServerApp. self.init_logging() + self.init_eventlog() if self._dispatching: return self.init_configurables() self.init_components() + self.init_eventlog() self.init_webapp() if new_httpserver: self.init_httpserver() diff --git a/jupyter_server/services/contents/handlers.py b/jupyter_server/services/contents/handlers.py index 53aff09078..7bdf369f11 100644 --- a/jupyter_server/services/contents/handlers.py +++ b/jupyter_server/services/contents/handlers.py @@ -112,6 +112,10 @@ async def get(self, path=''): )) validate_model(model, expect_content=content) self._finish_model(model, location=False) + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + { 'action': 'get', 'path': model['path'] } + ) @web.authenticated async def patch(self, path=''): @@ -120,10 +124,19 @@ async def patch(self, path=''): model = self.get_json_body() if model is None: raise web.HTTPError(400, u'JSON body missing') - model = cm.update(model, path) + self.log.info(model) + model = yield maybe_future(cm.update(model, path)) validate_model(model, expect_content=False) self._finish_model(model) + self.log.info(model) + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + # FIXME: 'path' always has a leading slash, while model['path'] does not. + # What to do here for source_path? path munge manually? Eww + { 'action': 'rename', 'path': model['path'], 'source_path': path } + ) + @gen.coroutine async def _copy(self, copy_from, copy_to=None): """Copy a file, optionally specifying a target directory.""" self.log.info(u"Copying {copy_from} to {copy_to}".format( @@ -134,6 +147,10 @@ async def _copy(self, copy_from, copy_to=None): self.set_status(201) validate_model(model, expect_content=False) self._finish_model(model) + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + { 'action': 'copy', 'path': model['path'], 'source_path': copy_from } + ) async def _upload(self, model, path): """Handle upload of a new file to path""" @@ -142,6 +159,10 @@ async def _upload(self, model, path): self.set_status(201) validate_model(model, expect_content=False) self._finish_model(model) + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + { 'action': 'upload', 'path': model['path'] } + ) async def _new_untitled(self, path, type='', ext=''): """Create a new, empty untitled entity""" @@ -150,6 +171,11 @@ async def _new_untitled(self, path, type='', ext=''): self.set_status(201) validate_model(model, expect_content=False) self._finish_model(model) + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + # Set path to path of created object, not directory it was created in + { 'action': 'create', 'path': model['path'] } + ) async def _save(self, model, path): """Save an existing file.""" @@ -160,6 +186,11 @@ async def _save(self, model, path): validate_model(model, expect_content=False) self._finish_model(model) + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + { 'action': 'save', 'path': model['path'] } + ) + @web.authenticated async def post(self, path=''): """Create a new file in the specified path. @@ -228,6 +259,10 @@ async def delete(self, path=''): cm.delete(path) self.set_status(204) self.finish() + self.eventlog.record_event( + 'jupyter.org/contentsmanager-actions', 1, + { 'action': 'delete', 'path': path } + ) class CheckpointsHandler(APIHandler): From a0f40eab99a2c407ddd13ea5e6043830946977d2 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 7 Jul 2019 00:31:11 -0700 Subject: [PATCH 02/49] Install jupyter_telemetry from source --- .travis.yml | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..6346a3d78d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,105 @@ +# http://travis-ci.org/#!/ipython/ipython +language: python + +cache: + directories: + - $HOME/.cache/bower + - $HOME/.cache/pip +python: + - 3.6 + + +env: + global: + - PATH=$TRAVIS_BUILD_DIR/pandoc:$PATH + matrix: + - GROUP=js/notebook + - GROUP=python + - GROUP=js/base + - GROUP=js/services + +before_install: + - pip install --upgrade pip + # Manually install jupyter_telemetry, as of https://github.com/jupyter/telemetry/pull/10 + - pip install git+https://github.com/yuvipanda/telemetry@5789321 + - pip install --upgrade setuptools wheel nose coverage codecov + - nvm install 6.9.2 + - nvm use 6.9.2 + - node --version + - npm --version + - npm upgrade -g npm + - npm install + - | + if [[ $GROUP == js* ]]; then + npm install -g casperjs@1.1.3 phantomjs-prebuilt@2.1.7 + fi + - | + if [[ $GROUP == docs ]]; then + pip install -r docs/doc-requirements.txt + pip install --upgrade pytest + fi + - | + if [[ $GROUP == selenium ]]; then + pip install --upgrade selenium pytest + # Install Webdriver backend for Firefox: + wget https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz + mkdir geckodriver + tar -xzf geckodriver-v0.19.1-linux64.tar.gz -C geckodriver + export PATH=$PATH:$PWD/geckodriver + fi + - pip install "attrs>=17.4.0" + +install: + - pip install --pre .[test] $EXTRA_PIP + - pip freeze + - wget https://github.com/jgm/pandoc/releases/download/1.19.1/pandoc-1.19.1-1-amd64.deb && sudo dpkg -i pandoc-1.19.1-1-amd64.deb + + +script: + - jupyter kernelspec list + - | + symlinks=$(find . -type l| grep -v './node_modules/' | grep -v './git-hooks') + if [[ $(echo $symlinks) ]]; then + echo "Repository contains symlinks which won't work on windows:" + echo $symlinks + echo "" + false + else + true + fi + - 'if [[ $GROUP == js* ]]; then travis_retry python -m notebook.jstest ${GROUP:3}; fi' + - 'if [[ $GROUP == python ]]; then nosetests -v --exclude-dir notebook/tests/selenium --with-coverage --cover-package=notebook notebook; fi' + - 'if [[ $GROUP == selenium ]]; then py.test -sv notebook/tests/selenium; fi' + - | + if [[ $GROUP == docs ]]; then + EXIT_STATUS=0 + make -C docs/ html || EXIT_STATUS=$? + + if [[ $TRAVIS_EVENT_TYPE == cron ]]; then + make -C docs/ linkcheck || EXIT_STATUS=$?; + fi + + pytest --nbval --current-env docs || EXIT_STATUS=$? + exit $EXIT_STATUS + fi + + +matrix: + include: + - python: 3.6 + env: + - GROUP=selenium + - JUPYTER_TEST_BROWSER=firefox + - MOZ_HEADLESS=1 + addons: + firefox: 57.0 + - python: 3.5 + env: GROUP=python + - python: 3.7 + dist: xenial + env: GROUP=python + - python: 3.6 + env: GROUP=docs + +after_success: + - codecov From 96bf2f03f7ba7dc828ef64bb7ebf9016a5f82538 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 7 Jul 2019 00:57:01 -0700 Subject: [PATCH 03/49] Set up an eventlog API endpoint Bump telemetry extension commit as well --- .travis.yml | 105 ------------------- jupyter_server/services/eventlog/__init__.py | 0 jupyter_server/services/eventlog/handlers.py | 42 ++++++++ 3 files changed, 42 insertions(+), 105 deletions(-) delete mode 100644 .travis.yml create mode 100644 jupyter_server/services/eventlog/__init__.py create mode 100644 jupyter_server/services/eventlog/handlers.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6346a3d78d..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,105 +0,0 @@ -# http://travis-ci.org/#!/ipython/ipython -language: python - -cache: - directories: - - $HOME/.cache/bower - - $HOME/.cache/pip -python: - - 3.6 - - -env: - global: - - PATH=$TRAVIS_BUILD_DIR/pandoc:$PATH - matrix: - - GROUP=js/notebook - - GROUP=python - - GROUP=js/base - - GROUP=js/services - -before_install: - - pip install --upgrade pip - # Manually install jupyter_telemetry, as of https://github.com/jupyter/telemetry/pull/10 - - pip install git+https://github.com/yuvipanda/telemetry@5789321 - - pip install --upgrade setuptools wheel nose coverage codecov - - nvm install 6.9.2 - - nvm use 6.9.2 - - node --version - - npm --version - - npm upgrade -g npm - - npm install - - | - if [[ $GROUP == js* ]]; then - npm install -g casperjs@1.1.3 phantomjs-prebuilt@2.1.7 - fi - - | - if [[ $GROUP == docs ]]; then - pip install -r docs/doc-requirements.txt - pip install --upgrade pytest - fi - - | - if [[ $GROUP == selenium ]]; then - pip install --upgrade selenium pytest - # Install Webdriver backend for Firefox: - wget https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz - mkdir geckodriver - tar -xzf geckodriver-v0.19.1-linux64.tar.gz -C geckodriver - export PATH=$PATH:$PWD/geckodriver - fi - - pip install "attrs>=17.4.0" - -install: - - pip install --pre .[test] $EXTRA_PIP - - pip freeze - - wget https://github.com/jgm/pandoc/releases/download/1.19.1/pandoc-1.19.1-1-amd64.deb && sudo dpkg -i pandoc-1.19.1-1-amd64.deb - - -script: - - jupyter kernelspec list - - | - symlinks=$(find . -type l| grep -v './node_modules/' | grep -v './git-hooks') - if [[ $(echo $symlinks) ]]; then - echo "Repository contains symlinks which won't work on windows:" - echo $symlinks - echo "" - false - else - true - fi - - 'if [[ $GROUP == js* ]]; then travis_retry python -m notebook.jstest ${GROUP:3}; fi' - - 'if [[ $GROUP == python ]]; then nosetests -v --exclude-dir notebook/tests/selenium --with-coverage --cover-package=notebook notebook; fi' - - 'if [[ $GROUP == selenium ]]; then py.test -sv notebook/tests/selenium; fi' - - | - if [[ $GROUP == docs ]]; then - EXIT_STATUS=0 - make -C docs/ html || EXIT_STATUS=$? - - if [[ $TRAVIS_EVENT_TYPE == cron ]]; then - make -C docs/ linkcheck || EXIT_STATUS=$?; - fi - - pytest --nbval --current-env docs || EXIT_STATUS=$? - exit $EXIT_STATUS - fi - - -matrix: - include: - - python: 3.6 - env: - - GROUP=selenium - - JUPYTER_TEST_BROWSER=firefox - - MOZ_HEADLESS=1 - addons: - firefox: 57.0 - - python: 3.5 - env: GROUP=python - - python: 3.7 - dist: xenial - env: GROUP=python - - python: 3.6 - env: GROUP=docs - -after_success: - - codecov diff --git a/jupyter_server/services/eventlog/__init__.py b/jupyter_server/services/eventlog/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py new file mode 100644 index 0000000000..687b2591cc --- /dev/null +++ b/jupyter_server/services/eventlog/handlers.py @@ -0,0 +1,42 @@ +import json + +from tornado import web + +from notebook.utils import url_path_join +from notebook.base.handlers import APIHandler, json_errors +from jupyter_telemetry.eventlog import EventLog + + +class EventLoggingHandler(APIHandler): + """ + A handler that receives and stores telemetry data from the client. + """ + @json_errors + @web.authenticated + def post(self, *args, **kwargs): + try: + # Parse the data from the request body + raw_event = json.loads(self.request.body.strip().decode()) + except Exception as e: + raise web.HTTPError(400, str(e)) + + required_fields = {'schema', 'version', 'event'} + for rf in required_fields: + if rf not in raw_event: + raise web.HTTPError(400, f'{rf} is a required field') + + schema_name = raw_event['schema'] + version = raw_event['version'] + event = raw_event['event'] + + # Profile, and move to a background thread if this is problematic + # FIXME: Return a more appropriate error response if validation fails + self.eventlog.record_event(schema_name, version, event) + + self.set_status(204) + self.finish() + + +default_handlers = [ + (r"/api/eventlog", EventLoggingHandler), +] \ No newline at end of file From 06b91e0c34050bfab6072b829584a63e15fff60b Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 7 Jul 2019 13:00:59 -0700 Subject: [PATCH 04/49] Use different naming convention & add test for it Experiments here informed the schema naming recommendations in https://github.com/jupyter/telemetry/pull/11 --- .../contentsmanager-actions.json | 2 +- .../contentsmanager-actions/v1.json | 30 +++++++++++++++++++ jupyter_server/serverapp.py | 17 +++++------ jupyter_server/services/contents/handlers.py | 22 ++++++++------ jupyter_server/utils.py | 9 ++++++ 5 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 jupyter_server/event-schemas/contentsmanager-actions/v1.json diff --git a/jupyter_server/event-schemas/contentsmanager-actions.json b/jupyter_server/event-schemas/contentsmanager-actions.json index 242111722e..5da6d68b88 100644 --- a/jupyter_server/event-schemas/contentsmanager-actions.json +++ b/jupyter_server/event-schemas/contentsmanager-actions.json @@ -1,5 +1,5 @@ { - "$id": "jupyter.org/contentsmanager-actions", + "$id": "eventlogging.jupyter.org/notebook/contentsmanager-actions", "version": 1, "title": "Contents Manager activities", "description": "Notebook Server emits this event whenever a contentsmanager action happens", diff --git a/jupyter_server/event-schemas/contentsmanager-actions/v1.json b/jupyter_server/event-schemas/contentsmanager-actions/v1.json new file mode 100644 index 0000000000..5da6d68b88 --- /dev/null +++ b/jupyter_server/event-schemas/contentsmanager-actions/v1.json @@ -0,0 +1,30 @@ +{ + "$id": "eventlogging.jupyter.org/notebook/contentsmanager-actions", + "version": 1, + "title": "Contents Manager activities", + "description": "Notebook Server emits this event whenever a contentsmanager action happens", + "type": "object", + "required": ["action", "path"], + "properties": { + "action": { + "enum": [ + "get", + "create", + "save", + "upload", + "rename", + "create", + "copy" + ], + "description": "Action performed by contents manager" + }, + "path": { + "type": "string", + "description": "Logical path the action was performed in" + }, + "source_path": { + "type": "string", + "description": "If action is 'copy', this specifies the source path" + } + } +} \ No newline at end of file diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index e10bc80967..3e7e561b49 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1765,15 +1765,14 @@ def _init_asyncio_patch(): def init_eventlog(self): self.eventlog = EventLog(parent=self) - schemas_glob = os.path.join( - os.path.dirname(__file__), - 'event-schemas', - '*.json' - ) - - for schema_file in glob(schemas_glob): - with open(schema_file) as f: - self.eventlog.register_schema(json.load(f)) + event_schemas_dir = os.path.join(os.path.dirname(__file__), 'event-schemas') + # Recursively register all .json files under event-schemas + for dirname, _, files in os.walk(event_schemas_dir): + for file in files: + if file.endswith('.json'): + file_path = os.path.join(dirname, file) + with open(file_path) as f: + self.eventlog.register_schema(json.load(f)) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): diff --git a/jupyter_server/services/contents/handlers.py b/jupyter_server/services/contents/handlers.py index 7bdf369f11..d80ba9b768 100644 --- a/jupyter_server/services/contents/handlers.py +++ b/jupyter_server/services/contents/handlers.py @@ -9,10 +9,14 @@ import json from tornado import web - -from jupyter_server.utils import url_path_join, url_escape, ensure_async from jupyter_client.jsonutil import date_default +from jupyter_server.utils import ( + url_path_join, + url_escape, + ensure_async, + eventlogging_schema_fqn +) from jupyter_server.base.handlers import ( JupyterHandler, APIHandler, path_regex, ) @@ -113,7 +117,7 @@ async def get(self, path=''): validate_model(model, expect_content=content) self._finish_model(model, location=False) self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'get', 'path': model['path'] } ) @@ -130,7 +134,7 @@ async def patch(self, path=''): self._finish_model(model) self.log.info(model) self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, # FIXME: 'path' always has a leading slash, while model['path'] does not. # What to do here for source_path? path munge manually? Eww { 'action': 'rename', 'path': model['path'], 'source_path': path } @@ -148,7 +152,7 @@ async def _copy(self, copy_from, copy_to=None): validate_model(model, expect_content=False) self._finish_model(model) self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'copy', 'path': model['path'], 'source_path': copy_from } ) @@ -160,7 +164,7 @@ async def _upload(self, model, path): validate_model(model, expect_content=False) self._finish_model(model) self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'upload', 'path': model['path'] } ) @@ -172,7 +176,7 @@ async def _new_untitled(self, path, type='', ext=''): validate_model(model, expect_content=False) self._finish_model(model) self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, # Set path to path of created object, not directory it was created in { 'action': 'create', 'path': model['path'] } ) @@ -187,7 +191,7 @@ async def _save(self, model, path): self._finish_model(model) self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'save', 'path': model['path'] } ) @@ -260,7 +264,7 @@ async def delete(self, path=''): self.set_status(204) self.finish() self.eventlog.record_event( - 'jupyter.org/contentsmanager-actions', 1, + eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'delete', 'path': path } ) diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 42a6ae9278..54e112f97b 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -440,3 +440,12 @@ def wrapped(): result = asyncio.ensure_future(maybe_async) return result return wrapped() + + +def eventlogging_schema_fqn(name): + """ + Return fully qualified event schema name + + Matches convention for this particular repo + """ + return 'eventlogging.jupyter.org/notebook/{}'.format(name) From 716ff1b3ef6a4f7da446ff191a89e1e93047a615 Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Sun, 7 Jul 2019 15:07:45 -0700 Subject: [PATCH 05/49] Don't use f-strings python 3.5 is still supported --- jupyter_server/services/eventlog/handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py index 687b2591cc..4665e43e8b 100644 --- a/jupyter_server/services/eventlog/handlers.py +++ b/jupyter_server/services/eventlog/handlers.py @@ -23,7 +23,7 @@ def post(self, *args, **kwargs): required_fields = {'schema', 'version', 'event'} for rf in required_fields: if rf not in raw_event: - raise web.HTTPError(400, f'{rf} is a required field') + raise web.HTTPError(400, '{} is a required field'.format(rf)) schema_name = raw_event['schema'] version = raw_event['version'] From 8e122fcaaa4d348a4edc9c1ce780742bff788d7f Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 9 Jul 2019 08:08:04 -0700 Subject: [PATCH 06/49] Derive JSON Schema files from YAML files This lets us add detailed documentation & description to our schemas, which is very hard to do in JSON. We also add a lot of documentation to the one JSON schema we have --- jupyter_server/event-schemas/README.md | 19 +++++ .../contentsmanager-actions.json | 17 ++-- jupyter_server/event-schemas/generate-json.py | 39 +++++++++ jupyter_server/event-schemas/v1.yaml | 79 +++++++++++++++++++ 4 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 jupyter_server/event-schemas/README.md create mode 100755 jupyter_server/event-schemas/generate-json.py create mode 100644 jupyter_server/event-schemas/v1.yaml diff --git a/jupyter_server/event-schemas/README.md b/jupyter_server/event-schemas/README.md new file mode 100644 index 0000000000..541a9b0398 --- /dev/null +++ b/jupyter_server/event-schemas/README.md @@ -0,0 +1,19 @@ +# Event Schemas + +## Generating .json files + +Event Schemas are written in a human readable `.yaml` format. +This is primarily to get multi-line strings in our descriptions, +as documentation is very important. + +Every time you modify a `.yaml` file, you should run the following +commands. + +```bash +./generate-json.py +``` + +This needs the `ruamel.yaml` python package installed. + +Hopefully, this is extremely temporary, and we can just use YAML +with jupyter_telemetry. \ No newline at end of file diff --git a/jupyter_server/event-schemas/contentsmanager-actions.json b/jupyter_server/event-schemas/contentsmanager-actions.json index 5da6d68b88..065f1d5c2f 100644 --- a/jupyter_server/event-schemas/contentsmanager-actions.json +++ b/jupyter_server/event-schemas/contentsmanager-actions.json @@ -2,9 +2,12 @@ "$id": "eventlogging.jupyter.org/notebook/contentsmanager-actions", "version": 1, "title": "Contents Manager activities", - "description": "Notebook Server emits this event whenever a contentsmanager action happens", + "description": "Record actions on files via the ContentsManager REST API.\n\nThe notebook ContentsManager REST API is used by all frontends to retreive,\nsave, list, delete and perform other actions on notebooks, directories,\nand other files through the UI. This is pluggable - the default acts on\nthe file system, but can be replaced with a different ContentsManager\nimplementation - to work on S3, Postgres, other object stores, etc.\nThe events get recorded regardless of the ContentsManager implementation\nbeing used.\n\nLimitations:\n\n1. This does not record all filesystem access, just the ones that happen\n explicitly via the notebook server's REST API. Users can (and often do)\n trivially access the filesystem in many other ways (such as `open()` calls\n in their code), so this is usually never a complete record.\n2. As with all events recorded by the notebook server, users most likely\n have the ability to modify the code of the notebook server. Unless other\n security measures are in place, these events should be treated as user\n controlled and not used in high security areas.\n3. Events are only recorded when an action succeeds.\n", "type": "object", - "required": ["action", "path"], + "required": [ + "action", + "path" + ], "properties": { "action": { "enum": [ @@ -13,18 +16,18 @@ "save", "upload", "rename", - "create", - "copy" + "copy", + "delete" ], - "description": "Action performed by contents manager" + "description": "Action performed by the ContentsManager API.\n\nThis is a required field.\n\nPossible values:\n\n1. get\n Get contents of a particular file, or list contents of a directory.\n\n2. create\n Create a new directory or file at 'path'. Currently, name of the\n file or directory is auto generated by the ContentsManager implementation.\n\n3. save\n Save a file at path with contents from the client\n\n4. upload\n Upload a file at given path with contents from the client\n\n5. rename\n Rename a file or directory from value in source_path to\n value in path.\n\n5. copy\n Copy a file or directory from value in source_path to\n value in path.\n\n6. delete\n Delete a file or empty directory at given path\n" }, "path": { "type": "string", - "description": "Logical path the action was performed in" + "description": "Logical path on which the operation was performed.\n\nThis is a required field.\n" }, "source_path": { "type": "string", - "description": "If action is 'copy', this specifies the source path" + "description": "Source path of an operation when action is 'copy' or 'rename'" } } } \ No newline at end of file diff --git a/jupyter_server/event-schemas/generate-json.py b/jupyter_server/event-schemas/generate-json.py new file mode 100755 index 0000000000..a39fa0610b --- /dev/null +++ b/jupyter_server/event-schemas/generate-json.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import jsonschema +from ruamel.yaml import YAML + +from jupyter_telemetry.eventlog import EventLog + +yaml = YAML(typ='safe') + +def main(): + argparser = argparse.ArgumentParser() + argparser.add_argument( + 'directory', + help='Directory with Schema .yaml files' + ) + + args = argparser.parse_args() + + el = EventLog() + for dirname, _, files in os.walk(args.directory): + for file in files: + if not file.endswith('.yaml'): + continue + yaml_path = os.path.join(dirname, file) + print('Processing', yaml_path) + with open(yaml_path) as f: + schema = yaml.load(f) + + # validate schema + el.register_schema(schema) + + json_path = os.path.join(dirname, os.path.splitext(file)[0] + '.json') + with open(json_path, 'w') as f: + json.dump(schema, f, indent=4) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/jupyter_server/event-schemas/v1.yaml b/jupyter_server/event-schemas/v1.yaml new file mode 100644 index 0000000000..3d7e8f2fe9 --- /dev/null +++ b/jupyter_server/event-schemas/v1.yaml @@ -0,0 +1,79 @@ +"$id": eventlogging.jupyter.org/notebook/contentsmanager-actions +version: 1 +title: Contents Manager activities +description: | + Record actions on files via the ContentsManager REST API. + + The notebook ContentsManager REST API is used by all frontends to retreive, + save, list, delete and perform other actions on notebooks, directories, + and other files through the UI. This is pluggable - the default acts on + the file system, but can be replaced with a different ContentsManager + implementation - to work on S3, Postgres, other object stores, etc. + The events get recorded regardless of the ContentsManager implementation + being used. + + Limitations: + + 1. This does not record all filesystem access, just the ones that happen + explicitly via the notebook server's REST API. Users can (and often do) + trivially access the filesystem in many other ways (such as `open()` calls + in their code), so this is usually never a complete record. + 2. As with all events recorded by the notebook server, users most likely + have the ability to modify the code of the notebook server. Unless other + security measures are in place, these events should be treated as user + controlled and not used in high security areas. + 3. Events are only recorded when an action succeeds. +type: object +required: +- action +- path +properties: + action: + enum: + - get + - create + - save + - upload + - rename + - copy + - delete + description: | + Action performed by the ContentsManager API. + + This is a required field. + + Possible values: + + 1. get + Get contents of a particular file, or list contents of a directory. + + 2. create + Create a new directory or file at 'path'. Currently, name of the + file or directory is auto generated by the ContentsManager implementation. + + 3. save + Save a file at path with contents from the client + + 4. upload + Upload a file at given path with contents from the client + + 5. rename + Rename a file or directory from value in source_path to + value in path. + + 5. copy + Copy a file or directory from value in source_path to + value in path. + + 6. delete + Delete a file or empty directory at given path + path: + type: string + description: | + Logical path on which the operation was performed. + + This is a required field. + source_path: + type: string + description: | + Source path of an operation when action is 'copy' or 'rename' \ No newline at end of file From f9a0dfb6c3ad69b541de65e053b5354b17d21d1f Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 9 Jul 2019 11:10:07 -0700 Subject: [PATCH 07/49] Keep event schemas in YAML Primary advantage over JSON is that we can do multi-line strings for more detailed documentation. We also expect humans to read & write these, so YAML is a much better format there. All JSON is also valid YAML, so that helps. Depends on https://github.com/jupyter/telemetry/pull/13 --- jupyter_server/serverapp.py | 7 ++-- notebook/tests/test_eventlog.py | 57 +++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 notebook/tests/test_eventlog.py diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 3e7e561b49..76896b03d3 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -33,6 +33,7 @@ import warnings import webbrowser import urllib +from ruamel.yaml import YAML from glob import glob from types import ModuleType @@ -1765,14 +1766,14 @@ def _init_asyncio_patch(): def init_eventlog(self): self.eventlog = EventLog(parent=self) + yaml = YAML(typ='safe') event_schemas_dir = os.path.join(os.path.dirname(__file__), 'event-schemas') # Recursively register all .json files under event-schemas for dirname, _, files in os.walk(event_schemas_dir): for file in files: - if file.endswith('.json'): + if file.endswith('.yaml'): file_path = os.path.join(dirname, file) - with open(file_path) as f: - self.eventlog.register_schema(json.load(f)) + self.eventlog.register_schema_file(file_path) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): diff --git a/notebook/tests/test_eventlog.py b/notebook/tests/test_eventlog.py new file mode 100644 index 0000000000..c2f74a59c9 --- /dev/null +++ b/notebook/tests/test_eventlog.py @@ -0,0 +1,57 @@ +import os +import re +import jsonschema +from ruamel.yaml import YAML +from notebook.notebookapp import NotebookApp +from notebook.utils import eventlogging_schema_fqn +from unittest import TestCase + +yaml = YAML(typ='safe') + +class RegisteredSchemasTestCase(TestCase): + def schema_files(self): + event_schemas_dir = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', 'event-schemas') + ) + schemas = [] + for dirname, _, files in os.walk(event_schemas_dir): + for file in files: + if file.endswith('.yaml'): + yield os.path.join(dirname, file) + + def test_eventlogging_schema_fqn(self): + self.assertEqual( + eventlogging_schema_fqn('test'), + 'eventlogging.jupyter.org/notebook/test' + ) + def test_valid_schemas(self): + """ + All schemas must be valid json schemas + """ + for schema_file in self.schema_files(): + with open(schema_file) as f: + jsonschema.Draft7Validator.check_schema(yaml.load(f)) + + def test_schema_conventions(self): + """ + Test schema naming convention for this repo. + + 1. All schemas should be under event-schamas/{name}/v{version}.yaml + 2. Schema id should be eventlogging.jupyter.org/notebook/{name} + 3. Schema version should match version in file + """ + for schema_file in self.schema_files(): + filename = os.path.basename(schema_file) + match = re.match('v(\d+)\.yaml', filename) + # All schema locations must match the following pattern + # schema-name/v(version).yaml + self.assertIsNotNone(match) + + with open(schema_file) as f: + schema = yaml.load(f) + + self.assertEqual(schema['$id'], eventlogging_schema_fqn( + os.path.basename(os.path.dirname(schema_file)) + )) + self.assertEqual(schema['version'], int(match.groups()[0])) + \ No newline at end of file From c7428e8aa778b8e2352b88af3c98fbb424cfac2e Mon Sep 17 00:00:00 2001 From: yuvipanda Date: Tue, 9 Jul 2019 16:34:36 -0700 Subject: [PATCH 08/49] Depend on the jupyter_telemetry package We made a v0.0.1 release! --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2697fad271..e402e71669 100755 --- a/setup.py +++ b/setup.py @@ -94,7 +94,8 @@ 'Send2Trash', 'terminado>=0.8.3', 'prometheus_client', - "pywin32>=1.0 ; sys_platform == 'win32'" + "pywin32>=1.0 ; sys_platform == 'win32'", + 'jupyter_telemetry' ], extras_require = { 'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters', From 9437e88353d515b1b39a210321c92cb34667dc8c Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 12:02:13 -0700 Subject: [PATCH 09/49] read schemas from new utils function --- jupyter_server/serverapp.py | 21 +++++++++++---------- jupyter_server/utils.py | 14 ++++++++++++++ notebook/tests/test_eventlog.py | 20 ++++++-------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 76896b03d3..18f28967f0 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -106,7 +106,14 @@ from jupyter_server._sysinfo import get_sys_info from ._tz import utcnow, utcfromtimestamp -from .utils import url_path_join, check_pid, url_escape, urljoin, pathname2url +from .utils import ( + url_path_join, + check_pid, + url_escape, + urljoin, + pathname2url, + get_schema_files +) from jupyter_server.extension.serverextension import ( ServerExtensionApp, @@ -1765,15 +1772,9 @@ def _init_asyncio_patch(): asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) def init_eventlog(self): self.eventlog = EventLog(parent=self) - - yaml = YAML(typ='safe') - event_schemas_dir = os.path.join(os.path.dirname(__file__), 'event-schemas') - # Recursively register all .json files under event-schemas - for dirname, _, files in os.walk(event_schemas_dir): - for file in files: - if file.endswith('.yaml'): - file_path = os.path.join(dirname, file) - self.eventlog.register_schema_file(file_path) + # Register schemas for notebook services. + for file_path in get_schema_files(): + self.eventlog.register_schema_file(file_path) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True): diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 54e112f97b..55389f037a 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -449,3 +449,17 @@ def eventlogging_schema_fqn(name): Matches convention for this particular repo """ return 'eventlogging.jupyter.org/notebook/{}'.format(name) + + +def get_schema_files(): + """Yield a sequence of event schemas for jupyter services.""" + # Hardcode path to event schemas directory. + event_schemas_dir = os.path.join(os.path.dirname(__file__), 'event-schemas') + schema_files = [] + # Recursively register all .json files under event-schemas + for dirname, _, files in os.walk(event_schemas_dir): + for file in files: + if file.endswith('.yaml'): + file_path = os.path.join(dirname, file) + schema_files.append(file_path) + yield schema_files diff --git a/notebook/tests/test_eventlog.py b/notebook/tests/test_eventlog.py index c2f74a59c9..994181b73e 100644 --- a/notebook/tests/test_eventlog.py +++ b/notebook/tests/test_eventlog.py @@ -3,32 +3,25 @@ import jsonschema from ruamel.yaml import YAML from notebook.notebookapp import NotebookApp -from notebook.utils import eventlogging_schema_fqn +from notebook.utils import eventlogging_schema_fqn, get_schema_files from unittest import TestCase yaml = YAML(typ='safe') + class RegisteredSchemasTestCase(TestCase): - def schema_files(self): - event_schemas_dir = os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', 'event-schemas') - ) - schemas = [] - for dirname, _, files in os.walk(event_schemas_dir): - for file in files: - if file.endswith('.yaml'): - yield os.path.join(dirname, file) def test_eventlogging_schema_fqn(self): self.assertEqual( eventlogging_schema_fqn('test'), 'eventlogging.jupyter.org/notebook/test' ) + def test_valid_schemas(self): """ All schemas must be valid json schemas """ - for schema_file in self.schema_files(): + for schema_file in get_schema_files(): with open(schema_file) as f: jsonschema.Draft7Validator.check_schema(yaml.load(f)) @@ -40,7 +33,7 @@ def test_schema_conventions(self): 2. Schema id should be eventlogging.jupyter.org/notebook/{name} 3. Schema version should match version in file """ - for schema_file in self.schema_files(): + for schema_file in get_schema_files(): filename = os.path.basename(schema_file) match = re.match('v(\d+)\.yaml', filename) # All schema locations must match the following pattern @@ -53,5 +46,4 @@ def test_schema_conventions(self): self.assertEqual(schema['$id'], eventlogging_schema_fqn( os.path.basename(os.path.dirname(schema_file)) )) - self.assertEqual(schema['version'], int(match.groups()[0])) - \ No newline at end of file + self.assertEqual(schema['version'], int(match.groups()[0])) \ No newline at end of file From 6e3c80c622352fb007c89315fa5063e5e71b9241 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 14:56:24 -0700 Subject: [PATCH 10/49] Add fix for tables in RTD theme sphinx docs. Solution came from https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html --- docs/source/_static/theme_overrides.css | 13 +++ docs/source/conf.py | 127 +++++++++++++++++++++++- jupyter_server/utils.py | 5 +- 3 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 docs/source/_static/theme_overrides.css diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 0000000000..63ee6cc74c --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,13 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} diff --git a/docs/source/conf.py b/docs/source/conf.py index e105e82d40..fc538a1613 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,8 +70,7 @@ 'sphinx.ext.mathjax', 'IPython.sphinxext.ipython_console_highlighting', 'sphinxcontrib_github_alt', - 'sphinxcontrib.openapi', - 'sphinxemoji.sphinxemoji' + 'sphinx-jsonschema' ] # Add any paths that contain templates here, relative to this directory. @@ -208,6 +207,12 @@ # since it is needed to properly generate _static in the build directory html_static_path = ['_static'] +html_context = { + 'css_files': [ + '_static/theme_overrides.css', # override wide tables in RTD theme + ], + } + # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. @@ -370,4 +375,122 @@ spelling_word_list_filename='spelling_wordlist.txt' # import before any doc is built, so _ is guaranteed to be injected +<<<<<<< HEAD import jupyter_server.transutils +======= +import notebook.transutils + +# -- Autogenerate documentation for event schemas ------------------ + +from notebook.utils import get_schema_files + +# Build a dictionary that describes the event schema table of contents. +# toc = { +# schema_name : { +# src: # file path to schema +# dst: # file path to documentation +# ver: # latest version of schema +# } +# } +toc = {} + +# Iterate over schema directories and generate documentation. +# Generates documentation for the latest version of each schema. +for file_path in get_schema_files(): + # Make path relative. + file_path = os.path.relpath(file_path) + # Break apart path to its pieces + pieces = file_path.split(os.path.sep) + # Schema version. Outputs as a string that looks like "v#" + schema_ver = os.path.splitext(pieces[-1])[0] + # Strip "v" and make version an integer. + schema_int = int(schema_ver[1:]) + # Schema name. + schema_name = pieces[-2] + + # Add this version file to schema_dir + src = '../' + file_path + dst = os.path.join('events', os.path.join(schema_name + '.rst')) + + if schema_name in toc: + # If this is a later version, replace the old version. + if schema_int > toc[schema_name]['ver']: + toc[schema_name] = { + 'src': src, + 'dst': dst, + 'ver': schema_int + } + else: + toc[schema_name] = { + 'src': src, + 'dst': dst, + 'ver': schema_int + } + +# Write schema documentation +for schema_name, x in toc.items(): + with open(dst, 'w') as f: + f.write('.. jsonschema:: {}'.format(src)) + +# Write table of contents +events_index = """ +.. toctree:: + :maxdepth: 1 + :glob: + +""" + +with open(os.path.join('events', 'index.rst'), 'w') as f: + f.write(events_index) + for item in toc.keys(): + f.write(' {}'.format(item)) + + + + + + + + + + +# # create a directory for this schema if it doesn't exist: +# schema_dir = os.path.join('events', schema_name) +# if not os.path.exists(schema_dir): +# os.makedirs(schema_dir) + + +# toc[schema_name] + + + +# with open(dst, 'w') as f: +# f.write('.. jsonschema:: {}'.format(src)) + + + + + + + +# toc.append(schema_name) + + +# events_index = """ +# .. toctree:: +# :maxdepth: 1 +# :glob: + +# """ + + +# with open(os.path.join('events', 'index.rst'), 'w') as f: +# f.write(events_index) +# for item in set(toc): +# f.write(' {}/*'.format(item)) + + + + + +>>>>>>> 4fb0a0443... Add fix for tables in RTD theme sphinx docs. diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 55389f037a..8fc6e89479 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -455,11 +455,10 @@ def get_schema_files(): """Yield a sequence of event schemas for jupyter services.""" # Hardcode path to event schemas directory. event_schemas_dir = os.path.join(os.path.dirname(__file__), 'event-schemas') - schema_files = [] + #schema_files = [] # Recursively register all .json files under event-schemas for dirname, _, files in os.walk(event_schemas_dir): for file in files: if file.endswith('.yaml'): file_path = os.path.join(dirname, file) - schema_files.append(file_path) - yield schema_files + yield file_path From 4035fd557e99a7549e1aa53e5b7874f83a2587e3 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 15:02:22 -0700 Subject: [PATCH 11/49] add event schema auto-documentation to jupyter notebook docs --- .gitignore | 1 + docs/environment.yml | 3 +- docs/source/conf.py | 118 --------------------------------------- docs/source/eventlog.rst | 47 ++++++++++++++++ 4 files changed, 50 insertions(+), 119 deletions(-) create mode 100644 docs/source/eventlog.rst diff --git a/.gitignore b/.gitignore index d9fb5e0c6c..a69d2eeee6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ docs/man/*.gz docs/source/api/generated docs/source/config.rst docs/gh-pages +docs/source/events notebook/i18n/*/LC_MESSAGES/*.mo notebook/i18n/*/LC_MESSAGES/nbjs.json notebook/static/components diff --git a/docs/environment.yml b/docs/environment.yml index 5d77bc7bb4..1d9c9d3eb8 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -13,4 +13,5 @@ dependencies: - sphinxcontrib_github_alt - sphinxcontrib-openapi - sphinxemoji - - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master \ No newline at end of file + - git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master + - sphinx-jsonschema diff --git a/docs/source/conf.py b/docs/source/conf.py index fc538a1613..41b089cc07 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -375,122 +375,4 @@ spelling_word_list_filename='spelling_wordlist.txt' # import before any doc is built, so _ is guaranteed to be injected -<<<<<<< HEAD import jupyter_server.transutils -======= -import notebook.transutils - -# -- Autogenerate documentation for event schemas ------------------ - -from notebook.utils import get_schema_files - -# Build a dictionary that describes the event schema table of contents. -# toc = { -# schema_name : { -# src: # file path to schema -# dst: # file path to documentation -# ver: # latest version of schema -# } -# } -toc = {} - -# Iterate over schema directories and generate documentation. -# Generates documentation for the latest version of each schema. -for file_path in get_schema_files(): - # Make path relative. - file_path = os.path.relpath(file_path) - # Break apart path to its pieces - pieces = file_path.split(os.path.sep) - # Schema version. Outputs as a string that looks like "v#" - schema_ver = os.path.splitext(pieces[-1])[0] - # Strip "v" and make version an integer. - schema_int = int(schema_ver[1:]) - # Schema name. - schema_name = pieces[-2] - - # Add this version file to schema_dir - src = '../' + file_path - dst = os.path.join('events', os.path.join(schema_name + '.rst')) - - if schema_name in toc: - # If this is a later version, replace the old version. - if schema_int > toc[schema_name]['ver']: - toc[schema_name] = { - 'src': src, - 'dst': dst, - 'ver': schema_int - } - else: - toc[schema_name] = { - 'src': src, - 'dst': dst, - 'ver': schema_int - } - -# Write schema documentation -for schema_name, x in toc.items(): - with open(dst, 'w') as f: - f.write('.. jsonschema:: {}'.format(src)) - -# Write table of contents -events_index = """ -.. toctree:: - :maxdepth: 1 - :glob: - -""" - -with open(os.path.join('events', 'index.rst'), 'w') as f: - f.write(events_index) - for item in toc.keys(): - f.write(' {}'.format(item)) - - - - - - - - - - -# # create a directory for this schema if it doesn't exist: -# schema_dir = os.path.join('events', schema_name) -# if not os.path.exists(schema_dir): -# os.makedirs(schema_dir) - - -# toc[schema_name] - - - -# with open(dst, 'w') as f: -# f.write('.. jsonschema:: {}'.format(src)) - - - - - - - -# toc.append(schema_name) - - -# events_index = """ -# .. toctree:: -# :maxdepth: 1 -# :glob: - -# """ - - -# with open(os.path.join('events', 'index.rst'), 'w') as f: -# f.write(events_index) -# for item in set(toc): -# f.write(' {}/*'.format(item)) - - - - - ->>>>>>> 4fb0a0443... Add fix for tables in RTD theme sphinx docs. diff --git a/docs/source/eventlog.rst b/docs/source/eventlog.rst new file mode 100644 index 0000000000..fd77a1b9c8 --- /dev/null +++ b/docs/source/eventlog.rst @@ -0,0 +1,47 @@ +Eventlogging and Telemetry +========================== + +The Notebook Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that the Notebook Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. + + +.. _logging: https://docs.python.org/3/library/logging.html +.. _`Telemetry System`: https://github.com/jupyter/telemetry +.. _`JSON schemas`: https://json-schema.org/ + +How to emit events +------------------ + +Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. + + +To begin recording events, you'll need to set two configurations: + + 1. ``handlers``: tells the EventLog *where* to route your events. This trait is a list of Python logging handlers that route events to + 2. ``allows_schemas``: tells the EventLog *which* events should be recorded. No events are emitted by default; all recorded events must be listed here. + +Here's a basic example for emitting events from the `contents` service: + +.. code-block:: + + import logging + + c.EventLog.handlers = [ + logging.FileHandler('event.log'), + ] + + c.EventLog.allowed_schemas = [ + 'hub.jupyter.org/server-action' + ] + +The output is a file, ``"event.log"``, with events recorded as JSON data. + + +.. _below: + +Event schemas +------------- + +.. toctree:: + :maxdepth: 2 + + events/index From 23d50a38b16512503acbeee1204d2e245c0af0ac Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 15:14:04 -0700 Subject: [PATCH 12/49] format paths in recorded events --- jupyter_server/services/contents/handlers.py | 30 +++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/jupyter_server/services/contents/handlers.py b/jupyter_server/services/contents/handlers.py index d80ba9b768..85065b21cb 100644 --- a/jupyter_server/services/contents/handlers.py +++ b/jupyter_server/services/contents/handlers.py @@ -117,7 +117,8 @@ async def get(self, path=''): validate_model(model, expect_content=content) self._finish_model(model, location=False) self.eventlog.record_event( - eventlogging_schema_fqn('contentsmanager-actions'), 1, + eventlogging_schema_fqn('contentsmanager-actions'), + 1, { 'action': 'get', 'path': model['path'] } ) @@ -134,10 +135,13 @@ async def patch(self, path=''): self._finish_model(model) self.log.info(model) self.eventlog.record_event( - eventlogging_schema_fqn('contentsmanager-actions'), 1, - # FIXME: 'path' always has a leading slash, while model['path'] does not. - # What to do here for source_path? path munge manually? Eww - { 'action': 'rename', 'path': model['path'], 'source_path': path } + eventlogging_schema_fqn('contentsmanager-actions'), + 1, + { + 'action': 'rename', + 'path': model['path'], + 'source_path': path.lstrip(os.path.sep) + } ) @gen.coroutine @@ -152,8 +156,13 @@ async def _copy(self, copy_from, copy_to=None): validate_model(model, expect_content=False) self._finish_model(model) self.eventlog.record_event( - eventlogging_schema_fqn('contentsmanager-actions'), 1, - { 'action': 'copy', 'path': model['path'], 'source_path': copy_from } + eventlogging_schema_fqn('contentsmanager-actions'), + 1, + { + 'action': 'copy', + 'path': model['path'], + 'source_path': copy_from.lstrip(os.path.sep) + } ) async def _upload(self, model, path): @@ -164,7 +173,8 @@ async def _upload(self, model, path): validate_model(model, expect_content=False) self._finish_model(model) self.eventlog.record_event( - eventlogging_schema_fqn('contentsmanager-actions'), 1, + eventlogging_schema_fqn('contentsmanager-actions'), + 1, { 'action': 'upload', 'path': model['path'] } ) @@ -189,9 +199,9 @@ async def _save(self, model, path): model = self.contents_manager.save(model, path) validate_model(model, expect_content=False) self._finish_model(model) - self.eventlog.record_event( - eventlogging_schema_fqn('contentsmanager-actions'), 1, + eventlogging_schema_fqn('contentsmanager-actions'), + 1, { 'action': 'save', 'path': model['path'] } ) From 3c94970d5be68fd5b8f6bab0f5c2ae3311843f00 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 15:37:03 -0700 Subject: [PATCH 13/49] add documentation for eventlog endpoint --- docs/source/eventlog.rst | 24 ++++++++++++++++---- jupyter_server/services/eventlog/handlers.py | 11 +++++---- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/source/eventlog.rst b/docs/source/eventlog.rst index fd77a1b9c8..df5c153fb7 100644 --- a/docs/source/eventlog.rst +++ b/docs/source/eventlog.rst @@ -8,12 +8,11 @@ The Notebook Server can be configured to record structured events from a running .. _`Telemetry System`: https://github.com/jupyter/telemetry .. _`JSON schemas`: https://json-schema.org/ -How to emit events ------------------- +Emitting Server Events +---------------------- Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. - To begin recording events, you'll need to set two configurations: 1. ``handlers``: tells the EventLog *where* to route your events. This trait is a list of Python logging handlers that route events to @@ -35,11 +34,26 @@ Here's a basic example for emitting events from the `contents` service: The output is a file, ``"event.log"``, with events recorded as JSON data. +`eventlog` endpoint +------------------- + +The Notebook Server provides a public REST endpoint for external applications to validate and log events +through the Server's Event Log. + +To log events, send a `POST` request to the `/api/eventlog` endpoint. The body of the request should be a +JSON blog and is required to have the follow keys: + + 1. `'schema'` : the event's schema ID. + 2. `'version'` : the version of the event's schema. + 3. `'event'` : the event data in JSON format. + +Events that are validated by this endpoint must have their schema listed in the `allowed_schemas` trait listed above. .. _below: -Event schemas -------------- + +Server Event schemas +-------=======------ .. toctree:: :maxdepth: 2 diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py index 4665e43e8b..b27dd87304 100644 --- a/jupyter_server/services/eventlog/handlers.py +++ b/jupyter_server/services/eventlog/handlers.py @@ -29,14 +29,15 @@ def post(self, *args, **kwargs): version = raw_event['version'] event = raw_event['event'] - # Profile, and move to a background thread if this is problematic - # FIXME: Return a more appropriate error response if validation fails - self.eventlog.record_event(schema_name, version, event) - + # Profile, may need to move to a background thread if this is problematic + try: + self.eventlog.record_event(schema_name, version, event) + except: + raise web.HTTPError(500, "Event could not be validated.") + self.set_status(204) self.finish() - default_handlers = [ (r"/api/eventlog", EventLoggingHandler), ] \ No newline at end of file From e76c91b3ac264b20bf2aa6d7d468a1d8c3999fc2 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 15:46:40 -0700 Subject: [PATCH 14/49] return exception as 400 error in eventlog endpoint --- docs/source/eventlog.rst | 2 +- jupyter_server/services/eventlog/handlers.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/eventlog.rst b/docs/source/eventlog.rst index df5c153fb7..7229717f69 100644 --- a/docs/source/eventlog.rst +++ b/docs/source/eventlog.rst @@ -53,7 +53,7 @@ Events that are validated by this endpoint must have their schema listed in the Server Event schemas --------=======------ +-------------------- .. toctree:: :maxdepth: 2 diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py index b27dd87304..0c9b69815f 100644 --- a/jupyter_server/services/eventlog/handlers.py +++ b/jupyter_server/services/eventlog/handlers.py @@ -6,7 +6,6 @@ from notebook.base.handlers import APIHandler, json_errors from jupyter_telemetry.eventlog import EventLog - class EventLoggingHandler(APIHandler): """ A handler that receives and stores telemetry data from the client. @@ -32,8 +31,8 @@ def post(self, *args, **kwargs): # Profile, may need to move to a background thread if this is problematic try: self.eventlog.record_event(schema_name, version, event) - except: - raise web.HTTPError(500, "Event could not be validated.") + except Exception as e: + raise web.HTTPError(400, e) self.set_status(204) self.finish() From 2ce7c54efa056604c52c1bf725464ead6504b4ed Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 1 Oct 2019 15:55:54 -0700 Subject: [PATCH 15/49] normalize path in emitted event --- jupyter_server/services/contents/handlers.py | 2 +- notebook/tests/test_eventlog.py | 49 -------------------- 2 files changed, 1 insertion(+), 50 deletions(-) delete mode 100644 notebook/tests/test_eventlog.py diff --git a/jupyter_server/services/contents/handlers.py b/jupyter_server/services/contents/handlers.py index 85065b21cb..c2ba749c85 100644 --- a/jupyter_server/services/contents/handlers.py +++ b/jupyter_server/services/contents/handlers.py @@ -275,7 +275,7 @@ async def delete(self, path=''): self.finish() self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, - { 'action': 'delete', 'path': path } + { 'action': 'delete', 'path': path.lstrip(os.path.sep) } ) diff --git a/notebook/tests/test_eventlog.py b/notebook/tests/test_eventlog.py deleted file mode 100644 index 994181b73e..0000000000 --- a/notebook/tests/test_eventlog.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import re -import jsonschema -from ruamel.yaml import YAML -from notebook.notebookapp import NotebookApp -from notebook.utils import eventlogging_schema_fqn, get_schema_files -from unittest import TestCase - -yaml = YAML(typ='safe') - - -class RegisteredSchemasTestCase(TestCase): - - def test_eventlogging_schema_fqn(self): - self.assertEqual( - eventlogging_schema_fqn('test'), - 'eventlogging.jupyter.org/notebook/test' - ) - - def test_valid_schemas(self): - """ - All schemas must be valid json schemas - """ - for schema_file in get_schema_files(): - with open(schema_file) as f: - jsonschema.Draft7Validator.check_schema(yaml.load(f)) - - def test_schema_conventions(self): - """ - Test schema naming convention for this repo. - - 1. All schemas should be under event-schamas/{name}/v{version}.yaml - 2. Schema id should be eventlogging.jupyter.org/notebook/{name} - 3. Schema version should match version in file - """ - for schema_file in get_schema_files(): - filename = os.path.basename(schema_file) - match = re.match('v(\d+)\.yaml', filename) - # All schema locations must match the following pattern - # schema-name/v(version).yaml - self.assertIsNotNone(match) - - with open(schema_file) as f: - schema = yaml.load(f) - - self.assertEqual(schema['$id'], eventlogging_schema_fqn( - os.path.basename(os.path.dirname(schema_file)) - )) - self.assertEqual(schema['version'], int(match.groups()[0])) \ No newline at end of file From 5794d31efb62a1b1a1c5ed1a6e3816a25d223849 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 19 May 2020 15:31:03 -0700 Subject: [PATCH 16/49] initial tests --- jupyter_server/event-schemas/README.md | 19 --------- .../contentsmanager-actions.json | 33 ---------------- .../contentsmanager-actions/v1.json | 30 -------------- .../{ => contentsmanager-actions}/v1.yaml | 8 +++- jupyter_server/event-schemas/generate-json.py | 39 ------------------- jupyter_server/services/contents/handlers.py | 36 ++++++++--------- setup.py | 3 +- tests/test_eventlog.py | 4 ++ 8 files changed, 29 insertions(+), 143 deletions(-) delete mode 100644 jupyter_server/event-schemas/README.md delete mode 100644 jupyter_server/event-schemas/contentsmanager-actions.json delete mode 100644 jupyter_server/event-schemas/contentsmanager-actions/v1.json rename jupyter_server/event-schemas/{ => contentsmanager-actions}/v1.yaml (94%) delete mode 100755 jupyter_server/event-schemas/generate-json.py create mode 100644 tests/test_eventlog.py diff --git a/jupyter_server/event-schemas/README.md b/jupyter_server/event-schemas/README.md deleted file mode 100644 index 541a9b0398..0000000000 --- a/jupyter_server/event-schemas/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Event Schemas - -## Generating .json files - -Event Schemas are written in a human readable `.yaml` format. -This is primarily to get multi-line strings in our descriptions, -as documentation is very important. - -Every time you modify a `.yaml` file, you should run the following -commands. - -```bash -./generate-json.py -``` - -This needs the `ruamel.yaml` python package installed. - -Hopefully, this is extremely temporary, and we can just use YAML -with jupyter_telemetry. \ No newline at end of file diff --git a/jupyter_server/event-schemas/contentsmanager-actions.json b/jupyter_server/event-schemas/contentsmanager-actions.json deleted file mode 100644 index 065f1d5c2f..0000000000 --- a/jupyter_server/event-schemas/contentsmanager-actions.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$id": "eventlogging.jupyter.org/notebook/contentsmanager-actions", - "version": 1, - "title": "Contents Manager activities", - "description": "Record actions on files via the ContentsManager REST API.\n\nThe notebook ContentsManager REST API is used by all frontends to retreive,\nsave, list, delete and perform other actions on notebooks, directories,\nand other files through the UI. This is pluggable - the default acts on\nthe file system, but can be replaced with a different ContentsManager\nimplementation - to work on S3, Postgres, other object stores, etc.\nThe events get recorded regardless of the ContentsManager implementation\nbeing used.\n\nLimitations:\n\n1. This does not record all filesystem access, just the ones that happen\n explicitly via the notebook server's REST API. Users can (and often do)\n trivially access the filesystem in many other ways (such as `open()` calls\n in their code), so this is usually never a complete record.\n2. As with all events recorded by the notebook server, users most likely\n have the ability to modify the code of the notebook server. Unless other\n security measures are in place, these events should be treated as user\n controlled and not used in high security areas.\n3. Events are only recorded when an action succeeds.\n", - "type": "object", - "required": [ - "action", - "path" - ], - "properties": { - "action": { - "enum": [ - "get", - "create", - "save", - "upload", - "rename", - "copy", - "delete" - ], - "description": "Action performed by the ContentsManager API.\n\nThis is a required field.\n\nPossible values:\n\n1. get\n Get contents of a particular file, or list contents of a directory.\n\n2. create\n Create a new directory or file at 'path'. Currently, name of the\n file or directory is auto generated by the ContentsManager implementation.\n\n3. save\n Save a file at path with contents from the client\n\n4. upload\n Upload a file at given path with contents from the client\n\n5. rename\n Rename a file or directory from value in source_path to\n value in path.\n\n5. copy\n Copy a file or directory from value in source_path to\n value in path.\n\n6. delete\n Delete a file or empty directory at given path\n" - }, - "path": { - "type": "string", - "description": "Logical path on which the operation was performed.\n\nThis is a required field.\n" - }, - "source_path": { - "type": "string", - "description": "Source path of an operation when action is 'copy' or 'rename'" - } - } -} \ No newline at end of file diff --git a/jupyter_server/event-schemas/contentsmanager-actions/v1.json b/jupyter_server/event-schemas/contentsmanager-actions/v1.json deleted file mode 100644 index 5da6d68b88..0000000000 --- a/jupyter_server/event-schemas/contentsmanager-actions/v1.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$id": "eventlogging.jupyter.org/notebook/contentsmanager-actions", - "version": 1, - "title": "Contents Manager activities", - "description": "Notebook Server emits this event whenever a contentsmanager action happens", - "type": "object", - "required": ["action", "path"], - "properties": { - "action": { - "enum": [ - "get", - "create", - "save", - "upload", - "rename", - "create", - "copy" - ], - "description": "Action performed by contents manager" - }, - "path": { - "type": "string", - "description": "Logical path the action was performed in" - }, - "source_path": { - "type": "string", - "description": "If action is 'copy', this specifies the source path" - } - } -} \ No newline at end of file diff --git a/jupyter_server/event-schemas/v1.yaml b/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml similarity index 94% rename from jupyter_server/event-schemas/v1.yaml rename to jupyter_server/event-schemas/contentsmanager-actions/v1.yaml index 3d7e8f2fe9..31a5f293a9 100644 --- a/jupyter_server/event-schemas/v1.yaml +++ b/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml @@ -1,6 +1,7 @@ "$id": eventlogging.jupyter.org/notebook/contentsmanager-actions version: 1 title: Contents Manager activities +personal-data: true description: | Record actions on files via the ContentsManager REST API. @@ -37,6 +38,7 @@ properties: - rename - copy - delete + category: unrestricted description: | Action performed by the ContentsManager API. @@ -60,20 +62,22 @@ properties: 5. rename Rename a file or directory from value in source_path to value in path. - + 5. copy Copy a file or directory from value in source_path to value in path. - + 6. delete Delete a file or empty directory at given path path: + category: personally-identifiable-information type: string description: | Logical path on which the operation was performed. This is a required field. source_path: + category: personally-identifiable-information type: string description: | Source path of an operation when action is 'copy' or 'rename' \ No newline at end of file diff --git a/jupyter_server/event-schemas/generate-json.py b/jupyter_server/event-schemas/generate-json.py deleted file mode 100755 index a39fa0610b..0000000000 --- a/jupyter_server/event-schemas/generate-json.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -import jsonschema -from ruamel.yaml import YAML - -from jupyter_telemetry.eventlog import EventLog - -yaml = YAML(typ='safe') - -def main(): - argparser = argparse.ArgumentParser() - argparser.add_argument( - 'directory', - help='Directory with Schema .yaml files' - ) - - args = argparser.parse_args() - - el = EventLog() - for dirname, _, files in os.walk(args.directory): - for file in files: - if not file.endswith('.yaml'): - continue - yaml_path = os.path.join(dirname, file) - print('Processing', yaml_path) - with open(yaml_path) as f: - schema = yaml.load(f) - - # validate schema - el.register_schema(schema) - - json_path = os.path.join(dirname, os.path.splitext(file)[0] + '.json') - with open(json_path, 'w') as f: - json.dump(schema, f, indent=4) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/jupyter_server/services/contents/handlers.py b/jupyter_server/services/contents/handlers.py index c2ba749c85..9b7802ff2a 100644 --- a/jupyter_server/services/contents/handlers.py +++ b/jupyter_server/services/contents/handlers.py @@ -5,7 +5,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. - +import os import json from tornado import web @@ -115,12 +115,12 @@ async def get(self, path=''): path=path, type=type, format=format, content=content, )) validate_model(model, expect_content=content) - self._finish_model(model, location=False) self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'get', 'path': model['path'] } ) + self._finish_model(model, location=False) @web.authenticated async def patch(self, path=''): @@ -130,21 +130,20 @@ async def patch(self, path=''): if model is None: raise web.HTTPError(400, u'JSON body missing') self.log.info(model) - model = yield maybe_future(cm.update(model, path)) + model = await ensure_async(cm.update(model, path)) validate_model(model, expect_content=False) - self._finish_model(model) - self.log.info(model) self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, - { - 'action': 'rename', - 'path': model['path'], - 'source_path': path.lstrip(os.path.sep) + { + 'action': 'rename', + 'path': model['path'], + 'source_path': path.lstrip(os.path.sep) } ) + self._finish_model(model) + - @gen.coroutine async def _copy(self, copy_from, copy_to=None): """Copy a file, optionally specifying a target directory.""" self.log.info(u"Copying {copy_from} to {copy_to}".format( @@ -154,16 +153,16 @@ async def _copy(self, copy_from, copy_to=None): model = self.contents_manager.copy(copy_from, copy_to) self.set_status(201) validate_model(model, expect_content=False) - self._finish_model(model) self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, { - 'action': 'copy', - 'path': model['path'], - 'source_path': copy_from.lstrip(os.path.sep) + 'action': 'copy', + 'path': model['path'], + 'source_path': copy_from.lstrip(os.path.sep) } ) + self._finish_model(model) async def _upload(self, model, path): """Handle upload of a new file to path""" @@ -171,12 +170,12 @@ async def _upload(self, model, path): model = self.contents_manager.new(model, path) self.set_status(201) validate_model(model, expect_content=False) - self._finish_model(model) self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'upload', 'path': model['path'] } ) + self._finish_model(model) async def _new_untitled(self, path, type='', ext=''): """Create a new, empty untitled entity""" @@ -184,12 +183,12 @@ async def _new_untitled(self, path, type='', ext=''): model = self.contents_manager.new_untitled(path=path, type=type, ext=ext) self.set_status(201) validate_model(model, expect_content=False) - self._finish_model(model) self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, # Set path to path of created object, not directory it was created in { 'action': 'create', 'path': model['path'] } ) + self._finish_model(model) async def _save(self, model, path): """Save an existing file.""" @@ -198,12 +197,12 @@ async def _save(self, model, path): self.log.info(u"Saving file at %s", path) model = self.contents_manager.save(model, path) validate_model(model, expect_content=False) - self._finish_model(model) self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'save', 'path': model['path'] } ) + self._finish_model(model) @web.authenticated async def post(self, path=''): @@ -272,12 +271,11 @@ async def delete(self, path=''): self.log.warning('delete %s', path) cm.delete(path) self.set_status(204) - self.finish() self.eventlog.record_event( eventlogging_schema_fqn('contentsmanager-actions'), 1, { 'action': 'delete', 'path': path.lstrip(os.path.sep) } ) - + self.finish() class CheckpointsHandler(APIHandler): diff --git a/setup.py b/setup.py index e402e71669..3ccc6b5fcb 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,8 @@ 'terminado>=0.8.3', 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", - 'jupyter_telemetry' + # Install teh + 'git+https://github.com/Zsailer/telemetry.git@personal-data' ], extras_require = { 'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters', diff --git a/tests/test_eventlog.py b/tests/test_eventlog.py new file mode 100644 index 0000000000..1f7b587327 --- /dev/null +++ b/tests/test_eventlog.py @@ -0,0 +1,4 @@ + + +def test_eventlog(serverapp): + pass \ No newline at end of file From 7c9d3d51f4e4b8f03d984819ddbee6f48202d2cf Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 19 May 2020 16:05:05 -0700 Subject: [PATCH 17/49] add initial telemetry docs --- .gitignore | 10 +---- docs/doc-requirements.txt | 3 +- docs/source/conf.py | 11 ++++-- docs/source/operators/index.rst | 3 +- docs/source/operators/telemetry.rst | 61 +++++++++++++++++++++++++++++ docs/source/other/full-config.rst | 2 +- jupyter_server/utils.py | 2 +- setup.py | 2 +- 8 files changed, 77 insertions(+), 17 deletions(-) create mode 100644 docs/source/operators/telemetry.rst diff --git a/.gitignore b/.gitignore index a69d2eeee6..0ab0672302 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ MANIFEST +docs/source/operators/events build dist _build @@ -7,15 +8,6 @@ docs/source/api/generated docs/source/config.rst docs/gh-pages docs/source/events -notebook/i18n/*/LC_MESSAGES/*.mo -notebook/i18n/*/LC_MESSAGES/nbjs.json -notebook/static/components -notebook/static/style/*.min.css* -notebook/static/*/js/built/ -notebook/static/*/built/ -notebook/static/built/ -notebook/static/*/js/main.min.js* -notebook/static/lab/*bundle.js node_modules *.py[co] __pycache__ diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt index 48b3eda1d0..4167aabf6d 100644 --- a/docs/doc-requirements.txt +++ b/docs/doc-requirements.txt @@ -8,4 +8,5 @@ prometheus_client sphinxcontrib_github_alt sphinxcontrib-openapi sphinxemoji -git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master \ No newline at end of file +git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master +jupyter_telemetry_sphinxext \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 41b089cc07..4add156c81 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -70,7 +70,7 @@ 'sphinx.ext.mathjax', 'IPython.sphinxext.ipython_console_highlighting', 'sphinxcontrib_github_alt', - 'sphinx-jsonschema' + 'jupyter_telemetry_sphinxext' ] # Add any paths that contain templates here, relative to this directory. @@ -209,9 +209,9 @@ html_context = { 'css_files': [ - '_static/theme_overrides.css', # override wide tables in RTD theme + '_static/theme_overrides.css', # override wide tables in RTD theme ], - } +} # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -376,3 +376,8 @@ # import before any doc is built, so _ is guaranteed to be injected import jupyter_server.transutils + +# Jupyter telemetry configuration values. +jupyter_telemetry_schema_source = "../jupyter_server/event-schemas" # Path is relative to conf.py +jupyter_telemetry_schema_output = "source/operators/events" # Path is relative to conf.py +jupyter_telemetry_index_title = "Telemetry Event Schemas" # Title of the index page that lists all found schemas. \ No newline at end of file diff --git a/docs/source/operators/index.rst b/docs/source/operators/index.rst index a654be1a0c..a6d2e212fd 100644 --- a/docs/source/operators/index.rst +++ b/docs/source/operators/index.rst @@ -12,4 +12,5 @@ These pages are targeted at people using, configuring, and/or deploying multiple configuring-extensions migrate-from-nbserver public-server - security \ No newline at end of file + security + telemetry \ No newline at end of file diff --git a/docs/source/operators/telemetry.rst b/docs/source/operators/telemetry.rst new file mode 100644 index 0000000000..2c94e99a7c --- /dev/null +++ b/docs/source/operators/telemetry.rst @@ -0,0 +1,61 @@ +Telemetry and Eventlogging +========================== + +Jupyter Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that the Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. + + +.. _logging: https://docs.python.org/3/library/logging.html +.. _`Telemetry System`: https://github.com/jupyter/telemetry +.. _`JSON schemas`: https://json-schema.org/ + +Emitting Server Events +---------------------- + +Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. + +To begin recording events, you'll need to set two configurations: + + 1. ``handlers``: tells the EventLog *where* to route your events. This trait is a list of Python logging handlers that route events to + 2. ``allows_schemas``: tells the EventLog *which* events should be recorded. No events are emitted by default; all recorded events must be listed here. + +Here's a basic example for emitting events from the `contents` service: + +.. code-block:: + + import logging + + c.EventLog.handlers = [ + logging.FileHandler('event.log'), + ] + + c.EventLog.allowed_schemas = [ + 'hub.jupyter.org/server-action' + ] + +The output is a file, ``"event.log"``, with events recorded as JSON data. + +`eventlog` endpoint +------------------- + +The Notebook Server provides a public REST endpoint for external applications to validate and log events +through the Server's Event Log. + +To log events, send a `POST` request to the `/api/eventlog` endpoint. The body of the request should be a +JSON blog and is required to have the follow keys: + + 1. `'schema'` : the event's schema ID. + 2. `'version'` : the version of the event's schema. + 3. `'event'` : the event data in JSON format. + +Events that are validated by this endpoint must have their schema listed in the `allowed_schemas` trait listed above. + +.. _below: + + +Server Event schemas +-------------------- + +.. toctree:: + :maxdepth: 2 + + events/index diff --git a/docs/source/other/full-config.rst b/docs/source/other/full-config.rst index f7f0cab4ba..70852ea40f 100644 --- a/docs/source/other/full-config.rst +++ b/docs/source/other/full-config.rst @@ -897,7 +897,7 @@ FileContentsManager.root_dir : Unicode No description -NotebookNotary.algorithm : 'md5'|'sha3_384'|'sha3_512'|'sha256'|'sha1'|'blake2s'|'sha3_256'|'sha3_224'|'sha384'|'sha512'|'blake2b'|'sha224' +NotebookNotary.algorithm : 'sha1'|'sha3_224'|'blake2s'|'sha384'|'sha224'|'sha3_256'|'sha3_384'|'sha3_512'|'sha512'|'sha256'|'md5'|'blake2b' Default: ``'sha256'`` The hashing algorithm used to sign notebooks. diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 8fc6e89479..ec44e13b75 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -448,7 +448,7 @@ def eventlogging_schema_fqn(name): Matches convention for this particular repo """ - return 'eventlogging.jupyter.org/notebook/{}'.format(name) + return 'eventlogging.jupyter.org/jupyter_server/{}'.format(name) def get_schema_files(): diff --git a/setup.py b/setup.py index 3ccc6b5fcb..bfedef2528 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,7 @@ 'terminado>=0.8.3', 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", - # Install teh + # Install the working branch of telemetry. 'git+https://github.com/Zsailer/telemetry.git@personal-data' ], extras_require = { From ef8573d82d407ddd56f3ebe7ab1d37729e685117 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 19 May 2020 16:16:31 -0700 Subject: [PATCH 18/49] fix jupyter_telemetry dependency --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bfedef2528..9f3932fac7 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", # Install the working branch of telemetry. - 'git+https://github.com/Zsailer/telemetry.git@personal-data' + 'jupyter_telemetry@git+https://github.com/Zsailer/telemetry.git@master' ], extras_require = { 'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters', From ea9e352d2906ace72c32158b3abb8976c5637a54 Mon Sep 17 00:00:00 2001 From: Zsailer Date: Tue, 19 May 2020 16:18:22 -0700 Subject: [PATCH 19/49] point telemetry at correct dev branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9f3932fac7..6123e8f64f 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", # Install the working branch of telemetry. - 'jupyter_telemetry@git+https://github.com/Zsailer/telemetry.git@master' + 'jupyter_telemetry@git+https://github.com/Zsailer/telemetry.git@personal-data' ], extras_require = { 'test': ['nose', 'coverage', 'requests', 'nose_warnings_filters', From b06f7d6f324ea8a29b2a351c631d3109be0a8971 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 20 Oct 2020 15:23:47 +0800 Subject: [PATCH 20/49] add tests for eventlog --- .../contentsmanager-actions/v1.yaml | 4 +- tests/test_eventlog.py | 44 ++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml b/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml index 31a5f293a9..d95de1d282 100644 --- a/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml +++ b/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml @@ -1,4 +1,4 @@ -"$id": eventlogging.jupyter.org/notebook/contentsmanager-actions +"$id": eventlogging.jupyter.org/jupyter_server/contentsmanager-actions version: 1 title: Contents Manager activities personal-data: true @@ -80,4 +80,4 @@ properties: category: personally-identifiable-information type: string description: | - Source path of an operation when action is 'copy' or 'rename' \ No newline at end of file + Source path of an operation when action is 'copy' or 'rename' diff --git a/tests/test_eventlog.py b/tests/test_eventlog.py index 1f7b587327..cef3915d43 100644 --- a/tests/test_eventlog.py +++ b/tests/test_eventlog.py @@ -1,4 +1,44 @@ +import io +import json +import logging +import jsonschema +import pytest +from traitlets.config import Config -def test_eventlog(serverapp): - pass \ No newline at end of file +from jupyter_server.utils import eventlogging_schema_fqn +from .services.contents.test_api import contents, contents_dir, dirs + + +@pytest.fixture +def eventlog_sink(configurable_serverapp): + """Return eventlog and sink objects""" + sink = io.StringIO() + handler = logging.StreamHandler(sink) + + cfg = Config() + cfg.EventLog.handlers = [handler] + serverapp = configurable_serverapp(config=cfg) + yield serverapp, sink + + +@pytest.mark.parametrize('path, name', dirs) +async def test_eventlog_list_notebooks(eventlog_sink, fetch, contents, path, name): + schema, version = (eventlogging_schema_fqn('contentsmanager-actions'), 1) + serverapp, sink = eventlog_sink + serverapp.eventlog.allowed_schemas = [schema] + + r = await fetch( + 'api', + 'contents', + path, + method='GET', + ) + assert r.code == 200 + + output = sink.getvalue() + assert output + data = json.loads(output) + jsonschema.validate(data, serverapp.eventlog.schemas[(schema, version)]) + expected = {'action': 'get', 'path': path} + assert expected.items() <= data.items() From 73023961ed15975c4ddaf60ad0b87482e20bda34 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 17 Dec 2020 13:32:06 +0800 Subject: [PATCH 21/49] Use correct fixture names --- tests/test_eventlog.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_eventlog.py b/tests/test_eventlog.py index cef3915d43..2462262c26 100644 --- a/tests/test_eventlog.py +++ b/tests/test_eventlog.py @@ -11,24 +11,24 @@ @pytest.fixture -def eventlog_sink(configurable_serverapp): +def eventlog_sink(jp_configurable_serverapp): """Return eventlog and sink objects""" sink = io.StringIO() handler = logging.StreamHandler(sink) cfg = Config() cfg.EventLog.handlers = [handler] - serverapp = configurable_serverapp(config=cfg) + serverapp = jp_configurable_serverapp(config=cfg) yield serverapp, sink @pytest.mark.parametrize('path, name', dirs) -async def test_eventlog_list_notebooks(eventlog_sink, fetch, contents, path, name): +async def test_eventlog_list_notebooks(eventlog_sink, jp_fetch, contents, path, name): schema, version = (eventlogging_schema_fqn('contentsmanager-actions'), 1) serverapp, sink = eventlog_sink serverapp.eventlog.allowed_schemas = [schema] - r = await fetch( + r = await jp_fetch( 'api', 'contents', path, From 0ac16dc2a74ccfae15394cb0a6900f00378b186c Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Wed, 10 Mar 2021 13:28:34 +0800 Subject: [PATCH 22/49] Fix import --- jupyter_server/services/eventlog/handlers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py index 0c9b69815f..b7e36ac8c3 100644 --- a/jupyter_server/services/eventlog/handlers.py +++ b/jupyter_server/services/eventlog/handlers.py @@ -2,8 +2,7 @@ from tornado import web -from notebook.utils import url_path_join -from notebook.base.handlers import APIHandler, json_errors +from jupyter_server.base.handlers import APIHandler, json_errors from jupyter_telemetry.eventlog import EventLog class EventLoggingHandler(APIHandler): @@ -18,25 +17,25 @@ def post(self, *args, **kwargs): raw_event = json.loads(self.request.body.strip().decode()) except Exception as e: raise web.HTTPError(400, str(e)) - + required_fields = {'schema', 'version', 'event'} for rf in required_fields: if rf not in raw_event: raise web.HTTPError(400, '{} is a required field'.format(rf)) - schema_name = raw_event['schema'] + schema_name = raw_event['schema'] version = raw_event['version'] event = raw_event['event'] - + # Profile, may need to move to a background thread if this is problematic - try: + try: self.eventlog.record_event(schema_name, version, event) except Exception as e: raise web.HTTPError(400, e) - + self.set_status(204) self.finish() default_handlers = [ (r"/api/eventlog", EventLoggingHandler), -] \ No newline at end of file +] From 7c81b238d494d2fbafde0089acd94e66379fcb6a Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 18 Mar 2021 23:19:59 +0800 Subject: [PATCH 23/49] Remove redundant call --- jupyter_server/serverapp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index a1e95a2496..78f1fce153 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1840,7 +1840,6 @@ def initialize(self, argv=None, find_extensions=True, new_httpserver=True, start self.init_resources() self.init_configurables() self.init_components() - self.init_eventlog() self.init_webapp() if new_httpserver: self.init_httpserver() From b8ca48406da2d21f8d9c25d0d6c40783abaa87eb Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 20 Mar 2021 23:17:13 +0800 Subject: [PATCH 24/49] Update telemetry --- .../contentsmanager-actions/v1.yaml | 109 +++++++++--------- jupyter_server/tests/test_eventlog.py | 9 +- setup.py | 2 +- 3 files changed, 65 insertions(+), 55 deletions(-) diff --git a/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml b/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml index d95de1d282..0f89d7ae42 100644 --- a/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml +++ b/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml @@ -3,81 +3,84 @@ version: 1 title: Contents Manager activities personal-data: true description: | - Record actions on files via the ContentsManager REST API. + Record actions on files via the ContentsManager REST API. - The notebook ContentsManager REST API is used by all frontends to retreive, - save, list, delete and perform other actions on notebooks, directories, - and other files through the UI. This is pluggable - the default acts on - the file system, but can be replaced with a different ContentsManager - implementation - to work on S3, Postgres, other object stores, etc. - The events get recorded regardless of the ContentsManager implementation - being used. + The notebook ContentsManager REST API is used by all frontends to retreive, + save, list, delete and perform other actions on notebooks, directories, + and other files through the UI. This is pluggable - the default acts on + the file system, but can be replaced with a different ContentsManager + implementation - to work on S3, Postgres, other object stores, etc. + The events get recorded regardless of the ContentsManager implementation + being used. - Limitations: + Limitations: - 1. This does not record all filesystem access, just the ones that happen - explicitly via the notebook server's REST API. Users can (and often do) - trivially access the filesystem in many other ways (such as `open()` calls - in their code), so this is usually never a complete record. - 2. As with all events recorded by the notebook server, users most likely - have the ability to modify the code of the notebook server. Unless other - security measures are in place, these events should be treated as user - controlled and not used in high security areas. - 3. Events are only recorded when an action succeeds. + 1. This does not record all filesystem access, just the ones that happen + explicitly via the notebook server's REST API. Users can (and often do) + trivially access the filesystem in many other ways (such as `open()` calls + in their code), so this is usually never a complete record. + 2. As with all events recorded by the notebook server, users most likely + have the ability to modify the code of the notebook server. Unless other + security measures are in place, these events should be treated as user + controlled and not used in high security areas. + 3. Events are only recorded when an action succeeds. type: object required: -- action -- path + - action + - path properties: action: enum: - - get - - create - - save - - upload - - rename - - copy - - delete - category: unrestricted + - get + - create + - save + - upload + - rename + - copy + - delete + categories: + - category.jupyter.org/unrestricted description: | - Action performed by the ContentsManager API. + Action performed by the ContentsManager API. - This is a required field. + This is a required field. - Possible values: + Possible values: - 1. get - Get contents of a particular file, or list contents of a directory. + 1. get + Get contents of a particular file, or list contents of a directory. - 2. create - Create a new directory or file at 'path'. Currently, name of the - file or directory is auto generated by the ContentsManager implementation. + 2. create + Create a new directory or file at 'path'. Currently, name of the + file or directory is auto generated by the ContentsManager implementation. - 3. save - Save a file at path with contents from the client + 3. save + Save a file at path with contents from the client - 4. upload - Upload a file at given path with contents from the client + 4. upload + Upload a file at given path with contents from the client - 5. rename - Rename a file or directory from value in source_path to - value in path. + 5. rename + Rename a file or directory from value in source_path to + value in path. - 5. copy - Copy a file or directory from value in source_path to - value in path. + 5. copy + Copy a file or directory from value in source_path to + value in path. - 6. delete - Delete a file or empty directory at given path + 6. delete + Delete a file or empty directory at given path path: - category: personally-identifiable-information + categories: + - category.jupyter.org/user-identifiable-information type: string description: | - Logical path on which the operation was performed. + Logical path on which the operation was performed. - This is a required field. + This is a required field. source_path: - category: personally-identifiable-information + categories: + - category.jupyter.org/user-identifiable-information type: string description: | - Source path of an operation when action is 'copy' or 'rename' + Source path of an operation when action is 'copy' or 'rename' diff --git a/jupyter_server/tests/test_eventlog.py b/jupyter_server/tests/test_eventlog.py index 2462262c26..37b0d6d243 100644 --- a/jupyter_server/tests/test_eventlog.py +++ b/jupyter_server/tests/test_eventlog.py @@ -26,7 +26,14 @@ def eventlog_sink(jp_configurable_serverapp): async def test_eventlog_list_notebooks(eventlog_sink, jp_fetch, contents, path, name): schema, version = (eventlogging_schema_fqn('contentsmanager-actions'), 1) serverapp, sink = eventlog_sink - serverapp.eventlog.allowed_schemas = [schema] + serverapp.eventlog.allowed_schemas = { + serverapp.eventlog.schemas[(schema, version)]['$id']: { + 'allowed_categories': [ + 'category.jupyter.org/unrestricted', + 'category.jupyter.org/user-identifiable-information' + ] + } + } r = await jp_fetch( 'api', diff --git a/setup.py b/setup.py index fcb10cd713..ffe9c25939 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ "pywin32>=1.0 ; sys_platform == 'win32'", 'anyio>=2.0.2', # Install the working branch of telemetry. - 'jupyter_telemetry@git+https://github.com/Zsailer/telemetry.git@personal-data' + 'jupyter_telemetry@git+https://github.com/jupyter/telemetry.git@d2c19ffa03fde7d1903759de4a265ef711b333ef' ], extras_require = { 'test': ['coverage', 'requests', From ac452cd7b5bd7ddecc1681ed5ab21490ec0d28e0 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 23 Mar 2021 02:54:57 +0800 Subject: [PATCH 25/49] Add note about security --- docs/source/eventlog.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/eventlog.rst b/docs/source/eventlog.rst index 7229717f69..6308f672d0 100644 --- a/docs/source/eventlog.rst +++ b/docs/source/eventlog.rst @@ -8,14 +8,17 @@ The Notebook Server can be configured to record structured events from a running .. _`Telemetry System`: https://github.com/jupyter/telemetry .. _`JSON schemas`: https://json-schema.org/ +.. warning:: + Do NOT rely on this feature for security or auditing purposes. Neither `server <#emitting-server-events>`_ nor `client <#eventlog-endpoint>`_ events are protected against meddling. For server events, those who have access to the environment can change the server code to emit whatever they want. The same goes for client events where nothing prevents users from sending spurious data to the `eventlog` endpoint. + Emitting Server Events ---------------------- -Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. +Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. To begin recording events, you'll need to set two configurations: - 1. ``handlers``: tells the EventLog *where* to route your events. This trait is a list of Python logging handlers that route events to + 1. ``handlers``: tells the EventLog *where* to route your events. This trait is a list of Python logging handlers that route events to 2. ``allows_schemas``: tells the EventLog *which* events should be recorded. No events are emitted by default; all recorded events must be listed here. Here's a basic example for emitting events from the `contents` service: @@ -38,9 +41,9 @@ The output is a file, ``"event.log"``, with events recorded as JSON data. ------------------- The Notebook Server provides a public REST endpoint for external applications to validate and log events -through the Server's Event Log. +through the Server's Event Log. -To log events, send a `POST` request to the `/api/eventlog` endpoint. The body of the request should be a +To log events, send a `POST` request to the `/api/eventlog` endpoint. The body of the request should be a JSON blog and is required to have the follow keys: 1. `'schema'` : the event's schema ID. From 70f92754f3abd8607f1513c5e10ac2802d743240 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Wed, 31 Mar 2021 15:25:43 +0800 Subject: [PATCH 26/49] Register client telemetry schemas using entry_points --- jupyter_server/serverapp.py | 9 +++++++- jupyter_server/utils.py | 46 +++++++++++++++++++++++++++++++++++++ setup.py | 2 ++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 78f1fce153..fa368e1d00 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -33,6 +33,7 @@ import urllib import inspect import pathlib +import importlib_resources from ruamel.yaml import YAML from glob import glob @@ -114,7 +115,8 @@ url_escape, urljoin, pathname2url, - get_schema_files + get_schema_files, + get_client_schema_files ) from jupyter_server.extension.serverextension import ServerExtensionApp @@ -1790,6 +1792,11 @@ def init_eventlog(self): for file_path in get_schema_files(): self.eventlog.register_schema_file(file_path) + for file in get_client_schema_files(): + with importlib_resources.as_file(file) as f: + self.eventlog.register_schema_file_object(f) + + @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True, starter_extension=None): """Initialize the Server application class, configurables, web application, and http server. diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 31e4446c7c..1a26673eff 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -10,7 +10,10 @@ import inspect import os import sys +import importlib_resources from distutils.version import LooseVersion +from importlib.metadata import entry_points +from itertools import chain from urllib.parse import quote, unquote, urlparse, urljoin from urllib.request import pathname2url @@ -248,3 +251,46 @@ def get_schema_files(): if file.endswith('.yaml'): file_path = os.path.join(dirname, file) yield file_path + + +JUPYTER_TELEMETRY_ENTRY_POINT = 'jupyter_telemetry' + + +def get_client_schema_files(): + telemetry_entry_points = entry_points().get(JUPYTER_TELEMETRY_ENTRY_POINT, []) + + dirs = (_safe_entry_point_load(ep) for ep in telemetry_entry_points) + dirs = chain.from_iterable(d for d in dirs if d is not None) + dirs = (_safe_load_resource(d) for d in dirs) + + files = chain.from_iterable(d.iterdir() for d in dirs if d is not None) + + return (f for f in files if f.suffix in ('.json', '.yaml')) + + +def _is_iterable(x): + try: + iter(x) + return True + except TypeError: + return False + + +def _safe_entry_point_load(ep): + try: + v = ep.load() + if isinstance(v, str): + return [v] + elif _is_iterable(v): + return v + return None + except: + return None + + +def _safe_load_resource(x): + try: + fs = importlib_resources.files(x) + return fs if fs.exists() and fs.is_dir() else None + except ModuleNotFoundError: + return None diff --git a/setup.py b/setup.py index ffe9c25939..0960122708 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,8 @@ 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", 'anyio>=2.0.2', + "importlib-metadata ; python_version < '3.8'", + "importlib-resources ; python_version < '3.9'", # Install the working branch of telemetry. 'jupyter_telemetry@git+https://github.com/jupyter/telemetry.git@d2c19ffa03fde7d1903759de4a265ef711b333ef' ], From 7f50c850bfe0310221269512e5043958977ac07c Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 6 Apr 2021 15:42:57 +0800 Subject: [PATCH 27/49] Add working telemetry commit for testing --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0960122708..b0b29e8bc2 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ "importlib-metadata ; python_version < '3.8'", "importlib-resources ; python_version < '3.9'", # Install the working branch of telemetry. - 'jupyter_telemetry@git+https://github.com/jupyter/telemetry.git@d2c19ffa03fde7d1903759de4a265ef711b333ef' + 'jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@a4fbf987da063f9e0b337abd21803c968ac64f27' ], extras_require = { 'test': ['coverage', 'requests', From 99439b617163e4473ba702c6a90db5fdb8825369 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 6 Apr 2021 16:16:54 +0800 Subject: [PATCH 28/49] Use backported importlib_metadata --- jupyter_server/utils.py | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 1a26673eff..ef8546a5b3 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -12,7 +12,7 @@ import sys import importlib_resources from distutils.version import LooseVersion -from importlib.metadata import entry_points +from importlib_metadata import entry_points from itertools import chain from urllib.parse import quote, unquote, urlparse, urljoin diff --git a/setup.py b/setup.py index b0b29e8bc2..2598423ba4 100644 --- a/setup.py +++ b/setup.py @@ -53,8 +53,8 @@ 'prometheus_client', "pywin32>=1.0 ; sys_platform == 'win32'", 'anyio>=2.0.2', - "importlib-metadata ; python_version < '3.8'", - "importlib-resources ; python_version < '3.9'", + 'importlib-metadata', + 'importlib-resources', # Install the working branch of telemetry. 'jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@a4fbf987da063f9e0b337abd21803c968ac64f27' ], From 8e69ab077145dcb3005141089fff350606514cf4 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 23 Apr 2021 14:46:01 +0800 Subject: [PATCH 29/49] Ignore errors while registering client events --- jupyter_server/serverapp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 98d209de34..f8fa0719ad 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1830,7 +1830,10 @@ def init_eventlog(self): for file in get_client_schema_files(): with importlib_resources.as_file(file) as f: - self.eventlog.register_schema_file_object(f) + try: + self.eventlog.register_schema_file_object(f) + except: + pass @catch_config_error From e2db0ad38b19a7aada415cf2596b09ba8cb2ee03 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 23 Apr 2021 18:30:35 +0800 Subject: [PATCH 30/49] Add client eventlog to list services --- jupyter_server/serverapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index f8fa0719ad..ca579f24fd 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -144,6 +144,7 @@ config=['jupyter_server.services.config.handlers'], contents=['jupyter_server.services.contents.handlers'], files=['jupyter_server.files.handlers'], + eventlog=['jupyter_server.services.eventlog.handlers'], kernels=['jupyter_server.services.kernels.handlers'], kernelspecs=[ 'jupyter_server.kernelspecs.handlers', @@ -623,6 +624,7 @@ class ServerApp(JupyterApp): 'config', 'contents', 'files', + 'eventlog', 'kernels', 'kernelspecs', 'nbconvert', From 66accdceabcd5ec497569f48f9ec7467cd8bb97e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 23 Apr 2021 18:31:38 +0800 Subject: [PATCH 31/49] Add tests for client telemetry events --- .../client_eventlog/__init__.py | 1 + .../client_eventlog/schemas/__init__.py | 0 .../client_eventlog/schemas/schema_yaml.yaml | 21 +++++++++ examples/client_eventlog/setup.py | 16 +++++++ examples/client_eventlog/tests/conftest.py | 3 ++ .../tests/test_client_eventlog.py | 44 +++++++++++++++++++ jupyter_server/pytest_plugin.py | 14 ++++++ jupyter_server/tests/test_eventlog.py | 16 +------ 8 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 examples/client_eventlog/client_eventlog/__init__.py create mode 100644 examples/client_eventlog/client_eventlog/schemas/__init__.py create mode 100644 examples/client_eventlog/client_eventlog/schemas/schema_yaml.yaml create mode 100644 examples/client_eventlog/setup.py create mode 100644 examples/client_eventlog/tests/conftest.py create mode 100644 examples/client_eventlog/tests/test_client_eventlog.py diff --git a/examples/client_eventlog/client_eventlog/__init__.py b/examples/client_eventlog/client_eventlog/__init__.py new file mode 100644 index 0000000000..f7b20ba32b --- /dev/null +++ b/examples/client_eventlog/client_eventlog/__init__.py @@ -0,0 +1 @@ +JUPYTER_TELEMETRY_SCHEMAS = ['client_eventlog.schemas'] diff --git a/examples/client_eventlog/client_eventlog/schemas/__init__.py b/examples/client_eventlog/client_eventlog/schemas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/client_eventlog/client_eventlog/schemas/schema_yaml.yaml b/examples/client_eventlog/client_eventlog/schemas/schema_yaml.yaml new file mode 100644 index 0000000000..43746184f8 --- /dev/null +++ b/examples/client_eventlog/client_eventlog/schemas/schema_yaml.yaml @@ -0,0 +1,21 @@ +$id: https://example.jupyter.org/client-event +version: 1 +title: Client event +description: | + An example client event +type: object +properties: + thing: + title: Thing + categories: + - category.jupyter.org/unrestricted + description: A random thing + user: + title: User name + type: string + categories: + - category.jupyter.org/user-identifier + description: Name of user who initiated event +required: + - thing + - user diff --git a/examples/client_eventlog/setup.py b/examples/client_eventlog/setup.py new file mode 100644 index 0000000000..e0b109092f --- /dev/null +++ b/examples/client_eventlog/setup.py @@ -0,0 +1,16 @@ +# Adapted from https://github.com/jasongrout/jupyter_core/blob/0310f4a199ba7da60abc54bd9115f7da9a9cec25/examples/scale/template/setup.py # noqa +from setuptools import setup + +name = 'client_eventlog' + +setup( + name=name, + version="1.0.0", + packages=[name], + include_package_data=True, + entry_points= { + 'jupyter_telemetry': [ + f'{name}.sample_entry_point = {name}:JUPYTER_TELEMETRY_SCHEMAS' + ] + } +) diff --git a/examples/client_eventlog/tests/conftest.py b/examples/client_eventlog/tests/conftest.py new file mode 100644 index 0000000000..87c6aff30a --- /dev/null +++ b/examples/client_eventlog/tests/conftest.py @@ -0,0 +1,3 @@ +pytest_plugins = [ + 'jupyter_server.pytest_plugin' +] diff --git a/examples/client_eventlog/tests/test_client_eventlog.py b/examples/client_eventlog/tests/test_client_eventlog.py new file mode 100644 index 0000000000..f7e156462b --- /dev/null +++ b/examples/client_eventlog/tests/test_client_eventlog.py @@ -0,0 +1,44 @@ +import io +import json +import logging + +import jsonschema +import pytest +from traitlets.config import Config + +from jupyter_server.utils import get_client_schema_files + + +EVENT = { + 'schema': 'https://example.jupyter.org/client-event', + 'version': 1.0, + 'event': { + 'user': 'user', + 'thing': 'thing' + } +} + + +async def test_client_eventlog(jp_eventlog_sink, jp_fetch): + serverapp, sink = jp_eventlog_sink + serverapp.eventlog.allowed_schemas = { + EVENT['schema']: { + 'allowed_categories': [ + 'category.jupyter.org/unrestricted', + 'category.jupyter.org/user-identifier' + ] + } + } + + r = await jp_fetch( + 'api', + 'eventlog', + method='POST', + body=json.dumps(EVENT) + ) + assert r.code == 204 + + output = sink.getvalue() + assert output + data = json.loads(output) + assert EVENT['event'].items() <= data.items() diff --git a/jupyter_server/pytest_plugin.py b/jupyter_server/pytest_plugin.py index 849cdc469b..f8580825f0 100644 --- a/jupyter_server/pytest_plugin.py +++ b/jupyter_server/pytest_plugin.py @@ -3,7 +3,9 @@ import os import sys +import io import json +import logging import pytest import shutil import urllib.parse @@ -432,3 +434,15 @@ def inner(nbpath): def jp_server_cleanup(): yield ServerApp.clear_instance() + + +@pytest.fixture +def jp_eventlog_sink(jp_configurable_serverapp): + """Return eventlog and sink objects""" + sink = io.StringIO() + handler = logging.StreamHandler(sink) + + cfg = Config() + cfg.EventLog.handlers = [handler] + serverapp = jp_configurable_serverapp(config=cfg) + yield serverapp, sink diff --git a/jupyter_server/tests/test_eventlog.py b/jupyter_server/tests/test_eventlog.py index 37b0d6d243..20e5958bea 100644 --- a/jupyter_server/tests/test_eventlog.py +++ b/jupyter_server/tests/test_eventlog.py @@ -10,22 +10,10 @@ from .services.contents.test_api import contents, contents_dir, dirs -@pytest.fixture -def eventlog_sink(jp_configurable_serverapp): - """Return eventlog and sink objects""" - sink = io.StringIO() - handler = logging.StreamHandler(sink) - - cfg = Config() - cfg.EventLog.handlers = [handler] - serverapp = jp_configurable_serverapp(config=cfg) - yield serverapp, sink - - @pytest.mark.parametrize('path, name', dirs) -async def test_eventlog_list_notebooks(eventlog_sink, jp_fetch, contents, path, name): +async def test_eventlog_list_notebooks(jp_eventlog_sink, jp_fetch, contents, path, name): schema, version = (eventlogging_schema_fqn('contentsmanager-actions'), 1) - serverapp, sink = eventlog_sink + serverapp, sink = jp_eventlog_sink serverapp.eventlog.allowed_schemas = { serverapp.eventlog.schemas[(schema, version)]['$id']: { 'allowed_categories': [ From 4dcd258826f774d9bd7e9978c67864ccc58177f9 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 23 Apr 2021 18:40:30 +0800 Subject: [PATCH 32/49] Add client telemetry eventlog tests to CI --- .github/workflows/python-linux.yml | 6 ++++++ .github/workflows/python-macos.yml | 6 ++++++ .github/workflows/python-windows.yml | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/.github/workflows/python-linux.yml b/.github/workflows/python-linux.yml index ffa45ccf9e..691ae7f94e 100644 --- a/.github/workflows/python-linux.yml +++ b/.github/workflows/python-linux.yml @@ -56,6 +56,12 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py + - name: Install the Python dependencies for the client telemetry eventlog example + run: | + cd examples/client_eventlog && pip install -e . + - name: Run the tests for the client telemetry eventlog example + run: | + pytest examples/client_eventlog/tests - name: Coverage if: ${{ matrix.python-version != 'pypy3' }} run: | diff --git a/.github/workflows/python-macos.yml b/.github/workflows/python-macos.yml index 94a7800fdc..cadbb90704 100644 --- a/.github/workflows/python-macos.yml +++ b/.github/workflows/python-macos.yml @@ -56,6 +56,12 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py + - name: Install the Python dependencies for the client telemetry eventlog example + run: | + cd examples/client_eventlog && pip install -e . + - name: Run the tests for the client telemetry eventlog example + run: | + pytest examples/client_eventlog/tests - name: Coverage if: ${{ matrix.python-version != 'pypy3' }} run: | diff --git a/.github/workflows/python-windows.yml b/.github/workflows/python-windows.yml index 347f487fc0..e964bdbc22 100644 --- a/.github/workflows/python-windows.yml +++ b/.github/workflows/python-windows.yml @@ -56,3 +56,9 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py + - name: Install the Python dependencies for the client telemetry eventlog example + run: | + cd examples/client_eventlog && pip install -e . + - name: Run the tests for the client telemetry eventlog example + run: | + pytest examples/client_eventlog/tests From 9fd2a5fe25a43772780599dc8b9b8aaee95e183e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 23 Apr 2021 20:24:47 +0800 Subject: [PATCH 33/49] Clean up --- examples/client_eventlog/tests/test_client_eventlog.py | 8 -------- jupyter_server/serverapp.py | 3 --- jupyter_server/services/contents/handlers.py | 1 + jupyter_server/services/eventlog/handlers.py | 3 ++- jupyter_server/tests/test_eventlog.py | 7 ++----- 5 files changed, 5 insertions(+), 17 deletions(-) diff --git a/examples/client_eventlog/tests/test_client_eventlog.py b/examples/client_eventlog/tests/test_client_eventlog.py index f7e156462b..57af5e311f 100644 --- a/examples/client_eventlog/tests/test_client_eventlog.py +++ b/examples/client_eventlog/tests/test_client_eventlog.py @@ -1,12 +1,4 @@ -import io import json -import logging - -import jsonschema -import pytest -from traitlets.config import Config - -from jupyter_server.utils import get_client_schema_files EVENT = { diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 1808689dd5..ee646d027b 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -32,9 +32,6 @@ import pathlib import importlib_resources -from ruamel.yaml import YAML -from glob import glob - from base64 import encodebytes try: import resource diff --git a/jupyter_server/services/contents/handlers.py b/jupyter_server/services/contents/handlers.py index 915cdbd76e..6bbbd08649 100644 --- a/jupyter_server/services/contents/handlers.py +++ b/jupyter_server/services/contents/handlers.py @@ -277,6 +277,7 @@ async def delete(self, path=''): ) self.finish() + class CheckpointsHandler(APIHandler): @web.authenticated diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py index b7e36ac8c3..581548c3c1 100644 --- a/jupyter_server/services/eventlog/handlers.py +++ b/jupyter_server/services/eventlog/handlers.py @@ -3,7 +3,7 @@ from tornado import web from jupyter_server.base.handlers import APIHandler, json_errors -from jupyter_telemetry.eventlog import EventLog + class EventLoggingHandler(APIHandler): """ @@ -36,6 +36,7 @@ def post(self, *args, **kwargs): self.set_status(204) self.finish() + default_handlers = [ (r"/api/eventlog", EventLoggingHandler), ] diff --git a/jupyter_server/tests/test_eventlog.py b/jupyter_server/tests/test_eventlog.py index 20e5958bea..e41f28a88b 100644 --- a/jupyter_server/tests/test_eventlog.py +++ b/jupyter_server/tests/test_eventlog.py @@ -1,17 +1,14 @@ -import io import json -import logging import jsonschema import pytest -from traitlets.config import Config from jupyter_server.utils import eventlogging_schema_fqn -from .services.contents.test_api import contents, contents_dir, dirs +from .services.contents.test_api import dirs @pytest.mark.parametrize('path, name', dirs) -async def test_eventlog_list_notebooks(jp_eventlog_sink, jp_fetch, contents, path, name): +async def test_eventlog_list_notebooks(jp_eventlog_sink, jp_fetch, path, name): schema, version = (eventlogging_schema_fqn('contentsmanager-actions'), 1) serverapp, sink = jp_eventlog_sink serverapp.eventlog.allowed_schemas = { From f692488792593a60aee11902751d8f6b31397b10 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 23 Apr 2021 21:39:26 +0800 Subject: [PATCH 34/49] Fix eventlog test --- jupyter_server/tests/test_eventlog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_server/tests/test_eventlog.py b/jupyter_server/tests/test_eventlog.py index e41f28a88b..749f889bd1 100644 --- a/jupyter_server/tests/test_eventlog.py +++ b/jupyter_server/tests/test_eventlog.py @@ -4,11 +4,11 @@ import pytest from jupyter_server.utils import eventlogging_schema_fqn -from .services.contents.test_api import dirs +from .services.contents.test_api import contents, contents_dir, dirs @pytest.mark.parametrize('path, name', dirs) -async def test_eventlog_list_notebooks(jp_eventlog_sink, jp_fetch, path, name): +async def test_eventlog_list_notebooks(jp_eventlog_sink, jp_fetch, contents, path, name): schema, version = (eventlogging_schema_fqn('contentsmanager-actions'), 1) serverapp, sink = jp_eventlog_sink serverapp.eventlog.allowed_schemas = { From 21117a6adf15eb9d875946312273ef8cd7d7eec0 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 24 Apr 2021 14:01:10 +0800 Subject: [PATCH 35/49] Use standard lib instead of backport when possible --- jupyter_server/serverapp.py | 6 +++++- jupyter_server/utils.py | 12 ++++++++++-- setup.cfg | 4 ++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index ee646d027b..f0955b35c3 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -30,7 +30,11 @@ import urllib import inspect import pathlib -import importlib_resources + +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources from base64 import encodebytes try: diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 7dea1ddd95..98bc623ccb 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -8,14 +8,22 @@ import inspect import os import sys -import importlib_resources from distutils.version import LooseVersion -from importlib_metadata import entry_points from itertools import chain from urllib.parse import quote, unquote, urlparse, urljoin from urllib.request import pathname2url +if sys.version_info >= (3, 8): + from importlib.metadata import entry_points +else: + from importlib_metadata import entry_points + +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + from ipython_genutils import py3compat diff --git a/setup.cfg b/setup.cfg index 584489b43a..4cf5088303 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,8 +43,8 @@ install_requires = prometheus_client pywin32>=1.0 ; sys_platform == 'win32' anyio>=2.0.2,<3 - importlib-metadata - importlib-resources + importlib-metadata ; python_version < '3.8' + importlib-resources ; python_version < '3.9' jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@a4fbf987da063f9e0b337abd21803c968ac64f27 [options.extras_require] From 193650356bb64f875fcc93319d8ad0b4d7f8c8a5 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 25 Apr 2021 15:25:20 +0800 Subject: [PATCH 36/49] Fix docs --- docs/doc-requirements.txt | 1 + docs/source/conf.py | 1 + docs/source/eventlog.rst | 64 --- docs/source/operators/telemetry.rst | 7 +- docs/source/other/full-config.rst | 587 +++++++++++++++++++++++----- 5 files changed, 503 insertions(+), 157 deletions(-) delete mode 100644 docs/source/eventlog.rst diff --git a/docs/doc-requirements.txt b/docs/doc-requirements.txt index d6c6014179..619f2f792c 100644 --- a/docs/doc-requirements.txt +++ b/docs/doc-requirements.txt @@ -11,4 +11,5 @@ sphinxemoji myst-parser pydata_sphinx_theme git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master +sphinx-jsonschema jupyter_telemetry_sphinxext diff --git a/docs/source/conf.py b/docs/source/conf.py index cb5cde4c09..b1d14e42c9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -76,6 +76,7 @@ 'sphinx.ext.mathjax', 'IPython.sphinxext.ipython_console_highlighting', 'sphinxcontrib_github_alt', + 'sphinx-jsonschema', 'jupyter_telemetry_sphinxext' ] diff --git a/docs/source/eventlog.rst b/docs/source/eventlog.rst deleted file mode 100644 index 6308f672d0..0000000000 --- a/docs/source/eventlog.rst +++ /dev/null @@ -1,64 +0,0 @@ -Eventlogging and Telemetry -========================== - -The Notebook Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that the Notebook Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. - - -.. _logging: https://docs.python.org/3/library/logging.html -.. _`Telemetry System`: https://github.com/jupyter/telemetry -.. _`JSON schemas`: https://json-schema.org/ - -.. warning:: - Do NOT rely on this feature for security or auditing purposes. Neither `server <#emitting-server-events>`_ nor `client <#eventlog-endpoint>`_ events are protected against meddling. For server events, those who have access to the environment can change the server code to emit whatever they want. The same goes for client events where nothing prevents users from sending spurious data to the `eventlog` endpoint. - -Emitting Server Events ----------------------- - -Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. - -To begin recording events, you'll need to set two configurations: - - 1. ``handlers``: tells the EventLog *where* to route your events. This trait is a list of Python logging handlers that route events to - 2. ``allows_schemas``: tells the EventLog *which* events should be recorded. No events are emitted by default; all recorded events must be listed here. - -Here's a basic example for emitting events from the `contents` service: - -.. code-block:: - - import logging - - c.EventLog.handlers = [ - logging.FileHandler('event.log'), - ] - - c.EventLog.allowed_schemas = [ - 'hub.jupyter.org/server-action' - ] - -The output is a file, ``"event.log"``, with events recorded as JSON data. - -`eventlog` endpoint -------------------- - -The Notebook Server provides a public REST endpoint for external applications to validate and log events -through the Server's Event Log. - -To log events, send a `POST` request to the `/api/eventlog` endpoint. The body of the request should be a -JSON blog and is required to have the follow keys: - - 1. `'schema'` : the event's schema ID. - 2. `'version'` : the version of the event's schema. - 3. `'event'` : the event data in JSON format. - -Events that are validated by this endpoint must have their schema listed in the `allowed_schemas` trait listed above. - -.. _below: - - -Server Event schemas --------------------- - -.. toctree:: - :maxdepth: 2 - - events/index diff --git a/docs/source/operators/telemetry.rst b/docs/source/operators/telemetry.rst index 2c94e99a7c..6308f672d0 100644 --- a/docs/source/operators/telemetry.rst +++ b/docs/source/operators/telemetry.rst @@ -1,13 +1,16 @@ -Telemetry and Eventlogging +Eventlogging and Telemetry ========================== -Jupyter Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that the Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. +The Notebook Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that the Notebook Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. .. _logging: https://docs.python.org/3/library/logging.html .. _`Telemetry System`: https://github.com/jupyter/telemetry .. _`JSON schemas`: https://json-schema.org/ +.. warning:: + Do NOT rely on this feature for security or auditing purposes. Neither `server <#emitting-server-events>`_ nor `client <#eventlog-endpoint>`_ events are protected against meddling. For server events, those who have access to the environment can change the server code to emit whatever they want. The same goes for client events where nothing prevents users from sending spurious data to the `eventlog` endpoint. + Emitting Server Events ---------------------- diff --git a/docs/source/other/full-config.rst b/docs/source/other/full-config.rst index 52d0bcf3ff..8fc9b60a56 100644 --- a/docs/source/other/full-config.rst +++ b/docs/source/other/full-config.rst @@ -110,9 +110,9 @@ ServerApp.allow_origin : Unicode Default: ``''`` Set the Access-Control-Allow-Origin header - + Use '*' to allow any origin to access your server. - + Takes precedence over allow_origin_pat. @@ -120,13 +120,13 @@ ServerApp.allow_origin_pat : Unicode Default: ``''`` Use a regular expression for the Access-Control-Allow-Origin header - + Requests from an origin matching the expression will get replies with: - + Access-Control-Allow-Origin: origin - + where `origin` is the origin of the request. - + Ignored if allow_origin is set. @@ -134,11 +134,11 @@ ServerApp.allow_password_change : Bool Default: ``True`` Allow password to be changed at login for the Jupyter server. - + While loggin in with a token, the Jupyter server UI will give the opportunity to the user to enter a new password at the same time that will replace the token login mechanism. - + This can be set to false to prevent changing password from the UI/API. @@ -146,15 +146,15 @@ ServerApp.allow_remote_access : Bool Default: ``False`` Allow requests where the Host header doesn't point to a local server - + By default, requests get a 403 forbidden response if the 'Host' header shows that the browser thinks it's on a non-local domain. Setting this option to True disables this check. - + This protects against 'DNS rebinding' attacks, where a remote web server serves you a page and then changes its DNS to send later requests to a local IP, bypassing same-origin checks. - + Local IP addresses (such as 127.0.0.1 and ::1) are allowed as local, along with hostnames configured in local_hostnames. @@ -169,11 +169,23 @@ ServerApp.answer_yes : Bool Answer yes to any prompts. +ServerApp.authenticate_prometheus : Bool + Default: ``True`` + + " + Require authentication to access prometheus metrics. + + +ServerApp.autoreload : Bool + Default: ``False`` + + Reload the webapp when changes are made to any Python src files. + ServerApp.base_url : Unicode Default: ``'/'`` The base URL for the Jupyter server. - + Leading and trailing slashes can be omitted, and will automatically be added. @@ -213,7 +225,7 @@ ServerApp.config_manager_class : Type The config manager class to use -ServerApp.contents_manager_class : Type +ServerApp.contents_manager_class : TypeFromClasses Default: ``'jupyter_server.services.contents.largefilemanager.LargeFileM...`` The content manager class to use. @@ -229,7 +241,7 @@ ServerApp.cookie_secret : Bytes The random bytes used to secure cookies. By default this is a new random number every time you start the server. Set it to a value in a config file to enable logins to persist across server sessions. - + Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). @@ -243,12 +255,12 @@ ServerApp.custom_display_url : Unicode Default: ``''`` Override URL shown to users. - + Replace actual URL, including protocol, address, port and base URL, with the given value when displaying URL to the users. Do not change the actual connection URL. If authentication token is enabled, the token is added to the custom URL automatically. - + This option is intended to be used when the URL to display to the user cannot be determined reliably by the Jupyter server (proxified or containerized setups for example). @@ -262,13 +274,13 @@ ServerApp.disable_check_xsrf : Bool Default: ``False`` Disable cross-site-request-forgery protection - + Jupyter notebook 4.3.1 introduces protection from cross-site request forgeries, requiring API requests to either: - + - originate from pages served by this server (validated with XSRF cookie and token), or - authenticate with a token - + Some anonymous compute resources still desire the ability to run code, completely without authentication. These services can disable all authentication and security checks, @@ -284,7 +296,7 @@ ServerApp.extra_static_paths : List Default: ``[]`` Extra paths to search for serving static files. - + This allows adding javascript/css to be available from the Jupyter server machine, or overriding individual files in the IPython @@ -292,13 +304,18 @@ ServerApp.extra_template_paths : List Default: ``[]`` Extra paths to search for serving jinja templates. - + Can be used to override templates from jupyter_server.templates. ServerApp.file_to_run : Unicode Default: ``''`` - No description + Open the named file when the application is launched. + +ServerApp.file_url_prefix : Unicode + Default: ``'notebooks'`` + + The URL prefix where files are opened directly. ServerApp.generate_config : Bool Default: ``False`` @@ -345,17 +362,17 @@ ServerApp.jpserver_extensions : Dict Dict of Python modules to load as notebook server extensions.Entry values can be used to enable and disable the loading ofthe extensions. The extensions will be loaded in alphabetical order. ServerApp.kernel_manager_class : Type - Default: ``'jupyter_server.services.kernels.kernelmanager.MappingKernelM...`` + Default: ``'jupyter_server.services.kernels.kernelmanager.AsyncMappingKe...`` The kernel manager class to use. ServerApp.kernel_spec_manager_class : Type Default: ``'jupyter_client.kernelspec.KernelSpecManager'`` - + The kernel spec manager class to use. Should be a subclass of `jupyter_client.kernelspec.KernelSpecManager`. - + The Api of KernelSpecManager is provisional and might change without warning between this version of Jupyter and the next stable one. @@ -369,7 +386,7 @@ ServerApp.local_hostnames : List Default: ``['localhost']`` Hostnames to allow as local when allow_remote_access is False. - + Local IP addresses (such as 127.0.0.1 and ::1) are automatically accepted as local as well. @@ -402,23 +419,33 @@ ServerApp.logout_handler_class : Type ServerApp.max_body_size : Int Default: ``536870912`` - + Sets the maximum allowed size of the client request body, specified in the Content-Length request header field. If the size in a request exceeds the configured value, a malformed HTTP message is returned to the client. - + Note: max_body_size is applied even in streaming mode. ServerApp.max_buffer_size : Int Default: ``536870912`` - + Gets or sets the maximum amount of memory, in bytes, that is allocated for use by the buffer manager. +ServerApp.min_open_files_limit : Int + Default: ``0`` + + + Gets or sets a lower bound on the open file handles process resource + limit. This may need to be increased if you run into an + OSError: [Errno 24] Too many open files. + This is not applicable when running on Windows. + + ServerApp.notebook_dir : Unicode Default: ``''`` @@ -438,11 +465,11 @@ ServerApp.password : Unicode Default: ``''`` Hashed password to use for web authentication. - + To generate, type in a python/IPython shell: - + from jupyter_server.auth import passwd; passwd() - + The string should be of the form type:salt:hashed-password. @@ -452,26 +479,26 @@ ServerApp.password_required : Bool Forces users to use a password for the Jupyter server. This is useful in a multi user environment, for instance when everybody in the LAN can access each other's machine through ssh. - + In such a case, serving on localhost is not secure since any user can connect to the Jupyter server via ssh. - + ServerApp.port : Int Default: ``8888`` - The port the Jupyter server will listen on. + The port the server will listen on (env: JUPYTER_PORT). ServerApp.port_retries : Int Default: ``50`` - The number of additional ports to try if the specified port is not available. + The number of additional ports to try if the specified port is not available (env: JUPYTER_PORT_RETRIES). ServerApp.pylab : Unicode Default: ``'disabled'`` - + DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. @@ -531,10 +558,10 @@ ServerApp.terminals_enabled : Bool Default: ``True`` Set to False to disable terminals. - + This does *not* make the server more secure by itself. Anything the user can in a terminal, they can also do in a notebook. - + Terminals may also be automatically disabled if the terminado package is not available. @@ -543,10 +570,13 @@ ServerApp.token : Unicode Default: ``''`` Token used for authenticating first-time connections to the server. - + + The token can be read from the file referenced by JUPYTER_TOKEN_FILE or set directly + with the JUPYTER_TOKEN environment variable. + When no password is enabled, the default is to generate a new, random token. - + Setting to an empty string disables authentication altogether, which is NOT RECOMMENDED. @@ -560,6 +590,23 @@ ServerApp.trust_xheaders : Bool Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headerssent by the upstream reverse proxy. Necessary if the proxy handles SSL +ServerApp.use_redirect_file : Bool + Default: ``True`` + + Disable launching browser by redirect file + For versions of notebook > 5.7.2, a security feature measure was added that + prevented the authentication token used to launch the browser from being visible. + This feature makes it difficult for other users on a multi-user system from + running code in your Jupyter session as you. + However, some environments (like Windows Subsystem for Linux (WSL) and Chromebooks), + launching a browser using a redirect file can lead the browser failing to load. + This is because of the difference in file structures/paths between the runtime and + the browser. + + Disabling this setting to False will disable this behavior, allowing the browser + to launch by using a URL and visible token (as before). + + ServerApp.webbrowser_open_new : Int Default: ``2`` @@ -567,24 +614,24 @@ ServerApp.webbrowser_open_new : Int `new` argument passed to the standard library method `webbrowser.open`. The behaviour is not guaranteed, but depends on browser support. Valid values are: - + - 2 opens a new tab, - 1 opens a new window, - 0 opens in an existing window. - + See the `webbrowser.open` documentation for details. ServerApp.websocket_compression_options : Any Default: ``None`` - + Set the tornado compression options for websocket connections. - + This value will be returned from :meth:`WebSocketHandler.get_compression_options`. None (default) will disable compression. A dict (even an empty one) will enable compression. - + See the tornado docs for WebSocketHandler.get_compression_options for details. @@ -593,7 +640,7 @@ ServerApp.websocket_url : Unicode The base URL for websockets, if it differs from the HTTP server (hint: it almost certainly doesn't). - + Should be in the form of an HTTP origin: ws[s]://hostname[:port] @@ -601,7 +648,7 @@ ConnectionFileMixin.connection_file : Unicode Default: ``''`` JSON file in which to store connection info [default: kernel-.json] - + This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. @@ -654,7 +701,7 @@ KernelManager.connection_file : Unicode Default: ``''`` JSON file in which to store connection info [default: kernel-.json] - + This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. @@ -687,7 +734,7 @@ KernelManager.kernel_cmd : List Default: ``[]`` DEPRECATED: Use kernel_name instead. - + The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, @@ -727,7 +774,7 @@ Session.check_pid : Bool Default: ``True`` Whether to check PID to protect against calls after fork. - + This check can be disabled if fork-safety is handled elsewhere. @@ -745,7 +792,7 @@ Session.digest_history_size : Int Default: ``65536`` The maximum number of digests to remember. - + The digest history will be culled when it exceeds this value. @@ -796,7 +843,7 @@ Session.unpacker : DottedObjectName Only used with custom functions for `packer`. Session.username : Unicode - Default: ``'username'`` + Default: ``'kien'`` Username for the Session. Default is your system username. @@ -833,10 +880,10 @@ MappingKernelManager.buffer_offline_messages : Bool Default: ``True`` Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - + When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. - + Disable if long-running kernels will produce too much output while no frontends are connected. @@ -874,7 +921,7 @@ MappingKernelManager.kernel_info_timeout : Float Default: ``60`` Timeout for giving up on a kernel (in seconds). - + On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take @@ -923,10 +970,115 @@ KernelSpecManager.whitelist : Set Default: ``set()`` Whitelist of allowed kernel names. - + By default, all installed kernels are allowed. +AsyncMultiKernelManager.default_kernel_name : Unicode + Default: ``'python3'`` + + The name of the default kernel to start + +AsyncMultiKernelManager.kernel_manager_class : DottedObjectName + Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` + + The kernel manager class. This is configurable to allow + subclassing of the AsyncKernelManager for customized behavior. + + +AsyncMultiKernelManager.shared_context : Bool + Default: ``True`` + + Share a single zmq.Context to talk to all my kernels + +AsyncMappingKernelManager.allow_tracebacks : Bool + Default: ``True`` + + Whether to send tracebacks to clients on exceptions. + +AsyncMappingKernelManager.allowed_message_types : List + Default: ``[]`` + + White list of allowed kernel message types. + When the list is empty, all message types are allowed. + + +AsyncMappingKernelManager.buffer_offline_messages : Bool + Default: ``True`` + + Whether messages from kernels whose frontends have disconnected should be buffered in-memory. + + When True (default), messages are buffered and replayed on reconnect, + avoiding lost messages due to interrupted connectivity. + + Disable if long-running kernels will produce too much output while + no frontends are connected. + + +AsyncMappingKernelManager.cull_busy : Bool + Default: ``False`` + + Whether to consider culling kernels which are busy. + Only effective if cull_idle_timeout > 0. + +AsyncMappingKernelManager.cull_connected : Bool + Default: ``False`` + + Whether to consider culling kernels which have one or more connections. + Only effective if cull_idle_timeout > 0. + +AsyncMappingKernelManager.cull_idle_timeout : Int + Default: ``0`` + + Timeout (in seconds) after which a kernel is considered idle and ready to be culled. + Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled + for users with poor network connections. + +AsyncMappingKernelManager.cull_interval : Int + Default: ``300`` + + The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value. + +AsyncMappingKernelManager.default_kernel_name : Unicode + Default: ``'python3'`` + + The name of the default kernel to start + +AsyncMappingKernelManager.kernel_info_timeout : Float + Default: ``60`` + + Timeout for giving up on a kernel (in seconds). + + On starting and restarting kernels, we check whether the + kernel is running and responsive by sending kernel_info_requests. + This sets the timeout in seconds for how long the kernel can take + before being presumed dead. + This affects the MappingKernelManager (which handles kernel restarts) + and the ZMQChannelsHandler (which handles the startup). + + +AsyncMappingKernelManager.kernel_manager_class : DottedObjectName + Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` + + The kernel manager class. This is configurable to allow + subclassing of the AsyncKernelManager for customized behavior. + + +AsyncMappingKernelManager.root_dir : Unicode + Default: ``''`` + + No description + +AsyncMappingKernelManager.shared_context : Bool + Default: ``True`` + + Share a single zmq.Context to talk to all my kernels + +AsyncMappingKernelManager.traceback_replacement_message : Unicode + Default: ``'An exception occurred at runtime, which is not shown due to ...`` + + Message to print when allow_tracebacks is False, and an exception occurs + ContentsManager.allow_hidden : Bool Default: ``False`` @@ -951,13 +1103,13 @@ ContentsManager.files_handler_class : Type Default: ``'jupyter_server.files.handlers.FilesHandler'`` handler class to use when serving raw file requests. - + Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. - + Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. - + Access to these files should be Authenticated. @@ -965,7 +1117,7 @@ ContentsManager.files_handler_params : Dict Default: ``{}`` Extra parameters to pass to files_handler_class. - + For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. @@ -973,7 +1125,7 @@ ContentsManager.files_handler_params : Dict ContentsManager.hide_globs : List Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - + Glob patterns to hide in file and directory listings. @@ -981,17 +1133,17 @@ ContentsManager.pre_save_hook : Any Default: ``None`` Python callable or importstring thereof - + To be called on a contents model prior to save. - + This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. - + It will be called as (all arguments passed by keyword):: - + hook(path=path, model=model, contents_manager=self) - + - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination @@ -1056,13 +1208,13 @@ FileContentsManager.files_handler_class : Type Default: ``'jupyter_server.files.handlers.FilesHandler'`` handler class to use when serving raw file requests. - + Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. - + Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. - + Access to these files should be Authenticated. @@ -1070,7 +1222,7 @@ FileContentsManager.files_handler_params : Dict Default: ``{}`` Extra parameters to pass to files_handler_class. - + For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. @@ -1078,7 +1230,7 @@ FileContentsManager.files_handler_params : Dict FileContentsManager.hide_globs : List Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - + Glob patterns to hide in file and directory listings. @@ -1086,16 +1238,16 @@ FileContentsManager.post_save_hook : Any Default: ``None`` Python callable or importstring thereof - + to be called on the path of a file just saved. - + This can be used to process the file on disk, such as converting the notebook to a script or HTML via nbconvert. - + It will be called as (all arguments passed by keyword):: - + hook(os_path=os_path, model=model, contents_manager=instance) - + - path: the filesystem path to the file just written - model: the model representing the file - contents_manager: this ContentsManager instance @@ -1105,17 +1257,17 @@ FileContentsManager.pre_save_hook : Any Default: ``None`` Python callable or importstring thereof - + To be called on a contents model prior to save. - + This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. - + It will be called as (all arguments passed by keyword):: - + hook(path=path, model=model, contents_manager=self) - + - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination @@ -1149,7 +1301,229 @@ FileContentsManager.use_atomic_writing : Bool This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) -NotebookNotary.algorithm : any of ``'blake2s'``|``'sha512'``|``'md5'``|``'sha3_512'``|``'sha3_224'``|``'blake2b'``|``'sha384'``|``'sha1'``|``'sha3_256'``|``'sha256'``|``'sha224'``|``'sha3_384'`` +AsyncContentsManager.allow_hidden : Bool + Default: ``False`` + + Allow access to hidden files + +AsyncContentsManager.checkpoints : Instance + Default: ``None`` + + No description + +AsyncContentsManager.checkpoints_class : Type + Default: ``'jupyter_server.services.contents.checkpoints.AsyncCheckpoints'`` + + No description + +AsyncContentsManager.checkpoints_kwargs : Dict + Default: ``{}`` + + No description + +AsyncContentsManager.files_handler_class : Type + Default: ``'jupyter_server.files.handlers.FilesHandler'`` + + handler class to use when serving raw file requests. + + Default is a fallback that talks to the ContentsManager API, + which may be inefficient, especially for large files. + + Local files-based ContentsManagers can use a StaticFileHandler subclass, + which will be much more efficient. + + Access to these files should be Authenticated. + + +AsyncContentsManager.files_handler_params : Dict + Default: ``{}`` + + Extra parameters to pass to files_handler_class. + + For example, StaticFileHandlers generally expect a `path` argument + specifying the root directory from which to serve files. + + +AsyncContentsManager.hide_globs : List + Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` + + + Glob patterns to hide in file and directory listings. + + +AsyncContentsManager.pre_save_hook : Any + Default: ``None`` + + Python callable or importstring thereof + + To be called on a contents model prior to save. + + This can be used to process the structure, + such as removing notebook outputs or other side effects that + should not be saved. + + It will be called as (all arguments passed by keyword):: + + hook(path=path, model=model, contents_manager=self) + + - model: the model to be saved. Includes file contents. + Modifying this dict will affect the file that is stored. + - path: the API path of the save destination + - contents_manager: this ContentsManager instance + + +AsyncContentsManager.root_dir : Unicode + Default: ``'/'`` + + No description + +AsyncContentsManager.untitled_directory : Unicode + Default: ``'Untitled Folder'`` + + The base name used when creating untitled directories. + +AsyncContentsManager.untitled_file : Unicode + Default: ``'untitled'`` + + The base name used when creating untitled files. + +AsyncContentsManager.untitled_notebook : Unicode + Default: ``'Untitled'`` + + The base name used when creating untitled notebooks. + +AsyncFileManagerMixin.use_atomic_writing : Bool + Default: ``True`` + + By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. + This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). + If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) + +AsyncFileContentsManager.allow_hidden : Bool + Default: ``False`` + + Allow access to hidden files + +AsyncFileContentsManager.checkpoints : Instance + Default: ``None`` + + No description + +AsyncFileContentsManager.checkpoints_class : Type + Default: ``'jupyter_server.services.contents.checkpoints.AsyncCheckpoints'`` + + No description + +AsyncFileContentsManager.checkpoints_kwargs : Dict + Default: ``{}`` + + No description + +AsyncFileContentsManager.delete_to_trash : Bool + Default: ``True`` + + If True (default), deleting files will send them to the + platform's trash/recycle bin, where they can be recovered. If False, + deleting files really deletes them. + +AsyncFileContentsManager.files_handler_class : Type + Default: ``'jupyter_server.files.handlers.FilesHandler'`` + + handler class to use when serving raw file requests. + + Default is a fallback that talks to the ContentsManager API, + which may be inefficient, especially for large files. + + Local files-based ContentsManagers can use a StaticFileHandler subclass, + which will be much more efficient. + + Access to these files should be Authenticated. + + +AsyncFileContentsManager.files_handler_params : Dict + Default: ``{}`` + + Extra parameters to pass to files_handler_class. + + For example, StaticFileHandlers generally expect a `path` argument + specifying the root directory from which to serve files. + + +AsyncFileContentsManager.hide_globs : List + Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` + + + Glob patterns to hide in file and directory listings. + + +AsyncFileContentsManager.post_save_hook : Any + Default: ``None`` + + Python callable or importstring thereof + + to be called on the path of a file just saved. + + This can be used to process the file on disk, + such as converting the notebook to a script or HTML via nbconvert. + + It will be called as (all arguments passed by keyword):: + + hook(os_path=os_path, model=model, contents_manager=instance) + + - path: the filesystem path to the file just written + - model: the model representing the file + - contents_manager: this ContentsManager instance + + +AsyncFileContentsManager.pre_save_hook : Any + Default: ``None`` + + Python callable or importstring thereof + + To be called on a contents model prior to save. + + This can be used to process the structure, + such as removing notebook outputs or other side effects that + should not be saved. + + It will be called as (all arguments passed by keyword):: + + hook(path=path, model=model, contents_manager=self) + + - model: the model to be saved. Includes file contents. + Modifying this dict will affect the file that is stored. + - path: the API path of the save destination + - contents_manager: this ContentsManager instance + + +AsyncFileContentsManager.root_dir : Unicode + Default: ``''`` + + No description + +AsyncFileContentsManager.untitled_directory : Unicode + Default: ``'Untitled Folder'`` + + The base name used when creating untitled directories. + +AsyncFileContentsManager.untitled_file : Unicode + Default: ``'untitled'`` + + The base name used when creating untitled files. + +AsyncFileContentsManager.untitled_notebook : Unicode + Default: ``'Untitled'`` + + The base name used when creating untitled notebooks. + +AsyncFileContentsManager.use_atomic_writing : Bool + Default: ``True`` + + By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. + This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). + If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) + +NotebookNotary.algorithm : any of ``'sha3_512'``|``'sha224'``|``'sha1'``|``'sha256'``|``'sha3_384'``|``'sha512'``|``'sha3_256'``|``'blake2s'``|``'sha3_224'``|``'md5'``|``'sha384'``|``'blake2b'`` Default: ``'sha256'`` The hashing algorithm used to sign notebooks. @@ -1194,10 +1568,10 @@ GatewayKernelManager.buffer_offline_messages : Bool Default: ``True`` Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - + When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. - + Disable if long-running kernels will produce too much output while no frontends are connected. @@ -1235,7 +1609,7 @@ GatewayKernelManager.kernel_info_timeout : Float Default: ``60`` Timeout for giving up on a kernel (in seconds). - + On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take @@ -1245,10 +1619,10 @@ GatewayKernelManager.kernel_info_timeout : Float GatewayKernelManager.kernel_manager_class : DottedObjectName - Default: ``'jupyter_client.ioloop.IOLoopKernelManager'`` + Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` The kernel manager class. This is configurable to allow - subclassing of the KernelManager for customized behavior. + subclassing of the AsyncKernelManager for customized behavior. GatewayKernelManager.root_dir : Unicode @@ -1284,7 +1658,7 @@ GatewayKernelSpecManager.whitelist : Set Default: ``set()`` Whitelist of allowed kernel names. - + By default, all installed kernels are allowed. @@ -1315,7 +1689,7 @@ GatewayClient.client_key : Unicode GatewayClient.connect_timeout : Float - Default: ``60.0`` + Default: ``40.0`` The time allowed for HTTP connection establishment with the Gateway server. (JUPYTER_GATEWAY_CONNECT_TIMEOUT env var) @@ -1328,6 +1702,26 @@ GatewayClient.env_whitelist : Unicode value must also be set on the Gateway server - since that configuration value indicates which environmental values to make available to the kernel. (JUPYTER_GATEWAY_ENV_WHITELIST env var) +GatewayClient.gateway_retry_interval : Float + Default: ``1.0`` + + The time allowed for HTTP reconnection with the Gateway server for the first time. + Next will be JUPYTER_GATEWAY_RETRY_INTERVAL multiplied by two in factor of numbers of retries + but less than JUPYTER_GATEWAY_RETRY_INTERVAL_MAX. + (JUPYTER_GATEWAY_RETRY_INTERVAL env var) + +GatewayClient.gateway_retry_interval_max : Float + Default: ``30.0`` + + The maximum time allowed for HTTP reconnection retry with the Gateway server. + (JUPYTER_GATEWAY_RETRY_INTERVAL_MAX env var) + +GatewayClient.gateway_retry_max : Int + Default: ``5`` + + The maximum retries allowed for HTTP reconnection with the Gateway server. + (JUPYTER_GATEWAY_RETRY_MAX env var) + GatewayClient.headers : Unicode Default: ``'{}'`` @@ -1364,7 +1758,7 @@ GatewayClient.kernelspecs_resource_endpoint : Unicode (JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT env var) GatewayClient.request_timeout : Float - Default: ``60.0`` + Default: ``40.0`` The time allowed for HTTP request completion. (JUPYTER_GATEWAY_REQUEST_TIMEOUT env var) @@ -1389,3 +1783,14 @@ GatewayClient.ws_url : Unicode The websocket url of the Kernel or Enterprise Gateway server. If not provided, this value will correspond to the value of the Gateway url with 'ws' in place of 'http'. (JUPYTER_GATEWAY_WS_URL env var) + +TerminalManager.cull_inactive_timeout : Int + Default: ``0`` + + Timeout (in seconds) in which a terminal has been inactive and ready to be culled. + Values of 0 or lower disable culling. + +TerminalManager.cull_interval : Int + Default: ``300`` + + The interval (in seconds) on which to check for terminals exceeding the inactive timeout value. From bc94f13ada0dbd276c8f5f807e63bd39b2f57098 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 25 Apr 2021 16:42:57 +0800 Subject: [PATCH 37/49] Fix docs --- docs/source/conf.py | 9 ++++++--- docs/source/other/full-config.rst | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index b1d14e42c9..1d53b6cacd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -76,6 +76,8 @@ 'sphinx.ext.mathjax', 'IPython.sphinxext.ipython_console_highlighting', 'sphinxcontrib_github_alt', + 'sphinxcontrib.openapi', + 'sphinxemoji.sphinxemoji', 'sphinx-jsonschema', 'jupyter_telemetry_sphinxext' ] @@ -387,9 +389,10 @@ import jupyter_server.transutils # Jupyter telemetry configuration values. -jupyter_telemetry_schema_source = "../jupyter_server/event-schemas" # Path is relative to conf.py -jupyter_telemetry_schema_output = "source/operators/events" # Path is relative to conf.py -jupyter_telemetry_index_title = "Telemetry Event Schemas" # Title of the index page that lists all found schemas. +jupyter_telemetry_schema_source = osp.join(HERE, '../../jupyter_server/event-schemas') +jupyter_telemetry_schema_output = osp.join(HERE, 'operators/events') +# Title of the index page that lists all found schemas +jupyter_telemetry_index_title = 'Telemetry Event Schemas' def setup(app): dest = osp.join(HERE, 'other', 'changelog.md') diff --git a/docs/source/other/full-config.rst b/docs/source/other/full-config.rst index 8fc9b60a56..8bc6e680bc 100644 --- a/docs/source/other/full-config.rst +++ b/docs/source/other/full-config.rst @@ -1523,7 +1523,7 @@ AsyncFileContentsManager.use_atomic_writing : Bool This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) -NotebookNotary.algorithm : any of ``'sha3_512'``|``'sha224'``|``'sha1'``|``'sha256'``|``'sha3_384'``|``'sha512'``|``'sha3_256'``|``'blake2s'``|``'sha3_224'``|``'md5'``|``'sha384'``|``'blake2b'`` +NotebookNotary.algorithm : any of ``'sha512'``|``'md5'``|``'sha224'``|``'sha256'``|``'blake2s'``|``'blake2b'``|``'sha3_224'``|``'sha384'``|``'sha1'``|``'sha3_256'``|``'sha3_512'``|``'sha3_384'`` Default: ``'sha256'`` The hashing algorithm used to sign notebooks. From bfbdd17d01f77087c6f9b5fcd1f4b4da23dd85a5 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 25 Apr 2021 20:09:05 +0800 Subject: [PATCH 38/49] Refine example --- .../client_eventlog/__init__.py | 1 - examples/client_eventlog/setup.py | 16 ------------ .../client_eventlog_example/__init__.py | 3 +++ .../schemas/__init__.py | 0 .../schemas/schema_yaml.yaml | 0 examples/client_eventlog_example/setup.cfg | 26 +++++++++++++++++++ examples/client_eventlog_example/setup.py | 2 ++ .../tests/conftest.py | 0 .../tests/test_client_eventlog.py | 0 9 files changed, 31 insertions(+), 17 deletions(-) delete mode 100644 examples/client_eventlog/client_eventlog/__init__.py delete mode 100644 examples/client_eventlog/setup.py create mode 100644 examples/client_eventlog_example/client_eventlog_example/__init__.py rename examples/{client_eventlog/client_eventlog => client_eventlog_example/client_eventlog_example}/schemas/__init__.py (100%) rename examples/{client_eventlog/client_eventlog => client_eventlog_example/client_eventlog_example}/schemas/schema_yaml.yaml (100%) create mode 100644 examples/client_eventlog_example/setup.cfg create mode 100644 examples/client_eventlog_example/setup.py rename examples/{client_eventlog => client_eventlog_example}/tests/conftest.py (100%) rename examples/{client_eventlog => client_eventlog_example}/tests/test_client_eventlog.py (100%) diff --git a/examples/client_eventlog/client_eventlog/__init__.py b/examples/client_eventlog/client_eventlog/__init__.py deleted file mode 100644 index f7b20ba32b..0000000000 --- a/examples/client_eventlog/client_eventlog/__init__.py +++ /dev/null @@ -1 +0,0 @@ -JUPYTER_TELEMETRY_SCHEMAS = ['client_eventlog.schemas'] diff --git a/examples/client_eventlog/setup.py b/examples/client_eventlog/setup.py deleted file mode 100644 index e0b109092f..0000000000 --- a/examples/client_eventlog/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -# Adapted from https://github.com/jasongrout/jupyter_core/blob/0310f4a199ba7da60abc54bd9115f7da9a9cec25/examples/scale/template/setup.py # noqa -from setuptools import setup - -name = 'client_eventlog' - -setup( - name=name, - version="1.0.0", - packages=[name], - include_package_data=True, - entry_points= { - 'jupyter_telemetry': [ - f'{name}.sample_entry_point = {name}:JUPYTER_TELEMETRY_SCHEMAS' - ] - } -) diff --git a/examples/client_eventlog_example/client_eventlog_example/__init__.py b/examples/client_eventlog_example/client_eventlog_example/__init__.py new file mode 100644 index 0000000000..7bb0d6990c --- /dev/null +++ b/examples/client_eventlog_example/client_eventlog_example/__init__.py @@ -0,0 +1,3 @@ +__version__ = '0.1.0' + +JUPYTER_TELEMETRY_SCHEMAS = ['client_eventlog_example.schemas'] diff --git a/examples/client_eventlog/client_eventlog/schemas/__init__.py b/examples/client_eventlog_example/client_eventlog_example/schemas/__init__.py similarity index 100% rename from examples/client_eventlog/client_eventlog/schemas/__init__.py rename to examples/client_eventlog_example/client_eventlog_example/schemas/__init__.py diff --git a/examples/client_eventlog/client_eventlog/schemas/schema_yaml.yaml b/examples/client_eventlog_example/client_eventlog_example/schemas/schema_yaml.yaml similarity index 100% rename from examples/client_eventlog/client_eventlog/schemas/schema_yaml.yaml rename to examples/client_eventlog_example/client_eventlog_example/schemas/schema_yaml.yaml diff --git a/examples/client_eventlog_example/setup.cfg b/examples/client_eventlog_example/setup.cfg new file mode 100644 index 0000000000..a6be0296bb --- /dev/null +++ b/examples/client_eventlog_example/setup.cfg @@ -0,0 +1,26 @@ +[metadata] +name = client_eventlog_example +version = attr: client_eventlog_example.__version__ +description = a dummy module for testing client telemetry eventlog entrypoint +long_description = file: README.md +long_description_content_type = text/markdown +url = https://jupyter.org +author = Jupyter Development Team +author_email = jupyter@googlegroups.org +license = BSD +license_file = COPYING.md +classifiers = + Intended Audience :: Developers + Intended Audience :: System Administrators + Intended Audience :: Science/Research + License :: OSI Approved :: BSD License + Programming Language :: Python + +zip_safe = False +include_package_data = True +packages = find: +python_requires = >=3.6 + +[options.entry_points] +jupyter_telemetry = + example-client-eventlog-entry-point = client_eventlog_example:JUPYTER_TELEMETRY_SCHEMAS diff --git a/examples/client_eventlog_example/setup.py b/examples/client_eventlog_example/setup.py new file mode 100644 index 0000000000..a4f49f9202 --- /dev/null +++ b/examples/client_eventlog_example/setup.py @@ -0,0 +1,2 @@ +import setuptools +setuptools.setup() diff --git a/examples/client_eventlog/tests/conftest.py b/examples/client_eventlog_example/tests/conftest.py similarity index 100% rename from examples/client_eventlog/tests/conftest.py rename to examples/client_eventlog_example/tests/conftest.py diff --git a/examples/client_eventlog/tests/test_client_eventlog.py b/examples/client_eventlog_example/tests/test_client_eventlog.py similarity index 100% rename from examples/client_eventlog/tests/test_client_eventlog.py rename to examples/client_eventlog_example/tests/test_client_eventlog.py From 0974231a19e358aa9c4a9d4f674e336f1a096fa8 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 25 Apr 2021 20:29:07 +0800 Subject: [PATCH 39/49] Use same interface for registering file and file object --- jupyter_server/serverapp.py | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index bbd529ec1c..2094962f16 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -1838,7 +1838,7 @@ def init_eventlog(self): for file in get_client_schema_files(): with importlib_resources.as_file(file) as f: try: - self.eventlog.register_schema_file_object(f) + self.eventlog.register_schema_file(f) except: pass diff --git a/setup.cfg b/setup.cfg index 4cf5088303..fb42dfe416 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,7 +45,7 @@ install_requires = anyio>=2.0.2,<3 importlib-metadata ; python_version < '3.8' importlib-resources ; python_version < '3.9' - jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@a4fbf987da063f9e0b337abd21803c968ac64f27 + jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@4f4b5650fc86ffb6fc097741f3523e433b1c4967 [options.extras_require] test = coverage; pytest; pytest-cov; pytest-mock; requests; pytest-tornasync; pytest-console-scripts; ipykernel From 035eb6ece70cdd53a2ace80281092d9488d99feb Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 25 Apr 2021 20:29:22 +0800 Subject: [PATCH 40/49] Fix client test ci --- .github/workflows/python-linux.yml | 4 ++-- .github/workflows/python-macos.yml | 4 ++-- .github/workflows/python-windows.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-linux.yml b/.github/workflows/python-linux.yml index f071ca9df2..0cf3f43528 100644 --- a/.github/workflows/python-linux.yml +++ b/.github/workflows/python-linux.yml @@ -58,10 +58,10 @@ jobs: pytest examples/simple/tests/test_handlers.py - name: Install the Python dependencies for the client telemetry eventlog example run: | - cd examples/client_eventlog && pip install -e . + cd examples/client_eventlog_example && pip install -e . - name: Run the tests for the client telemetry eventlog example run: | - pytest examples/client_eventlog/tests + pytest examples/client_eventlog_example/tests - name: Coverage if: ${{ matrix.python-version != 'pypy3' }} run: | diff --git a/.github/workflows/python-macos.yml b/.github/workflows/python-macos.yml index baaecc5d7c..fdb1ecfeb4 100644 --- a/.github/workflows/python-macos.yml +++ b/.github/workflows/python-macos.yml @@ -58,10 +58,10 @@ jobs: pytest examples/simple/tests/test_handlers.py - name: Install the Python dependencies for the client telemetry eventlog example run: | - cd examples/client_eventlog && pip install -e . + cd examples/client_eventlog_example && pip install -e . - name: Run the tests for the client telemetry eventlog example run: | - pytest examples/client_eventlog/tests + pytest examples/client_eventlog_example/tests - name: Coverage if: ${{ matrix.python-version != 'pypy3' }} run: | diff --git a/.github/workflows/python-windows.yml b/.github/workflows/python-windows.yml index e1198c0eb5..0b988f306b 100644 --- a/.github/workflows/python-windows.yml +++ b/.github/workflows/python-windows.yml @@ -58,7 +58,7 @@ jobs: pytest examples/simple/tests/test_handlers.py - name: Install the Python dependencies for the client telemetry eventlog example run: | - cd examples/client_eventlog && pip install -e . + cd examples/client_eventlog_example && pip install -e . - name: Run the tests for the client telemetry eventlog example run: | - pytest examples/client_eventlog/tests + pytest examples/client_eventlog_example/tests From 8179c475f062b26d3336fa34479fa42f54756780 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 26 Apr 2021 17:46:47 +0800 Subject: [PATCH 41/49] Remove redundant check --- jupyter_server/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 8010a1f554..f93321cfc7 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -293,7 +293,6 @@ def _safe_entry_point_load(ep): def _safe_load_resource(x): try: - fs = importlib_resources.files(x) - return fs if fs.exists() and fs.is_dir() else None + return importlib_resources.files(x) except ModuleNotFoundError: return None From c307a8de8e17a56858abdb283524cb8a7e0a973b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 26 Apr 2021 17:47:31 +0800 Subject: [PATCH 42/49] Add docs on registering client events --- docs/source/operators/telemetry.rst | 50 ++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/docs/source/operators/telemetry.rst b/docs/source/operators/telemetry.rst index 6308f672d0..c21c5df019 100644 --- a/docs/source/operators/telemetry.rst +++ b/docs/source/operators/telemetry.rst @@ -11,7 +11,7 @@ The Notebook Server can be configured to record structured events from a running .. warning:: Do NOT rely on this feature for security or auditing purposes. Neither `server <#emitting-server-events>`_ nor `client <#eventlog-endpoint>`_ events are protected against meddling. For server events, those who have access to the environment can change the server code to emit whatever they want. The same goes for client events where nothing prevents users from sending spurious data to the `eventlog` endpoint. -Emitting Server Events +Emitting server events ---------------------- Event logging is handled by its ``Eventlog`` object. This leverages Python's standing logging_ library to emit, filter, and collect event data. @@ -37,8 +37,16 @@ Here's a basic example for emitting events from the `contents` service: The output is a file, ``"event.log"``, with events recorded as JSON data. -`eventlog` endpoint -------------------- +Server event schemas +-------------------- + +.. toctree:: + :maxdepth: 2 + + events/index + +The ``eventlog`` endpoint +------------------------- The Notebook Server provides a public REST endpoint for external applications to validate and log events through the Server's Event Log. @@ -54,11 +62,37 @@ Events that are validated by this endpoint must have their schema listed in the .. _below: +Register client event schemas +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Server Event schemas --------------------- +``jupyter_server`` looks for locations of schema files provided by external packages by looking into the ``jupyter_telemetry`` entry point and then loads the files using the ``importlib.resources`` standard library. -.. toctree:: - :maxdepth: 2 +For example, suppose there is a ``client_events`` package which wants to send events with schemas ``schema1.yaml``, ``schema2.yaml`` and ``extra_schema.yaml`` to the ``eventlog`` endpoint and has the following package structure: - events/index +.. code-block:: text + + client_events/ + __init__.py + schemas/ + __init__.py + schema1.yaml + schema2.yaml + extras/ + __init__.py + extra_schema.yaml + +``schema1.yaml`` and ``schema2.yaml`` are resources under ``client_events.schemas`` and ``extra_schema.yaml`` under ``client_events.extras``. To make these schemas discoverable by ``jupyter_server``, create an entry point under the ``jupyter_telemetry`` group which resolves to a list containing their locations, in this case ``['client_events.schemas', 'client_events.extras']``: + +In :file:`setup.cfg` + +.. code-block:: yaml + + [options.entry_points] + jupyter_telemetry = + my-event-entry-point = client_events:JUPYTER_TELEMETRY_SCHEMAS + +In :file:`client_events/__init__.py` + +.. code-block:: python + + JUPYTER_TELEMETRY_SCHEMAS = ['client_events.schemas', 'client_events.extras'] From e206188bb4716fdcc800b2730fd1d5fd5b98c488 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 26 Apr 2021 17:57:31 +0800 Subject: [PATCH 43/49] Remove unrelated doc change --- docs/source/other/full-config.rst | 587 +++++------------------------- 1 file changed, 91 insertions(+), 496 deletions(-) diff --git a/docs/source/other/full-config.rst b/docs/source/other/full-config.rst index 8bc6e680bc..52d0bcf3ff 100644 --- a/docs/source/other/full-config.rst +++ b/docs/source/other/full-config.rst @@ -110,9 +110,9 @@ ServerApp.allow_origin : Unicode Default: ``''`` Set the Access-Control-Allow-Origin header - + Use '*' to allow any origin to access your server. - + Takes precedence over allow_origin_pat. @@ -120,13 +120,13 @@ ServerApp.allow_origin_pat : Unicode Default: ``''`` Use a regular expression for the Access-Control-Allow-Origin header - + Requests from an origin matching the expression will get replies with: - + Access-Control-Allow-Origin: origin - + where `origin` is the origin of the request. - + Ignored if allow_origin is set. @@ -134,11 +134,11 @@ ServerApp.allow_password_change : Bool Default: ``True`` Allow password to be changed at login for the Jupyter server. - + While loggin in with a token, the Jupyter server UI will give the opportunity to the user to enter a new password at the same time that will replace the token login mechanism. - + This can be set to false to prevent changing password from the UI/API. @@ -146,15 +146,15 @@ ServerApp.allow_remote_access : Bool Default: ``False`` Allow requests where the Host header doesn't point to a local server - + By default, requests get a 403 forbidden response if the 'Host' header shows that the browser thinks it's on a non-local domain. Setting this option to True disables this check. - + This protects against 'DNS rebinding' attacks, where a remote web server serves you a page and then changes its DNS to send later requests to a local IP, bypassing same-origin checks. - + Local IP addresses (such as 127.0.0.1 and ::1) are allowed as local, along with hostnames configured in local_hostnames. @@ -169,23 +169,11 @@ ServerApp.answer_yes : Bool Answer yes to any prompts. -ServerApp.authenticate_prometheus : Bool - Default: ``True`` - - " - Require authentication to access prometheus metrics. - - -ServerApp.autoreload : Bool - Default: ``False`` - - Reload the webapp when changes are made to any Python src files. - ServerApp.base_url : Unicode Default: ``'/'`` The base URL for the Jupyter server. - + Leading and trailing slashes can be omitted, and will automatically be added. @@ -225,7 +213,7 @@ ServerApp.config_manager_class : Type The config manager class to use -ServerApp.contents_manager_class : TypeFromClasses +ServerApp.contents_manager_class : Type Default: ``'jupyter_server.services.contents.largefilemanager.LargeFileM...`` The content manager class to use. @@ -241,7 +229,7 @@ ServerApp.cookie_secret : Bytes The random bytes used to secure cookies. By default this is a new random number every time you start the server. Set it to a value in a config file to enable logins to persist across server sessions. - + Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). @@ -255,12 +243,12 @@ ServerApp.custom_display_url : Unicode Default: ``''`` Override URL shown to users. - + Replace actual URL, including protocol, address, port and base URL, with the given value when displaying URL to the users. Do not change the actual connection URL. If authentication token is enabled, the token is added to the custom URL automatically. - + This option is intended to be used when the URL to display to the user cannot be determined reliably by the Jupyter server (proxified or containerized setups for example). @@ -274,13 +262,13 @@ ServerApp.disable_check_xsrf : Bool Default: ``False`` Disable cross-site-request-forgery protection - + Jupyter notebook 4.3.1 introduces protection from cross-site request forgeries, requiring API requests to either: - + - originate from pages served by this server (validated with XSRF cookie and token), or - authenticate with a token - + Some anonymous compute resources still desire the ability to run code, completely without authentication. These services can disable all authentication and security checks, @@ -296,7 +284,7 @@ ServerApp.extra_static_paths : List Default: ``[]`` Extra paths to search for serving static files. - + This allows adding javascript/css to be available from the Jupyter server machine, or overriding individual files in the IPython @@ -304,18 +292,13 @@ ServerApp.extra_template_paths : List Default: ``[]`` Extra paths to search for serving jinja templates. - + Can be used to override templates from jupyter_server.templates. ServerApp.file_to_run : Unicode Default: ``''`` - Open the named file when the application is launched. - -ServerApp.file_url_prefix : Unicode - Default: ``'notebooks'`` - - The URL prefix where files are opened directly. + No description ServerApp.generate_config : Bool Default: ``False`` @@ -362,17 +345,17 @@ ServerApp.jpserver_extensions : Dict Dict of Python modules to load as notebook server extensions.Entry values can be used to enable and disable the loading ofthe extensions. The extensions will be loaded in alphabetical order. ServerApp.kernel_manager_class : Type - Default: ``'jupyter_server.services.kernels.kernelmanager.AsyncMappingKe...`` + Default: ``'jupyter_server.services.kernels.kernelmanager.MappingKernelM...`` The kernel manager class to use. ServerApp.kernel_spec_manager_class : Type Default: ``'jupyter_client.kernelspec.KernelSpecManager'`` - + The kernel spec manager class to use. Should be a subclass of `jupyter_client.kernelspec.KernelSpecManager`. - + The Api of KernelSpecManager is provisional and might change without warning between this version of Jupyter and the next stable one. @@ -386,7 +369,7 @@ ServerApp.local_hostnames : List Default: ``['localhost']`` Hostnames to allow as local when allow_remote_access is False. - + Local IP addresses (such as 127.0.0.1 and ::1) are automatically accepted as local as well. @@ -419,33 +402,23 @@ ServerApp.logout_handler_class : Type ServerApp.max_body_size : Int Default: ``536870912`` - + Sets the maximum allowed size of the client request body, specified in the Content-Length request header field. If the size in a request exceeds the configured value, a malformed HTTP message is returned to the client. - + Note: max_body_size is applied even in streaming mode. ServerApp.max_buffer_size : Int Default: ``536870912`` - + Gets or sets the maximum amount of memory, in bytes, that is allocated for use by the buffer manager. -ServerApp.min_open_files_limit : Int - Default: ``0`` - - - Gets or sets a lower bound on the open file handles process resource - limit. This may need to be increased if you run into an - OSError: [Errno 24] Too many open files. - This is not applicable when running on Windows. - - ServerApp.notebook_dir : Unicode Default: ``''`` @@ -465,11 +438,11 @@ ServerApp.password : Unicode Default: ``''`` Hashed password to use for web authentication. - + To generate, type in a python/IPython shell: - + from jupyter_server.auth import passwd; passwd() - + The string should be of the form type:salt:hashed-password. @@ -479,26 +452,26 @@ ServerApp.password_required : Bool Forces users to use a password for the Jupyter server. This is useful in a multi user environment, for instance when everybody in the LAN can access each other's machine through ssh. - + In such a case, serving on localhost is not secure since any user can connect to the Jupyter server via ssh. - + ServerApp.port : Int Default: ``8888`` - The port the server will listen on (env: JUPYTER_PORT). + The port the Jupyter server will listen on. ServerApp.port_retries : Int Default: ``50`` - The number of additional ports to try if the specified port is not available (env: JUPYTER_PORT_RETRIES). + The number of additional ports to try if the specified port is not available. ServerApp.pylab : Unicode Default: ``'disabled'`` - + DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. @@ -558,10 +531,10 @@ ServerApp.terminals_enabled : Bool Default: ``True`` Set to False to disable terminals. - + This does *not* make the server more secure by itself. Anything the user can in a terminal, they can also do in a notebook. - + Terminals may also be automatically disabled if the terminado package is not available. @@ -570,13 +543,10 @@ ServerApp.token : Unicode Default: ``''`` Token used for authenticating first-time connections to the server. - - The token can be read from the file referenced by JUPYTER_TOKEN_FILE or set directly - with the JUPYTER_TOKEN environment variable. - + When no password is enabled, the default is to generate a new, random token. - + Setting to an empty string disables authentication altogether, which is NOT RECOMMENDED. @@ -590,23 +560,6 @@ ServerApp.trust_xheaders : Bool Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headerssent by the upstream reverse proxy. Necessary if the proxy handles SSL -ServerApp.use_redirect_file : Bool - Default: ``True`` - - Disable launching browser by redirect file - For versions of notebook > 5.7.2, a security feature measure was added that - prevented the authentication token used to launch the browser from being visible. - This feature makes it difficult for other users on a multi-user system from - running code in your Jupyter session as you. - However, some environments (like Windows Subsystem for Linux (WSL) and Chromebooks), - launching a browser using a redirect file can lead the browser failing to load. - This is because of the difference in file structures/paths between the runtime and - the browser. - - Disabling this setting to False will disable this behavior, allowing the browser - to launch by using a URL and visible token (as before). - - ServerApp.webbrowser_open_new : Int Default: ``2`` @@ -614,24 +567,24 @@ ServerApp.webbrowser_open_new : Int `new` argument passed to the standard library method `webbrowser.open`. The behaviour is not guaranteed, but depends on browser support. Valid values are: - + - 2 opens a new tab, - 1 opens a new window, - 0 opens in an existing window. - + See the `webbrowser.open` documentation for details. ServerApp.websocket_compression_options : Any Default: ``None`` - + Set the tornado compression options for websocket connections. - + This value will be returned from :meth:`WebSocketHandler.get_compression_options`. None (default) will disable compression. A dict (even an empty one) will enable compression. - + See the tornado docs for WebSocketHandler.get_compression_options for details. @@ -640,7 +593,7 @@ ServerApp.websocket_url : Unicode The base URL for websockets, if it differs from the HTTP server (hint: it almost certainly doesn't). - + Should be in the form of an HTTP origin: ws[s]://hostname[:port] @@ -648,7 +601,7 @@ ConnectionFileMixin.connection_file : Unicode Default: ``''`` JSON file in which to store connection info [default: kernel-.json] - + This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. @@ -701,7 +654,7 @@ KernelManager.connection_file : Unicode Default: ``''`` JSON file in which to store connection info [default: kernel-.json] - + This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. @@ -734,7 +687,7 @@ KernelManager.kernel_cmd : List Default: ``[]`` DEPRECATED: Use kernel_name instead. - + The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, @@ -774,7 +727,7 @@ Session.check_pid : Bool Default: ``True`` Whether to check PID to protect against calls after fork. - + This check can be disabled if fork-safety is handled elsewhere. @@ -792,7 +745,7 @@ Session.digest_history_size : Int Default: ``65536`` The maximum number of digests to remember. - + The digest history will be culled when it exceeds this value. @@ -843,7 +796,7 @@ Session.unpacker : DottedObjectName Only used with custom functions for `packer`. Session.username : Unicode - Default: ``'kien'`` + Default: ``'username'`` Username for the Session. Default is your system username. @@ -880,10 +833,10 @@ MappingKernelManager.buffer_offline_messages : Bool Default: ``True`` Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - + When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. - + Disable if long-running kernels will produce too much output while no frontends are connected. @@ -921,7 +874,7 @@ MappingKernelManager.kernel_info_timeout : Float Default: ``60`` Timeout for giving up on a kernel (in seconds). - + On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take @@ -970,114 +923,9 @@ KernelSpecManager.whitelist : Set Default: ``set()`` Whitelist of allowed kernel names. - - By default, all installed kernels are allowed. - - -AsyncMultiKernelManager.default_kernel_name : Unicode - Default: ``'python3'`` - - The name of the default kernel to start - -AsyncMultiKernelManager.kernel_manager_class : DottedObjectName - Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` - - The kernel manager class. This is configurable to allow - subclassing of the AsyncKernelManager for customized behavior. - - -AsyncMultiKernelManager.shared_context : Bool - Default: ``True`` - - Share a single zmq.Context to talk to all my kernels - -AsyncMappingKernelManager.allow_tracebacks : Bool - Default: ``True`` - - Whether to send tracebacks to clients on exceptions. - -AsyncMappingKernelManager.allowed_message_types : List - Default: ``[]`` - - White list of allowed kernel message types. - When the list is empty, all message types are allowed. + By default, all installed kernels are allowed. -AsyncMappingKernelManager.buffer_offline_messages : Bool - Default: ``True`` - - Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - - When True (default), messages are buffered and replayed on reconnect, - avoiding lost messages due to interrupted connectivity. - - Disable if long-running kernels will produce too much output while - no frontends are connected. - - -AsyncMappingKernelManager.cull_busy : Bool - Default: ``False`` - - Whether to consider culling kernels which are busy. - Only effective if cull_idle_timeout > 0. - -AsyncMappingKernelManager.cull_connected : Bool - Default: ``False`` - - Whether to consider culling kernels which have one or more connections. - Only effective if cull_idle_timeout > 0. - -AsyncMappingKernelManager.cull_idle_timeout : Int - Default: ``0`` - - Timeout (in seconds) after which a kernel is considered idle and ready to be culled. - Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled - for users with poor network connections. - -AsyncMappingKernelManager.cull_interval : Int - Default: ``300`` - - The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value. - -AsyncMappingKernelManager.default_kernel_name : Unicode - Default: ``'python3'`` - - The name of the default kernel to start - -AsyncMappingKernelManager.kernel_info_timeout : Float - Default: ``60`` - - Timeout for giving up on a kernel (in seconds). - - On starting and restarting kernels, we check whether the - kernel is running and responsive by sending kernel_info_requests. - This sets the timeout in seconds for how long the kernel can take - before being presumed dead. - This affects the MappingKernelManager (which handles kernel restarts) - and the ZMQChannelsHandler (which handles the startup). - - -AsyncMappingKernelManager.kernel_manager_class : DottedObjectName - Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` - - The kernel manager class. This is configurable to allow - subclassing of the AsyncKernelManager for customized behavior. - - -AsyncMappingKernelManager.root_dir : Unicode - Default: ``''`` - - No description - -AsyncMappingKernelManager.shared_context : Bool - Default: ``True`` - - Share a single zmq.Context to talk to all my kernels - -AsyncMappingKernelManager.traceback_replacement_message : Unicode - Default: ``'An exception occurred at runtime, which is not shown due to ...`` - - Message to print when allow_tracebacks is False, and an exception occurs ContentsManager.allow_hidden : Bool Default: ``False`` @@ -1103,13 +951,13 @@ ContentsManager.files_handler_class : Type Default: ``'jupyter_server.files.handlers.FilesHandler'`` handler class to use when serving raw file requests. - + Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. - + Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. - + Access to these files should be Authenticated. @@ -1117,7 +965,7 @@ ContentsManager.files_handler_params : Dict Default: ``{}`` Extra parameters to pass to files_handler_class. - + For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. @@ -1125,7 +973,7 @@ ContentsManager.files_handler_params : Dict ContentsManager.hide_globs : List Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - + Glob patterns to hide in file and directory listings. @@ -1133,17 +981,17 @@ ContentsManager.pre_save_hook : Any Default: ``None`` Python callable or importstring thereof - + To be called on a contents model prior to save. - + This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. - + It will be called as (all arguments passed by keyword):: - + hook(path=path, model=model, contents_manager=self) - + - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination @@ -1208,13 +1056,13 @@ FileContentsManager.files_handler_class : Type Default: ``'jupyter_server.files.handlers.FilesHandler'`` handler class to use when serving raw file requests. - + Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. - + Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. - + Access to these files should be Authenticated. @@ -1222,7 +1070,7 @@ FileContentsManager.files_handler_params : Dict Default: ``{}`` Extra parameters to pass to files_handler_class. - + For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. @@ -1230,7 +1078,7 @@ FileContentsManager.files_handler_params : Dict FileContentsManager.hide_globs : List Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - + Glob patterns to hide in file and directory listings. @@ -1238,16 +1086,16 @@ FileContentsManager.post_save_hook : Any Default: ``None`` Python callable or importstring thereof - + to be called on the path of a file just saved. - + This can be used to process the file on disk, such as converting the notebook to a script or HTML via nbconvert. - + It will be called as (all arguments passed by keyword):: - + hook(os_path=os_path, model=model, contents_manager=instance) - + - path: the filesystem path to the file just written - model: the model representing the file - contents_manager: this ContentsManager instance @@ -1257,17 +1105,17 @@ FileContentsManager.pre_save_hook : Any Default: ``None`` Python callable or importstring thereof - + To be called on a contents model prior to save. - + This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. - + It will be called as (all arguments passed by keyword):: - + hook(path=path, model=model, contents_manager=self) - + - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination @@ -1301,229 +1149,7 @@ FileContentsManager.use_atomic_writing : Bool This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) -AsyncContentsManager.allow_hidden : Bool - Default: ``False`` - - Allow access to hidden files - -AsyncContentsManager.checkpoints : Instance - Default: ``None`` - - No description - -AsyncContentsManager.checkpoints_class : Type - Default: ``'jupyter_server.services.contents.checkpoints.AsyncCheckpoints'`` - - No description - -AsyncContentsManager.checkpoints_kwargs : Dict - Default: ``{}`` - - No description - -AsyncContentsManager.files_handler_class : Type - Default: ``'jupyter_server.files.handlers.FilesHandler'`` - - handler class to use when serving raw file requests. - - Default is a fallback that talks to the ContentsManager API, - which may be inefficient, especially for large files. - - Local files-based ContentsManagers can use a StaticFileHandler subclass, - which will be much more efficient. - - Access to these files should be Authenticated. - - -AsyncContentsManager.files_handler_params : Dict - Default: ``{}`` - - Extra parameters to pass to files_handler_class. - - For example, StaticFileHandlers generally expect a `path` argument - specifying the root directory from which to serve files. - - -AsyncContentsManager.hide_globs : List - Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - - - Glob patterns to hide in file and directory listings. - - -AsyncContentsManager.pre_save_hook : Any - Default: ``None`` - - Python callable or importstring thereof - - To be called on a contents model prior to save. - - This can be used to process the structure, - such as removing notebook outputs or other side effects that - should not be saved. - - It will be called as (all arguments passed by keyword):: - - hook(path=path, model=model, contents_manager=self) - - - model: the model to be saved. Includes file contents. - Modifying this dict will affect the file that is stored. - - path: the API path of the save destination - - contents_manager: this ContentsManager instance - - -AsyncContentsManager.root_dir : Unicode - Default: ``'/'`` - - No description - -AsyncContentsManager.untitled_directory : Unicode - Default: ``'Untitled Folder'`` - - The base name used when creating untitled directories. - -AsyncContentsManager.untitled_file : Unicode - Default: ``'untitled'`` - - The base name used when creating untitled files. - -AsyncContentsManager.untitled_notebook : Unicode - Default: ``'Untitled'`` - - The base name used when creating untitled notebooks. - -AsyncFileManagerMixin.use_atomic_writing : Bool - Default: ``True`` - - By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. - This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). - If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) - -AsyncFileContentsManager.allow_hidden : Bool - Default: ``False`` - - Allow access to hidden files - -AsyncFileContentsManager.checkpoints : Instance - Default: ``None`` - - No description - -AsyncFileContentsManager.checkpoints_class : Type - Default: ``'jupyter_server.services.contents.checkpoints.AsyncCheckpoints'`` - - No description - -AsyncFileContentsManager.checkpoints_kwargs : Dict - Default: ``{}`` - - No description - -AsyncFileContentsManager.delete_to_trash : Bool - Default: ``True`` - - If True (default), deleting files will send them to the - platform's trash/recycle bin, where they can be recovered. If False, - deleting files really deletes them. - -AsyncFileContentsManager.files_handler_class : Type - Default: ``'jupyter_server.files.handlers.FilesHandler'`` - - handler class to use when serving raw file requests. - - Default is a fallback that talks to the ContentsManager API, - which may be inefficient, especially for large files. - - Local files-based ContentsManagers can use a StaticFileHandler subclass, - which will be much more efficient. - - Access to these files should be Authenticated. - - -AsyncFileContentsManager.files_handler_params : Dict - Default: ``{}`` - - Extra parameters to pass to files_handler_class. - - For example, StaticFileHandlers generally expect a `path` argument - specifying the root directory from which to serve files. - - -AsyncFileContentsManager.hide_globs : List - Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - - - Glob patterns to hide in file and directory listings. - - -AsyncFileContentsManager.post_save_hook : Any - Default: ``None`` - - Python callable or importstring thereof - - to be called on the path of a file just saved. - - This can be used to process the file on disk, - such as converting the notebook to a script or HTML via nbconvert. - - It will be called as (all arguments passed by keyword):: - - hook(os_path=os_path, model=model, contents_manager=instance) - - - path: the filesystem path to the file just written - - model: the model representing the file - - contents_manager: this ContentsManager instance - - -AsyncFileContentsManager.pre_save_hook : Any - Default: ``None`` - - Python callable or importstring thereof - - To be called on a contents model prior to save. - - This can be used to process the structure, - such as removing notebook outputs or other side effects that - should not be saved. - - It will be called as (all arguments passed by keyword):: - - hook(path=path, model=model, contents_manager=self) - - - model: the model to be saved. Includes file contents. - Modifying this dict will affect the file that is stored. - - path: the API path of the save destination - - contents_manager: this ContentsManager instance - - -AsyncFileContentsManager.root_dir : Unicode - Default: ``''`` - - No description - -AsyncFileContentsManager.untitled_directory : Unicode - Default: ``'Untitled Folder'`` - - The base name used when creating untitled directories. - -AsyncFileContentsManager.untitled_file : Unicode - Default: ``'untitled'`` - - The base name used when creating untitled files. - -AsyncFileContentsManager.untitled_notebook : Unicode - Default: ``'Untitled'`` - - The base name used when creating untitled notebooks. - -AsyncFileContentsManager.use_atomic_writing : Bool - Default: ``True`` - - By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. - This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). - If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) - -NotebookNotary.algorithm : any of ``'sha512'``|``'md5'``|``'sha224'``|``'sha256'``|``'blake2s'``|``'blake2b'``|``'sha3_224'``|``'sha384'``|``'sha1'``|``'sha3_256'``|``'sha3_512'``|``'sha3_384'`` +NotebookNotary.algorithm : any of ``'blake2s'``|``'sha512'``|``'md5'``|``'sha3_512'``|``'sha3_224'``|``'blake2b'``|``'sha384'``|``'sha1'``|``'sha3_256'``|``'sha256'``|``'sha224'``|``'sha3_384'`` Default: ``'sha256'`` The hashing algorithm used to sign notebooks. @@ -1568,10 +1194,10 @@ GatewayKernelManager.buffer_offline_messages : Bool Default: ``True`` Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - + When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. - + Disable if long-running kernels will produce too much output while no frontends are connected. @@ -1609,7 +1235,7 @@ GatewayKernelManager.kernel_info_timeout : Float Default: ``60`` Timeout for giving up on a kernel (in seconds). - + On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take @@ -1619,10 +1245,10 @@ GatewayKernelManager.kernel_info_timeout : Float GatewayKernelManager.kernel_manager_class : DottedObjectName - Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` + Default: ``'jupyter_client.ioloop.IOLoopKernelManager'`` The kernel manager class. This is configurable to allow - subclassing of the AsyncKernelManager for customized behavior. + subclassing of the KernelManager for customized behavior. GatewayKernelManager.root_dir : Unicode @@ -1658,7 +1284,7 @@ GatewayKernelSpecManager.whitelist : Set Default: ``set()`` Whitelist of allowed kernel names. - + By default, all installed kernels are allowed. @@ -1689,7 +1315,7 @@ GatewayClient.client_key : Unicode GatewayClient.connect_timeout : Float - Default: ``40.0`` + Default: ``60.0`` The time allowed for HTTP connection establishment with the Gateway server. (JUPYTER_GATEWAY_CONNECT_TIMEOUT env var) @@ -1702,26 +1328,6 @@ GatewayClient.env_whitelist : Unicode value must also be set on the Gateway server - since that configuration value indicates which environmental values to make available to the kernel. (JUPYTER_GATEWAY_ENV_WHITELIST env var) -GatewayClient.gateway_retry_interval : Float - Default: ``1.0`` - - The time allowed for HTTP reconnection with the Gateway server for the first time. - Next will be JUPYTER_GATEWAY_RETRY_INTERVAL multiplied by two in factor of numbers of retries - but less than JUPYTER_GATEWAY_RETRY_INTERVAL_MAX. - (JUPYTER_GATEWAY_RETRY_INTERVAL env var) - -GatewayClient.gateway_retry_interval_max : Float - Default: ``30.0`` - - The maximum time allowed for HTTP reconnection retry with the Gateway server. - (JUPYTER_GATEWAY_RETRY_INTERVAL_MAX env var) - -GatewayClient.gateway_retry_max : Int - Default: ``5`` - - The maximum retries allowed for HTTP reconnection with the Gateway server. - (JUPYTER_GATEWAY_RETRY_MAX env var) - GatewayClient.headers : Unicode Default: ``'{}'`` @@ -1758,7 +1364,7 @@ GatewayClient.kernelspecs_resource_endpoint : Unicode (JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT env var) GatewayClient.request_timeout : Float - Default: ``40.0`` + Default: ``60.0`` The time allowed for HTTP request completion. (JUPYTER_GATEWAY_REQUEST_TIMEOUT env var) @@ -1783,14 +1389,3 @@ GatewayClient.ws_url : Unicode The websocket url of the Kernel or Enterprise Gateway server. If not provided, this value will correspond to the value of the Gateway url with 'ws' in place of 'http'. (JUPYTER_GATEWAY_WS_URL env var) - -TerminalManager.cull_inactive_timeout : Int - Default: ``0`` - - Timeout (in seconds) in which a terminal has been inactive and ready to be culled. - Values of 0 or lower disable culling. - -TerminalManager.cull_interval : Int - Default: ``300`` - - The interval (in seconds) on which to check for terminals exceeding the inactive timeout value. From a67bb25e142953ca2fd96882d0f14df99275ad68 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 3 May 2021 17:41:47 +0800 Subject: [PATCH 44/49] Remove eventlog endpoint Move to a separate branch --- .github/workflows/python-linux.yml | 6 - .github/workflows/python-macos.yml | 6 - .github/workflows/python-windows.yml | 6 - docs/source/operators/telemetry.rst | 35 -- docs/source/other/full-config.rst | 587 +++++++++++++++--- .../client_eventlog_example/__init__.py | 3 - .../schemas/__init__.py | 0 .../schemas/schema_yaml.yaml | 21 - examples/client_eventlog_example/setup.cfg | 26 - examples/client_eventlog_example/setup.py | 2 - .../client_eventlog_example/tests/conftest.py | 3 - .../tests/test_client_eventlog.py | 36 -- jupyter_server/serverapp.py | 16 - jupyter_server/services/eventlog/__init__.py | 0 jupyter_server/services/eventlog/handlers.py | 42 -- jupyter_server/utils.py | 53 -- setup.cfg | 2 - 17 files changed, 496 insertions(+), 348 deletions(-) delete mode 100644 examples/client_eventlog_example/client_eventlog_example/__init__.py delete mode 100644 examples/client_eventlog_example/client_eventlog_example/schemas/__init__.py delete mode 100644 examples/client_eventlog_example/client_eventlog_example/schemas/schema_yaml.yaml delete mode 100644 examples/client_eventlog_example/setup.cfg delete mode 100644 examples/client_eventlog_example/setup.py delete mode 100644 examples/client_eventlog_example/tests/conftest.py delete mode 100644 examples/client_eventlog_example/tests/test_client_eventlog.py delete mode 100644 jupyter_server/services/eventlog/__init__.py delete mode 100644 jupyter_server/services/eventlog/handlers.py diff --git a/.github/workflows/python-linux.yml b/.github/workflows/python-linux.yml index 0cf3f43528..8fedfc3ecc 100644 --- a/.github/workflows/python-linux.yml +++ b/.github/workflows/python-linux.yml @@ -56,12 +56,6 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py - - name: Install the Python dependencies for the client telemetry eventlog example - run: | - cd examples/client_eventlog_example && pip install -e . - - name: Run the tests for the client telemetry eventlog example - run: | - pytest examples/client_eventlog_example/tests - name: Coverage if: ${{ matrix.python-version != 'pypy3' }} run: | diff --git a/.github/workflows/python-macos.yml b/.github/workflows/python-macos.yml index fdb1ecfeb4..163e74a525 100644 --- a/.github/workflows/python-macos.yml +++ b/.github/workflows/python-macos.yml @@ -56,12 +56,6 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py - - name: Install the Python dependencies for the client telemetry eventlog example - run: | - cd examples/client_eventlog_example && pip install -e . - - name: Run the tests for the client telemetry eventlog example - run: | - pytest examples/client_eventlog_example/tests - name: Coverage if: ${{ matrix.python-version != 'pypy3' }} run: | diff --git a/.github/workflows/python-windows.yml b/.github/workflows/python-windows.yml index 0b988f306b..525aa9d7ed 100644 --- a/.github/workflows/python-windows.yml +++ b/.github/workflows/python-windows.yml @@ -56,9 +56,3 @@ jobs: - name: Run the tests for the examples run: | pytest examples/simple/tests/test_handlers.py - - name: Install the Python dependencies for the client telemetry eventlog example - run: | - cd examples/client_eventlog_example && pip install -e . - - name: Run the tests for the client telemetry eventlog example - run: | - pytest examples/client_eventlog_example/tests diff --git a/docs/source/operators/telemetry.rst b/docs/source/operators/telemetry.rst index c21c5df019..293e32cbf9 100644 --- a/docs/source/operators/telemetry.rst +++ b/docs/source/operators/telemetry.rst @@ -61,38 +61,3 @@ JSON blog and is required to have the follow keys: Events that are validated by this endpoint must have their schema listed in the `allowed_schemas` trait listed above. .. _below: - -Register client event schemas -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``jupyter_server`` looks for locations of schema files provided by external packages by looking into the ``jupyter_telemetry`` entry point and then loads the files using the ``importlib.resources`` standard library. - -For example, suppose there is a ``client_events`` package which wants to send events with schemas ``schema1.yaml``, ``schema2.yaml`` and ``extra_schema.yaml`` to the ``eventlog`` endpoint and has the following package structure: - -.. code-block:: text - - client_events/ - __init__.py - schemas/ - __init__.py - schema1.yaml - schema2.yaml - extras/ - __init__.py - extra_schema.yaml - -``schema1.yaml`` and ``schema2.yaml`` are resources under ``client_events.schemas`` and ``extra_schema.yaml`` under ``client_events.extras``. To make these schemas discoverable by ``jupyter_server``, create an entry point under the ``jupyter_telemetry`` group which resolves to a list containing their locations, in this case ``['client_events.schemas', 'client_events.extras']``: - -In :file:`setup.cfg` - -.. code-block:: yaml - - [options.entry_points] - jupyter_telemetry = - my-event-entry-point = client_events:JUPYTER_TELEMETRY_SCHEMAS - -In :file:`client_events/__init__.py` - -.. code-block:: python - - JUPYTER_TELEMETRY_SCHEMAS = ['client_events.schemas', 'client_events.extras'] diff --git a/docs/source/other/full-config.rst b/docs/source/other/full-config.rst index 52d0bcf3ff..2b0be1640a 100644 --- a/docs/source/other/full-config.rst +++ b/docs/source/other/full-config.rst @@ -110,9 +110,9 @@ ServerApp.allow_origin : Unicode Default: ``''`` Set the Access-Control-Allow-Origin header - + Use '*' to allow any origin to access your server. - + Takes precedence over allow_origin_pat. @@ -120,13 +120,13 @@ ServerApp.allow_origin_pat : Unicode Default: ``''`` Use a regular expression for the Access-Control-Allow-Origin header - + Requests from an origin matching the expression will get replies with: - + Access-Control-Allow-Origin: origin - + where `origin` is the origin of the request. - + Ignored if allow_origin is set. @@ -134,11 +134,11 @@ ServerApp.allow_password_change : Bool Default: ``True`` Allow password to be changed at login for the Jupyter server. - + While loggin in with a token, the Jupyter server UI will give the opportunity to the user to enter a new password at the same time that will replace the token login mechanism. - + This can be set to false to prevent changing password from the UI/API. @@ -146,15 +146,15 @@ ServerApp.allow_remote_access : Bool Default: ``False`` Allow requests where the Host header doesn't point to a local server - + By default, requests get a 403 forbidden response if the 'Host' header shows that the browser thinks it's on a non-local domain. Setting this option to True disables this check. - + This protects against 'DNS rebinding' attacks, where a remote web server serves you a page and then changes its DNS to send later requests to a local IP, bypassing same-origin checks. - + Local IP addresses (such as 127.0.0.1 and ::1) are allowed as local, along with hostnames configured in local_hostnames. @@ -169,11 +169,23 @@ ServerApp.answer_yes : Bool Answer yes to any prompts. +ServerApp.authenticate_prometheus : Bool + Default: ``True`` + + " + Require authentication to access prometheus metrics. + + +ServerApp.autoreload : Bool + Default: ``False`` + + Reload the webapp when changes are made to any Python src files. + ServerApp.base_url : Unicode Default: ``'/'`` The base URL for the Jupyter server. - + Leading and trailing slashes can be omitted, and will automatically be added. @@ -213,7 +225,7 @@ ServerApp.config_manager_class : Type The config manager class to use -ServerApp.contents_manager_class : Type +ServerApp.contents_manager_class : TypeFromClasses Default: ``'jupyter_server.services.contents.largefilemanager.LargeFileM...`` The content manager class to use. @@ -229,7 +241,7 @@ ServerApp.cookie_secret : Bytes The random bytes used to secure cookies. By default this is a new random number every time you start the server. Set it to a value in a config file to enable logins to persist across server sessions. - + Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). @@ -243,12 +255,12 @@ ServerApp.custom_display_url : Unicode Default: ``''`` Override URL shown to users. - + Replace actual URL, including protocol, address, port and base URL, with the given value when displaying URL to the users. Do not change the actual connection URL. If authentication token is enabled, the token is added to the custom URL automatically. - + This option is intended to be used when the URL to display to the user cannot be determined reliably by the Jupyter server (proxified or containerized setups for example). @@ -262,13 +274,13 @@ ServerApp.disable_check_xsrf : Bool Default: ``False`` Disable cross-site-request-forgery protection - + Jupyter notebook 4.3.1 introduces protection from cross-site request forgeries, requiring API requests to either: - + - originate from pages served by this server (validated with XSRF cookie and token), or - authenticate with a token - + Some anonymous compute resources still desire the ability to run code, completely without authentication. These services can disable all authentication and security checks, @@ -284,7 +296,7 @@ ServerApp.extra_static_paths : List Default: ``[]`` Extra paths to search for serving static files. - + This allows adding javascript/css to be available from the Jupyter server machine, or overriding individual files in the IPython @@ -292,13 +304,18 @@ ServerApp.extra_template_paths : List Default: ``[]`` Extra paths to search for serving jinja templates. - + Can be used to override templates from jupyter_server.templates. ServerApp.file_to_run : Unicode Default: ``''`` - No description + Open the named file when the application is launched. + +ServerApp.file_url_prefix : Unicode + Default: ``'notebooks'`` + + The URL prefix where files are opened directly. ServerApp.generate_config : Bool Default: ``False`` @@ -345,17 +362,17 @@ ServerApp.jpserver_extensions : Dict Dict of Python modules to load as notebook server extensions.Entry values can be used to enable and disable the loading ofthe extensions. The extensions will be loaded in alphabetical order. ServerApp.kernel_manager_class : Type - Default: ``'jupyter_server.services.kernels.kernelmanager.MappingKernelM...`` + Default: ``'jupyter_server.services.kernels.kernelmanager.AsyncMappingKe...`` The kernel manager class to use. ServerApp.kernel_spec_manager_class : Type Default: ``'jupyter_client.kernelspec.KernelSpecManager'`` - + The kernel spec manager class to use. Should be a subclass of `jupyter_client.kernelspec.KernelSpecManager`. - + The Api of KernelSpecManager is provisional and might change without warning between this version of Jupyter and the next stable one. @@ -369,7 +386,7 @@ ServerApp.local_hostnames : List Default: ``['localhost']`` Hostnames to allow as local when allow_remote_access is False. - + Local IP addresses (such as 127.0.0.1 and ::1) are automatically accepted as local as well. @@ -402,23 +419,33 @@ ServerApp.logout_handler_class : Type ServerApp.max_body_size : Int Default: ``536870912`` - + Sets the maximum allowed size of the client request body, specified in the Content-Length request header field. If the size in a request exceeds the configured value, a malformed HTTP message is returned to the client. - + Note: max_body_size is applied even in streaming mode. ServerApp.max_buffer_size : Int Default: ``536870912`` - + Gets or sets the maximum amount of memory, in bytes, that is allocated for use by the buffer manager. +ServerApp.min_open_files_limit : Int + Default: ``0`` + + + Gets or sets a lower bound on the open file handles process resource + limit. This may need to be increased if you run into an + OSError: [Errno 24] Too many open files. + This is not applicable when running on Windows. + + ServerApp.notebook_dir : Unicode Default: ``''`` @@ -438,11 +465,11 @@ ServerApp.password : Unicode Default: ``''`` Hashed password to use for web authentication. - + To generate, type in a python/IPython shell: - + from jupyter_server.auth import passwd; passwd() - + The string should be of the form type:salt:hashed-password. @@ -452,26 +479,26 @@ ServerApp.password_required : Bool Forces users to use a password for the Jupyter server. This is useful in a multi user environment, for instance when everybody in the LAN can access each other's machine through ssh. - + In such a case, serving on localhost is not secure since any user can connect to the Jupyter server via ssh. - + ServerApp.port : Int Default: ``8888`` - The port the Jupyter server will listen on. + The port the server will listen on (env: JUPYTER_PORT). ServerApp.port_retries : Int Default: ``50`` - The number of additional ports to try if the specified port is not available. + The number of additional ports to try if the specified port is not available (env: JUPYTER_PORT_RETRIES). ServerApp.pylab : Unicode Default: ``'disabled'`` - + DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. @@ -531,10 +558,10 @@ ServerApp.terminals_enabled : Bool Default: ``True`` Set to False to disable terminals. - + This does *not* make the server more secure by itself. Anything the user can in a terminal, they can also do in a notebook. - + Terminals may also be automatically disabled if the terminado package is not available. @@ -543,10 +570,13 @@ ServerApp.token : Unicode Default: ``''`` Token used for authenticating first-time connections to the server. - + + The token can be read from the file referenced by JUPYTER_TOKEN_FILE or set directly + with the JUPYTER_TOKEN environment variable. + When no password is enabled, the default is to generate a new, random token. - + Setting to an empty string disables authentication altogether, which is NOT RECOMMENDED. @@ -560,6 +590,23 @@ ServerApp.trust_xheaders : Bool Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headerssent by the upstream reverse proxy. Necessary if the proxy handles SSL +ServerApp.use_redirect_file : Bool + Default: ``True`` + + Disable launching browser by redirect file + For versions of notebook > 5.7.2, a security feature measure was added that + prevented the authentication token used to launch the browser from being visible. + This feature makes it difficult for other users on a multi-user system from + running code in your Jupyter session as you. + However, some environments (like Windows Subsystem for Linux (WSL) and Chromebooks), + launching a browser using a redirect file can lead the browser failing to load. + This is because of the difference in file structures/paths between the runtime and + the browser. + + Disabling this setting to False will disable this behavior, allowing the browser + to launch by using a URL and visible token (as before). + + ServerApp.webbrowser_open_new : Int Default: ``2`` @@ -567,24 +614,24 @@ ServerApp.webbrowser_open_new : Int `new` argument passed to the standard library method `webbrowser.open`. The behaviour is not guaranteed, but depends on browser support. Valid values are: - + - 2 opens a new tab, - 1 opens a new window, - 0 opens in an existing window. - + See the `webbrowser.open` documentation for details. ServerApp.websocket_compression_options : Any Default: ``None`` - + Set the tornado compression options for websocket connections. - + This value will be returned from :meth:`WebSocketHandler.get_compression_options`. None (default) will disable compression. A dict (even an empty one) will enable compression. - + See the tornado docs for WebSocketHandler.get_compression_options for details. @@ -593,7 +640,7 @@ ServerApp.websocket_url : Unicode The base URL for websockets, if it differs from the HTTP server (hint: it almost certainly doesn't). - + Should be in the form of an HTTP origin: ws[s]://hostname[:port] @@ -601,7 +648,7 @@ ConnectionFileMixin.connection_file : Unicode Default: ``''`` JSON file in which to store connection info [default: kernel-.json] - + This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. @@ -654,7 +701,7 @@ KernelManager.connection_file : Unicode Default: ``''`` JSON file in which to store connection info [default: kernel-.json] - + This file will contain the IP, ports, and authentication key needed to connect clients to this kernel. By default, this file will be created in the security dir of the current profile, but can be specified by absolute path. @@ -687,7 +734,7 @@ KernelManager.kernel_cmd : List Default: ``[]`` DEPRECATED: Use kernel_name instead. - + The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, @@ -727,7 +774,7 @@ Session.check_pid : Bool Default: ``True`` Whether to check PID to protect against calls after fork. - + This check can be disabled if fork-safety is handled elsewhere. @@ -745,7 +792,7 @@ Session.digest_history_size : Int Default: ``65536`` The maximum number of digests to remember. - + The digest history will be culled when it exceeds this value. @@ -796,7 +843,7 @@ Session.unpacker : DottedObjectName Only used with custom functions for `packer`. Session.username : Unicode - Default: ``'username'`` + Default: ``'kien'`` Username for the Session. Default is your system username. @@ -833,10 +880,10 @@ MappingKernelManager.buffer_offline_messages : Bool Default: ``True`` Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - + When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. - + Disable if long-running kernels will produce too much output while no frontends are connected. @@ -874,7 +921,7 @@ MappingKernelManager.kernel_info_timeout : Float Default: ``60`` Timeout for giving up on a kernel (in seconds). - + On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take @@ -923,10 +970,115 @@ KernelSpecManager.whitelist : Set Default: ``set()`` Whitelist of allowed kernel names. - + By default, all installed kernels are allowed. +AsyncMultiKernelManager.default_kernel_name : Unicode + Default: ``'python3'`` + + The name of the default kernel to start + +AsyncMultiKernelManager.kernel_manager_class : DottedObjectName + Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` + + The kernel manager class. This is configurable to allow + subclassing of the AsyncKernelManager for customized behavior. + + +AsyncMultiKernelManager.shared_context : Bool + Default: ``True`` + + Share a single zmq.Context to talk to all my kernels + +AsyncMappingKernelManager.allow_tracebacks : Bool + Default: ``True`` + + Whether to send tracebacks to clients on exceptions. + +AsyncMappingKernelManager.allowed_message_types : List + Default: ``[]`` + + White list of allowed kernel message types. + When the list is empty, all message types are allowed. + + +AsyncMappingKernelManager.buffer_offline_messages : Bool + Default: ``True`` + + Whether messages from kernels whose frontends have disconnected should be buffered in-memory. + + When True (default), messages are buffered and replayed on reconnect, + avoiding lost messages due to interrupted connectivity. + + Disable if long-running kernels will produce too much output while + no frontends are connected. + + +AsyncMappingKernelManager.cull_busy : Bool + Default: ``False`` + + Whether to consider culling kernels which are busy. + Only effective if cull_idle_timeout > 0. + +AsyncMappingKernelManager.cull_connected : Bool + Default: ``False`` + + Whether to consider culling kernels which have one or more connections. + Only effective if cull_idle_timeout > 0. + +AsyncMappingKernelManager.cull_idle_timeout : Int + Default: ``0`` + + Timeout (in seconds) after which a kernel is considered idle and ready to be culled. + Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled + for users with poor network connections. + +AsyncMappingKernelManager.cull_interval : Int + Default: ``300`` + + The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value. + +AsyncMappingKernelManager.default_kernel_name : Unicode + Default: ``'python3'`` + + The name of the default kernel to start + +AsyncMappingKernelManager.kernel_info_timeout : Float + Default: ``60`` + + Timeout for giving up on a kernel (in seconds). + + On starting and restarting kernels, we check whether the + kernel is running and responsive by sending kernel_info_requests. + This sets the timeout in seconds for how long the kernel can take + before being presumed dead. + This affects the MappingKernelManager (which handles kernel restarts) + and the ZMQChannelsHandler (which handles the startup). + + +AsyncMappingKernelManager.kernel_manager_class : DottedObjectName + Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` + + The kernel manager class. This is configurable to allow + subclassing of the AsyncKernelManager for customized behavior. + + +AsyncMappingKernelManager.root_dir : Unicode + Default: ``''`` + + No description + +AsyncMappingKernelManager.shared_context : Bool + Default: ``True`` + + Share a single zmq.Context to talk to all my kernels + +AsyncMappingKernelManager.traceback_replacement_message : Unicode + Default: ``'An exception occurred at runtime, which is not shown due to ...`` + + Message to print when allow_tracebacks is False, and an exception occurs + ContentsManager.allow_hidden : Bool Default: ``False`` @@ -951,13 +1103,13 @@ ContentsManager.files_handler_class : Type Default: ``'jupyter_server.files.handlers.FilesHandler'`` handler class to use when serving raw file requests. - + Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. - + Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. - + Access to these files should be Authenticated. @@ -965,7 +1117,7 @@ ContentsManager.files_handler_params : Dict Default: ``{}`` Extra parameters to pass to files_handler_class. - + For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. @@ -973,7 +1125,7 @@ ContentsManager.files_handler_params : Dict ContentsManager.hide_globs : List Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - + Glob patterns to hide in file and directory listings. @@ -981,17 +1133,17 @@ ContentsManager.pre_save_hook : Any Default: ``None`` Python callable or importstring thereof - + To be called on a contents model prior to save. - + This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. - + It will be called as (all arguments passed by keyword):: - + hook(path=path, model=model, contents_manager=self) - + - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination @@ -1056,13 +1208,13 @@ FileContentsManager.files_handler_class : Type Default: ``'jupyter_server.files.handlers.FilesHandler'`` handler class to use when serving raw file requests. - + Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. - + Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. - + Access to these files should be Authenticated. @@ -1070,7 +1222,7 @@ FileContentsManager.files_handler_params : Dict Default: ``{}`` Extra parameters to pass to files_handler_class. - + For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. @@ -1078,7 +1230,7 @@ FileContentsManager.files_handler_params : Dict FileContentsManager.hide_globs : List Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` - + Glob patterns to hide in file and directory listings. @@ -1086,16 +1238,16 @@ FileContentsManager.post_save_hook : Any Default: ``None`` Python callable or importstring thereof - + to be called on the path of a file just saved. - + This can be used to process the file on disk, such as converting the notebook to a script or HTML via nbconvert. - + It will be called as (all arguments passed by keyword):: - + hook(os_path=os_path, model=model, contents_manager=instance) - + - path: the filesystem path to the file just written - model: the model representing the file - contents_manager: this ContentsManager instance @@ -1105,17 +1257,17 @@ FileContentsManager.pre_save_hook : Any Default: ``None`` Python callable or importstring thereof - + To be called on a contents model prior to save. - + This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. - + It will be called as (all arguments passed by keyword):: - + hook(path=path, model=model, contents_manager=self) - + - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination @@ -1149,7 +1301,229 @@ FileContentsManager.use_atomic_writing : Bool This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) -NotebookNotary.algorithm : any of ``'blake2s'``|``'sha512'``|``'md5'``|``'sha3_512'``|``'sha3_224'``|``'blake2b'``|``'sha384'``|``'sha1'``|``'sha3_256'``|``'sha256'``|``'sha224'``|``'sha3_384'`` +AsyncContentsManager.allow_hidden : Bool + Default: ``False`` + + Allow access to hidden files + +AsyncContentsManager.checkpoints : Instance + Default: ``None`` + + No description + +AsyncContentsManager.checkpoints_class : Type + Default: ``'jupyter_server.services.contents.checkpoints.AsyncCheckpoints'`` + + No description + +AsyncContentsManager.checkpoints_kwargs : Dict + Default: ``{}`` + + No description + +AsyncContentsManager.files_handler_class : Type + Default: ``'jupyter_server.files.handlers.FilesHandler'`` + + handler class to use when serving raw file requests. + + Default is a fallback that talks to the ContentsManager API, + which may be inefficient, especially for large files. + + Local files-based ContentsManagers can use a StaticFileHandler subclass, + which will be much more efficient. + + Access to these files should be Authenticated. + + +AsyncContentsManager.files_handler_params : Dict + Default: ``{}`` + + Extra parameters to pass to files_handler_class. + + For example, StaticFileHandlers generally expect a `path` argument + specifying the root directory from which to serve files. + + +AsyncContentsManager.hide_globs : List + Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` + + + Glob patterns to hide in file and directory listings. + + +AsyncContentsManager.pre_save_hook : Any + Default: ``None`` + + Python callable or importstring thereof + + To be called on a contents model prior to save. + + This can be used to process the structure, + such as removing notebook outputs or other side effects that + should not be saved. + + It will be called as (all arguments passed by keyword):: + + hook(path=path, model=model, contents_manager=self) + + - model: the model to be saved. Includes file contents. + Modifying this dict will affect the file that is stored. + - path: the API path of the save destination + - contents_manager: this ContentsManager instance + + +AsyncContentsManager.root_dir : Unicode + Default: ``'/'`` + + No description + +AsyncContentsManager.untitled_directory : Unicode + Default: ``'Untitled Folder'`` + + The base name used when creating untitled directories. + +AsyncContentsManager.untitled_file : Unicode + Default: ``'untitled'`` + + The base name used when creating untitled files. + +AsyncContentsManager.untitled_notebook : Unicode + Default: ``'Untitled'`` + + The base name used when creating untitled notebooks. + +AsyncFileManagerMixin.use_atomic_writing : Bool + Default: ``True`` + + By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. + This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). + If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) + +AsyncFileContentsManager.allow_hidden : Bool + Default: ``False`` + + Allow access to hidden files + +AsyncFileContentsManager.checkpoints : Instance + Default: ``None`` + + No description + +AsyncFileContentsManager.checkpoints_class : Type + Default: ``'jupyter_server.services.contents.checkpoints.AsyncCheckpoints'`` + + No description + +AsyncFileContentsManager.checkpoints_kwargs : Dict + Default: ``{}`` + + No description + +AsyncFileContentsManager.delete_to_trash : Bool + Default: ``True`` + + If True (default), deleting files will send them to the + platform's trash/recycle bin, where they can be recovered. If False, + deleting files really deletes them. + +AsyncFileContentsManager.files_handler_class : Type + Default: ``'jupyter_server.files.handlers.FilesHandler'`` + + handler class to use when serving raw file requests. + + Default is a fallback that talks to the ContentsManager API, + which may be inefficient, especially for large files. + + Local files-based ContentsManagers can use a StaticFileHandler subclass, + which will be much more efficient. + + Access to these files should be Authenticated. + + +AsyncFileContentsManager.files_handler_params : Dict + Default: ``{}`` + + Extra parameters to pass to files_handler_class. + + For example, StaticFileHandlers generally expect a `path` argument + specifying the root directory from which to serve files. + + +AsyncFileContentsManager.hide_globs : List + Default: ``['__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dyl...`` + + + Glob patterns to hide in file and directory listings. + + +AsyncFileContentsManager.post_save_hook : Any + Default: ``None`` + + Python callable or importstring thereof + + to be called on the path of a file just saved. + + This can be used to process the file on disk, + such as converting the notebook to a script or HTML via nbconvert. + + It will be called as (all arguments passed by keyword):: + + hook(os_path=os_path, model=model, contents_manager=instance) + + - path: the filesystem path to the file just written + - model: the model representing the file + - contents_manager: this ContentsManager instance + + +AsyncFileContentsManager.pre_save_hook : Any + Default: ``None`` + + Python callable or importstring thereof + + To be called on a contents model prior to save. + + This can be used to process the structure, + such as removing notebook outputs or other side effects that + should not be saved. + + It will be called as (all arguments passed by keyword):: + + hook(path=path, model=model, contents_manager=self) + + - model: the model to be saved. Includes file contents. + Modifying this dict will affect the file that is stored. + - path: the API path of the save destination + - contents_manager: this ContentsManager instance + + +AsyncFileContentsManager.root_dir : Unicode + Default: ``''`` + + No description + +AsyncFileContentsManager.untitled_directory : Unicode + Default: ``'Untitled Folder'`` + + The base name used when creating untitled directories. + +AsyncFileContentsManager.untitled_file : Unicode + Default: ``'untitled'`` + + The base name used when creating untitled files. + +AsyncFileContentsManager.untitled_notebook : Unicode + Default: ``'Untitled'`` + + The base name used when creating untitled notebooks. + +AsyncFileContentsManager.use_atomic_writing : Bool + Default: ``True`` + + By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. + This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). + If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota ) + +NotebookNotary.algorithm : any of ``'sha3_512'``|``'sha224'``|``'sha1'``|``'sha256'``|``'sha384'``|``'md5'``|``'blake2b'``|``'sha3_224'``|``'sha3_256'``|``'blake2s'``|``'sha512'``|``'sha3_384'`` Default: ``'sha256'`` The hashing algorithm used to sign notebooks. @@ -1194,10 +1568,10 @@ GatewayKernelManager.buffer_offline_messages : Bool Default: ``True`` Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - + When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. - + Disable if long-running kernels will produce too much output while no frontends are connected. @@ -1235,7 +1609,7 @@ GatewayKernelManager.kernel_info_timeout : Float Default: ``60`` Timeout for giving up on a kernel (in seconds). - + On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take @@ -1245,10 +1619,10 @@ GatewayKernelManager.kernel_info_timeout : Float GatewayKernelManager.kernel_manager_class : DottedObjectName - Default: ``'jupyter_client.ioloop.IOLoopKernelManager'`` + Default: ``'jupyter_client.ioloop.AsyncIOLoopKernelManager'`` The kernel manager class. This is configurable to allow - subclassing of the KernelManager for customized behavior. + subclassing of the AsyncKernelManager for customized behavior. GatewayKernelManager.root_dir : Unicode @@ -1284,7 +1658,7 @@ GatewayKernelSpecManager.whitelist : Set Default: ``set()`` Whitelist of allowed kernel names. - + By default, all installed kernels are allowed. @@ -1315,7 +1689,7 @@ GatewayClient.client_key : Unicode GatewayClient.connect_timeout : Float - Default: ``60.0`` + Default: ``40.0`` The time allowed for HTTP connection establishment with the Gateway server. (JUPYTER_GATEWAY_CONNECT_TIMEOUT env var) @@ -1328,6 +1702,26 @@ GatewayClient.env_whitelist : Unicode value must also be set on the Gateway server - since that configuration value indicates which environmental values to make available to the kernel. (JUPYTER_GATEWAY_ENV_WHITELIST env var) +GatewayClient.gateway_retry_interval : Float + Default: ``1.0`` + + The time allowed for HTTP reconnection with the Gateway server for the first time. + Next will be JUPYTER_GATEWAY_RETRY_INTERVAL multiplied by two in factor of numbers of retries + but less than JUPYTER_GATEWAY_RETRY_INTERVAL_MAX. + (JUPYTER_GATEWAY_RETRY_INTERVAL env var) + +GatewayClient.gateway_retry_interval_max : Float + Default: ``30.0`` + + The maximum time allowed for HTTP reconnection retry with the Gateway server. + (JUPYTER_GATEWAY_RETRY_INTERVAL_MAX env var) + +GatewayClient.gateway_retry_max : Int + Default: ``5`` + + The maximum retries allowed for HTTP reconnection with the Gateway server. + (JUPYTER_GATEWAY_RETRY_MAX env var) + GatewayClient.headers : Unicode Default: ``'{}'`` @@ -1364,7 +1758,7 @@ GatewayClient.kernelspecs_resource_endpoint : Unicode (JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT env var) GatewayClient.request_timeout : Float - Default: ``60.0`` + Default: ``40.0`` The time allowed for HTTP request completion. (JUPYTER_GATEWAY_REQUEST_TIMEOUT env var) @@ -1389,3 +1783,14 @@ GatewayClient.ws_url : Unicode The websocket url of the Kernel or Enterprise Gateway server. If not provided, this value will correspond to the value of the Gateway url with 'ws' in place of 'http'. (JUPYTER_GATEWAY_WS_URL env var) + +TerminalManager.cull_inactive_timeout : Int + Default: ``0`` + + Timeout (in seconds) in which a terminal has been inactive and ready to be culled. + Values of 0 or lower disable culling. + +TerminalManager.cull_interval : Int + Default: ``300`` + + The interval (in seconds) on which to check for terminals exceeding the inactive timeout value. diff --git a/examples/client_eventlog_example/client_eventlog_example/__init__.py b/examples/client_eventlog_example/client_eventlog_example/__init__.py deleted file mode 100644 index 7bb0d6990c..0000000000 --- a/examples/client_eventlog_example/client_eventlog_example/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -__version__ = '0.1.0' - -JUPYTER_TELEMETRY_SCHEMAS = ['client_eventlog_example.schemas'] diff --git a/examples/client_eventlog_example/client_eventlog_example/schemas/__init__.py b/examples/client_eventlog_example/client_eventlog_example/schemas/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/examples/client_eventlog_example/client_eventlog_example/schemas/schema_yaml.yaml b/examples/client_eventlog_example/client_eventlog_example/schemas/schema_yaml.yaml deleted file mode 100644 index 43746184f8..0000000000 --- a/examples/client_eventlog_example/client_eventlog_example/schemas/schema_yaml.yaml +++ /dev/null @@ -1,21 +0,0 @@ -$id: https://example.jupyter.org/client-event -version: 1 -title: Client event -description: | - An example client event -type: object -properties: - thing: - title: Thing - categories: - - category.jupyter.org/unrestricted - description: A random thing - user: - title: User name - type: string - categories: - - category.jupyter.org/user-identifier - description: Name of user who initiated event -required: - - thing - - user diff --git a/examples/client_eventlog_example/setup.cfg b/examples/client_eventlog_example/setup.cfg deleted file mode 100644 index a6be0296bb..0000000000 --- a/examples/client_eventlog_example/setup.cfg +++ /dev/null @@ -1,26 +0,0 @@ -[metadata] -name = client_eventlog_example -version = attr: client_eventlog_example.__version__ -description = a dummy module for testing client telemetry eventlog entrypoint -long_description = file: README.md -long_description_content_type = text/markdown -url = https://jupyter.org -author = Jupyter Development Team -author_email = jupyter@googlegroups.org -license = BSD -license_file = COPYING.md -classifiers = - Intended Audience :: Developers - Intended Audience :: System Administrators - Intended Audience :: Science/Research - License :: OSI Approved :: BSD License - Programming Language :: Python - -zip_safe = False -include_package_data = True -packages = find: -python_requires = >=3.6 - -[options.entry_points] -jupyter_telemetry = - example-client-eventlog-entry-point = client_eventlog_example:JUPYTER_TELEMETRY_SCHEMAS diff --git a/examples/client_eventlog_example/setup.py b/examples/client_eventlog_example/setup.py deleted file mode 100644 index a4f49f9202..0000000000 --- a/examples/client_eventlog_example/setup.py +++ /dev/null @@ -1,2 +0,0 @@ -import setuptools -setuptools.setup() diff --git a/examples/client_eventlog_example/tests/conftest.py b/examples/client_eventlog_example/tests/conftest.py deleted file mode 100644 index 87c6aff30a..0000000000 --- a/examples/client_eventlog_example/tests/conftest.py +++ /dev/null @@ -1,3 +0,0 @@ -pytest_plugins = [ - 'jupyter_server.pytest_plugin' -] diff --git a/examples/client_eventlog_example/tests/test_client_eventlog.py b/examples/client_eventlog_example/tests/test_client_eventlog.py deleted file mode 100644 index 57af5e311f..0000000000 --- a/examples/client_eventlog_example/tests/test_client_eventlog.py +++ /dev/null @@ -1,36 +0,0 @@ -import json - - -EVENT = { - 'schema': 'https://example.jupyter.org/client-event', - 'version': 1.0, - 'event': { - 'user': 'user', - 'thing': 'thing' - } -} - - -async def test_client_eventlog(jp_eventlog_sink, jp_fetch): - serverapp, sink = jp_eventlog_sink - serverapp.eventlog.allowed_schemas = { - EVENT['schema']: { - 'allowed_categories': [ - 'category.jupyter.org/unrestricted', - 'category.jupyter.org/user-identifier' - ] - } - } - - r = await jp_fetch( - 'api', - 'eventlog', - method='POST', - body=json.dumps(EVENT) - ) - assert r.code == 204 - - output = sink.getvalue() - assert output - data = json.loads(output) - assert EVENT['event'].items() <= data.items() diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 2094962f16..f38b891584 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -31,11 +31,6 @@ import inspect import pathlib -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources - from base64 import encodebytes try: import resource @@ -113,7 +108,6 @@ urljoin, pathname2url, get_schema_files, - get_client_schema_files ) from jupyter_server.extension.serverextension import ServerExtensionApp @@ -144,7 +138,6 @@ config=['jupyter_server.services.config.handlers'], contents=['jupyter_server.services.contents.handlers'], files=['jupyter_server.files.handlers'], - eventlog=['jupyter_server.services.eventlog.handlers'], kernels=['jupyter_server.services.kernels.handlers'], kernelspecs=[ 'jupyter_server.kernelspecs.handlers', @@ -624,7 +617,6 @@ class ServerApp(JupyterApp): 'config', 'contents', 'files', - 'eventlog', 'kernels', 'kernelspecs', 'nbconvert', @@ -1835,14 +1827,6 @@ def init_eventlog(self): for file_path in get_schema_files(): self.eventlog.register_schema_file(file_path) - for file in get_client_schema_files(): - with importlib_resources.as_file(file) as f: - try: - self.eventlog.register_schema_file(f) - except: - pass - - @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True, starter_extension=None): """Initialize the Server application class, configurables, web application, and http server. diff --git a/jupyter_server/services/eventlog/__init__.py b/jupyter_server/services/eventlog/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/jupyter_server/services/eventlog/handlers.py b/jupyter_server/services/eventlog/handlers.py deleted file mode 100644 index 581548c3c1..0000000000 --- a/jupyter_server/services/eventlog/handlers.py +++ /dev/null @@ -1,42 +0,0 @@ -import json - -from tornado import web - -from jupyter_server.base.handlers import APIHandler, json_errors - - -class EventLoggingHandler(APIHandler): - """ - A handler that receives and stores telemetry data from the client. - """ - @json_errors - @web.authenticated - def post(self, *args, **kwargs): - try: - # Parse the data from the request body - raw_event = json.loads(self.request.body.strip().decode()) - except Exception as e: - raise web.HTTPError(400, str(e)) - - required_fields = {'schema', 'version', 'event'} - for rf in required_fields: - if rf not in raw_event: - raise web.HTTPError(400, '{} is a required field'.format(rf)) - - schema_name = raw_event['schema'] - version = raw_event['version'] - event = raw_event['event'] - - # Profile, may need to move to a background thread if this is problematic - try: - self.eventlog.record_event(schema_name, version, event) - except Exception as e: - raise web.HTTPError(400, e) - - self.set_status(204) - self.finish() - - -default_handlers = [ - (r"/api/eventlog", EventLoggingHandler), -] diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index f93321cfc7..895a001009 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -9,21 +9,10 @@ import os import sys from distutils.version import LooseVersion -from itertools import chain from urllib.parse import quote, unquote, urlparse, urljoin from urllib.request import pathname2url -if sys.version_info >= (3, 8): - from importlib.metadata import entry_points -else: - from importlib_metadata import entry_points - -if sys.version_info >= (3, 9): - import importlib.resources as importlib_resources -else: - import importlib_resources - def url_path_join(*pieces): """Join components of url into a relative url @@ -254,45 +243,3 @@ def get_schema_files(): if file.endswith('.yaml'): file_path = os.path.join(dirname, file) yield file_path - - -JUPYTER_TELEMETRY_ENTRY_POINT = 'jupyter_telemetry' - - -def get_client_schema_files(): - telemetry_entry_points = entry_points().get(JUPYTER_TELEMETRY_ENTRY_POINT, []) - - dirs = (_safe_entry_point_load(ep) for ep in telemetry_entry_points) - dirs = chain.from_iterable(d for d in dirs if d is not None) - dirs = (_safe_load_resource(d) for d in dirs) - - files = chain.from_iterable(d.iterdir() for d in dirs if d is not None) - - return (f for f in files if f.suffix in ('.json', '.yaml')) - - -def _is_iterable(x): - try: - iter(x) - return True - except TypeError: - return False - - -def _safe_entry_point_load(ep): - try: - v = ep.load() - if isinstance(v, str): - return [v] - elif _is_iterable(v): - return v - return None - except: - return None - - -def _safe_load_resource(x): - try: - return importlib_resources.files(x) - except ModuleNotFoundError: - return None diff --git a/setup.cfg b/setup.cfg index fb42dfe416..1f9edaefb2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,8 +43,6 @@ install_requires = prometheus_client pywin32>=1.0 ; sys_platform == 'win32' anyio>=2.0.2,<3 - importlib-metadata ; python_version < '3.8' - importlib-resources ; python_version < '3.9' jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@4f4b5650fc86ffb6fc097741f3523e433b1c4967 [options.extras_require] From 7d906af85d52f0beb4c6da716419644fbf86e6ac Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 3 May 2021 18:12:02 +0800 Subject: [PATCH 45/49] Use telemetry upstream --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1f9edaefb2..3cd81889a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,7 +43,7 @@ install_requires = prometheus_client pywin32>=1.0 ; sys_platform == 'win32' anyio>=2.0.2,<3 - jupyter_telemetry@git+https://github.com/kiendang/telemetry.git@4f4b5650fc86ffb6fc097741f3523e433b1c4967 + jupyter_telemetry@git+https://github.com/jupyter/telemetry.git@6f1933ca88349fbcfb02dbbe35028ffa930cf836 [options.extras_require] test = coverage; pytest; pytest-cov; pytest-mock; requests; pytest-tornasync; pytest-console-scripts; ipykernel From 82e70b81519789b1e5af690cf87340c769fc7e3b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 3 May 2021 19:05:44 +0800 Subject: [PATCH 46/49] Use importlib.resources to read event schemas instead of raw paths since raw paths do not work if e.g. install package from a zip file https://importlib-resources.readthedocs.io/en/latest/using.html#example --- jupyter_server/event_schemas/__init__.py | 0 .../contentsmanager-actions/v1.yaml | 0 jupyter_server/serverapp.py | 10 +++++-- jupyter_server/utils.py | 27 ++++++++++++------- setup.cfg | 1 + 5 files changed, 27 insertions(+), 11 deletions(-) create mode 100644 jupyter_server/event_schemas/__init__.py rename jupyter_server/{event-schemas => event_schemas}/contentsmanager-actions/v1.yaml (100%) diff --git a/jupyter_server/event_schemas/__init__.py b/jupyter_server/event_schemas/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/jupyter_server/event-schemas/contentsmanager-actions/v1.yaml b/jupyter_server/event_schemas/contentsmanager-actions/v1.yaml similarity index 100% rename from jupyter_server/event-schemas/contentsmanager-actions/v1.yaml rename to jupyter_server/event_schemas/contentsmanager-actions/v1.yaml diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index f38b891584..11866fa1f3 100755 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -31,6 +31,11 @@ import inspect import pathlib +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + from base64 import encodebytes try: import resource @@ -1824,8 +1829,9 @@ def _init_asyncio_patch(): def init_eventlog(self): self.eventlog = EventLog(parent=self) # Register schemas for notebook services. - for file_path in get_schema_files(): - self.eventlog.register_schema_file(file_path) + for file in get_schema_files(): + with importlib_resources.as_file(file) as f: + self.eventlog.register_schema_file(f) @catch_config_error def initialize(self, argv=None, find_extensions=True, new_httpserver=True, starter_extension=None): diff --git a/jupyter_server/utils.py b/jupyter_server/utils.py index 895a001009..5163e63f9b 100644 --- a/jupyter_server/utils.py +++ b/jupyter_server/utils.py @@ -10,6 +10,11 @@ import sys from distutils.version import LooseVersion +if sys.version_info >= (3, 9): + import importlib.resources as importlib_resources +else: + import importlib_resources + from urllib.parse import quote, unquote, urlparse, urljoin from urllib.request import pathname2url @@ -232,14 +237,18 @@ def eventlogging_schema_fqn(name): return 'eventlogging.jupyter.org/jupyter_server/{}'.format(name) +def list_resources(resources): + for entry in resources.iterdir(): + if entry.is_dir(): + yield from list_resources(entry) + else: + yield entry + + def get_schema_files(): """Yield a sequence of event schemas for jupyter services.""" - # Hardcode path to event schemas directory. - event_schemas_dir = os.path.join(os.path.dirname(__file__), 'event-schemas') - #schema_files = [] - # Recursively register all .json files under event-schemas - for dirname, _, files in os.walk(event_schemas_dir): - for file in files: - if file.endswith('.yaml'): - file_path = os.path.join(dirname, file) - yield file_path + return ( + entry for entry in list_resources( + importlib_resources.files('jupyter_server.event_schemas') + ) if os.path.splitext(entry.name)[1] == '.yaml' + ) diff --git a/setup.cfg b/setup.cfg index 3cd81889a6..8c0e3c52f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,6 +43,7 @@ install_requires = prometheus_client pywin32>=1.0 ; sys_platform == 'win32' anyio>=2.0.2,<3 + importlib-resources ; python_version < '3.9' jupyter_telemetry@git+https://github.com/jupyter/telemetry.git@6f1933ca88349fbcfb02dbbe35028ffa930cf836 [options.extras_require] From 22b682f8e7c41d876c14e3399052e73b978e980c Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 3 May 2021 19:13:20 +0800 Subject: [PATCH 47/49] Fix docs --- MANIFEST.in | 2 +- docs/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 796479e393..ce12c048d0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,7 +10,7 @@ include package.json recursive-include jupyter_server * # Event Schemas -graft jupyter_server/event-schemas +graft jupyter_server/event_schemas # Documentation graft docs diff --git a/docs/source/conf.py b/docs/source/conf.py index 1d53b6cacd..7e2fdd31dd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -389,7 +389,7 @@ import jupyter_server.transutils # Jupyter telemetry configuration values. -jupyter_telemetry_schema_source = osp.join(HERE, '../../jupyter_server/event-schemas') +jupyter_telemetry_schema_source = osp.join(HERE, '../../jupyter_server/event_schemas') jupyter_telemetry_schema_output = osp.join(HERE, 'operators/events') # Title of the index page that lists all found schemas jupyter_telemetry_index_title = 'Telemetry Event Schemas' From e0c38d2a1fbc6fed9b1819219d6fc669e2d478bb Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 3 May 2021 23:15:36 +0800 Subject: [PATCH 48/49] Doc change notebook server to jupyter server --- docs/source/operators/telemetry.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/operators/telemetry.rst b/docs/source/operators/telemetry.rst index 293e32cbf9..33584d13fe 100644 --- a/docs/source/operators/telemetry.rst +++ b/docs/source/operators/telemetry.rst @@ -1,7 +1,7 @@ Eventlogging and Telemetry ========================== -The Notebook Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that the Notebook Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. +Jupyter Server can be configured to record structured events from a running server using Jupyter's `Telemetry System`_. The types of events that Jupyter Server emits are defined by `JSON schemas`_ listed below_ emitted as JSON data, defined and validated by the JSON schemas listed below. .. _logging: https://docs.python.org/3/library/logging.html From 31585e41c38899d1db68e4014afb564b4e4f376e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 3 May 2021 23:37:34 +0800 Subject: [PATCH 49/49] Remove eventlog API endpoint doc --- docs/source/operators/telemetry.rst | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docs/source/operators/telemetry.rst b/docs/source/operators/telemetry.rst index 33584d13fe..0ab41a34d4 100644 --- a/docs/source/operators/telemetry.rst +++ b/docs/source/operators/telemetry.rst @@ -9,7 +9,7 @@ Jupyter Server can be configured to record structured events from a running serv .. _`JSON schemas`: https://json-schema.org/ .. warning:: - Do NOT rely on this feature for security or auditing purposes. Neither `server <#emitting-server-events>`_ nor `client <#eventlog-endpoint>`_ events are protected against meddling. For server events, those who have access to the environment can change the server code to emit whatever they want. The same goes for client events where nothing prevents users from sending spurious data to the `eventlog` endpoint. + Do NOT rely on this feature for security or auditing purposes. Neither `server <#emitting-server-events>`_ nor `client <#the-eventlog-endpoint>`_ events are protected against meddling. For server events, those who have access to the environment can change the server code to emit whatever they want. The same goes for client events where nothing prevents users from sending spurious data to the `eventlog` endpoint. Emitting server events ---------------------- @@ -48,16 +48,7 @@ Server event schemas The ``eventlog`` endpoint ------------------------- -The Notebook Server provides a public REST endpoint for external applications to validate and log events -through the Server's Event Log. - -To log events, send a `POST` request to the `/api/eventlog` endpoint. The body of the request should be a -JSON blog and is required to have the follow keys: - - 1. `'schema'` : the event's schema ID. - 2. `'version'` : the version of the event's schema. - 3. `'event'` : the event data in JSON format. - -Events that are validated by this endpoint must have their schema listed in the `allowed_schemas` trait listed above. +.. note:: + This has not yet been implemented. .. _below: