Serialize/deserialize binary values to the serialization format

docs/source/embedding.md

@@ -30,92 +30,15 @@ This HTML snippet is composed of multiple `<script>` tags:
  - The second script tag contains the state of all the widget models currently
    in use. It has the mime type `application/vnd.jupyter.widget-state+json`.
 
-   The JSON schema for the content of that script tag is:
-
-    ```json
-    {
-        "$schema": "http://json-schema.org/draft-04/schema#",
-        "description": "Jupyter Interactive Widget State JSON schema.",
-        "type": "object",
-        "properties" : {
-            "version_major" : {
-                "description": "Format version (major)",
-                "type": "number",
-                "minimum": 1,
-                "maximum": 1
-            },
-            "version_minor" : {
-                "description": "Format version (minor)",
-                "type": "number"
-            },
-            "state": {
-                "description": "Model State for All Widget Models",
-                "type": "object",
-                "additionalProperties": true,
-                "additionalProperties" : {
-                    "type": "object",
-                    "properties": {
-                        "model_name": {
-                            "description" : "Name of the JavaScript class holding the model implementation",
-                            "type": "string"
-                        },
-                        "model_module": {
-                            "description" : "Name of the JavaScript module holding the model implementation",
-                            "type": "string"
-                        },
-                        "model_module_version": {
-                            "description" : "Semver range for the JavaScript module holding the model implementation",
-                            "type": "string"
-                        },
-                        "state": {
-                            "description" : "Serialized state of the model",
-                            "type": "object",
-                            "additional_properties": true
-                        }
-                    },
-                    "required": [ "model_name", "model_module", "state" ],
-                    "additionalProperties": false
-                }
-            }
-        },
-        "required": [ "version_major", "version_minor", "state" ],
-        "additionalProperties": false
-    }
-    ```
+   The JSON schema for the content of that script tag is found in the jupyter-widgets-schema npm package.
 
 - The following script tags correspond to the views which you want to display
   in the web page. They have the mime type `application/vnd.jupyter.widget-view+json`.
 
   The *Embed Widgets* action currently creates such a tag for each view
   displayed in the notebook at this time.
 
-  The JSON schema for the content of that script tag is:
-
-    ```json
-    {
-        "$schema": "http://json-schema.org/draft-04/schema#",
-        "description": "Jupyter Interactive Widget View JSON schema.",
-        "type": "object",
-        "properties" : {
-            "version_major" : {
-                "description": "Format version (major)",
-                "type": "number",
-                "minimum": 1,
-                "maximum": 1
-            },
-            "version_minor" : {
-                "description": "Format version (minor)",
-                "type": "number"
-            },
-            "model_id": {
-                "description": "Unique identifier of the widget model to be displayed",
-                "type": "string"
-            },
-            "required": [ "model_id" ]
-        },
-        "additionalProperties": false
-    }
-    ```
+  The JSON schema for the content of that script tag is found in the jupyter-widgets-schema npm package.
 
   If you want to lay out these script tags in a custom fashion or only keep
   some of them, you can change their location in the DOM when including the

ipywidgets/state.schema.json

@@ -6,17 +6,18 @@
         "version_major" : {
             "description": "Format version (major)",
             "type": "number",
-            "minimum": 1,
-            "maximum": 1
+            "minimum": 2,
+            "maximum": 2
         },
         "version_minor" : {
             "description": "Format version (minor)",
-            "type": "number"
+            "type": "number",
+            "minimum": 0,
+            "maximum": 0
         },
         "state": {
-            "description": "Model State for All Widget Models",
+            "description": "Model State for All Widget Models - keys are model ids, values are model state",
             "type": "object",
-            "additionalProperties": true,
             "additionalProperties" : {
                 "type": "object",
                 "properties": {
@@ -34,15 +35,44 @@
                     },
                     "state": {
                         "description" : "Serialized state of the model",
-                        "type": "object",
-                        "additional_properties": true
+                        "type": "object"
+                    },
+                    "buffers": {
+                        "description": "Binary buffers in the state",
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "path": {
+                                    "description": "A path for a binary buffer value.",
+                                    "type": "array",
+                                    "items": {
+                                        "description": "An object key or array index",
+                                        "type": ["string", "number"]
+                                    }
+                                },
+                                "data": {
+                                    "description" : "A binary buffer encoded as specified in the 'encoding' property",
+                                    "type": "string"
+                                },
+                                "encoding": {
+                                    "description": "The encoding of the buffer data",
+                                    "type": "string",
+                                    "oneOf": [
+                                        {"enum": ["hex"], "description": "Base 16 encoding, as specified in RFC 4648, section 8 (https://tools.ietf.org/html/rfc4648#section-8)"},
+                                        {"enum": ["base64"], "description": "Base 64 encoding, as specified in RFC 4648, section 4 (https://tools.ietf.org/html/rfc4648#section-4)"}
+
+                                    ]
+                                }
+
+                            },
+                            "required": ["path", "data", "encoding"]
+                        }
                     }
                 },
-                "required": [ "model_name", "model_module", "state" ],
-                "additionalProperties": false
+                "required": [ "model_name", "model_module", "state" ]
             }
         }
     },
-    "required": [ "version_major", "version_minor", "state" ],
-    "additionalProperties": false
+    "required": [ "version_major", "version_minor", "state" ]
 }

ipywidgets/widgets/widget.py

@@ -16,6 +16,8 @@
 from ipython_genutils.py3compat import string_types, PY3
 from IPython.display import display
 
+from base64 import standard_b64decode, standard_b64encode
+
 from .._version import __protocol_version__
 
 
@@ -279,17 +281,26 @@ def handle_comm_opened(comm, msg):
 
     @staticmethod
     def get_manager_state(drop_defaults=False):
-        return dict(version_major=1, version_minor=0, state={
-            k: {
-                'model_name': Widget.widgets[k]._model_name,
-                'model_module': Widget.widgets[k]._model_module,
-                'model_module_version': Widget.widgets[k]._model_module_version,
-                'state': Widget.widgets[k].get_state(drop_defaults=drop_defaults)
-            } for k in Widget.widgets
-        })
+        manager_state = {}
+        for model_id, model in Widget.widgets.items():
+            state = {
+                'model_name': model._model_name,
+                'model_module': model.model_module,
+                'model_module_version': model._model_module_version
+            }
+            model_state, buffer_paths, buffers = _remove_buffers(model.get_state(drop_defaults=drop_defaults))
+            state['state'] = model_state
+            if len(buffers) > 0:
+                state['buffers'] = {
+                    'buffers': [{'path': p, 'data': standard_b64encode(d).decode('ascii')}
+                                for p, d in zip(buffer_paths, buffers)],
+                    'encoding': 'base64'
+                }
+            manager_state[model_id] = state
+        return {'version_major': 2, 'version_minor': 0, 'state': manager_state}
 
     def get_view_spec(self):
-        return dict(version_major=1, version_minor=0, model_id=self._model_id)
+        return dict(version_major=2, version_minor=0, model_id=self._model_id)
 
     #-------------------------------------------------------------------------
     # Traits
@@ -619,6 +630,8 @@ def _ipython_display_(self, **kwargs):
             data = {
                 'text/plain': "A Jupyter Widget",
                 'application/vnd.jupyter.widget-view+json': {
+                    'version_major': '2',
+                    'version_minor': '0',
                     'model_id': self._model_id
                 }
             }

jupyter-js-widgets/package.json

@@ -91,11 +91,12 @@
     "@types/semver": "^5.3.30",
     "ajv": "^4.9.0",
     "backbone": "1.2.0",
+    "base64-js": "1.2.0",
     "d3-format": "^0.5.1",
     "font-awesome": "^4.5.0",
     "jquery": "^3.1.1",
     "jquery-ui": "^1.12.1",
-    "jupyter-widgets-schema": "^0.1.1",
+    "jupyter-widgets-schema": "^0.3.0-beta.2",
     "lolex": "^1.4.0",
     "scriptjs": "^2.5.8",
     "semver": "^5.1.0",

jupyter-js-widgets/src-embed/embed-webpack.ts

@@ -19,8 +19,8 @@ require('../css/widgets.css');
 
 // Load json schema validator
 var Ajv = require('ajv');
-var widget_state_schema = require('jupyter-widgets-schema').v1.state;
-var widget_view_schema = require('jupyter-widgets-schema').v1.view;
+var widget_state_schema = require('jupyter-widgets-schema').v2.state;
+var widget_view_schema = require('jupyter-widgets-schema').v2.view;
 
 // Magic global widget rendering function:
 import * as widgets from '../../jupyter-js-widgets/lib/index';
@@ -90,7 +90,7 @@ function renderManager(element, tag) {
         console.log(model_validate.errors);
     }
     var manager = new embed.EmbedManager();
-    manager.set_state(widgetStateObject.state, {}).then(function(models) {
+    manager.set_state(widgetStateObject, {}).then(function(models) {
         var tags = element.querySelectorAll('script[type="application/vnd.jupyter.widget-view+json"]');
         for (var i=0; i!=tags.length; ++i) {
             // TODO: validate view schema