Automatically configure wrapper-kernel endpoints in widget (#362)

* Add pre-configured endpoints to endpoint widget automatically

* Fix crash on partially-defined kernel configurations

* Use LANGS_SUPPORTED constant to get list of possible kernel config sections

* Rename is_default attr to implicitly_added

* Adding blank line between imports and class declaration

* Log failure to connect to implicitly-defined endpoints

* Adding comment explaining implicitly_added

* Pass auth parameter through

Author: apetresc

sparkmagic/sparkmagic/controllerwidget/magicscontrollerwidget.py

@@ -5,21 +5,40 @@
 from sparkmagic.controllerwidget.manageendpointwidget import ManageEndpointWidget
 from sparkmagic.controllerwidget.managesessionwidget import ManageSessionWidget
 from sparkmagic.controllerwidget.createsessionwidget import CreateSessionWidget
+from sparkmagic.livyclientlib.endpoint import Endpoint
+from sparkmagic.utils.constants import LANGS_SUPPORTED
+import sparkmagic.utils.configuration as conf
 
 
 class MagicsControllerWidget(AbstractMenuWidget):
     def __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoints=None):
         super(MagicsControllerWidget, self).__init__(spark_controller, ipywidget_factory, ipython_display)
 
         if endpoints is None:
-            endpoints = {}
+            endpoints = {endpoint.url: endpoint for endpoint in self._get_default_endpoints()}
         self.endpoints = endpoints
 
         self._refresh()
 
     def run(self):
         pass
 
+    @staticmethod
+    def _get_default_endpoints():
+        default_endpoints = set()
+
+        for kernel_type in LANGS_SUPPORTED:
+            endpoint_config = getattr(conf, 'kernel_%s_credentials' % kernel_type)()
+            if all([p in endpoint_config for p in ["url", "password", "username"]]) and endpoint_config["url"] != "":
+                default_endpoints.add(Endpoint(
+                    username=endpoint_config["username"],
+                    password=endpoint_config["password"],
+                    auth=endpoint_config.get("auth", None),
+                    url=endpoint_config["url"],
+                    implicitly_added=True))
+
+        return default_endpoints
+
     def _refresh(self):
         self.endpoints_dropdown_widget = self.ipywidget_factory.get_dropdown(
                 description="Endpoint:",

sparkmagic/sparkmagic/controllerwidget/manageendpointwidget.py

@@ -1,13 +1,16 @@
 # Copyright (c) 2015  [email protected]
 # Distributed under the terms of the Modified BSD License.
 from sparkmagic.controllerwidget.abstractmenuwidget import AbstractMenuWidget
+from sparkmagic.livyclientlib.exceptions import HttpClientException
+from sparkmagic.utils.sparklogger import SparkLog
 
 
 class ManageEndpointWidget(AbstractMenuWidget):
     def __init__(self, spark_controller, ipywidget_factory, ipython_display, endpoints, refresh_method):
         # This is nested
         super(ManageEndpointWidget, self).__init__(spark_controller, ipywidget_factory, ipython_display, True)
 
+        self.logger = SparkLog("ManageEndpointWidget")
         self.endpoints = endpoints
         self.refresh_method = refresh_method
 
@@ -30,7 +33,14 @@ def get_existing_endpoint_widgets(self):
 
             # Endpoints
             for url, endpoint in self.endpoints.items():
-                endpoint_widgets.append(self.get_endpoint_widget(url, endpoint))
+                try:
+                    endpoint_widgets.append(self.get_endpoint_widget(url, endpoint))
+                except HttpClientException:
+                    # If we can't reach one of the default endpoints, just skip over it
+                    if not endpoint.implicitly_added:
+                        raise
+                    else:
+                        self.logger.info("Failed to connect to implicitly-defined endpoint at: %s" % url)
 
             endpoint_widgets.append(self.ipywidget_factory.get_html(value="<br/>", width="600px"))
         else:

sparkmagic/sparkmagic/livyclientlib/endpoint.py

@@ -2,19 +2,26 @@
 
 
 class Endpoint(object):
-    def __init__(self, url, auth, username="", password=""):
+    def __init__(self, url, auth, username="", password="", implicitly_added=False):
         if not url:
             raise BadUserDataException(u"URL must not be empty")
         self.url = url.rstrip(u"/")
         self.username = username
         self.password = password
         self.auth = auth
+        # implicitly_added is set to True only if the endpoint wasn't configured manually by the user through
+        # a widget, but was instead implicitly defined as an endpoint to a wrapper kernel in the configuration
+        # JSON file.
+        self.implicitly_added = implicitly_added
 
     def __eq__(self, other):
         if type(other) is not Endpoint:
             return False
         return self.url == other.url and self.username == other.username and self.password == other.password
 
+    def __hash__(self):
+        return hash((self.url, self.username, self.password))
+
     def __ne__(self, other):
         return not self == other
 

sparkmagic/sparkmagic/livyclientlib/linearretrypolicy.py

@@ -11,6 +11,8 @@ def __init__(self, seconds_to_sleep, max_retries):
         self._max_retries = max_retries
 
     def should_retry(self, status_code, error, retry_count):
+        if None in (status_code, retry_count):
+            return False
         return (status_code >= 500 and retry_count <= self._max_retries) or error
 
     def seconds_to_sleep(self, retry_count):

sparkmagic/sparkmagic/utils/configuration.py

@@ -220,7 +220,7 @@ def server_extension_default_kernel_name():
 def _credentials_override(f):
     """Provides special handling for credentials. It still calls _override().
     If 'base64_password' in config is set, it will base64 decode it and returned in return value's 'password' field.
-    If 'base64_password' is not set, it will fallback to to 'password' in config.
+    If 'base64_password' is not set, it will fallback to 'password' in config.
     """
     credentials = f()
     base64_decoded_credentials = {k: credentials.get(k) for k in ('username', 'password', 'url', 'auth')}