Updated as per comments.

Author: praveen-kanamarlapudi

sparkmagic/example_config.json

@@ -2,13 +2,15 @@
   "kernel_python_credentials" : {
     "username": "",
     "password": "",
-    "url": "http://localhost:8998"
+    "url": "http://localhost:8998",
+    "auth": "None"
   },
 
   "kernel_scala_credentials" : {
     "username": "",
     "password": "",
-    "url": "http://localhost:8998"
+    "url": "http://localhost:8998",
+    "auth": "None"
   },
 
   "logging_config": {

sparkmagic/requirements.txt

@@ -9,4 +9,5 @@ requests
 ipykernel>=4.2.2,<5
 ipywidgets>5.0.0,<7.0
 notebook>=4.2,<5.0
-tornado>=4
+tornado>=4
+requests_kerberos>=0.8.0

sparkmagic/setup.py

@@ -89,6 +89,6 @@ def version(path):
           'ipywidgets>5.0.0,<7.0',
           'notebook>=4.2,<5.0',
           'tornado>=4',
-          'requests_kerberos'
+          'requests_kerberos>=0.8.0'
       ])
 

sparkmagic/sparkmagic/kernels/kernelmagics.py

@@ -349,14 +349,14 @@ def _do_not_call_change_language(self, line, cell="", local_ns=None):
     @argument("-u", "--username", type=str, help="Username to use.")
     @argument("-p", "--password", type=str, help="Password to use.")
     @argument("-s", "--server", type=str, help="Url of server to use.")
-    @argument("-t", "--auth_type", type=str, help="Auth type for authentication")
+    @argument("-t", "--auth", type=str, help="Auth type for authentication")
     @_event
     def _do_not_call_change_endpoint(self, line, cell="", local_ns=None):
         args = parse_argstring_or_throw(self._do_not_call_change_endpoint, line)
         username = args.username
         password = args.password
         server = args.server
-        auth_type = args.auth_type
+        auth_type = args.auth
 
         if self.session_started:
             error = u"Cannot change the endpoint if a session has been started."

sparkmagic/sparkmagic/livyclientlib/endpoint.py

@@ -1,9 +1,8 @@
 from .exceptions import BadUserDataException
-import sparkmagic.utils.configuration as conf
 
 
 class Endpoint(object):
-    def __init__(self, url, auth_type=conf.default_livy_endpoint_auth_type(), username="", password=""):
+    def __init__(self, url, auth_type, username="", password=""):
         if not url:
             raise BadUserDataException(u"URL must not be empty")
         self.url = url.rstrip(u"/")

sparkmagic/sparkmagic/livyclientlib/reliablehttpclient.py

@@ -24,6 +24,8 @@ def __init__(self, endpoint, headers, retry_policy):
             self._auth = HTTPKerberosAuth(mutual_authentication=REQUIRED)
         elif self._endpoint.auth_type == constants.AUTH_BASIC:
             self._auth = (self._endpoint.username, self._endpoint.password)
+        elif self._endpoint.auth_type != constants.NO_AUTH:
+            raise ValueError("Unsupported auth_type.")
 
         self.logger = SparkLog(u"ReliableHttpClient")
 

sparkmagic/sparkmagic/magics/remotesparkmagics.py

@@ -79,7 +79,7 @@ def spark(self, line, cell="", local_ns=None):
            add
                Add a Livy session given a session name (-s), language (-l), and endpoint credentials.
                The -k argument, if present, will skip adding this session if it already exists.
-               e.g. `%spark add -s test -l python -u https://sparkcluster.net/livy -a u -p -k`
+               e.g. `%spark add -s test -l python -u https://sparkcluster.net/livy -t Kerberos -a u -p -k`
            config
                Override the livy session properties sent to Livy on session creation. All session creations will
                contain these config settings from then on.

sparkmagic/sparkmagic/serverextension/handlers.py

@@ -8,6 +8,7 @@
 
 from sparkmagic.kernels.kernelmagics import KernelMagics
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 from sparkmagic.utils.sparkevents import SparkEvents
 from sparkmagic.utils.sparklogger import SparkLog
 
@@ -38,6 +39,12 @@ def post(self):
             username = self._get_argument_or_raise(data, 'username')
             password = self._get_argument_or_raise(data, 'password')
             endpoint = self._get_argument_or_raise(data, 'endpoint')
+            auth_type = self._get_argument_if_exists(data, 'auth_type')
+            if auth_type is None:
+                if username == '' and password == '':
+                    auth_type = constants.NO_AUTH
+                else:
+                    auth_type = constants.AUTH_BASIC
         except MissingArgumentError as e:
             self.set_status(400)
             self.finish(str(e))
@@ -52,7 +59,7 @@ def post(self):
 
         # Execute code
         client = kernel_manager.client()
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, endpoint, username, password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, endpoint, username, password, auth_type)
         response_id = client.execute(code, silent=False, store_history=False)
         msg = client.get_shell_msg(response_id)
 

sparkmagic/sparkmagic/tests/test_configuration.py

@@ -59,6 +59,6 @@ def test_configuration_raise_error_for_bad_base64_password():
 
 @with_setup(_setup)
 def test_share_config_between_pyspark_and_pyspark3():
-    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L' }
+    kpc = { 'username': 'U', 'password': 'P', 'base64_password': 'cGFzc3dvcmQ=', 'url': 'L', 'auth_type': AUTH_BASIC }
     overrides = { conf.kernel_python_credentials.__name__: kpc }
     assert_equals(conf.base64_kernel_python3_credentials(), conf.base64_kernel_python_credentials())

sparkmagic/sparkmagic/tests/test_handlers.py

@@ -9,6 +9,7 @@
 from sparkmagic.serverextension.handlers import ReconnectHandler
 from sparkmagic.kernels.kernelmagics import KernelMagics
 import sparkmagic.utils.configuration as conf
+from sparkmagic.utils import constants
 
 
 class SimpleObject(object):
@@ -30,6 +31,7 @@ class TestSparkMagicHandler(AsyncTestCase):
     username = 'username'
     password = 'password'
     endpoint = 'http://endpoint.com'
+    auth_type = constants.AUTH_BASIC
     response_id = '0'
     good_msg = dict(content=dict(status='ok'))
     bad_msg = dict(content=dict(status='error', ename='SyntaxError', evalue='oh no!'))
@@ -62,7 +64,7 @@ def setUp(self):
 
         # Mock request
         self.request = MagicMock()
-        self.request.body = json.dumps({"path": self.path, "username": self.username, "password": self.password, "endpoint": self.endpoint})
+        self.request.body = json.dumps({"path": self.path, "username": self.username, "password": self.password, "endpoint": self.endpoint, "auth_type": self.auth_type})
 
         # Create mocked reconnect_handler
         ReconnectHandler.__bases__ = (SimpleObject,)
@@ -124,7 +126,7 @@ def test_post_existing_kernel(self, _get_kernel_manager):
         res = yield self.reconnect_handler.post()
         assert_equals(res, None)
 
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password, self.auth_type)
         self.client.execute.assert_called_once_with(code, silent=False, store_history=False)
         self.reconnect_handler.set_status.assert_called_once_with(200)
         self.reconnect_handler.finish.assert_called_once_with('{"error": null, "success": true}')
@@ -141,7 +143,7 @@ def test_post_existing_kernel_failed(self, _get_kernel_manager):
         res = yield self.reconnect_handler.post()
         assert_equals(res, None)
 
-        code = '%{} -s {} -u {} -p {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password)
+        code = '%{} -s {} -u {} -p {} -t {}'.format(KernelMagics._do_not_call_change_endpoint.__name__, self.endpoint, self.username, self.password, self.auth_type)
         self.client.execute.assert_called_once_with(code, silent=False, store_history=False)
         self.reconnect_handler.set_status.assert_called_once_with(500)
         self.reconnect_handler.finish.assert_called_once_with('{"error": "SyntaxError:\\noh no!", "success": false}')