Support credentials and SSL with SQLAlchemy via HTTP/DB URIs

.github/workflows/tests.yml

@@ -18,7 +18,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
-        cratedb-version: ['4.5.0']
+        cratedb-version: ['4.7.1']
         sqla-version: ['1.1.18', '1.2.19', '1.3.23']
       fail-fast: true
 

CHANGES.txt

@@ -11,6 +11,11 @@ Unreleased
   HTTP by default. Previously, this setting defaulted to false. This setting
   can be changed via the ``verify_ssl_cert`` connection parameter.
 
+- Adjusted connect arguments to accept credentials within the HTTP URI.
+
+- Added support for enabling SSL using SQLAlchemy DB URI with parameter
+  ``?ssl=true``.
+
 2020/09/28 0.26.0
 =================
 

docs/appendices/data-types.rst

@@ -48,19 +48,19 @@ CrateDB       Python
 
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#boolean
 __ https://docs.python.org/3/library/stdtypes.html#boolean-values
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data
 __ https://docs.python.org/3/library/stdtypes.html#str
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ https://docs.python.org/3/library/functions.html#int
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ https://docs.python.org/3/library/functions.html#int
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ https://docs.python.org/3/library/functions.html#int
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ https://docs.python.org/3/library/functions.html#float
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ https://docs.python.org/3/library/functions.html#float
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ https://docs.python.org/3/library/functions.html#int
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#geo-point
 __ https://docs.python.org/3/library/stdtypes.html#list
@@ -82,15 +82,15 @@ Python        CrateDB
 ============= ====================================
 
 __ https://docs.python.org/3/library/decimal.html
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data
 __ https://docs.python.org/3/library/datetime.html#date-objects
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data
 __ https://docs.python.org/3/library/datetime.html#datetime-objects
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data
 
 .. NOTE::
 
@@ -134,21 +134,21 @@ CrateDB           SQLAlchemy
 
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#boolean
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.Boolean
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.SmallInteger
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.SmallInteger
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.Integer
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.NUMERIC
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.Float
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.DECIMAL
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#date-time-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#dates-and-times
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.TIMESTAMP
-__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types
+__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.String
 __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#array
 __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.ARRAY

docs/connect.rst

@@ -29,7 +29,7 @@ Connect to CrateDB
 Connect to a single node
 ========================
 
-To connect to a single CrateDB node, use the ``connect()`` function, like so:
+To connect to a single CrateDB node, use the ``connect()`` function, like so::
 
     >>> connection = client.connect("<NODE_URL>", username="<USERNAME>")
 
@@ -69,7 +69,7 @@ Connect to multiple nodes
 =========================
 
 To connect to one of multiple nodes, pass a list of database URLs to the
-connect() function, like so:
+connect() function, like so::
 
     >>> connection = client.connect(["<NODE_1_URL>", "<NODE_2_URL>"], ...)
 
@@ -220,10 +220,14 @@ Authentication
 
    See the :ref:`compatibility notes <cratedb-versions>` for more information.
 
-You can authenticate with CrateDB like so:
+You can authenticate with CrateDB like so::
 
     >>> connection = client.connect(..., username="<USERNAME>", password="<PASSWORD>")
 
+At your disposal, you can also embed the credentials into the URI, like so::
+
+    >>> connection = client.connect("https://<USERNAME>:<PASSWORD>@cratedb.example.org:4200")
+
 Here, replace ``<USERNAME>`` and ``<PASSWORD>`` with the appropriate username
 and password.
 
@@ -238,7 +242,7 @@ and password.
 Schema selection
 ================
 
-You can select a schema using the optional ``schema`` argument, like so:
+You can select a schema using the optional ``schema`` argument, like so::
 
     >>> connection = client.connect(..., schema="<SCHEMA>")
 

docs/sqlalchemy.rst

@@ -11,7 +11,7 @@ The CrateDB Python client library provides support for SQLAlchemy. A CrateDB
 configuration.
 
 The CrateDB Python client library works with SQLAlchemy versions ``1.0``,
-``1.1`` and ``1.2``.
+``1.1``, ``1.2`` and ``1.3``.
 
 .. NOTE::
 
@@ -49,27 +49,33 @@ Locator* (URL) called a `database URL`_.
 
 The simplest database URL for CrateDB looks like this::
 
-    crate://<HOST>
+    crate://<HOST>/[?option=value]
 
-Here, ``<HOST>`` is the node *host string*.
+Here, ``<HOST>`` is the node *host string*. After the host, additional query
+parameters can be specified to adjust some connection settings.
 
 A host string looks like this::
 
-    <HOST_ADDR>:<PORT>
+    [<USERNAME>:<PASSWORD>@]<HOST_ADDR>:<PORT>
 
 Here, ``<HOST_ADDR>`` is the hostname or IP address of the CrateDB node and
 ``<PORT>`` is a valid `psql.port`_ number.
 
-Example host strings:
+When authentication is needed, the credentials can be optionally supplied using
+``<USERNAME>:<PASSWORD>@``. For connecting to an SSL-secured HTTP endpoint, you
+can add the query parameter ``?ssl=true`` to the database URI.
 
-- ``localhost:4200``
-- ``crate-1.vm.example.com:4200``
-- ``198.51.100.1:4200``
+Example database URIs:
+
+- ``crate://localhost:4200``
+- ``crate://crate-1.vm.example.com:4200``
+- ``crate://username:[email protected]:4200/?ssl=true``
+- ``crate://198.51.100.1:4200``
 
 .. TIP::
 
-    If ``<HOST>`` is blank (i.e. just ``crate://``) then ``localhost:4200`` will
-    be assumed.
+    If ``<HOST>`` is blank (i.e. the database URI is just ``crate://``), then
+    ``localhost:4200`` will be assumed.
 
 Getting a connection
 --------------------

src/crate/client/doctests/blob.txt

@@ -39,9 +39,10 @@ Store from a file::
     >>> file_blob = container.put(f)
     >>> file_blob
     'ea6e03a4a4ee8a2366fe5a88af2bde61797973ea'
+    >>> f.close()
 
 If the blob data is not provided as a seekable stream the hash must be
-provided explicetely::
+provided explicitly::
 
     >>> import hashlib
     >>> string_data = b'String data'

src/crate/client/doctests/client.txt

@@ -13,6 +13,7 @@ The client provides a ``connect()`` function which is used to establish a
 connection, the first argument is the url of the server to connect to::
 
     >>> connection = client.connect(crate_host)
+    >>> connection.close()
 
 CrateDB is a clustered database providing high availability through
 replication. In order for clients to make use of this property it is
@@ -21,22 +22,26 @@ respond, the request is automatically routed to the next server::
 
     >>> invalid_host = 'http://not_responding_host:4200'
     >>> connection = client.connect([invalid_host, crate_host])
+    >>> connection.close()
 
 If no ``servers`` are given, the default one ``http://127.0.0.1:4200`` is used::
 
     >>> connection = client.connect()
     >>> connection.client._active_servers
     ['http://127.0.0.1:4200']
+    >>> connection.close()
 
 If the option ``error_trace`` is set to ``True``, the client will print a whole
 traceback if a server error occurs::
 
     >>> connection = client.connect([crate_host], error_trace=True)
+    >>> connection.close()
 
 It's possible to define a default timeout value in seconds for all servers
 using the optional parameter ``timeout``::
 
     >>> connection = client.connect([crate_host, invalid_host], timeout=5)
+    >>> connection.close()
 
 Authentication
 --------------
@@ -47,13 +52,40 @@ connect::
 
     >>> connection = client.connect([crate_host],
     ...                             username='trusted_me')
+    >>> connection.client.username
+    'trusted_me'
+    >>> connection.client.password
+    >>> connection.close()
+
+The username for trusted users can also be provided in the URL::
+
+    >>> connection = client.connect(['http://trusted_me@' + crate_host])
+    >>> connection.client.username
+    'trusted_me'
+    >>> connection.client.password
+    >>> connection.close()
 
 To connect to CrateDB with as a user that requires password authentication, you
 also need to provide ``password`` as argument for the ``connect()`` call::
 
     >>> connection = client.connect([crate_host],
     ...                             username='me',
     ...                             password='my_secret_pw')
+    >>> connection.client.username
+    'me'
+    >>> connection.client.password
+    'my_secret_pw'
+    >>> connection.close()
+
+The authentication credentials can also be provided in the URL::
+
+    >>> connection = client.connect(['http://me:my_secret_pw@' + crate_host])
+    >>> connection.client.username
+    'me'
+    >>> connection.client.password
+    'my_secret_pw'
+    >>> connection.close()
+
 
 Default Schema
 --------------
@@ -63,15 +95,16 @@ provide the ``schema`` keyword argument in the ``connect()`` method, like so::
 
     >>> connection = client.connect([crate_host],
     ...                             schema='custom_schema')
+    >>> connection.close()
 
 Inserting Data
 ==============
 
 Use user "crate" for rest of the tests::
 
-    >>> connection = client.connect([crate_host], timeout=2)
+    >>> connection = client.connect([crate_host])
 
-Before executing any statement a cursor has to be opened to perform
+Before executing any statement, a cursor has to be opened to perform
 database operations::
 
     >>> cursor = connection.cursor()
@@ -88,7 +121,7 @@ To bulk insert data you can use the ``executemany`` function::
     [{'rowcount': 1}, {'rowcount': 1}]
 
 ``executemany`` returns a list of results for every parameter. Each result
-contains a rowcount. If an error occurs the rowcount is -2 and the result
+contains a rowcount. If an error occurs, the rowcount is ``-2`` and the result
 may contain an ``error_message`` depending on the error.
 
 Refresh locations:

src/crate/client/doctests/cursor.txt

@@ -263,6 +263,9 @@ For completeness' sake the cursor description is updated nonetheless::
     ...     "duration":123
     ... })
 
+.. Hidden: close connection
+
+    >>> connection.close()
 
 Usually ``executemany`` sends the ``bulk_args`` parameter to the crate sql 
 endpoint which was introduced with Crate 0.42.0.
@@ -313,3 +316,6 @@ closed connection an ``ProgrammingError`` exception will be raised::
     ...
     crate.client.exceptions.ProgrammingError: Cursor closed
 
+.. Hidden: close connection
+
+    >>> connection.close()

src/crate/client/http.py

@@ -346,6 +346,22 @@ def __init__(self,
             servers = [self.default_server]
         else:
             servers = _to_server_list(servers)
+
+        # Try to derive credentials from first server argument if not
+        # explicitly given.
+        if servers and not username:
+            try:
+                url = urlparse(servers[0])
+                if url.username is not None:
+                    username = url.username
+                if url.password is not None:
+                    password = url.password
+            except Exception as ex:
+                logger.warning("Unable to decode credentials from database "
+                               "URI, so connecting to CrateDB without "
+                               "authentication: {ex}"
+                               .format(ex=ex))
+
         self._active_servers = servers
         self._inactive_servers = []
         pool_kw = _pool_kw_args(

src/crate/client/sqlalchemy/dialect.py

@@ -25,6 +25,7 @@
 from sqlalchemy import types as sqltypes
 from sqlalchemy.engine import default, reflection
 from sqlalchemy.sql import functions
+from sqlalchemy.util import asbool, to_list
 
 from .compiler import (
     CrateCompiler,
@@ -194,8 +195,12 @@ def connect(self, host=None, port=None, *args, **kwargs):
             server = '{0}:{1}'.format(host, port or '4200')
         if 'servers' in kwargs:
             server = kwargs.pop('servers')
-        if server:
-            return self.dbapi.connect(servers=server, **kwargs)
+        servers = to_list(server)
+        if servers:
+            use_ssl = asbool(kwargs.pop("ssl", False))
+            if use_ssl:
+                servers = ["https://" + server for server in servers]
+            return self.dbapi.connect(servers=servers, **kwargs)
         return self.dbapi.connect(**kwargs)
 
     def _get_default_schema_name(self, connection):

src/crate/client/sqlalchemy/doctests/itests.txt

@@ -220,3 +220,7 @@ Refresh "characters" table:
     >>> import pprint
     >>> pprint.pprint(char.details)
     {'name': {'first': 'Trillian', 'last': 'Dent'}, 'size': 45}
+
+.. Hidden: close connection
+
+    >>> connection.close()