diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8ba0b83..c7e914c1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] python-version: [3.5, 3.6, 3.7, 3.8, 3.9] - cratedb-version: ['4.5.0'] + cratedb-version: ['4.7.1'] sqla-version: ['1.1.18', '1.2.19', '1.3.23'] fail-fast: true diff --git a/CHANGES.txt b/CHANGES.txt index e0e66bda..eba40376 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -11,6 +11,11 @@ Unreleased HTTP by default. Previously, this setting defaulted to false. This setting can be changed via the ``verify_ssl_cert`` connection parameter. +- Adjusted connect arguments to accept credentials within the HTTP URI. + +- Added support for enabling SSL using SQLAlchemy DB URI with parameter + ``?ssl=true``. + 2020/09/28 0.26.0 ================= diff --git a/docs/appendices/data-types.rst b/docs/appendices/data-types.rst index 057b15e7..b0d15e16 100644 --- a/docs/appendices/data-types.rst +++ b/docs/appendices/data-types.rst @@ -48,19 +48,19 @@ CrateDB Python __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#boolean __ https://docs.python.org/3/library/stdtypes.html#boolean-values -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data __ https://docs.python.org/3/library/stdtypes.html#str -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ https://docs.python.org/3/library/functions.html#int -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ https://docs.python.org/3/library/functions.html#float -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ https://docs.python.org/3/library/functions.html#float -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ https://docs.python.org/3/library/functions.html#int __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#geo-point __ https://docs.python.org/3/library/stdtypes.html#list @@ -82,15 +82,15 @@ Python CrateDB ============= ==================================== __ https://docs.python.org/3/library/decimal.html -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data __ https://docs.python.org/3/library/datetime.html#date-objects -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data __ https://docs.python.org/3/library/datetime.html#datetime-objects -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data .. NOTE:: @@ -134,21 +134,21 @@ CrateDB SQLAlchemy __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#boolean __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.Boolean -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.SmallInteger -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.SmallInteger -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.Integer -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.NUMERIC -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.Float -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#numeric-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.DECIMAL -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#date-time-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#dates-and-times __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.TIMESTAMP -__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data-types +__ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#character-data __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.String __ https://crate.io/docs/crate/reference/en/latest/general/ddl/data-types.html#array __ http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.ARRAY diff --git a/docs/connect.rst b/docs/connect.rst index 43d2a88e..7649a027 100644 --- a/docs/connect.rst +++ b/docs/connect.rst @@ -29,7 +29,7 @@ Connect to CrateDB Connect to a single node ======================== -To connect to a single CrateDB node, use the ``connect()`` function, like so: +To connect to a single CrateDB node, use the ``connect()`` function, like so:: >>> connection = client.connect("", username="") @@ -69,7 +69,7 @@ Connect to multiple nodes ========================= To connect to one of multiple nodes, pass a list of database URLs to the -connect() function, like so: +connect() function, like so:: >>> connection = client.connect(["", ""], ...) @@ -220,10 +220,14 @@ Authentication See the :ref:`compatibility notes ` for more information. -You can authenticate with CrateDB like so: +You can authenticate with CrateDB like so:: >>> connection = client.connect(..., username="", password="") +At your disposal, you can also embed the credentials into the URI, like so:: + + >>> connection = client.connect("https://:@cratedb.example.org:4200") + Here, replace ```` and ```` with the appropriate username and password. @@ -238,7 +242,7 @@ and password. Schema selection ================ -You can select a schema using the optional ``schema`` argument, like so: +You can select a schema using the optional ``schema`` argument, like so:: >>> connection = client.connect(..., schema="") diff --git a/docs/sqlalchemy.rst b/docs/sqlalchemy.rst index 2b98517b..ad67ff76 100644 --- a/docs/sqlalchemy.rst +++ b/docs/sqlalchemy.rst @@ -11,7 +11,7 @@ The CrateDB Python client library provides support for SQLAlchemy. A CrateDB configuration. The CrateDB Python client library works with SQLAlchemy versions ``1.0``, -``1.1`` and ``1.2``. +``1.1``, ``1.2`` and ``1.3``. .. NOTE:: @@ -49,27 +49,33 @@ Locator* (URL) called a `database URL`_. The simplest database URL for CrateDB looks like this:: - crate:// + crate:///[?option=value] -Here, ```` is the node *host string*. +Here, ```` is the node *host string*. After the host, additional query +parameters can be specified to adjust some connection settings. A host string looks like this:: - : + [:@]: Here, ```` is the hostname or IP address of the CrateDB node and ```` is a valid `psql.port`_ number. -Example host strings: +When authentication is needed, the credentials can be optionally supplied using +``:@``. For connecting to an SSL-secured HTTP endpoint, you +can add the query parameter ``?ssl=true`` to the database URI. -- ``localhost:4200`` -- ``crate-1.vm.example.com:4200`` -- ``198.51.100.1:4200`` +Example database URIs: + +- ``crate://localhost:4200`` +- ``crate://crate-1.vm.example.com:4200`` +- ``crate://username:password@crate-2.vm.example.com:4200/?ssl=true`` +- ``crate://198.51.100.1:4200`` .. TIP:: - If ```` is blank (i.e. just ``crate://``) then ``localhost:4200`` will - be assumed. + If ```` is blank (i.e. the database URI is just ``crate://``), then + ``localhost:4200`` will be assumed. Getting a connection -------------------- diff --git a/src/crate/client/doctests/blob.txt b/src/crate/client/doctests/blob.txt index e4256d9d..f04b27f9 100644 --- a/src/crate/client/doctests/blob.txt +++ b/src/crate/client/doctests/blob.txt @@ -39,9 +39,10 @@ Store from a file:: >>> file_blob = container.put(f) >>> file_blob 'ea6e03a4a4ee8a2366fe5a88af2bde61797973ea' + >>> f.close() If the blob data is not provided as a seekable stream the hash must be -provided explicetely:: +provided explicitly:: >>> import hashlib >>> string_data = b'String data' diff --git a/src/crate/client/doctests/client.txt b/src/crate/client/doctests/client.txt index b564c04d..9761642d 100644 --- a/src/crate/client/doctests/client.txt +++ b/src/crate/client/doctests/client.txt @@ -13,6 +13,7 @@ The client provides a ``connect()`` function which is used to establish a connection, the first argument is the url of the server to connect to:: >>> connection = client.connect(crate_host) + >>> connection.close() CrateDB is a clustered database providing high availability through replication. In order for clients to make use of this property it is @@ -21,22 +22,26 @@ respond, the request is automatically routed to the next server:: >>> invalid_host = 'http://not_responding_host:4200' >>> connection = client.connect([invalid_host, crate_host]) + >>> connection.close() If no ``servers`` are given, the default one ``http://127.0.0.1:4200`` is used:: >>> connection = client.connect() >>> connection.client._active_servers ['http://127.0.0.1:4200'] + >>> connection.close() If the option ``error_trace`` is set to ``True``, the client will print a whole traceback if a server error occurs:: >>> connection = client.connect([crate_host], error_trace=True) + >>> connection.close() It's possible to define a default timeout value in seconds for all servers using the optional parameter ``timeout``:: >>> connection = client.connect([crate_host, invalid_host], timeout=5) + >>> connection.close() Authentication -------------- @@ -47,6 +52,18 @@ connect:: >>> connection = client.connect([crate_host], ... username='trusted_me') + >>> connection.client.username + 'trusted_me' + >>> connection.client.password + >>> connection.close() + +The username for trusted users can also be provided in the URL:: + + >>> connection = client.connect(['http://trusted_me@' + crate_host]) + >>> connection.client.username + 'trusted_me' + >>> connection.client.password + >>> connection.close() To connect to CrateDB with as a user that requires password authentication, you also need to provide ``password`` as argument for the ``connect()`` call:: @@ -54,6 +71,21 @@ also need to provide ``password`` as argument for the ``connect()`` call:: >>> connection = client.connect([crate_host], ... username='me', ... password='my_secret_pw') + >>> connection.client.username + 'me' + >>> connection.client.password + 'my_secret_pw' + >>> connection.close() + +The authentication credentials can also be provided in the URL:: + + >>> connection = client.connect(['http://me:my_secret_pw@' + crate_host]) + >>> connection.client.username + 'me' + >>> connection.client.password + 'my_secret_pw' + >>> connection.close() + Default Schema -------------- @@ -63,15 +95,16 @@ provide the ``schema`` keyword argument in the ``connect()`` method, like so:: >>> connection = client.connect([crate_host], ... schema='custom_schema') + >>> connection.close() Inserting Data ============== Use user "crate" for rest of the tests:: - >>> connection = client.connect([crate_host], timeout=2) + >>> connection = client.connect([crate_host]) -Before executing any statement a cursor has to be opened to perform +Before executing any statement, a cursor has to be opened to perform database operations:: >>> cursor = connection.cursor() @@ -88,7 +121,7 @@ To bulk insert data you can use the ``executemany`` function:: [{'rowcount': 1}, {'rowcount': 1}] ``executemany`` returns a list of results for every parameter. Each result -contains a rowcount. If an error occurs the rowcount is -2 and the result +contains a rowcount. If an error occurs, the rowcount is ``-2`` and the result may contain an ``error_message`` depending on the error. Refresh locations: diff --git a/src/crate/client/doctests/cursor.txt b/src/crate/client/doctests/cursor.txt index b1ecd7e0..41ad720a 100644 --- a/src/crate/client/doctests/cursor.txt +++ b/src/crate/client/doctests/cursor.txt @@ -263,6 +263,9 @@ For completeness' sake the cursor description is updated nonetheless:: ... "duration":123 ... }) +.. Hidden: close connection + + >>> connection.close() Usually ``executemany`` sends the ``bulk_args`` parameter to the crate sql endpoint which was introduced with Crate 0.42.0. @@ -313,3 +316,6 @@ closed connection an ``ProgrammingError`` exception will be raised:: ... crate.client.exceptions.ProgrammingError: Cursor closed +.. Hidden: close connection + + >>> connection.close() diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 02aa80bd..3c3ae734 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -346,6 +346,22 @@ def __init__(self, servers = [self.default_server] else: servers = _to_server_list(servers) + + # Try to derive credentials from first server argument if not + # explicitly given. + if servers and not username: + try: + url = urlparse(servers[0]) + if url.username is not None: + username = url.username + if url.password is not None: + password = url.password + except Exception as ex: + logger.warning("Unable to decode credentials from database " + "URI, so connecting to CrateDB without " + "authentication: {ex}" + .format(ex=ex)) + self._active_servers = servers self._inactive_servers = [] pool_kw = _pool_kw_args( diff --git a/src/crate/client/sqlalchemy/dialect.py b/src/crate/client/sqlalchemy/dialect.py index 8c3bc91a..4cb16e62 100644 --- a/src/crate/client/sqlalchemy/dialect.py +++ b/src/crate/client/sqlalchemy/dialect.py @@ -25,6 +25,7 @@ from sqlalchemy import types as sqltypes from sqlalchemy.engine import default, reflection from sqlalchemy.sql import functions +from sqlalchemy.util import asbool, to_list from .compiler import ( CrateCompiler, @@ -194,8 +195,12 @@ def connect(self, host=None, port=None, *args, **kwargs): server = '{0}:{1}'.format(host, port or '4200') if 'servers' in kwargs: server = kwargs.pop('servers') - if server: - return self.dbapi.connect(servers=server, **kwargs) + servers = to_list(server) + if servers: + use_ssl = asbool(kwargs.pop("ssl", False)) + if use_ssl: + servers = ["https://" + server for server in servers] + return self.dbapi.connect(servers=servers, **kwargs) return self.dbapi.connect(**kwargs) def _get_default_schema_name(self, connection): diff --git a/src/crate/client/sqlalchemy/doctests/itests.txt b/src/crate/client/sqlalchemy/doctests/itests.txt index 1b07c085..1697f291 100644 --- a/src/crate/client/sqlalchemy/doctests/itests.txt +++ b/src/crate/client/sqlalchemy/doctests/itests.txt @@ -220,3 +220,7 @@ Refresh "characters" table: >>> import pprint >>> pprint.pprint(char.details) {'name': {'first': 'Trillian', 'last': 'Dent'}, 'size': 45} + +.. Hidden: close connection + + >>> connection.close() diff --git a/src/crate/client/sqlalchemy/tests/connection_test.py b/src/crate/client/sqlalchemy/tests/connection_test.py index 4bdfef4e..6e6cb613 100644 --- a/src/crate/client/sqlalchemy/tests/connection_test.py +++ b/src/crate/client/sqlalchemy/tests/connection_test.py @@ -21,13 +21,14 @@ from unittest import TestCase import sqlalchemy as sa +from sqlalchemy.exc import NoSuchModuleError class SqlAlchemyConnectionTest(TestCase): - def setUp(self): - self.engine = sa.create_engine('crate://') - self.connection = self.engine.connect() + def test_connection_server_uri_unknown_sa_plugin(self): + with self.assertRaises(NoSuchModuleError): + sa.create_engine("foobar://otherhost:19201") def test_default_connection(self): engine = sa.create_engine('crate://') @@ -35,14 +36,44 @@ def test_default_connection(self): self.assertEqual(">", repr(conn.connection)) - def test_connection_server(self): + def test_connection_server_uri_http(self): engine = sa.create_engine( "crate://otherhost:19201") conn = engine.raw_connection() self.assertEqual(">", repr(conn.connection)) - def test_connection_multiple_server(self): + def test_connection_server_uri_https(self): + engine = sa.create_engine( + "crate://otherhost:19201/?ssl=true") + conn = engine.raw_connection() + self.assertEqual(">", + repr(conn.connection)) + + def test_connection_server_uri_invalid_port(self): + with self.assertRaises(ValueError) as context: + sa.create_engine("crate://foo:bar") + self.assertTrue("invalid literal for int() with base 10: 'bar'" in str(context.exception)) + + def test_connection_server_uri_https_with_trusted_user(self): + engine = sa.create_engine( + "crate://foo@otherhost:19201/?ssl=true") + conn = engine.raw_connection() + self.assertEqual(">", + repr(conn.connection)) + self.assertEqual(conn.connection.client.username, "foo") + self.assertEqual(conn.connection.client.password, None) + + def test_connection_server_uri_https_with_credentials(self): + engine = sa.create_engine( + "crate://foo:bar@otherhost:19201/?ssl=true") + conn = engine.raw_connection() + self.assertEqual(">", + repr(conn.connection)) + self.assertEqual(conn.connection.client.username, "foo") + self.assertEqual(conn.connection.client.password, "bar") + + def test_connection_multiple_server_http(self): engine = sa.create_engine( "crate://", connect_args={ 'servers': ['localhost:4201', 'localhost:4202'] @@ -53,3 +84,16 @@ def test_connection_multiple_server(self): ">", repr(conn.connection)) + + def test_connection_multiple_server_https(self): + engine = sa.create_engine( + "crate://", connect_args={ + 'servers': ['localhost:4201', 'localhost:4202'], + 'ssl': True, + } + ) + conn = engine.raw_connection() + self.assertEqual( + ">", + repr(conn.connection)) diff --git a/src/crate/client/test_http.py b/src/crate/client/test_http.py index 5ef8203a..4a073099 100644 --- a/src/crate/client/test_http.py +++ b/src/crate/client/test_http.py @@ -116,6 +116,7 @@ def test_connection_reset_exception(self): def test_no_connection_exception(self): client = Client() self.assertRaises(ConnectionError, client.sql, 'select foo') + client.close() @patch(REQUEST) def test_http_error_is_re_raised(self, request): @@ -123,6 +124,7 @@ def test_http_error_is_re_raised(self, request): client = Client() self.assertRaises(ProgrammingError, client.sql, 'select foo') + client.close() @patch(REQUEST) def test_programming_error_contains_http_error_response_content(self, request): @@ -283,6 +285,7 @@ def test_socket_options_contain_keepalive(self): self.assertTrue( (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) in conn_kw['socket_options'] ) + client.close() @patch(REQUEST, fail_sometimes) @@ -583,8 +586,8 @@ def setUp(self): self.client = self.clientWithKwargs(schema='my_custom_schema') def tearDown(self): - super().tearDown() self.client.close() + super().tearDown() def test_default_schema(self): self.client.sql('SELECT 1') diff --git a/src/crate/client/tests.py b/src/crate/client/tests.py index 574f1a7e..b2c56750 100644 --- a/src/crate/client/tests.py +++ b/src/crate/client/tests.py @@ -264,13 +264,11 @@ def do_GET(self): self.end_headers() self.wfile.write(payload) - def __init__(self): + def setUp(self): self.server = self.HttpsServer( (self.HOST, self.PORT), self.HttpsHandler ) - - def setUp(self): thread = threading.Thread(target=self.serve_forever) thread.daemon = True # quit interpreter when only thread exists thread.start() @@ -283,6 +281,7 @@ def serve_forever(self): def tearDown(self): self.server.shutdown() + self.server.server_close() def isUp(self): """