diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5102f9b5..ff303ed4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.0 +current_version = 2.0.0 commit = True tag = False diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ee3aafd7..fa660888 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,171 @@ Changelog ========= +2.0.0 (2024-03-19) +------------------ + +This is a major release with new features and breaking changes. + +Features +~~~~~~~~ + +* Support for ILP over HTTP. The sender can now send data to QuestDB via HTTP + instead of TCP. This provides error feedback from the server and new features. + + .. code-block:: python + + conf = 'http::addr=localhost:9000;' + with Sender.from_conf(conf) as sender: + sender.row(...) + sender.dataframe(...) + + # Will raise `IngressError` if there is an error from the server. + sender.flush() + +* New configuration string construction. The sender can now be also constructed + from a :ref:`configuration string ` in addition to the + constructor arguments. + This allows for more flexible configuration and is the recommended way to + construct a sender. + The same string can also be loaded from the ``QDB_CLIENT_CONF`` environment + variable. + The constructor arguments have been updated and some options have changed. + +* Explicit transaction support over HTTP. A set of rows for a single table can + now be committed via the sender transactionally. You can do this using a + ``with sender.transaction('table_name') as txn:`` block. + + .. code-block:: python + + conf = 'http::addr=localhost:9000;' + with Sender.from_conf(conf) as sender: + with sender.transaction('test_table') as txn: + # Same arguments as the sender methods, minus the table name. + txn.row(...) + txn.dataframe(...) + +* A number of documentation improvements. + + +Breaking Changes +~~~~~~~~~~~~~~~~ + +* New ``protocol`` parameter in the + :ref:`Sender ` constructor. + + In previous version the protocol was always TCP. + In this new version you must specify the protocol explicitly. + +* New auto-flush defaults. In previous versions + :ref:`auto-flushing ` was enabled by + default and triggered by a maximum buffer size. In this new version + auto-flushing is enabled by row count (600 rows by default) and interval + (1 second by default), while auto-flushing by buffer size is disabled by + default. + + The old behaviour can be still be achieved by tweaking the auto-flush + settings. + + .. list-table:: + :header-rows: 1 + + * - Setting + - Old default + - New default + * - **auto_flush_rows** + - off + - 600 + * - **auto_flush_interval** + - off + - 1000 + * - **auto_flush_bytes** + - 64512 + - off + +* The ``at=..`` argument of :func:`row ` and + :func:`dataframe ` methods is now mandatory. + Omitting it would previously use a server-generated timestamp for the row. + Now if you want a server generated timestamp, you can pass the :ref:`ServerTimestamp ` + singleton to this parameter. _The ``ServerTimestamp`` behaviour is considered legacy._ + +* The ``auth=(u, t, x, y)`` argument of the ``Sender`` constructor has now been + broken up into multiple arguments: ``username``, ``token``, ``token_x``, ``token_y``. + +* The ``tls`` argument of the ``Sender`` constructor has been removed and + replaced with the ``protocol`` argument. Use ``Protocol.Tcps`` + (or ``Protocol.Https``) to enable TLS. + The ``tls`` values have been moved to new ``tls_ca`` and ``tls_roots`` + :ref:`configuration settings `. + +* The ``net_interface`` argument of the ``Sender`` constructor has been renamed + to ``bind_interface`` and is now only available for TCP connections. + +The following example shows how to migrate to the new API. + +**Old questdb 1.x code** + +.. code-block:: python + + from questdb.ingress import Sender + + auth = ( + 'testUser1', + '5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48', + 'token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU', + 'token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac') + with Sender('localhost', 9009, auth=auth, tls=True) as sender: + sender.row( + 'test_table', + symbols={'sym': 'AAPL'}, + columns={'price': 100.0}) # `at=None` was defaulted for server time + +**Equivalent questdb 2.x code** + +.. code-block:: python + + from questdb.ingress import Sender, Protocol, ServerTimestamp + + sender = Sender( + Protocol.Tcps, + 'localhost', + 9009, + username='testUser1', + token='5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48', + token_x='token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU', + token_y='token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac', + auto_flush_rows='off', + auto_flush_interval='off', + auto_flush_bytes=64512) + with sender: + sender.row( + 'test_table', + symbols={'sym': 'AAPL'}, + columns={'price': 100.0}, + at=ServerTimestamp) + +**Equivalent questdb 2.x code with configuration string** + +.. code-block:: python + + from questdb.ingress import Sender + + conf = ( + 'tcp::addr=localhost:9009;' + + 'username=testUser1;' + + 'token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;' + + 'token_x=token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;' + + 'token_y=token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;' + + 'auto_flush_rows=off;' + + 'auto_flush_interval=off;' + + 'auto_flush_bytes=64512') + with Sender.from_conf(conf) as sender: + sender.row( + 'test_table', + symbols={'sym': 'AAPL'}, + columns={'price': 100.0}, + at=ServerTimestamp) + + 1.2.0 (2023-11-23) ------------------ diff --git a/README.rst b/README.rst index 0f17c28b..5705fb4d 100644 --- a/README.rst +++ b/README.rst @@ -2,27 +2,30 @@ QuestDB Client Library for Python ================================= -This library makes it easy to insert data into `QuestDB `_. +This is the official Python client library for `QuestDB `_. This client library implements QuestDB's variant of the `InfluxDB Line Protocol `_ -(ILP) over TCP. +(ILP) over HTTP and TCP. ILP provides the fastest way to insert data into QuestDB. This implementation supports `authentication -`_ and full-connection -encryption with TLS. +`_ +and full-connection encryption with +`TLS `_. Quickstart ========== -The latest version of the library is 1.2.0. +The latest version of the library is 2.0.0. :: python3 -m pip install -U questdb +Please start by `setting up QuestDB `_ . Once set up, you can use this library to insert data. + .. code-block:: python from questdb.ingress import Sender, TimestampNanos @@ -68,29 +71,25 @@ You can continue by reading the `Sending Data Over ILP `_ guide. -Docs -==== - -https://py-questdb-client.readthedocs.io/ - +Links +===== -Code -==== +* `Core database documentation `_ -https://github.com/questdb/py-questdb-client +* `Python library documentation `_ +* `GitHub repository `_ -Package on PyPI -=============== - -https://pypi.org/project/questdb/ - +* `Package on PyPI `_ Community ========= -If you need help, have additional questions or want to provide feedback, you -may find us on `Slack `_. +If you need help, you can ask on `Stack Overflow +`_: +We monitor the ``#questdb`` and ``#py-questdb-client`` tags. + +Alternatively, you may find us on `Slack `_. You can also `sign up to our mailing list `_ to get notified of new releases. diff --git a/docs/conf.py b/docs/conf.py index 700aef7f..18bd2c47 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,10 +17,10 @@ source_suffix = '.rst' master_doc = 'index' project = 'questdb' -year = '2023' +year = '2024' author = 'QuestDB' copyright = '{0}, {1}'.format(year, author) -version = release = '1.2.0' +version = release = '2.0.0' github_repo_url = 'https://github.com/questdb/py-questdb-client' diff --git a/docs/conf.rst b/docs/conf.rst index 3c8a2db8..15ff6ff9 100644 --- a/docs/conf.rst +++ b/docs/conf.rst @@ -36,6 +36,21 @@ If you're unsure which protocol to use, see :ref:`sender_which_protocol`. Only the ``addr=host:port`` key is mandatory. It specifies the hostname and port of the QuestDB server. +The same configuration string can also be loaded from the ``QDB_CLIENT_CONF`` +environment variable. This is useful for keeping sensitive information out of +your code. + +.. code-block:: bash + + export QDB_CLIENT_CONF="http::addr=localhost:9009;username=admin;password=quest;" + +.. code-block:: python + + from questdb.ingress import Sender + + with Sender.from_env() as sender: + ... + Connection ========== diff --git a/docs/examples.rst b/docs/examples.rst index e021b4c4..455a98b3 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -5,22 +5,23 @@ Examples Basics ====== -Row-by-row Insertion +HTTP with Token Auth -------------------- The following example connects to the database and sends two rows (lines). -The connection is unauthenticated and the data is sent at the end of the -``with`` block. +The connection is made via HTTPS and uses token based authentication. -.. literalinclude:: ../examples/basic.py +The data is sent at the end of the ``with`` block. + +.. literalinclude:: ../examples/http.py :language: python .. _auth_and_tls_example: -Authentication and TLS ----------------------- +TCP Authentication and TLS +-------------------------- Continuing from the previous example, the connection is authenticated and also uses TLS. @@ -44,13 +45,13 @@ Note that this bypasses :ref:`auto-flushing `. :language: python -Ticking Random Data and Timer-based Flush ------------------------------------------ +Ticking Data and Auto-Flush +--------------------------- The following example somewhat mimics the behavior of a loop in an application. -It creates random ticking data at a random interval and flushes it explicitly -based on a timer if the auto-flushing logic was not triggered recently. +It creates random ticking data at a random interval and uses non-default +auto-flush settings. .. literalinclude:: ../examples/random_data.py :language: python diff --git a/docs/sender.rst b/docs/sender.rst index ad89827f..1a39beab 100644 --- a/docs/sender.rst +++ b/docs/sender.rst @@ -168,6 +168,8 @@ Note that all timestamps in QuestDB are stored as microseconds since the epoch, without timezone information. Any timezone information is dropped when the data is appended to the ILP buffer. +.. _sender_server_timestamp: + Set by server ~~~~~~~~~~~~~ @@ -591,15 +593,15 @@ Python type mappings: * Parameters that require numbers can also take an ``int``. -* Millisecond durations can take an ``int`` or a ```datetime.timedelta``. +* Millisecond durations can take an ``int`` or a ``datetime.timedelta``. * Any ``'on'`` / ``'off'`` / ``'unsafe_off'`` parameters can also be specified as a ``bool``. * Paths can also be specified as a ``pathlib.Path``. -Customising `.from_conf()` and `.from_env()` --------------------------------------------- +Customising ``.from_conf()`` and ``.from_env()`` +------------------------------------------------ If you want to further customise the behaviour of the ``.from_conf()`` or ``.from_env()`` methods, you can pass additional parameters to these methods. diff --git a/examples/http.py b/examples/http.py new file mode 100644 index 00000000..ee976cb4 --- /dev/null +++ b/examples/http.py @@ -0,0 +1,47 @@ +from questdb.ingress import Sender, IngressError, TimestampNanos +import sys +import datetime + + +def example(host: str = 'localhost', port: int = 9009): + try: + conf = f'https::addr={host}:{port};token=the_secure_token;' + with Sender.from_conf(conf) as sender: + # Record with provided designated timestamp (using the 'at' param) + # Notice the designated timestamp is expected in Nanoseconds, + # but timestamps in other columns are expected in Microseconds. + # The API provides convenient functions + sender.row( + 'trades', + symbols={ + 'pair': 'USDGBP', + 'type': 'buy'}, + columns={ + 'traded_price': 0.83, + 'limit_price': 0.84, + 'qty': 100, + 'traded_ts': datetime.datetime( + 2022, 8, 6, 7, 35, 23, 189062, + tzinfo=datetime.timezone.utc)}, + at=TimestampNanos.now()) + + # You can call `sender.row` multiple times inside the same `with` + # block. The client will buffer the rows and send them in batches. + + # You can flush manually at any point. + sender.flush() + + # If you don't flush manually, the client will flush automatically + # when a row is added and either: + # * The buffer contains 75000 rows (if HTTP) or 600 rows (if TCP) + # * The last flush was more than 1000ms ago. + # Auto-flushing can be customized via the `auto_flush_..` params. + + # Any remaining pending rows will be sent when the `with` block ends. + + except IngressError as e: + sys.stderr.write(f'Got error: {e}\n') + + +if __name__ == '__main__': + example() diff --git a/examples/random_data.py b/examples/random_data.py index df266e49..86da88b1 100644 --- a/examples/random_data.py +++ b/examples/random_data.py @@ -6,11 +6,13 @@ def example(host: str = 'localhost', port: int = 9009): table_name: str = str(uuid.uuid1()) - # Flush if the internal buffer exceeds 1KiB - conf: str = f"tcp::addr={host}:{port};auto_flush_bytes=1024;" + conf: str = ( + f"tcp::addr={host}:{port};" + + "auto_flush_bytes=1024;" + # Flush if the internal buffer exceeds 1KiB + "auto_flush_rows=off;" # Disable auto-flushing based on row count + "auto_flush_interval=5000;") # Flush if last flushed more than 5s ago with Sender.from_conf(conf) as sender: total_rows = 0 - last_flush = time.monotonic() try: print("Ctrl^C to terminate...") while True: @@ -31,17 +33,9 @@ def example(host: str = 'localhost', port: int = 9009): # If the internal buffer is empty, then auto-flush triggered. if len(sender) == 0: print('Auto-flush triggered.') - last_flush = time.monotonic() - - # Flush at least once every five seconds. - if time.monotonic() - last_flush > 5: - print('Timer-flushing triggered.') - sender.flush() - last_flush = time.monotonic() except KeyboardInterrupt: print(f"table: {table_name}, total rows sent: {total_rows}") - print("(wait commitLag for all rows to be available)") print("bye!") diff --git a/pyproject.toml b/pyproject.toml index ec5f481a..2ccfc764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # See: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ name = "questdb" requires-python = ">=3.8" -version = "1.2.0" +version = "2.0.0" description = "QuestDB client library for Python" readme = "README.rst" classifiers = [ diff --git a/setup.py b/setup.py index 03b5eea9..cbff3274 100755 --- a/setup.py +++ b/setup.py @@ -165,7 +165,7 @@ def readme(): setup( name='questdb', - version='1.2.0', + version='2.0.0', platforms=['any'], python_requires='>=3.8', install_requires=[], diff --git a/src/questdb/__init__.py b/src/questdb/__init__.py index 58d478ab..afced147 100644 --- a/src/questdb/__init__.py +++ b/src/questdb/__init__.py @@ -1 +1 @@ -__version__ = '1.2.0' +__version__ = '2.0.0' diff --git a/src/questdb/ingress.pyx b/src/questdb/ingress.pyx index f6c0d650..01eff5a9 100644 --- a/src/questdb/ingress.pyx +++ b/src/questdb/ingress.pyx @@ -86,7 +86,7 @@ import os # This value is automatically updated by the `bump2version` tool. # If you need to update it, also update the search definition in # .bumpversion.cfg. -VERSION = '1.2.0' +VERSION = '2.0.0' cdef bint _has_gil(PyThreadState** gs): @@ -740,7 +740,7 @@ cdef class Buffer: from questdb.ingress import Sender, Buffer - sender = Sender(host='localhost', port=9009, + sender = Sender('http', 'localhost', 9009, init_buf_size=16384, max_name_len=64) buf = sender.new_buffer() assert buf.init_buf_size == 16384 @@ -2020,6 +2020,15 @@ cdef class Sender: object auto_flush_interval=None, # Default 1000 milliseconds object init_buf_size=None, # 64KiB object max_name_len=None): # 127 + """ + Construct a sender from a :ref:`configuration string `. + + The additional arguments are used to specify additional parameters + which are not present in the configuration string. + + Note that any parameters already present in the configuration string + cannot be overridden. + """ cdef line_sender_error* err = NULL cdef object protocol @@ -2134,6 +2143,18 @@ cdef class Sender: object auto_flush_interval=None, # Default 1000 milliseconds object init_buf_size=None, # 64KiB object max_name_len=None): # 127 + """ + Construct a sender from the ``QDB_CLIENT_CONF`` environment variable. + + The environment variable must be set to a valid + :ref:`configuration string `. + + The additional arguments are used to specify additional parameters + which are not present in the configuration string. + + Note that any parameters already present in the configuration string + cannot be overridden. + """ cdef str conf_str = os.environ.get('QDB_CLIENT_CONF') if conf_str is None: raise IngressError( @@ -2301,7 +2322,7 @@ cdef class Sender: pd.Timestamp('2022-08-09 13:56:02'), pd.Timestamp('2022-08-09 13:56:03')]}) - with qi.Sender('localhost', 9000) as sender: + with qi.Sender.from_env() as sender: sender.dataframe(df, table_name='race_metrics', at='ts') This method builds on top of the :func:`Buffer.dataframe` method.