QDB-9440 - Format.

Author: vianneyPL

quasardb/__init__.py

@@ -37,5 +37,5 @@
 
 try:
     from quasardb.quasardb import *
-except:
+except BaseException:
     from quasardb import *

quasardb/firehose.py

@@ -8,30 +8,36 @@
 
 logger = logging.getLogger('quasardb.firehose')
 
+
 def _init():
     """
     Initialize our internal state.
     """
     return {'last': None,
             'seen': set()}
 
+
 def _get_transactions_since(conn, table_name, last):
     """
     Retrieve all transactions since a certain timestamp. `last` is expected to be a dict
     firehose row with at least a $timestamp attached.
     """
     if last is None:
-        q = "SELECT $timestamp, transaction_id, begin, end FROM \"{}\" WHERE table = '{}' ORDER BY $timestamp".format(FIREHOSE_TABLE, table_name)
+        q = "SELECT $timestamp, transaction_id, begin, end FROM \"{}\" WHERE table = '{}' ORDER BY $timestamp".format(
+            FIREHOSE_TABLE, table_name)
     else:
-        q = "SELECT $timestamp, transaction_id, begin, end FROM \"{}\" IN RANGE ({}, +1y) WHERE table = '{}' ORDER BY $timestamp".format(FIREHOSE_TABLE, last['$timestamp'], table_name)
+        q = "SELECT $timestamp, transaction_id, begin, end FROM \"{}\" IN RANGE ({}, +1y) WHERE table = '{}' ORDER BY $timestamp".format(
+            FIREHOSE_TABLE, last['$timestamp'], table_name)
 
     return conn.query(q)
 
+
 def _get_transaction_data(conn, table_name, begin, end):
     """
     Gets all data from a certain table.
     """
-    q = "SELECT * FROM \"{}\" IN RANGE ({}, {}) ".format(table_name, begin, end)
+    q = "SELECT * FROM \"{}\" IN RANGE ({}, {}) ".format(
+        table_name, begin, end)
     return conn.query(q)
 
 
@@ -50,13 +56,13 @@ def _get_next(conn, table_name, state):
     for tx in txs:
         txid = tx['transaction_id']
 
-        if state['last'] != None and tx['$timestamp'] > state['last']['$timestamp']:
+        if state['last'] is not None and tx['$timestamp'] > state['last']['$timestamp']:
             # At this point we are guaranteed that the transaction we encounter is
             # 'new', will not conflict with any other transaction ids. It is thus
             # safe to reset the txid set.
             state['seen'] = set()
 
-        if not txid in state['seen']:
+        if txid not in state['seen']:
             xs = xs + _get_transaction_data(conn,
                                             table_name,
                                             tx['begin'],
@@ -66,10 +72,10 @@ def _get_next(conn, table_name, state):
                                             tx['end'] + np.timedelta64(1, 'ns'))
 
             # Because it is possible that multiple firehose changes are stored with the
-            # exact same $timestamp, we also keep track of the actually seen transaction ids.
+            # exact same $timestamp, we also keep track of the actually seen
+            # transaction ids.
             state['seen'].add(txid)
 
-
         state['last'] = tx
 
     return (state, xs)

quasardb/stats.py

@@ -6,8 +6,9 @@
 
 
 stats_prefix = '$qdb.statistics.'
-user_pattern = re.compile('\$qdb.statistics.(.*).uid_([0-9]+)$')
-total_pattern = re.compile('\$qdb.statistics.(.*)$(?<!uid_[0-9])')
+user_pattern = re.compile(r'\$qdb.statistics.(.*).uid_([0-9]+)$')
+total_pattern = re.compile(r'\$qdb.statistics.(.*)$(?<!uid_[0-9])')
+
 
 def by_node(conn):
     """
@@ -19,6 +20,7 @@ def by_node(conn):
     """
     return {x: of_node(conn, x) for x in conn.endpoints()}
 
+
 def of_node(conn, uri):
     """
     Returns statistic for a single node.
@@ -32,36 +34,45 @@ def of_node(conn, uri):
     """
     dconn = conn.node(uri)
 
-    raw = {k: _get_stat(dconn, k) for k in dconn.prefix_get(stats_prefix, 1000)}
+    raw = {
+        k: _get_stat(
+            dconn,
+            k) for k in dconn.prefix_get(
+            stats_prefix,
+            1000)}
     return {'by_uid': _by_uid(raw),
             'cumulative': _cumulative(raw)}
 
+
 def _get_stat(dconn, k):
-    # Ugly, but works: try to retrieve as integer, if not an int, retrieve as blob
+    # Ugly, but works: try to retrieve as integer, if not an int, retrieve as
+    # blob
     try:
         return dconn.integer(k).get()
     except quasardb.quasardb.Error:
         blob = dconn.blob(k).get()
         return blob.decode('utf-8', 'replace')
 
+
 def _by_uid(stats):
     xs = {}
-    for k,v in stats.items():
+    for k, v in stats.items():
         matches = user_pattern.match(k)
         if matches:
             (metric, uid_str) = matches.groups()
             uid = int(uid_str)
-            if not uid in xs:
+            if uid not in xs:
                 xs[uid] = {}
 
             xs[uid][metric] = v
 
     return xs
 
+
 def _cumulative(stats):
     xs = {}
 
-    for k,v in stats.items():
+    for k, v in stats.items():
         matches = total_pattern.match(k)
         if matches:
             metric = matches.groups()[0]

tests/conftest.py

@@ -11,9 +11,11 @@
 def connect(uri):
     return quasardb.Cluster(uri)
 
+
 def direct_connect(conn, node_uri):
     return conn.node(node_uri)
 
+
 def config():
     return {"uri":
             {"insecure": "qdb://127.0.0.1:2836",
@@ -43,6 +45,7 @@ def qdbd_settings(scope="module"):
 def qdbd_connection(qdbd_settings):
     return connect(qdbd_settings.get("uri").get("insecure"))
 
+
 @pytest.fixture
 def qdbd_secure_connection(qdbd_settings):
     return quasardb.Cluster(
@@ -54,13 +57,17 @@ def qdbd_secure_connection(qdbd_settings):
 
 @pytest.fixture
 def qdbd_direct_connection(qdbd_settings, qdbd_connection):
-    return direct_connect(qdbd_connection, qdbd_settings.get("uri").get("insecure").replace("qdb://", ""))
+    return direct_connect(qdbd_connection, qdbd_settings.get(
+        "uri").get("insecure").replace("qdb://", ""))
 
 
 @pytest.fixture
 def qdbd_direct_secure_connection(qdbd_settings, qdbd_secure_connection):
-    return direct_connect(qdbd_secure_connection,
-                          qdbd_settings.get("uri").get("secure").replace("qdb://", ""))
+    return direct_connect(
+        qdbd_secure_connection,
+        qdbd_settings.get("uri").get("secure").replace(
+            "qdb://",
+            ""))
 
 
 @pytest.fixture
@@ -101,6 +108,7 @@ def entry_name(random_string):
 def random_blob(random_string):
     return np.random.bytes(16)
 
+
 @pytest.fixture
 def random_integer():
     return random.randint(-1000000000, 1000000000)
@@ -120,14 +128,16 @@ def integer_entry(qdbd_connection, entry_name):
 def table_name(entry_name):
     return entry_name
 
+
 def _create_table(c, table_name):
     t = c.table(table_name)
     double_col = quasardb.ColumnInfo(quasardb.ColumnType.Double, "the_double")
     blob_col = quasardb.ColumnInfo(quasardb.ColumnType.Blob, "the_blob")
     string_col = quasardb.ColumnInfo(quasardb.ColumnType.String, "the_string")
     int64_col = quasardb.ColumnInfo(quasardb.ColumnType.Int64, "the_int64")
     ts_col = quasardb.ColumnInfo(quasardb.ColumnType.Timestamp, "the_ts")
-    symbol_col = quasardb.ColumnInfo(quasardb.ColumnType.Symbol, "the_symbol", "symtable")
+    symbol_col = quasardb.ColumnInfo(
+        quasardb.ColumnType.Symbol, "the_symbol", "symtable")
 
     t.create([double_col, blob_col, string_col, int64_col, ts_col, symbol_col])
     return t
@@ -142,6 +152,7 @@ def table(qdbd_connection, entry_name):
 def secure_table(qdbd_secure_connection, entry_name):
     return _create_table(qdbd_secure_connection, entry_name)
 
+
 @pytest.fixture
 def intervals():
     start_time = np.datetime64('2017-01-01', 'ns')
@@ -152,6 +163,8 @@ def create_many_intervals():
     start_time = np.datetime64('2017-01-01', 'ns')
     return np.array([(start_time + np.timedelta64(i, 's'))
                      for i in range(10000)]).astype('datetime64[ns]')
+
+
 @pytest.fixture
 def many_intervals():
     return create_many_intervals()

tests/test_batch_inserter.py

@@ -185,7 +185,10 @@ def test_successful_bulk_row_insert(qdbd_connection, table, many_intervals):
         _regular_push)
 
 
-def test_successful_secure_bulk_row_insert(qdbd_secure_connection, secure_table, many_intervals):
+def test_successful_secure_bulk_row_insert(
+        qdbd_secure_connection,
+        secure_table,
+        many_intervals):
     inserter = qdbd_secure_connection.inserter(
         _make_inserter_info(secure_table))
 
@@ -316,7 +319,8 @@ def test_push_truncate_explicit_range(qdbd_connection, table, many_intervals):
     np.testing.assert_array_equal(results[1], doubles[1:])
 
 
-def test_push_truncate_throws_error_on_invalid_range(qdbd_connection, table, many_intervals):
+def test_push_truncate_throws_error_on_invalid_range(
+        qdbd_connection, table, many_intervals):
     whole_range = (
         many_intervals[0], many_intervals[-1:][0] + np.timedelta64(2, 's'))
 

tests/test_buf_size.py

@@ -29,7 +29,8 @@ def test_client_query_buf_size_error(qdbd_connection, table, many_intervals):
     inserter = qdbd_connection.inserter(
         batchlib._make_inserter_info(table))
 
-    # doubles, blobs, strings, integers, timestamps = batchlib._test_with_table(
+    # doubles, blobs, strings, integers, timestamps =
+    # batchlib._test_with_table(
     batchlib._test_with_table(
         inserter,
         table,

tests/test_firehose.py

@@ -10,6 +10,7 @@
 import multiprocessing
 from sys import platform
 
+
 def _ensure_timeout(f, timeout=3):
     # HACKS(leon): apparently there is no reliable way to do this kind of stuff
     # on Windows or OSX, due to the way threading works.
@@ -59,16 +60,12 @@ def test_subscribe_single_table(qdbd_connection, table, many_intervals):
     xs = table.subscribe(qdbd_connection)
 
     doubles, blobs, strings, integers, timestamps, symbols = batchlib._test_with_table(
-        inserter,
-        table,
-        many_intervals,
-        batchlib._regular_push)
+        inserter, table, many_intervals, batchlib._regular_push)
 
     time.sleep(4)
 
     n = len(doubles)
 
-
     # After insertion, we expect exactly `n` items in the database, and not a
     # single element more.
     offset = 0
@@ -87,7 +84,7 @@ def next_row():
 
     # Ensure that acquiring the next row times out after 3 seconds, which ensures
     # that we have reached 'the end'.
-    assert _ensure_timeout(next_row, timeout=3) == True
+    assert _ensure_timeout(next_row, timeout=3)
 
     # Now, we insert additional data, which should wake up our subscription.
     # What we're going to do is just insert the exact same data, but this time
@@ -97,10 +94,7 @@ def next_row():
         many_intervals_.append(x + np.timedelta64(365, 'D'))
 
     doubles, blobs, strings, integers, timestamps, symbols = batchlib._test_with_table(
-        inserter,
-        table,
-        many_intervals_,
-        batchlib._regular_push)
+        inserter, table, many_intervals_, batchlib._regular_push)
 
     # Note that we only reset `offset`; since we just inserted exactly double
     # the data, `n` is still 10000 which is perfect!
@@ -115,7 +109,6 @@ def next_row():
         assert row['the_symbol'] == symbols[offset]
         offset = offset + 1
 
-
     # Ensure that acquiring the next row times out after 3 seconds, which ensures
     # that we have reached 'the end'.
-    assert _ensure_timeout(next_row, timeout=3) == True
+    assert _ensure_timeout(next_row, timeout=3)

tests/test_logging.py

@@ -71,4 +71,4 @@ def test_invalid_utf8_logs_qdb3361(qdbd_connection, caplog):
     for lr in caplog.records:
         seen = seen or '\x80' in lr.message
 
-    assert seen == True
+    assert seen

tests/test_node.py

@@ -24,11 +24,11 @@ def test_node_find_prefixes(qdbd_direct_connection):
 def test_node_blob_get(qdbd_direct_connection):
     entry = qdbd_direct_connection.blob("$qdb.statistics.node_id")
     got = entry.get()
-    assert type(got) is bytes and len(got) > 0
+    assert isinstance(got, bytes) and len(got) > 0
 
 
 def test_node_integer_get(qdbd_direct_connection):
     entry = qdbd_direct_connection.integer("$qdb.statistics.cpu.system")
     got = entry.get()
 
-    assert type(got) is int and got > 0
+    assert isinstance(got, int) and got > 0

tests/test_pandas.py

@@ -280,7 +280,12 @@ def _sparsify(xs):
                                          quasardb.ColumnType.Blob,
                                          quasardb.ColumnType.String,
                                          quasardb.ColumnType.Timestamp])
-def test_inference(qdbd_connection, table_name, input_gen, column_type, sparse):
+def test_inference(
+        qdbd_connection,
+        table_name,
+        input_gen,
+        column_type,
+        sparse):
 
     # Create table
     t = qdbd_connection.ts(table_name)