Merge branch 'master' into target_nodes

Author: barshaul

benchmarks/base.py

@@ -1,9 +1,10 @@
 import functools
 import itertools
-import redis
 import sys
 import timeit
 
+import redis
+
 
 class Benchmark:
     ARGUMENTS = ()
@@ -15,9 +16,7 @@ def get_client(self, **kwargs):
         # eventually make this more robust and take optional args from
         # argparse
         if self._client is None or kwargs:
-            defaults = {
-                'db': 9
-            }
+            defaults = {"db": 9}
             defaults.update(kwargs)
             pool = redis.ConnectionPool(**kwargs)
             self._client = redis.Redis(connection_pool=pool)
@@ -30,16 +29,16 @@ def run(self, **kwargs):
         pass
 
     def run_benchmark(self):
-        group_names = [group['name'] for group in self.ARGUMENTS]
-        group_values = [group['values'] for group in self.ARGUMENTS]
+        group_names = [group["name"] for group in self.ARGUMENTS]
+        group_values = [group["values"] for group in self.ARGUMENTS]
         for value_set in itertools.product(*group_values):
             pairs = list(zip(group_names, value_set))
-            arg_string = ', '.join(['%s=%s' % (p[0], p[1]) for p in pairs])
-            sys.stdout.write('Benchmark: %s... ' % arg_string)
+            arg_string = ", ".join(f"{p[0]}={p[1]}" for p in pairs)
+            sys.stdout.write(f"Benchmark: {arg_string}... ")
             sys.stdout.flush()
             kwargs = dict(pairs)
             setup = functools.partial(self.setup, **kwargs)
             run = functools.partial(self.run, **kwargs)
             t = timeit.timeit(stmt=run, setup=setup, number=1000)
-            sys.stdout.write('%f\n' % t)
+            sys.stdout.write(f"{t:f}\n")
             sys.stdout.flush()

benchmarks/basic_operations.py

@@ -1,24 +1,27 @@
-import redis
 import time
-from functools import wraps
 from argparse import ArgumentParser
+from functools import wraps
+
+import redis
 
 
 def parse_args():
     parser = ArgumentParser()
-    parser.add_argument('-n',
-                        type=int,
-                        help='Total number of requests (default 100000)',
-                        default=100000)
-    parser.add_argument('-P',
-                        type=int,
-                        help=('Pipeline <numreq> requests.'
-                              ' Default 1 (no pipeline).'),
-                        default=1)
-    parser.add_argument('-s',
-                        type=int,
-                        help='Data size of SET/GET value in bytes (default 2)',
-                        default=2)
+    parser.add_argument(
+        "-n", type=int, help="Total number of requests (default 100000)", default=100000
+    )
+    parser.add_argument(
+        "-P",
+        type=int,
+        help=("Pipeline <numreq> requests." " Default 1 (no pipeline)."),
+        default=1,
+    )
+    parser.add_argument(
+        "-s",
+        type=int,
+        help="Data size of SET/GET value in bytes (default 2)",
+        default=2,
+    )
 
     args = parser.parse_args()
     return args
@@ -45,15 +48,16 @@ def wrapper(*args, **kwargs):
         start = time.monotonic()
         ret = func(*args, **kwargs)
         duration = time.monotonic() - start
-        if 'num' in kwargs:
-            count = kwargs['num']
+        if "num" in kwargs:
+            count = kwargs["num"]
         else:
             count = args[1]
-        print('{} - {} Requests'.format(func.__name__, count))
-        print('Duration  = {}'.format(duration))
-        print('Rate = {}'.format(count/duration))
+        print(f"{func.__name__} - {count} Requests")
+        print(f"Duration  = {duration}")
+        print(f"Rate = {count/duration}")
         print()
         return ret
+
     return wrapper
 
 
@@ -62,10 +66,9 @@ def set_str(conn, num, pipeline_size, data_size):
     if pipeline_size > 1:
         conn = conn.pipeline()
 
-    format_str = '{:0<%d}' % data_size
-    set_data = format_str.format('a')
+    set_data = "a".ljust(data_size, "0")
     for i in range(num):
-        conn.set('set_str:%d' % i, set_data)
+        conn.set(f"set_str:{i}", set_data)
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -78,10 +81,9 @@ def set_int(conn, num, pipeline_size, data_size):
     if pipeline_size > 1:
         conn = conn.pipeline()
 
-    format_str = '{:0<%d}' % data_size
-    set_data = int(format_str.format('1'))
+    set_data = 10 ** (data_size - 1)
     for i in range(num):
-        conn.set('set_int:%d' % i, set_data)
+        conn.set(f"set_int:{i}", set_data)
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -95,7 +97,7 @@ def get_str(conn, num, pipeline_size, data_size):
         conn = conn.pipeline()
 
     for i in range(num):
-        conn.get('set_str:%d' % i)
+        conn.get(f"set_str:{i}")
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -109,7 +111,7 @@ def get_int(conn, num, pipeline_size, data_size):
         conn = conn.pipeline()
 
     for i in range(num):
-        conn.get('set_int:%d' % i)
+        conn.get(f"set_int:{i}")
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -123,7 +125,7 @@ def incr(conn, num, pipeline_size, *args, **kwargs):
         conn = conn.pipeline()
 
     for i in range(num):
-        conn.incr('incr_key')
+        conn.incr("incr_key")
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -136,10 +138,9 @@ def lpush(conn, num, pipeline_size, data_size):
     if pipeline_size > 1:
         conn = conn.pipeline()
 
-    format_str = '{:0<%d}' % data_size
-    set_data = int(format_str.format('1'))
+    set_data = 10 ** (data_size - 1)
     for i in range(num):
-        conn.lpush('lpush_key', set_data)
+        conn.lpush("lpush_key", set_data)
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -153,7 +154,7 @@ def lrange_300(conn, num, pipeline_size, data_size):
         conn = conn.pipeline()
 
     for i in range(num):
-        conn.lrange('lpush_key', i, i+300)
+        conn.lrange("lpush_key", i, i + 300)
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
@@ -166,7 +167,7 @@ def lpop(conn, num, pipeline_size, data_size):
     if pipeline_size > 1:
         conn = conn.pipeline()
     for i in range(num):
-        conn.lpop('lpush_key')
+        conn.lpop("lpush_key")
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
     if pipeline_size > 1:
@@ -178,17 +179,15 @@ def hmset(conn, num, pipeline_size, data_size):
     if pipeline_size > 1:
         conn = conn.pipeline()
 
-    set_data = {'str_value': 'string',
-                'int_value': 123456,
-                'float_value': 123456.0}
+    set_data = {"str_value": "string", "int_value": 123456, "float_value": 123456.0}
     for i in range(num):
-        conn.hmset('hmset_key', set_data)
+        conn.hmset("hmset_key", set_data)
         if pipeline_size > 1 and i % pipeline_size == 0:
             conn.execute()
 
     if pipeline_size > 1:
         conn.execute()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     run()

benchmarks/command_packer_benchmark.py

@@ -1,8 +1,7 @@
-import socket
-from redis.connection import (Connection, SYM_STAR, SYM_DOLLAR, SYM_EMPTY,
-                              SYM_CRLF)
 from base import Benchmark
 
+from redis.connection import SYM_CRLF, SYM_DOLLAR, SYM_EMPTY, SYM_STAR, Connection
+
 
 class StringJoiningConnection(Connection):
     def send_packed_command(self, command, check_health=True):
@@ -11,26 +10,30 @@ def send_packed_command(self, command, check_health=True):
             self.connect()
         try:
             self._sock.sendall(command)
-        except socket.error as e:
+        except OSError as e:
             self.disconnect()
             if len(e.args) == 1:
-                _errno, errmsg = 'UNKNOWN', e.args[0]
+                _errno, errmsg = "UNKNOWN", e.args[0]
             else:
                 _errno, errmsg = e.args
-            raise ConnectionError("Error %s while writing to socket. %s." %
-                                  (_errno, errmsg))
+            raise ConnectionError(f"Error {_errno} while writing to socket. {errmsg}.")
         except Exception:
             self.disconnect()
             raise
 
     def pack_command(self, *args):
         "Pack a series of arguments into a value Redis command"
-        args_output = SYM_EMPTY.join([
-            SYM_EMPTY.join(
-                (SYM_DOLLAR, str(len(k)).encode(), SYM_CRLF, k, SYM_CRLF))
-            for k in map(self.encoder.encode, args)])
+        args_output = SYM_EMPTY.join(
+            [
+                SYM_EMPTY.join(
+                    (SYM_DOLLAR, str(len(k)).encode(), SYM_CRLF, k, SYM_CRLF)
+                )
+                for k in map(self.encoder.encode, args)
+            ]
+        )
         output = SYM_EMPTY.join(
-            (SYM_STAR, str(len(args)).encode(), SYM_CRLF, args_output))
+            (SYM_STAR, str(len(args)).encode(), SYM_CRLF, args_output)
+        )
         return output
 
 
@@ -43,33 +46,33 @@ def send_packed_command(self, command, check_health=True):
                 command = [command]
             for item in command:
                 self._sock.sendall(item)
-        except socket.error as e:
+        except OSError as e:
             self.disconnect()
             if len(e.args) == 1:
-                _errno, errmsg = 'UNKNOWN', e.args[0]
+                _errno, errmsg = "UNKNOWN", e.args[0]
             else:
                 _errno, errmsg = e.args
-            raise ConnectionError("Error %s while writing to socket. %s." %
-                                  (_errno, errmsg))
+            raise ConnectionError(f"Error {_errno} while writing to socket. {errmsg}.")
         except Exception:
             self.disconnect()
             raise
 
     def pack_command(self, *args):
         output = []
-        buff = SYM_EMPTY.join(
-            (SYM_STAR, str(len(args)).encode(), SYM_CRLF))
+        buff = SYM_EMPTY.join((SYM_STAR, str(len(args)).encode(), SYM_CRLF))
 
         for k in map(self.encoder.encode, args):
             if len(buff) > 6000 or len(k) > 6000:
                 buff = SYM_EMPTY.join(
-                    (buff, SYM_DOLLAR, str(len(k)).encode(), SYM_CRLF))
+                    (buff, SYM_DOLLAR, str(len(k)).encode(), SYM_CRLF)
+                )
                 output.append(buff)
                 output.append(k)
                 buff = SYM_CRLF
             else:
-                buff = SYM_EMPTY.join((buff, SYM_DOLLAR, str(len(k)).encode(),
-                                       SYM_CRLF, k, SYM_CRLF))
+                buff = SYM_EMPTY.join(
+                    (buff, SYM_DOLLAR, str(len(k)).encode(), SYM_CRLF, k, SYM_CRLF)
+                )
         output.append(buff)
         return output
 
@@ -78,13 +81,12 @@ class CommandPackerBenchmark(Benchmark):
 
     ARGUMENTS = (
         {
-            'name': 'connection_class',
-            'values': [StringJoiningConnection, ListJoiningConnection]
+            "name": "connection_class",
+            "values": [StringJoiningConnection, ListJoiningConnection],
         },
         {
-            'name': 'value_size',
-            'values': [10, 100, 1000, 10000, 100000, 1000000, 10000000,
-                       100000000]
+            "name": "value_size",
+            "values": [10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000],
         },
     )
 
@@ -93,9 +95,9 @@ def setup(self, connection_class, value_size):
 
     def run(self, connection_class, value_size):
         r = self.get_client()
-        x = 'a' * value_size
-        r.set('benchmark', x)
+        x = "a" * value_size
+        r.set("benchmark", x)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     CommandPackerBenchmark().run_benchmark()

benchmarks/socket_read_size.py

@@ -1,34 +1,27 @@
-from redis.connection import PythonParser, HiredisParser
 from base import Benchmark
 
+from redis.connection import HiredisParser, PythonParser
+
 
 class SocketReadBenchmark(Benchmark):
 
     ARGUMENTS = (
+        {"name": "parser", "values": [PythonParser, HiredisParser]},
         {
-            'name': 'parser',
-            'values': [PythonParser, HiredisParser]
-        },
-        {
-            'name': 'value_size',
-            'values': [10, 100, 1000, 10000, 100000, 1000000, 10000000,
-                       100000000]
+            "name": "value_size",
+            "values": [10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000],
         },
-        {
-            'name': 'read_size',
-            'values': [4096, 8192, 16384, 32768, 65536, 131072]
-        }
+        {"name": "read_size", "values": [4096, 8192, 16384, 32768, 65536, 131072]},
     )
 
     def setup(self, value_size, read_size, parser):
-        r = self.get_client(parser_class=parser,
-                            socket_read_size=read_size)
-        r.set('benchmark', 'a' * value_size)
+        r = self.get_client(parser_class=parser, socket_read_size=read_size)
+        r.set("benchmark", "a" * value_size)
 
     def run(self, value_size, read_size, parser):
         r = self.get_client()
-        r.get('benchmark')
+        r.get("benchmark")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     SocketReadBenchmark().run_benchmark()

dev_requirements.txt

@@ -1,4 +1,7 @@
-flake8>=3.9.2
+black==21.11b1
+flake8==4.0.1
+flynt~=0.69.0
+isort==5.10.1
 pytest==6.2.5
 pytest-timeout==2.0.1
 tox==3.24.4