Extended HASH, LIST, and STRING benchmarks (#276)

* Added SET with EX test

* Added HELLO pipeline benchmarks

* Added LPUSH pipeline benchmarks

* Added LINDEX pipeline benchmarks

* Added ZREVRANGEBYSCORE pipeline benchmarks

* Added ZREVRANGE benchmark

* fixed memtier_benchmark-1Mkeys-string-get-10B-pipeline-10

* Included 2 listpack benchmarks (LRANGE command) focused on integer encoding

* Fixed missing pipeline 10 on the 2 new listpack tests

* Moved latency benchmark suite out of common folder

* Fixed memtier_benchmark-1key-hash-1K-fields-hgetall init step

* Extended io-threads use-case to 100% gets and mixed reads/writes

* applied fmt

Author: fcostaoliveira

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.247"
+version = "0.1.256"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__compare__/compare.py

@@ -804,21 +804,22 @@ def compute_regression_table(
     len_baseline_only_list = len(baseline_only_list)
     if len_baseline_only_list > 0:
         table_output += f"\n  WARNING: There were {len_baseline_only_list} benchmarks with datapoints only on baseline.\n\n"
-        baseline_only_test_names_str = "|".join([l[0] for l in baseline_only_list])
+        baseline_only_test_names_str = "|".join([l for l in baseline_only_list])
         table_output += (
             f"  Baseline only test regexp names: {baseline_only_test_names_str}\n\n"
         )
     len_comparison_only_list = len(comparison_only_list)
     if len_comparison_only_list > 0:
         table_output += f"\n  WARNING: There were {len_comparison_only_list} benchmarks with datapoints only on comparison.\n\n"
-        comparison_only_test_names_str = "|".join([l[0] for l in comparison_only_list])
+        comparison_only_test_names_str = "|".join([l for l in comparison_only_list])
         table_output += (
             f"  Comparison only test regexp names: {comparison_only_test_names_str}\n\n"
         )
     len_no_datapoints = len(no_datapoints_list)
     if len_no_datapoints > 0:
         table_output += f"\n  WARNING: There were {len_no_datapoints} benchmarks with NO datapoints for both baseline and comparison.\n\n"
-        no_datapoints_test_names_str = "|".join([l[0] for l in no_datapoints_list])
+        no_datapoints_test_names_str = "|".join([l for l in no_datapoints_list])
+
         table_output += (
             f"  NO DATAPOINTS test regexp names: {no_datapoints_test_names_str}\n\n"
         )

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-100Kkeys-load-hash-20-fields-with-1B-values-pipeline-30
+description: Runs memtier_benchmark, for a keyspace length of 100K keys loading HASHES with 50 fields each. Each field value has a data size of 10 Bytes.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 6g
+tested-groups:
+- hash
+tested-commands:
+- hset
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --distinct-client-seed "--data-size" "10" --command "HSET __key__ field:1 __data__ field:2 __data__ field:3 __data__ field:4 __data__ field:5 __data__ field:6 __data__ field:7 __data__ field:8 __data__ field:9 __data__ field:10 __data__ field:11 __data__ field:12 __data__ field:13 __data__ field:14 __data__ field:15 __data__ field:16 __data__ field:17 __data__ field:18 __data__ field:19 __data__ field:20 __data__ field:21 __data__ field:22 __data__ field:23 __data__ field:24 __data__ field:25 __data__ field:26 __data__ field:27 __data__ field:28 __data__ field:29 __data__ field:30 __data__ field:31 __data__ field:32 __data__ field:33 __data__ field:34 __data__ field:35 __data__ field:36 __data__ field:37 __data__ field:38 __data__ field:39 __data__ field:40 __data__ field:41 __data__ field:42 __data__ field:43 __data__ field:44 __data__ field:45 __data__ field:46 __data__ field:47 __data__ field:48 __data__ field:49 __data__ field:50 __data__" --command-key-pattern="R" --key-minimum=1 --key-maximum 100000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 5

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-1000B-values.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-10Kkeys-load-list-with-10B-values-pipeline-50
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading LISTs in which the value has a data size of 10 Bytes.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- list
+tested-commands:
+- lpush
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "10" --pipeline 50 --command "LPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 10000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 38

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-100B-values.yml

@@ -0,0 +1,30 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-load-list-with-10B-values-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys loading LISTs in which the value has a data size of 10 Bytes.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 0
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- list
+tested-commands:
+- lpush
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "10" --pipeline 10 --command "LPUSH __key__ __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 38

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-100Kkeys-load-hash-50-fields-with-10B-values.yml

@@ -9,7 +9,7 @@ dbconfig:
   preload_tool:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
-    arguments: '"--data-size" "10" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+    arguments: '"--key-maximum" "1000000" "-n" "allkeys" "--data-size" "10" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
   resources:
     requests:
       memory: 1g
@@ -23,7 +23,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: '"--pipeline" "10" "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180'
+  arguments: '"--key-minimum" "1" "--key-maximum" "1000000" "--pipeline" "10" "--data-size" "10" --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180'
   resources:
     requests:
       cpus: '4'

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10.yml

@@ -0,0 +1,33 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-string-set-with-ex-100B-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys with a data size of 100 Bytes for each key.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- set
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --test-time 120 --pipeline 10 --command "SET __key__ __value__ EX 3600" --command-key-pattern="R" --key-minimum=1 --key-maximum 1000000 -c 50 -t 4 --hide-histogram
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+tested-groups:
+- string
+priority: 1

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-list-100-elements-int-7bit-uint-lrange-all-elements-pipeline-10
+description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 100 integer elements (all of one encoding type: 7BIT_UINT) in it and we query it using LRANGE with a range of all elements. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"LPUSH" "list:100:ints:7BIT_UINT" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "20" "21" "22" "23" "24" "25" "26" "27" "28" "29" "30" "31" "32" "33" "34" "35" "36" "37" "38" "39" "40" "41" "42" "43" "44" "45" "46" "47" "48" "49" "50" "51" "52" "53" "54" "55" "56" "57" "58" "59" "60" "61" "62" "63" "64" "65" "66" "67" "68" "69" "70" "71" "72" "73" "74" "75" "76" "77" "78" "79" "80" "81" "82" "83" "84" "85" "86" "87" "88" "89" "90" "91" "92" "93" "94" "95" "96" "97" "98" "99"'
+tested-groups:
+- list
+tested-commands:
+- lrange
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --pipeline 10 --command="LRANGE list:100:ints:7BIT_UINT 0 -1"  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 34

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values-pipeline-10.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values-pipeline-10.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-list-100-elements-int-lrange-all-elements-pipeline-10
+description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 100 integer elements (across all 6 encoding types, meaning 7BIT_UINT 13BIT_INT 16BIT_INT 24BIT_INT 32BIT_INT 64BIT_INT) in it and we query it using LRANGE with a range of all elements. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"LPUSH" "list:100:ints" "0" "7" "14" "21" "28" "35" "42" "49" "56" "63" "70" "77" "84" "91" "98" "105" "112" "-4096" "-3585" "-3074" "-2563" "-2052" "-1541" "-1030" "-519" "-8" "503" "1014" "1525" "2036" "2547" "3058" "3569" "4080" "-32768" "-28673" "-24578" "-20483" "-16388" "-12293" "-8198" "-4103" "-8" "4087" "8182" "12277" "16372" "20467" "24562" "28657" "32752" "-8388608" "-7340033" "-6291458" "-5242883" "-4194308" "-3145733" "-2097158" "-1048583" "-8" "1048567" "2097142" "3145717" "4194292" "5242867" "6291442" "7340017" "8388592" "-2147483648" "-1861152495" "-1574821342" "-1288490189" "-1002159036" "-715827883" "-429496730" "-143165577" "143165576" "429496729" "715827882" "1002159035" "1288490188" "1574821341" "1861152494" "2147483647" "-9223372036854775808" "-7993589098607472367" "-6763806160360168926" "-5534023222112865485" "-4304240283865562044" "-3074457345618258603" "-1844674407370955162" "-614891469123651721" "614891469123651720" "1844674407370955161" "3074457345618258602" "4304240283865562043" "5534023222112865484" "6763806160360168925" "7993589098607472366" "9223372036854775807"'
+tested-groups:
+- list
+tested-commands:
+- lrange
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --pipeline 10 --command="LRANGE list:100:ints 0 -1"  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 34

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-list-10K-elements-lindex-string-pipeline-10
+description: 'Runs memtier_benchmark, for a keyspace length of 1 LIST key. The LIST contains 10000 string elements in it and we query it using LPOS of a element that we know it is in the middle of the list. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --hide-histogram --command "RPUSH strlist __key__" --key-maximum 10000 --key-minimum 1 -n allkeys --key-prefix "hello" --command-key-pattern S -c 1 -t 1 --pipeline 10
+  resources:
+    requests:
+      cpus: '2'
+      memory: 1g
+tested-groups:
+- list
+tested-commands:
+- lindex
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="LINDEX strlist 5000"  --hide-histogram --test-time 120 --pipeline 10
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+priority: 34

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-100B-expire-use-case.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-100B-expire-use-case.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zrank-100K-elements-pipeline-1
+description: Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key. The SORTED SET contains 100K elements and we query it using ZRANK.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --pipeline 50 --command="ZADD key1 __key__  __key__" --command-key-pattern=P --key-maximum 100000 --key-prefix "" -n allkeys --hide-histogram -t 1 -c 1
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- sorted-set
+tested-commands:
+- zrank
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="ZRANK key1 __key__" --key-maximum 100000 --command-key-pattern=R --key-prefix "" --hide-histogram --test-time 180 --pipeline 1 -c 1 -t 1
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 72

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-10B-expire-use-case.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-10B-expire-use-case.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zrank-10M-elements-pipeline-1
+description: Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key. The SORTED SET contains 10M elements and we query it using ZRANK.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --pipeline 50 --command="ZADD key1 __key__  __key__" --command-key-pattern=P --key-maximum 10000000 --key-prefix "" -n 1000000 --hide-histogram -t 10 -c 1
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- sorted-set
+tested-commands:
+- zrank
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="ZRANK key1 __key__" --key-maximum 10000000 --command-key-pattern=R --key-prefix "" --hide-histogram --test-time 180 --pipeline 1 -c 1 -t 1
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 72

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-1KiB-expire-use-case.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-1KiB-expire-use-case.yml

@@ -9,7 +9,7 @@ dbconfig:
   preload_tool:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
-    arguments: --command="ZADD key1 __key__  __key__" --command-key-pattern=P --key-maximum 1000000 --key-prefix "" -n 1000000 --hide-histogram -t 1 -c 1
+    arguments: --pipeline 50 --command="ZADD key1 __key__  __key__" --command-key-pattern=P --key-maximum 1000000 --key-prefix "" -n 100000 --hide-histogram -t 10 -c 1
   resources:
     requests:
       memory: 1g

redis_benchmarks_specification/test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-4KiB-expire-use-case.yml renamed to redis_benchmarks_specification/latency-test-suites/latency-rate-limited-10000_qps-memtier_benchmark-1Mkeys-4KiB-expire-use-case.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1key-zrevrangebyscore-256K-elements-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key. The SORTED SET contains 256K elements of 1KB each and we query it using ZREVRANGEBYSCORE.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: --command="ZADD key1 __key__  __data__" --command-key-pattern=P --key-maximum 262144 --key-prefix "" -R --data-size=1024 -n 262144 --hide-histogram -t 1 -c 1
+  resources:
+    requests:
+      memory: 1g
+tested-groups:
+- sorted-set
+tested-commands:
+- zrevrangebyscore
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="ZREVRANGEBYSCORE key1 262144 1 WITHSCORES LIMIT 1 10" --hide-histogram --test-time 180 --pipeline 10
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 16