Included XREAD, XADD, and ZRANGE benchmarks (#88)

* Removed test_commands.py given it's not ready yet

* Included XREAD, XADD, and ZRANGE benchmarks

* Fixes per PR review

Author: filipecosta90

redis_benchmarks_specification/__runner__/args.py

@@ -24,8 +24,8 @@ def create_client_runner_args(project_name):
     )
     parser.add_argument("--triggering_env", type=str, default="ci")
     parser.add_argument("--setup_type", type=str, default="oss-standalone")
-    parser.add_argument("--github_repo", type=str, required=True)
-    parser.add_argument("--github_org", type=str, required=True)
+    parser.add_argument("--github_repo", type=str, default="redis")
+    parser.add_argument("--github_org", type=str, default="redis")
     parser.add_argument("--github_version", type=str, default="NA")
     parser.add_argument(
         "--logname", type=str, default=None, help="logname to write the logs to"

redis_benchmarks_specification/__runner__/runner.py

@@ -15,6 +15,7 @@
 from redisbench_admin.run.common import (
     get_start_time_vars,
     prepare_benchmark_parameters,
+    execute_init_commands,
 )
 from redisbench_admin.run.redistimeseries import timeseries_test_sucess_flow
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
@@ -293,6 +294,10 @@ def process_self_contained_coordinator_stream(
                             test_tls_cacert,
                         )
 
+                    execute_init_commands(
+                        benchmark_config, r, dbconfig_keyname="dbconfig"
+                    )
+
                     benchmark_tool = extract_client_tool(benchmark_config)
                     # backwards compatible
                     if benchmark_tool is None:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -411,6 +411,9 @@ def process_self_contained_coordinator_stream(
                                     run_image, db_cpuset_cpus, command_str
                                 )
                             )
+                            # profiler_enabled = False
+                            # if "profile" in metadata:
+                            #     profiler_enabled = bool(metadata["profile"])
                             container = docker_client.containers.run(
                                 image=run_image,
                                 volumes={

redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-profilers.yml

@@ -0,0 +1,19 @@
+version: 0.1
+id: gcc:8.5.0-amd64-debian-buster-default
+os: debian-buster
+arch: amd64
+compiler: "gcc"
+cpp_compiler: "g++"
+kind: docker
+build_image: gcc:8.5.0-buster
+description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description)
+              pre-configured environment with all the tools required to build with gcc."
+metadata:
+  compiler: "gcc"
+  compiler_version: "8.5.0"
+  os: debian-buster
+  arch: amd64
+  profile: "true"
+
+env:
+  REDIS_CFLAGS: "-g -fno-omit-frame-pointer"

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values-pipeline-10.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values-pipeline-10"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              loading STREAMs in which the value has a data size of 100 Bytes."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+tested-commands:
+  - XADD
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--pipeline" "10" "--data-size" "100" --command "XADD __key__ * field __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-load-stream-1-fields-with-100B-values"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              loading STREAMs in which the value has a data size of 100 Bytes."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+tested-commands:
+  - XADD
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "100" --command "XADD __key__ * field __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values-pipeline-10.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values-pipeline-10"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              loading STREAMs in which the 5 field values have a data size of 100 Bytes."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+tested-commands:
+  - XADD
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--pipeline" "10" "--data-size" "100" --command "XADD __key__ * field __data__ field __data__ field __data__ field __data__ field __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: "memtier_benchmark-1Mkeys-load-stream-5-fields-with-100B-values"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              loading STREAMs in which the 5 field values have a data size of 100 Bytes."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+tested-commands:
+  - XADD
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--data-size" "100" --command "XADD __key__ * field __data__ field __data__ field __data__ field __data__ field __data__" --command-key-pattern="P" --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-10-elements-zrange-all-elements.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: "memtier_benchmark-1key-zset-10-elements-zrange-all-elements"
+description: "Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key.
+              The SORTED SET contains 10 elements in it and we query it using ZRANGE BYSCORE with a range of all elements.
+              "
+
+dbconfig:
+  - configuration-parameters:
+    save: '""'
+  - init_commands:
+    - '"ZADD" "zset:10" "0.306275" "lysbgqqfqw" "0.486004" "mtccjerdon" "0.941626" "jekkafodvk" "0.602656" "nmgxcctxpn" "0.630771" "vyqqkuszzh" "0.406379" "pytrnqdhvs" "0.521814" "oguwnmniig" "0.182854" "gekntrykfh" "0.657658" "nhfnbxqgol" "0.218066" "cgoeihlnei"'
+tested-commands:
+  - ZRANGE
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--command="ZRANGE zset:10 0 1 BYSCORE WITHSCORES"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-100-elements-zrange-all-elements.yml

@@ -0,0 +1,36 @@
+version: 0.4
+name: "memtier_benchmark-1key-zset-100-elements-zrange-all-elements"
+description: "Runs memtier_benchmark, for a keyspace length of 1 SORTED SET key.
+              The SORTED SET contains 100 elements in it and we query it using ZRANGE BYSCORE with a range of all elements.
+              "
+
+dbconfig:
+  - configuration-parameters:
+    save: '""'
+  - init_commands:
+    - '"ZADD" "zset:100" "0.306275" "lysbgqqfqw" "0.486004" "mtccjerdon" "0.941626" "jekkafodvk" "0.602656" "nmgxcctxpn" "0.630771" "vyqqkuszzh" "0.406379" "pytrnqdhvs" "0.521814" "oguwnmniig" "0.182854" "gekntrykfh" "0.657658" "nhfnbxqgol" "0.218066" "cgoeihlnei" "0.366328" "kgqrovsxce" "0.520723" "imyizmhzjk" "0.426630" "urvgxwbiiz" "0.878001" "onzjrteqmu" "0.126097" "alqlzsvuuz" "0.562698" "lgrkndhekf" "0.357487" "tcjvjofxtd" "0.395563" "ouuybhtvyb" "0.745796" "owbbdezgxn" "0.165356" "ksqpdywgdd" "0.757214" "qrosceblyo" "0.346153" "agsdggdghc" "0.297342" "gfdqdrondm" "0.668749" "oapbvnisyq" "0.281552" "htvbwmfyic" "0.013874" "vyoomgwuzv" "0.052023" "pcxdhdjwnf" "0.648518" "ibhohmfxzt" "0.967245" "erxulvebrj" "0.971791" "qwxrsgafzy" "0.154577" "whmotknaff" "0.404082" "ueyrvldzwl" "0.324634" "nsxaigrnje" "0.342459" "xsepfhdizi" "0.988925" "exvhmqxvvp" "0.345154" "owtkxlzaci" "0.728413" "ypfeltycpy" "0.406783" "qbiwbqiubb" "0.714527" "wqiwdbncmt" "0.760975" "dapacykoah" "0.212614" "hhjclfbbka" "0.445330" "ujrxcmpvsq" "0.576866" "lbtpbknelm" "0.764225" "wvcnqbvlnf" "0.281427" "mibvtmqxcy" "0.596978" "aovfsvbwjg" "0.650632" "qbyhorvill" "0.948063" "ypekeuutef" "0.028779" "xamjodnbpf" "0.678338" "hhniikmtsx" "0.757806" "sahqeskveq" "0.671949" "bqntlsaqjb" "0.190314" "luemuetmia" "0.219219" "zwnitejtpg" "0.212301" "ncjfckgkcl" "0.730613" "qjyeqcfhjj" "0.657992" "wkbasfyzqz" "0.556851" "ccagtnjilc" "0.543822" "pqyurbvifk" "0.101854" "djetcyfxuc" "0.878471" "vpmpffdoqz" "0.696931" "hlpdstpvzw" "0.662898" "imdqxmkzdj" "0.887214" "eouliovvra" "0.193298" "gxephxbdru" "0.978115" "dlsjwmqzhx" "0.743150" "uapsgmizgh" "0.181609" "gorgpnnqwr" "0.638726" "rojeolnopp" "0.561627" "mbxohpancs" "0.223001" "upodnpqenn" "0.957952" "dlbqcynhrn" "0.586659" "ichqzmiyot" "0.813623" "ulbrotlhze" "0.111318" "licotqplim" "0.525834" "mxwgdcutnb" "0.405858" "vpbkvwgxsf" "0.129768" "ijsmoyesvd" "0.591274" "mbgehjiauu" "0.718370" "vymjzlzqcn" "0.189343" "rjkknoigmu" "0.566307" "jxoxtnzujs" "0.563082" "eqlgkwosie" "0.040883" "ljcgdooafo" "0.326765" "cwaveajmcz" "0.423381" "ssjqrsluod" "0.865798" "fhuvzpxjbc" "0.784445" "perfwnpvkl" "0.659790" "mjjuylgssm" "0.918901" "vxcbagyymm" "0.282908" "nxcdcaqgit" "0.030110" "ewomnmugfa" "0.164237" "rkaznetutk" "0.281729" "rqvryfvlie" "0.877790" "holjcdpijr" "0.878293" "pquewclxuy" "0.674968" "ocrcszcznp" "0.520069" "xczfmrxrja" "0.314841" "nxzsnkmxvi" "0.719988" "hhjchwjlmc"'
+tested-commands:
+  - ZRANGE
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--command="ZRANGE zset:100 0 1 BYSCORE WITHSCORES"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-zset-1K-elements-zrange-all-elements.yml

@@ -0,0 +1,45 @@
+version: 0.4
+name: "memtier_benchmark-2keys-stream-5-entries-xread-all-entries-pipeline-10"
+description: "Runs memtier_benchmark, for a keyspace length of 2 STREAM keys with 5 entries on each stream.
+              We query both streams on the same command using XREAD with range of all entries.
+              "
+
+dbconfig:
+  - configuration-parameters:
+    save: '""'
+  - init_commands:
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+tested-commands:
+  - XREAD
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--pipeline 10 --command="XREAD STREAMS key-1 key-2 0 0"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'

redis_benchmarks_specification/test-suites/memtier_benchmark-2keys-stream-5-entries-xread-all-entries-pipeline-10.yml

@@ -0,0 +1,45 @@
+version: 0.4
+name: "memtier_benchmark-2keys-stream-5-entries-xread-all-entries"
+description: "Runs memtier_benchmark, for a keyspace length of 2 STREAM keys with 5 entries on each stream.
+              We query both streams on the same command using XREAD with range of all entries.
+              "
+
+dbconfig:
+  - configuration-parameters:
+    save: '""'
+  - init_commands:
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-1 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+      - 'xadd key-2 * field value'
+tested-commands:
+  - XREAD
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '--command="XREAD STREAMS key-1 key-2 0 0"  --hide-histogram --test-time 180'
+  resources:
+    requests:
+      cpus: "4"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".*."Ops/sec"'
+      - '$."ALL STATS".*."Latency"'