From 3aacfe433b502f00a57a324e27d041693f932924 Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 9 Feb 2022 08:36:36 +0000 Subject: [PATCH] Fixed multiple TLS tests fail on 2nd test due to tls cert/key filenames being replaced --- pyproject.toml | 2 +- .../__runner__/args.py | 27 ++++ .../__runner__/runner.py | 110 +++++++++++++- utils/tests/test_runner.py | 142 ++++++++++++++++++ 4 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 utils/tests/test_runner.py diff --git a/pyproject.toml b/pyproject.toml index 13d84a71..6aa4e4a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.19" +version = "0.1.20" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__runner__/args.py b/redis_benchmarks_specification/__runner__/args.py index a526397e..26bba291 100644 --- a/redis_benchmarks_specification/__runner__/args.py +++ b/redis_benchmarks_specification/__runner__/args.py @@ -76,4 +76,31 @@ def create_client_runner_args(project_name): action="store_true", help="At the end of every test send a FLUSHALL", ) + parser.add_argument( + "--tls", + default=False, + action="store_true", + help="Enable SSL/TLS transport security", + ) + parser.add_argument( + "--tls-skip-verify", + default=False, + action="store_true", + help="Skip verification of server certificate", + ) + parser.add_argument( + "--cert", + default="", + help="Use specified client certificate for TLS", + ) + parser.add_argument( + "--key", + default="", + help="Use specified private key for TLS", + ) + parser.add_argument( + "--cacert", + default="", + help="Use specified CA certs bundle for TLS", + ) return parser diff --git a/redis_benchmarks_specification/__runner__/runner.py b/redis_benchmarks_specification/__runner__/runner.py index 60cc5f94..9c2fc835 100644 --- a/redis_benchmarks_specification/__runner__/runner.py +++ b/redis_benchmarks_specification/__runner__/runner.py @@ -3,11 +3,11 @@ import logging import os import pathlib -import shutil import sys import tempfile import traceback from pathlib import Path +import shutil import docker import redis @@ -110,6 +110,11 @@ def main(): exit(1) running_platform = args.platform_name + tls_enabled = args.tls + tls_skip_verify = args.tls_skip_verify + tls_cert = args.cert + tls_key = args.key + tls_cacert = args.cacert docker_client = docker.from_env() home = str(Path.home()) logging.info("Running the benchmark specs.") @@ -124,6 +129,11 @@ def main(): testsuite_spec_files, {}, running_platform, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + tls_cacert, ) @@ -134,6 +144,11 @@ def prepare_memtier_benchmark_parameters( server, local_benchmark_output_filename, oss_cluster_api_enabled, + tls_enabled=False, + tls_skip_verify=False, + tls_cert=None, + tls_key=None, + tls_cacert=None, ): benchmark_command = [ full_benchmark_path, @@ -144,6 +159,17 @@ def prepare_memtier_benchmark_parameters( "--json-out-file", local_benchmark_output_filename, ] + if tls_enabled: + benchmark_command.append("--tls") + if tls_cert is not None and tls_cert != "": + benchmark_command.extend(["--cert", tls_cert]) + if tls_key is not None and tls_key != "": + benchmark_command.extend(["--key", tls_key]) + if tls_cacert is not None and tls_cacert != "": + benchmark_command.extend(["--cacert", tls_cacert]) + if tls_skip_verify: + benchmark_command.append("--tls-skip-verify") + if oss_cluster_api_enabled is True: benchmark_command.append("--cluster-mode") benchmark_command_str = " ".join(benchmark_command) @@ -163,6 +189,11 @@ def process_self_contained_coordinator_stream( testsuite_spec_files, topologies_map, running_platform, + tls_enabled=False, + tls_skip_verify=False, + tls_cert=None, + tls_key=None, + tls_cacert=None, ): overall_result = True total_test_suite_runs = 0 @@ -170,7 +201,17 @@ def process_self_contained_coordinator_stream( client_containers = [] with open(test_file, "r") as stream: - benchmark_config, test_name = get_final_benchmark_config(None, stream, "") + _, benchmark_config, test_name = get_final_benchmark_config( + None, stream, "" + ) + + if tls_enabled: + test_name = test_name + "-tls" + logging.info( + "Given that TLS is enabled, appending -tls to the testname: {}.".format( + test_name + ) + ) for topology_spec_name in benchmark_config["redis-topologies"]: test_result = False @@ -189,8 +230,22 @@ def process_self_contained_coordinator_stream( port = args.db_server_port host = args.db_server_host - r = redis.StrictRedis(host=host, port=port) + + ssl_cert_reqs = "required" + if tls_skip_verify: + ssl_cert_reqs = None + r = redis.StrictRedis( + host=host, + port=port, + ssl=tls_enabled, + ssl_cert_reqs=ssl_cert_reqs, + ssl_keyfile=tls_key, + ssl_certfile=tls_cert, + ssl_ca_certs=tls_cacert, + ssl_check_hostname=False, + ) r.ping() + ceil_client_cpu_limit = extract_client_cpu_limit(benchmark_config) client_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( ceil_client_cpu_limit, current_cpu_pos @@ -202,6 +257,21 @@ def process_self_contained_coordinator_stream( benchmark_tool_workdir = client_mnt_point metadata = {} + if tls_enabled: + metadata["tls"] = "true" + if tls_cert is not None and tls_cert != "": + _, test_tls_cert = cp_to_workdir( + temporary_dir_client, tls_cert + ) + if tls_cacert is not None and tls_cacert != "": + _, test_tls_cacert = cp_to_workdir( + temporary_dir_client, tls_cacert + ) + if tls_key is not None and tls_key != "": + _, test_tls_key = cp_to_workdir( + temporary_dir_client, tls_key + ) + if "preload_tool" in benchmark_config["dbconfig"]: data_prepopulation_step( benchmark_config, @@ -213,6 +283,11 @@ def process_self_contained_coordinator_stream( temporary_dir_client, test_name, host, + tls_enabled, + tls_skip_verify, + test_tls_cert, + test_tls_key, + test_tls_cacert, ) benchmark_tool = extract_client_tool(benchmark_config) @@ -263,7 +338,12 @@ def process_self_contained_coordinator_stream( port, host, local_benchmark_output_filename, - benchmark_tool_workdir, + False, + tls_enabled, + tls_skip_verify, + test_tls_cert, + test_tls_key, + test_tls_cacert, ) client_container_image = extract_client_container_image( @@ -389,6 +469,18 @@ def process_self_contained_coordinator_stream( overall_result &= test_result +def cp_to_workdir(benchmark_tool_workdir, srcfile): + head, filename = os.path.split(srcfile) + dstfile = "{}/{}".format(benchmark_tool_workdir, filename) + shutil.copyfile(srcfile, dstfile) + logging.info( + "Copying to workdir the following file {}. Final workdir file {}".format( + srcfile, dstfile + ) + ) + return dstfile, filename + + def data_prepopulation_step( benchmark_config, benchmark_tool_workdir, @@ -399,6 +491,11 @@ def data_prepopulation_step( temporary_dir, test_name, host, + tls_enabled=False, + tls_skip_verify=False, + tls_cert=None, + tls_key=None, + tls_cacert=None, ): # setup the benchmark ( @@ -426,6 +523,11 @@ def data_prepopulation_step( host, local_benchmark_output_filename, False, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + tls_cacert, ) logging.info( diff --git a/utils/tests/test_runner.py b/utils/tests/test_runner.py new file mode 100644 index 00000000..f6425f42 --- /dev/null +++ b/utils/tests/test_runner.py @@ -0,0 +1,142 @@ +import yaml + +from redis_benchmarks_specification.__common__.spec import extract_client_tool +from redis_benchmarks_specification.__runner__.runner import ( + prepare_memtier_benchmark_parameters, +) + + +def test_prepare_memtier_benchmark_parameters(): + with open( + "./redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml", + "r", + ) as yml_file: + benchmark_config = yaml.safe_load(yml_file) + client_tool = extract_client_tool(benchmark_config) + assert client_tool == "memtier_benchmark" + local_benchmark_output_filename = "1.json" + oss_api_enabled = False + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + ) + oss_api_enabled = True + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --cluster-mode "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + ) + + oss_api_enabled = False + tls_enabled = False + tls_skip_verify = True + tls_cert = None + tls_key = None + + # ensure that when tls is disabled we dont change the args + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + ) + + tls_enabled = True + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --tls-skip-verify "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + ) + + tls_skip_verify = False + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + ) + + tls_skip_verify = False + tls_cert = "cert.file" + tls_key = "key.file" + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + ) + + tls_cacert = "cacert.file" + (_, benchmark_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + client_tool, + 12000, + "localhost", + local_benchmark_output_filename, + oss_api_enabled, + tls_enabled, + tls_skip_verify, + tls_cert, + tls_key, + tls_cacert, + ) + assert ( + benchmark_command_str + == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file --cacert cacert.file "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 300' + )