diff --git a/pyproject.toml b/pyproject.toml index 0c0cddc..06755f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.41" +version = "0.1.50" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" @@ -19,7 +19,6 @@ docker = "^5.0.0" redisbench-admin = "^0.9.3" #redisbench-admin = {path = "../redisbench-admin", develop = true} psutil = "^5.8.0" -tox-docker = "^3.1.0" PyGithub = "^1.55" GitPython = "^3.1.20" semver = "^2.13.0" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index b77d046..6c3da0f 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -381,35 +381,48 @@ def build_spec_image_prefetch(builders_folder, different_build_specs): build_config, id = get_build_config(builders_folder + "/" + build_spec) if build_config["kind"] == "docker": build_image = build_config["build_image"] - if build_image not in already_checked_images: - logging.info( - "Build {} requirement: checking build image {} is available.".format( - id, build_image - ) - ) - local_images = [ - x.tags[0] - for x in client.images.list(filters={"reference": build_image}) - ] - if build_image not in local_images: - logging.info( - "Build {} requirement: build image {} is not available locally. Fetching it from hub".format( - id, build_image - ) - ) - client.images.pull(build_image) - hub_pulled_images = hub_pulled_images + 1 - else: - logging.info( - "Build {} requirement: build image {} is available locally.".format( - id, build_image - ) - ) - already_checked_images.append(build_image) - else: - logging.info( - "Build {} requirement: build image {} availability was already checked.".format( - id, build_image - ) + hub_pulled_images = check_docker_image_available( + already_checked_images, build_image, client, hub_pulled_images, id + ) + if "run_image" in build_config: + run_image = build_config["run_image"] + hub_pulled_images = check_docker_image_available( + already_checked_images, run_image, client, hub_pulled_images, id ) return already_checked_images, hub_pulled_images + + +def check_docker_image_available( + already_checked_images, build_image, client, hub_pulled_images, id +): + if build_image not in already_checked_images: + logging.info( + "Build {} requirement: checking docker image {} is available.".format( + id, build_image + ) + ) + local_images = [ + x.tags[0] for x in client.images.list(filters={"reference": build_image}) + ] + if build_image not in local_images: + logging.info( + "Build {} requirement: docker image {} is not available locally. Fetching it from hub".format( + id, build_image + ) + ) + client.images.pull(build_image) + hub_pulled_images = hub_pulled_images + 1 + else: + logging.info( + "Build {} requirement: docker image {} is available locally.".format( + id, build_image + ) + ) + already_checked_images.append(build_image) + else: + logging.info( + "Build {} requirement: docker image {} availability was already checked.".format( + id, build_image + ) + ) + return hub_pulled_images diff --git a/redis_benchmarks_specification/__cli__/args.py b/redis_benchmarks_specification/__cli__/args.py index b404e71..1635442 100644 --- a/redis_benchmarks_specification/__cli__/args.py +++ b/redis_benchmarks_specification/__cli__/args.py @@ -63,4 +63,10 @@ def spec_cli_args(parser): action="store_true", help="Only check how many benchmarks we would trigger. Don't request benchmark runs at the end.", ) + parser.add_argument( + "--last_n", + type=int, + default=-1, + help="Use the last N samples. by default will use all available values", + ) return parser diff --git a/redis_benchmarks_specification/__cli__/cli.py b/redis_benchmarks_specification/__cli__/cli.py index 470445c..846a719 100644 --- a/redis_benchmarks_specification/__cli__/cli.py +++ b/redis_benchmarks_specification/__cli__/cli.py @@ -89,6 +89,7 @@ def cli_command_logic(args, project_name, project_version): ) repo = git.Repo(redisDirPath) commits = [] + total_commits = 0 if args.use_branch: for commit in repo.iter_commits(): if ( @@ -98,8 +99,17 @@ def cli_command_logic(args, project_name, project_version): ) <= args.to_date ): - print(commit.summary) - commits.append({"git_hash": commit.hexsha, "git_branch": args.branch}) + if ( + args.last_n > 0 and total_commits < args.last_n + ) or args.last_n == -1: + total_commits = total_commits + 1 + print(commit.summary) + commits.append( + { + "git_hash": commit.hexsha, + "git_branch": repo.active_branch.name, + } + ) if args.use_tags: tags_regexp = args.tags_regexp if tags_regexp == ".*": @@ -150,7 +160,7 @@ def cli_command_logic(args, project_name, project_version): pass by_description = "n/a" if args.use_branch: - by_description = "from branch {}".format(args.branch) + by_description = "from branch {}".format(repo.active_branch.name) if args.use_tags: by_description = "by tags" logging.info( diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/args.py b/redis_benchmarks_specification/__self_contained_coordinator__/args.py index da91b9d..08696e8 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/args.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/args.py @@ -116,4 +116,10 @@ def create_self_contained_coordinator_args(project_name): action="store_true", help="Read the docker images from redis keys.", ) + parser.add_argument( + "--verbose", + default=False, + action="store_true", + help="Run in verbose mode.", + ) return parser diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/build_info.py b/redis_benchmarks_specification/__self_contained_coordinator__/build_info.py new file mode 100644 index 0000000..d77f150 --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/build_info.py @@ -0,0 +1,57 @@ +import json +import logging + + +def extract_build_info_from_streamdata(testDetails): + use_git_timestamp = False + git_timestamp_ms = None + git_version = None + git_branch = None + metadata = None + build_variant_name = None + fields = [fieldname.decode() for fieldname in testDetails.keys()] + logging.info("Fields on stream {}".format(fields)) + git_hash = testDetails[b"git_hash"] + if b"use_git_timestamp" in testDetails: + use_git_timestamp = bool(testDetails[b"use_git_timestamp"].decode()) + if b"git_timestamp_ms" in testDetails: + git_timestamp_ms = int(testDetails[b"git_timestamp_ms"].decode()) + if b"id" in testDetails: + build_variant_name = testDetails[b"id"] + if type(build_variant_name) == bytes: + build_variant_name = build_variant_name.decode() + if b"git_branch" in testDetails: + git_branch = testDetails[b"git_branch"] + if type(git_branch) == bytes: + git_branch = git_branch.decode() + if b"git_version" in testDetails: + git_version = testDetails[b"git_version"] + if type(git_version) == bytes: + git_version = git_version.decode() + if type(git_hash) == bytes: + git_hash = git_hash.decode() + logging.info("Received commit hash specifier {}.".format(git_hash)) + build_artifacts_str = "redis-server" + build_image = testDetails[b"build_image"].decode() + run_image = build_image + if b"run_image" in testDetails: + run_image = testDetails[b"run_image"].decode() + logging.info("detected run image info {}.".format(run_image)) + else: + logging.info("using build image info {}.".format(build_image)) + if b"build_artifacts" in testDetails: + build_artifacts_str = testDetails[b"build_artifacts"].decode() + build_artifacts = build_artifacts_str.split(",") + if b"metadata" in testDetails: + metadata = json.loads(testDetails[b"metadata"].decode()) + return ( + build_variant_name, + metadata, + build_artifacts, + git_hash, + git_branch, + git_version, + run_image, + use_git_timestamp, + git_timestamp_ms, + ) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/clients.py b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py new file mode 100644 index 0000000..e09df38 --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/clients.py @@ -0,0 +1,24 @@ +def prepare_memtier_benchmark_parameters( + clientconfig, + full_benchmark_path, + port, + server, + local_benchmark_output_filename, + oss_cluster_api_enabled, +): + benchmark_command = [ + full_benchmark_path, + "--port", + "{}".format(port), + "--server", + "{}".format(server), + "--json-out-file", + local_benchmark_output_filename, + ] + if oss_cluster_api_enabled is True: + benchmark_command.append("--cluster-mode") + benchmark_command_str = " ".join(benchmark_command) + if "arguments" in clientconfig: + benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"] + + return None, benchmark_command_str diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py b/redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py new file mode 100644 index 0000000..83adf04 --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py @@ -0,0 +1,17 @@ +import math + + +def generate_cpuset_cpus(ceil_db_cpu_limit, current_cpu_pos): + previous_cpu_pos = current_cpu_pos + current_cpu_pos = current_cpu_pos + int(ceil_db_cpu_limit) + db_cpuset_cpus = ",".join( + [str(x) for x in range(previous_cpu_pos, current_cpu_pos)] + ) + return db_cpuset_cpus, current_cpu_pos + + +def extract_db_cpu_limit(topologies_map, topology_spec_name): + topology_spec = topologies_map[topology_spec_name] + db_cpu_limit = topology_spec["resources"]["requests"]["cpus"] + ceil_db_cpu_limit = math.ceil(float(db_cpu_limit)) + return ceil_db_cpu_limit diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py new file mode 100644 index 0000000..94c6ded --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py @@ -0,0 +1,90 @@ +import logging + +import docker + +from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import ( + generate_cpuset_cpus, +) + + +def generate_standalone_redis_server_args( + binary, port, dbdir, configuration_parameters=None +): + added_params = ["port", "protected-mode", "dir"] + # start redis-server + command = [ + binary, + "--protected-mode", + "no", + "--port", + "{}".format(port), + "--dir", + dbdir, + ] + if configuration_parameters is not None: + for parameter, parameter_value in configuration_parameters.items(): + if parameter not in added_params: + command.extend( + [ + "--{}".format(parameter), + parameter_value, + ] + ) + return command + + +def teardown_containers(redis_containers, container_type): + for container in redis_containers: + try: + container.stop() + except docker.errors.NotFound: + logging.info( + "When trying to stop {} container with id {} and image {} it was already stopped".format( + container_type, container.id, container.image + ) + ) + pass + + +def spin_docker_standalone_redis( + ceil_db_cpu_limit, + current_cpu_pos, + docker_client, + redis_configuration_parameters, + redis_containers, + redis_proc_start_port, + run_image, + temporary_dir, +): + mnt_point = "/mnt/redis/" + command = generate_standalone_redis_server_args( + "{}redis-server".format(mnt_point), + redis_proc_start_port, + mnt_point, + redis_configuration_parameters, + ) + command_str = " ".join(command) + db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( + ceil_db_cpu_limit, current_cpu_pos + ) + logging.info( + "Running redis-server on docker image {} (cpuset={}) with the following args: {}".format( + run_image, db_cpuset_cpus, command_str + ) + ) + container = docker_client.containers.run( + image=run_image, + volumes={ + temporary_dir: {"bind": mnt_point, "mode": "rw"}, + }, + auto_remove=True, + privileged=True, + working_dir=mnt_point, + command=command_str, + network_mode="host", + detach=True, + cpuset_cpus=db_cpuset_cpus, + pid_mode="host", + ) + redis_containers.append(container) + return current_cpu_pos diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py b/redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py new file mode 100644 index 0000000..f706869 --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py @@ -0,0 +1,91 @@ +import datetime +import logging + +from redisbench_admin.run.common import get_start_time_vars +from redisbench_admin.run.run import calculate_client_tool_duration_and_check +from redisbench_admin.utils.local import get_local_run_full_filename + +from redis_benchmarks_specification.__common__.spec import ( + extract_client_container_image, + extract_client_tool, +) +from redis_benchmarks_specification.__self_contained_coordinator__.clients import ( + prepare_memtier_benchmark_parameters, +) + + +def data_prepopulation_step( + benchmark_config, + benchmark_tool_workdir, + client_cpuset_cpus, + docker_client, + git_hash, + port, + temporary_dir, + test_name, +): + # setup the benchmark + ( + start_time, + start_time_ms, + start_time_str, + ) = get_start_time_vars() + local_benchmark_output_filename = get_local_run_full_filename( + start_time_str, + git_hash, + "preload__" + test_name, + "oss-standalone", + ) + preload_image = extract_client_container_image( + benchmark_config["dbconfig"], "preload_tool" + ) + preload_tool = extract_client_tool(benchmark_config["dbconfig"], "preload_tool") + full_benchmark_path = "/usr/local/bin/{}".format(preload_tool) + client_mnt_point = "/mnt/client/" + if "memtier_benchmark" in preload_tool: + (_, preload_command_str,) = prepare_memtier_benchmark_parameters( + benchmark_config["dbconfig"]["preload_tool"], + full_benchmark_path, + port, + "localhost", + local_benchmark_output_filename, + False, + ) + + logging.info( + "Using docker image {} as benchmark PRELOAD image (cpuset={}) with the following args: {}".format( + preload_image, + client_cpuset_cpus, + preload_command_str, + ) + ) + # run the benchmark + preload_start_time = datetime.datetime.now() + + client_container_stdout = docker_client.containers.run( + image=preload_image, + volumes={ + temporary_dir: { + "bind": client_mnt_point, + "mode": "rw", + }, + }, + auto_remove=True, + privileged=True, + working_dir=benchmark_tool_workdir, + command=preload_command_str, + network_mode="host", + detach=False, + cpuset_cpus=client_cpuset_cpus, + ) + + preload_end_time = datetime.datetime.now() + preload_duration_seconds = calculate_client_tool_duration_and_check( + preload_end_time, preload_start_time, "Preload", False + ) + logging.info( + "Tool {} seconds to load data. Output {}".format( + preload_duration_seconds, + client_container_stdout, + ) + ) diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py new file mode 100644 index 0000000..0f1fecc --- /dev/null +++ b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py @@ -0,0 +1,572 @@ +import datetime +import json +import logging +import shutil +import sys +import tempfile +import traceback + +import redis +from redisbench_admin.environments.oss_cluster import generate_cluster_redis_server_args +from redisbench_admin.profilers.profilers_local import ( + local_profilers_platform_checks, + profilers_start_if_required, + profilers_stop_if_required, +) +from redisbench_admin.run.common import ( + get_start_time_vars, + prepare_benchmark_parameters, +) +from redisbench_admin.run.grafana import generate_artifacts_table_grafana_redis +from redisbench_admin.run.redistimeseries import ( + datasink_profile_tabular_data, + timeseries_test_sucess_flow, +) +from redisbench_admin.run.run import calculate_client_tool_duration_and_check +from redisbench_admin.utils.benchmark_config import ( + get_final_benchmark_config, + extract_redis_dbconfig_parameters, +) +from redisbench_admin.utils.local import get_local_run_full_filename +from redisbench_admin.utils.results import post_process_benchmark_results + +from redis_benchmarks_specification.__common__.env import ( + STREAM_KEYNAME_NEW_BUILD_EVENTS, + STREAM_GH_NEW_BUILD_RUNNERS_CG, + S3_BUCKET_NAME, +) +from redis_benchmarks_specification.__common__.spec import ( + extract_build_variant_variations, + extract_client_cpu_limit, + extract_client_tool, + extract_client_container_image, +) +from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import ( + restore_build_artifacts_from_test_details, +) +from redis_benchmarks_specification.__self_contained_coordinator__.build_info import ( + extract_build_info_from_streamdata, +) +from redis_benchmarks_specification.__self_contained_coordinator__.clients import ( + prepare_memtier_benchmark_parameters, +) +from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import ( + extract_db_cpu_limit, + generate_cpuset_cpus, +) +from redis_benchmarks_specification.__self_contained_coordinator__.docker import ( + spin_docker_standalone_redis, + teardown_containers, +) +from redis_benchmarks_specification.__self_contained_coordinator__.prepopulation import ( + data_prepopulation_step, +) + + +def build_runners_consumer_group_create(conn, running_platform, id="$"): + consumer_group_name = get_runners_consumer_group_name(running_platform) + logging.info("Will use consumer group named {}.".format(consumer_group_name)) + try: + conn.xgroup_create( + STREAM_KEYNAME_NEW_BUILD_EVENTS, + consumer_group_name, + mkstream=True, + id=id, + ) + logging.info( + "Created consumer group named {} to distribute work.".format( + consumer_group_name + ) + ) + except redis.exceptions.ResponseError: + logging.info( + "Consumer group named {} already existed.".format(consumer_group_name) + ) + + +def get_runners_consumer_group_name(running_platform): + consumer_group_name = "{}-{}".format( + STREAM_GH_NEW_BUILD_RUNNERS_CG, running_platform + ) + return consumer_group_name + + +def process_self_contained_coordinator_stream( + conn, + datasink_push_results_redistimeseries, + docker_client, + home, + newTestInfo, + datasink_conn, + testsuite_spec_files, + topologies_map, + running_platform, + profilers_enabled=False, + profilers_list=[], + grafana_profile_dashboard="", + cpuset_start_pos=0, + redis_proc_start_port=6379, + docker_air_gap=False, + verbose=False, +): + stream_id = "n/a" + overall_result = False + total_test_suite_runs = 0 + try: + stream_id, testDetails = newTestInfo[0][1][0] + stream_id = stream_id.decode() + logging.info("Received work . Stream id {}.".format(stream_id)) + + if b"git_hash" in testDetails: + ( + build_variant_name, + metadata, + build_artifacts, + git_hash, + git_branch, + git_version, + run_image, + use_git_timestamp, + git_timestamp_ms, + ) = extract_build_info_from_streamdata(testDetails) + + overall_result = True + profiler_dashboard_links = [] + if docker_air_gap: + airgap_key = "docker:air-gap:{}".format(run_image) + logging.info( + "Restoring docker image: {} from {}".format(run_image, airgap_key) + ) + if conn.exists(airgap_key): + airgap_docker_image_bin = conn.get(airgap_key) + images_loaded = docker_client.images.load(airgap_docker_image_bin) + logging.info("Successfully loaded images {}".format(images_loaded)) + else: + logging.error( + "docker image {} was not present on key {}".format( + run_image, airgap_key + ) + ) + + for test_file in testsuite_spec_files: + redis_containers = [] + client_containers = [] + + with open(test_file, "r") as stream: + result, benchmark_config, test_name = get_final_benchmark_config( + None, stream, "" + ) + if result is False: + logging.error( + "Skipping {} given there were errors while calling get_final_benchmark_config()".format( + test_file + ) + ) + continue + ( + _, + _, + redis_configuration_parameters, + _, + _, + ) = extract_redis_dbconfig_parameters(benchmark_config, "dbconfig") + build_variants = extract_build_variant_variations(benchmark_config) + if build_variants is not None: + logging.info("Detected build variant filter") + if build_variant_name not in build_variants: + logging.error( + "Skipping {} given it's not part of build-variants for this test-suite {}".format( + build_variant_name, build_variants + ) + ) + continue + else: + logging.error( + "Running build variant {} given it's present on the build-variants spec {}".format( + build_variant_name, build_variants + ) + ) + for topology_spec_name in benchmark_config["redis-topologies"]: + test_result = False + try: + current_cpu_pos = cpuset_start_pos + ceil_db_cpu_limit = extract_db_cpu_limit( + topologies_map, topology_spec_name + ) + + temporary_dir_client = tempfile.mkdtemp(dir=home) + temporary_dir = tempfile.mkdtemp(dir=home) + logging.info( + "Using local temporary dir to persist redis build artifacts. Path: {}".format( + temporary_dir + ) + ) + tf_github_org = "redis" + tf_github_repo = "redis" + setup_name = topology_spec_name + tf_triggering_env = "ci" + github_actor = "{}-{}".format( + tf_triggering_env, running_platform + ) + restore_build_artifacts_from_test_details( + build_artifacts, conn, temporary_dir, testDetails + ) + dso = "redis-server" + profilers_artifacts_matrix = [] + + collection_summary_str = "" + if profilers_enabled: + collection_summary_str = ( + local_profilers_platform_checks( + dso, + github_actor, + git_branch, + tf_github_repo, + git_hash, + ) + ) + logging.info( + "Using the following collection summary string for profiler description: {}".format( + collection_summary_str + ) + ) + if setup_name == "oss-standalone": + current_cpu_pos = spin_docker_standalone_redis( + ceil_db_cpu_limit, + current_cpu_pos, + docker_client, + redis_configuration_parameters, + redis_containers, + redis_proc_start_port, + run_image, + temporary_dir, + ) + else: + for master_shard_id in range(1, shard_count + 1): + shard_port = master_shard_id + start_port - 1 + + ( + command, + logfile, + ) = generate_cluster_redis_server_args( + "redis-server", + dbdir_folder, + remote_module_files, + server_private_ip, + shard_port, + redis_configuration_parameters, + "yes", + modules_configuration_parameters_map, + logname_prefix, + "yes", + redis_7, + ) + logging.error( + "Remote primary shard {} command: {}".format( + master_shard_id, " ".join(command) + ) + ) + + r = redis.StrictRedis(port=redis_proc_start_port) + r.ping() + redis_pids = [] + first_redis_pid = r.info()["process_id"] + redis_pids.append(first_redis_pid) + ceil_client_cpu_limit = extract_client_cpu_limit( + benchmark_config + ) + client_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( + ceil_client_cpu_limit, current_cpu_pos + ) + client_mnt_point = "/mnt/client/" + benchmark_tool_workdir = client_mnt_point + + if "preload_tool" in benchmark_config["dbconfig"]: + data_prepopulation_step( + benchmark_config, + benchmark_tool_workdir, + client_cpuset_cpus, + docker_client, + git_hash, + redis_proc_start_port, + temporary_dir, + test_name, + ) + + benchmark_tool = extract_client_tool(benchmark_config) + # backwards compatible + if benchmark_tool is None: + benchmark_tool = "redis-benchmark" + full_benchmark_path = "/usr/local/bin/{}".format( + benchmark_tool + ) + + # setup the benchmark + ( + start_time, + start_time_ms, + start_time_str, + ) = get_start_time_vars() + local_benchmark_output_filename = ( + get_local_run_full_filename( + start_time_str, + git_hash, + test_name, + topology_spec_name, + ) + ) + logging.info( + "Will store benchmark json output to local file {}".format( + local_benchmark_output_filename + ) + ) + if "memtier_benchmark" not in benchmark_tool: + # prepare the benchmark command + ( + benchmark_command, + benchmark_command_str, + ) = prepare_benchmark_parameters( + benchmark_config, + full_benchmark_path, + redis_proc_start_port, + "localhost", + local_benchmark_output_filename, + False, + benchmark_tool_workdir, + False, + ) + else: + ( + _, + benchmark_command_str, + ) = prepare_memtier_benchmark_parameters( + benchmark_config["clientconfig"], + full_benchmark_path, + redis_proc_start_port, + "localhost", + local_benchmark_output_filename, + benchmark_tool_workdir, + ) + + client_container_image = extract_client_container_image( + benchmark_config + ) + profiler_call_graph_mode = "dwarf" + profiler_frequency = 99 + # start the profile + ( + profiler_name, + profilers_map, + ) = profilers_start_if_required( + profilers_enabled, + profilers_list, + redis_pids, + setup_name, + start_time_str, + test_name, + profiler_frequency, + profiler_call_graph_mode, + ) + + logging.info( + "Using docker image {} as benchmark client image (cpuset={}) with the following args: {}".format( + client_container_image, + client_cpuset_cpus, + benchmark_command_str, + ) + ) + # run the benchmark + benchmark_start_time = datetime.datetime.now() + + client_container_stdout = docker_client.containers.run( + image=client_container_image, + volumes={ + temporary_dir_client: { + "bind": client_mnt_point, + "mode": "rw", + }, + }, + auto_remove=True, + privileged=True, + working_dir=benchmark_tool_workdir, + command=benchmark_command_str, + network_mode="host", + detach=False, + cpuset_cpus=client_cpuset_cpus, + ) + + benchmark_end_time = datetime.datetime.now() + benchmark_duration_seconds = ( + calculate_client_tool_duration_and_check( + benchmark_end_time, benchmark_start_time + ) + ) + if verbose: + logging.info( + "output {}".format(client_container_stdout) + ) + r.shutdown(save=False) + + (_, overall_tabular_data_map,) = profilers_stop_if_required( + datasink_push_results_redistimeseries, + benchmark_duration_seconds, + collection_summary_str, + dso, + tf_github_org, + tf_github_repo, + profiler_name, + profilers_artifacts_matrix, + profilers_enabled, + profilers_map, + redis_pids, + S3_BUCKET_NAME, + test_name, + ) + if ( + profilers_enabled + and datasink_push_results_redistimeseries + ): + datasink_profile_tabular_data( + git_branch, + tf_github_org, + tf_github_repo, + git_hash, + overall_tabular_data_map, + conn, + setup_name, + start_time_ms, + start_time_str, + test_name, + tf_triggering_env, + ) + if len(profilers_artifacts_matrix) == 0: + logging.error("No profiler artifact was retrieved") + else: + profilers_artifacts = [] + for line in profilers_artifacts_matrix: + artifact_name = line[2] + s3_link = line[4] + profilers_artifacts.append( + { + "artifact_name": artifact_name, + "s3_link": s3_link, + } + ) + https_link = generate_artifacts_table_grafana_redis( + datasink_push_results_redistimeseries, + grafana_profile_dashboard, + profilers_artifacts, + datasink_conn, + setup_name, + start_time_ms, + start_time_str, + test_name, + tf_github_org, + tf_github_repo, + git_hash, + git_branch, + ) + profiler_dashboard_links.append( + [ + setup_name, + test_name, + " {} ".format(https_link), + ] + ) + logging.info( + "Published new profile info for this testcase. Access it via: {}".format( + https_link + ) + ) + + datapoint_time_ms = start_time_ms + if ( + use_git_timestamp is True + and git_timestamp_ms is not None + ): + datapoint_time_ms = git_timestamp_ms + post_process_benchmark_results( + benchmark_tool, + local_benchmark_output_filename, + datapoint_time_ms, + start_time_str, + client_container_stdout, + None, + ) + full_result_path = local_benchmark_output_filename + if "memtier_benchmark" in benchmark_tool: + full_result_path = "{}/{}".format( + temporary_dir_client, + local_benchmark_output_filename, + ) + logging.critical( + "Reading results json from {}".format(full_result_path) + ) + + with open( + full_result_path, + "r", + ) as json_file: + results_dict = json.load(json_file) + dataset_load_duration_seconds = 0 + + logging.info( + "Using datapoint_time_ms: {}".format(datapoint_time_ms) + ) + + timeseries_test_sucess_flow( + datasink_push_results_redistimeseries, + git_version, + benchmark_config, + benchmark_duration_seconds, + dataset_load_duration_seconds, + None, + topology_spec_name, + setup_name, + None, + results_dict, + datasink_conn, + datapoint_time_ms, + test_name, + git_branch, + tf_github_org, + tf_github_repo, + tf_triggering_env, + metadata, + build_variant_name, + running_platform, + ) + test_result = True + total_test_suite_runs = total_test_suite_runs + 1 + + except: + logging.critical( + "Some unexpected exception was caught " + "during local work. Failing test...." + ) + logging.critical(sys.exc_info()[0]) + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + test_result = False + # tear-down + logging.info("Tearing down setup") + teardown_containers(redis_containers, "DB") + teardown_containers(client_containers, "CLIENT") + shutil.rmtree(temporary_dir, ignore_errors=True) + + overall_result &= test_result + + else: + logging.error("Missing commit information within received message.") + except: + logging.critical( + "Some unexpected exception was caught " + "during local work on stream {}. Failing test....".format(stream_id) + ) + logging.critical(sys.exc_info()[0]) + print("-" * 60) + traceback.print_exc(file=sys.stdout) + print("-" * 60) + overall_result = False + return stream_id, overall_result, total_test_suite_runs diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index c2c3f6d..1d33df0 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -1,12 +1,5 @@ -import datetime -import json import logging -import math import pathlib -import sys -import tempfile -import shutil -import traceback import docker import redis import os @@ -14,56 +7,27 @@ from redisbench_admin.profilers.profilers_local import ( check_compatible_system_and_kernel_and_prepare_profile, - profilers_start_if_required, - local_profilers_platform_checks, - profilers_stop_if_required, ) -from docker.models.containers import Container - -from redisbench_admin.run.common import ( - get_start_time_vars, - prepare_benchmark_parameters, -) -from redisbench_admin.run.grafana import generate_artifacts_table_grafana_redis -from redisbench_admin.utils.benchmark_config import ( - get_final_benchmark_config, -) -from redisbench_admin.run.redistimeseries import ( - timeseries_test_sucess_flow, - datasink_profile_tabular_data, -) -from redisbench_admin.run.run import calculate_client_tool_duration_and_check -from redisbench_admin.utils.benchmark_config import ( - extract_redis_dbconfig_parameters, -) -from redisbench_admin.utils.local import get_local_run_full_filename -from redisbench_admin.utils.results import post_process_benchmark_results from redis_benchmarks_specification.__common__.env import ( LOG_FORMAT, LOG_DATEFMT, LOG_LEVEL, STREAM_KEYNAME_NEW_BUILD_EVENTS, - STREAM_GH_NEW_BUILD_RUNNERS_CG, REDIS_HEALTH_CHECK_INTERVAL, REDIS_SOCKET_TIMEOUT, - S3_BUCKET_NAME, ) from redis_benchmarks_specification.__common__.package import ( get_version_string, populate_with_poetry_data, ) -from redis_benchmarks_specification.__common__.spec import ( - extract_client_cpu_limit, - extract_client_container_image, - extract_client_tool, - extract_build_variant_variations, -) from redis_benchmarks_specification.__self_contained_coordinator__.args import ( create_self_contained_coordinator_args, ) -from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import ( - restore_build_artifacts_from_test_details, +from redis_benchmarks_specification.__self_contained_coordinator__.runners import ( + build_runners_consumer_group_create, + get_runners_consumer_group_name, + process_self_contained_coordinator_stream, ) from redis_benchmarks_specification.__setups__.topologies import get_topologies @@ -230,34 +194,6 @@ def main(): ) -def build_runners_consumer_group_create(conn, running_platform, id="$"): - consumer_group_name = get_runners_consumer_group_name(running_platform) - logging.info("Will use consumer group named {}.".format(consumer_group_name)) - try: - conn.xgroup_create( - STREAM_KEYNAME_NEW_BUILD_EVENTS, - consumer_group_name, - mkstream=True, - id=id, - ) - logging.info( - "Created consumer group named {} to distribute work.".format( - consumer_group_name - ) - ) - except redis.exceptions.ResponseError: - logging.info( - "Consumer group named {} already existed.".format(consumer_group_name) - ) - - -def get_runners_consumer_group_name(running_platform): - consumer_group_name = "{}-{}".format( - STREAM_GH_NEW_BUILD_RUNNERS_CG, running_platform - ) - return consumer_group_name - - def self_contained_coordinator_blocking_read( conn, datasink_push_results_redistimeseries, @@ -938,95 +874,3 @@ def get_benchmark_specs(testsuites_folder): "Running all specified benchmarks: {}".format(" ".join([str(x) for x in files])) ) return files - - -def extract_build_info_from_streamdata(testDetails): - use_git_timestamp = False - git_timestamp_ms = None - git_version = None - git_branch = None - metadata = None - build_variant_name = None - git_hash = testDetails[b"git_hash"] - if b"use_git_timestamp" in testDetails: - use_git_timestamp = bool(testDetails[b"use_git_timestamp"].decode()) - if b"git_timestamp_ms" in testDetails: - git_timestamp_ms = int(testDetails[b"git_timestamp_ms"].decode()) - if b"id" in testDetails: - build_variant_name = testDetails[b"id"] - if type(build_variant_name) == bytes: - build_variant_name = build_variant_name.decode() - if b"git_branch" in testDetails: - git_branch = testDetails[b"git_branch"] - if type(git_branch) == bytes: - git_branch = git_branch.decode() - if b"git_version" in testDetails: - git_version = testDetails[b"git_version"] - if type(git_version) == bytes: - git_version = git_version.decode() - if type(git_hash) == bytes: - git_hash = git_hash.decode() - logging.info("Received commit hash specifier {}.".format(git_hash)) - build_artifacts_str = "redis-server" - build_image = testDetails[b"build_image"].decode() - run_image = build_image - if b"run_image" in testDetails[b"run_image"]: - run_image = testDetails[b"run_image"].decode() - if b"build_artifacts" in testDetails: - build_artifacts_str = testDetails[b"build_artifacts"].decode() - build_artifacts = build_artifacts_str.split(",") - if b"metadata" in testDetails: - metadata = json.loads(testDetails[b"metadata"].decode()) - return ( - build_variant_name, - metadata, - build_artifacts, - git_hash, - git_branch, - git_version, - run_image, - use_git_timestamp, - git_timestamp_ms, - ) - - -def generate_cpuset_cpus(ceil_db_cpu_limit, current_cpu_pos): - previous_cpu_pos = current_cpu_pos - current_cpu_pos = current_cpu_pos + int(ceil_db_cpu_limit) - db_cpuset_cpus = ",".join( - [str(x) for x in range(previous_cpu_pos, current_cpu_pos)] - ) - return db_cpuset_cpus, current_cpu_pos - - -def extract_db_cpu_limit(topologies_map, topology_spec_name): - topology_spec = topologies_map[topology_spec_name] - db_cpu_limit = topology_spec["resources"]["requests"]["cpus"] - ceil_db_cpu_limit = math.ceil(float(db_cpu_limit)) - return ceil_db_cpu_limit - - -def generate_standalone_redis_server_args( - binary, port, dbdir, configuration_parameters=None -): - added_params = ["port", "protected-mode", "dir"] - # start redis-server - command = [ - binary, - "--protected-mode", - "no", - "--port", - "{}".format(port), - "--dir", - dbdir, - ] - if configuration_parameters is not None: - for parameter, parameter_value in configuration_parameters.items(): - if parameter not in added_params: - command.extend( - [ - "--{}".format(parameter), - parameter_value, - ] - ) - return command diff --git a/redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-default.yml b/redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-default.yml index fa51e8e..6c40bcb 100644 --- a/redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-default.yml +++ b/redis_benchmarks_specification/setups/builders/gcc:8.5.0-amd64-debian-buster-default.yml @@ -6,6 +6,7 @@ compiler: "gcc" cpp_compiler: "g++" kind: docker build_image: gcc:8.5.0-buster +run_image: debian:buster description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description) pre-configured environment with all the tools required to build with gcc." metadata: @@ -14,6 +15,4 @@ metadata: os: debian-buster arch: amd64 -env: - REDIS_CFLAGS: "-g -fno-omit-frame-pointer" diff --git a/redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml b/redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml index 2ef5d7c..6febfce 100644 --- a/redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml +++ b/redis_benchmarks_specification/setups/builders/gcc:8.5.0-arm64-debian-buster-default.yml @@ -6,6 +6,7 @@ compiler: "gcc" cpp_compiler: "g++" kind: docker build_image: gcc:8.5.0-buster +run_image: debian:buster description: "Using GNU Compiler Containers (https://hub.docker.com/_/gcc?tab=description) pre-configured environment with all the tools required to build with gcc." metadata: @@ -14,6 +15,4 @@ metadata: os: debian-buster arch: arm64 -env: - REDIS_CFLAGS: "-g -fno-omit-frame-pointer" diff --git a/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-default.yml b/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-default.yml index ad279b3..8829bb1 100644 --- a/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-default.yml +++ b/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-default.yml @@ -6,6 +6,7 @@ compiler: "icc" cpp_compiler: "icc" kind: docker build_image: intel/oneapi-hpckit:2021.3-devel-ubuntu18.04 +run_image: debian:buster description: "Using Intel® oneAPI Containers (https://github.com/intel/oneapi-containers) pre-configured environment with all the tools required to build with Intel related products and targeting Intel® environments and their instruction set extensions." diff --git a/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-libc.yml b/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-libc.yml index 8103cbe..ce4bd2e 100644 --- a/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-libc.yml +++ b/redis_benchmarks_specification/setups/builders/icc-2021.3.0-amd64-ubuntu18.04-libc.yml @@ -6,6 +6,7 @@ compiler: "icc" cpp_compiler: "icc" kind: docker build_image: intel/oneapi-hpckit:2021.3-devel-ubuntu18.04 +run_image: debian:buster description: "Using Intel® oneAPI Containers (https://github.com/intel/oneapi-containers) pre-configured environment with all the tools required to build with Intel related products and targeting Intel® environments and their instruction set extensions." diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-100B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-100B-values.yml index 694997e..bda46dd 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-100B-values.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-string-with-100B-values.yml @@ -9,6 +9,11 @@ tested-commands: - SET redis-topologies: - oss-standalone + - oss-cluster-03-primaries + - oss-cluster-05-primaries + - oss-cluster-09-primaries + - oss-cluster-15-primaries + - oss-cluster-21-primaries build-variants: - gcc:8.5.0-amd64-debian-buster-default @@ -21,6 +26,9 @@ clientconfig: requests: cpus: "4" memory: "2g" + scalability: + mode: "1:1" + max_processes: 21 exporter: redistimeseries: break_by: diff --git a/utils/tests/test_self_contained_coordinator.py b/utils/tests/test_self_contained_coordinator.py index 5cdb7d6..71cf8bf 100644 --- a/utils/tests/test_self_contained_coordinator.py +++ b/utils/tests/test_self_contained_coordinator.py @@ -17,11 +17,17 @@ extract_client_tool, ) from redis_benchmarks_specification.__self_contained_coordinator__.self_contained_coordinator import ( - generate_cpuset_cpus, self_contained_coordinator_blocking_read, +) +from redis_benchmarks_specification.__self_contained_coordinator__.clients import ( + prepare_memtier_benchmark_parameters, +) +from redis_benchmarks_specification.__self_contained_coordinator__.runners import ( build_runners_consumer_group_create, get_runners_consumer_group_name, - prepare_memtier_benchmark_parameters, +) +from redis_benchmarks_specification.__self_contained_coordinator__.cpuset import ( + generate_cpuset_cpus, ) from redis_benchmarks_specification.__setups__.topologies import get_topologies from utils.tests.test_data.api_builder_common import flow_1_and_2_api_builder_checks diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index def776b..ecef9c4 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -17,9 +17,13 @@ from redis_benchmarks_specification.__self_contained_coordinator__.self_contained_coordinator import ( self_contained_coordinator_blocking_read, - build_runners_consumer_group_create, +) +from redis_benchmarks_specification.__self_contained_coordinator__.clients import ( prepare_memtier_benchmark_parameters, ) +from redis_benchmarks_specification.__self_contained_coordinator__.runners import ( + build_runners_consumer_group_create, +) from redis_benchmarks_specification.__setups__.topologies import get_topologies from utils.tests.test_data.api_builder_common import flow_1_and_2_api_builder_checks