diff --git a/pyproject.toml b/pyproject.toml index a15161d..0c0cddc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redis-benchmarks-specification" -version = "0.1.40" +version = "0.1.41" description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute." authors = ["filipecosta90 ","Redis Performance Group "] readme = "Readme.md" diff --git a/redis_benchmarks_specification/__builder__/builder.py b/redis_benchmarks_specification/__builder__/builder.py index b10b0f3..b77d046 100644 --- a/redis_benchmarks_specification/__builder__/builder.py +++ b/redis_benchmarks_specification/__builder__/builder.py @@ -71,6 +71,12 @@ def main(): type=str, default=">", ) + parser.add_argument( + "--docker-air-gap", + default=False, + action="store_true", + help="Store the docker images in redis keys.", + ) args = parser.parse_args() if args.logname is not None: print("Writting log to {}".format(args.logname)) @@ -130,7 +136,11 @@ def main(): previous_id = args.consumer_start_id while True: previous_id, new_builds_count = builder_process_stream( - builders_folder, conn, different_build_specs, previous_id + builders_folder, + conn, + different_build_specs, + previous_id, + args.docker_air_gap, ) @@ -155,7 +165,9 @@ def builder_consumer_group_create(conn, id="$"): ) -def builder_process_stream(builders_folder, conn, different_build_specs, previous_id): +def builder_process_stream( + builders_folder, conn, different_build_specs, previous_id, docker_air_gap=False +): new_builds_count = 0 logging.info("Entering blocking read waiting for work.") consumer_name = "{}-proc#{}".format(STREAM_GH_EVENTS_COMMIT_BUILDERS_CG, "1") @@ -212,6 +224,30 @@ def builder_process_stream(builders_folder, conn, different_build_specs, previou run_image = build_image if "run_image" in build_config: run_image = build_config["run_image"] + if docker_air_gap: + airgap_key = "docker:air-gap:{}".format(run_image) + logging.info( + "DOCKER AIR GAP: storing run image named: {} in redis key {}".format( + run_image, airgap_key + ) + ) + run_image_binary_stream = io.BytesIO() + run_image_docker = docker_client.images.get(run_image) + for chunk in run_image_docker.save(): + run_image_binary_stream.write(chunk) + # 7 days expire + binary_exp_secs = 24 * 60 * 60 * 7 + res_airgap = conn.set( + airgap_key, + run_image_binary_stream.getbuffer(), + ex=binary_exp_secs, + ) + logging.info( + "DOCKER AIR GAP: result of set bin data to {}: {}".format( + airgap_key, res_airgap + ) + ) + compiler = build_config["compiler"] cpp_compiler = build_config["cpp_compiler"] build_os = build_config["os"] diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/args.py b/redis_benchmarks_specification/__self_contained_coordinator__/args.py index c4cf753..da91b9d 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/args.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/args.py @@ -110,4 +110,10 @@ def create_self_contained_coordinator_args(project_name): type=str, default="https://benchmarksredisio.grafana.net/d/uRPZar57k/ci-profiler-viewer", ) + parser.add_argument( + "--docker-air-gap", + default=False, + action="store_true", + help="Read the docker images from redis keys.", + ) return parser diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 9361a24..c2c3f6d 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -186,6 +186,13 @@ def main(): consumer_pos = args.consumer_pos logging.info("Consumer pos {}".format(consumer_pos)) + # Docker air gap usage + docker_air_gap = args.docker_air_gap + if docker_air_gap: + logging.info( + "Using docker in an air-gapped way. Restoring running images from redis keys." + ) + profilers_list = [] profilers_enabled = args.enable_profilers if profilers_enabled: @@ -219,6 +226,7 @@ def main(): cpuset_start_pos, redis_proc_start_port, consumer_pos, + docker_air_gap, ) @@ -266,6 +274,7 @@ def self_contained_coordinator_blocking_read( cpuset_start_pos=0, redis_proc_start_port=6379, consumer_pos=1, + docker_air_gap=False, ): num_process_streams = 0 num_process_test_suites = 0 @@ -307,6 +316,7 @@ def self_contained_coordinator_blocking_read( grafana_profile_dashboard, cpuset_start_pos, redis_proc_start_port, + docker_air_gap, ) num_process_streams = num_process_streams + 1 num_process_test_suites = num_process_test_suites + total_test_suite_runs @@ -374,6 +384,7 @@ def process_self_contained_coordinator_stream( grafana_profile_dashboard="", cpuset_start_pos=0, redis_proc_start_port=6379, + docker_air_gap=False, ): stream_id = "n/a" overall_result = False @@ -398,6 +409,14 @@ def process_self_contained_coordinator_stream( overall_result = True profiler_dashboard_links = [] + if docker_air_gap: + airgap_key = "docker:air-gap:{}".format(run_image) + logging.info( + "Restoring docker image: {} from {}".format(run_image, airgap_key) + ) + airgap_docker_image_bin = conn.get(airgap_key) + images_loaded = docker_client.images.load(airgap_docker_image_bin) + logging.info("Successfully loaded images {}".format(images_loaded)) for test_file in testsuite_spec_files: redis_containers = []