From 99eaee91a3c4ca80a4c8ea0dd29ed8e2ad0d90d7 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Wed, 9 Apr 2025 15:18:57 -0700 Subject: [PATCH 01/16] Trim Env Layer --- src/core/src/bootstrap/EnvLayer.py | 329 ++++---------------- src/core/tests/library/RuntimeCompositor.py | 1 - 2 files changed, 63 insertions(+), 267 deletions(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 4f20677d..453bf5bb 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -15,10 +15,8 @@ # Requires Python 2.7+ from __future__ import print_function -import base64 import datetime import glob -import json import os import re import platform @@ -35,27 +33,10 @@ class EnvLayer(object): """ Environment related functions """ def __init__(self, real_record_path=None, recorder_enabled=False, emulator_enabled=False): - # Recorder / emulator storage - self.__real_record_path = real_record_path - self.__real_record_pointer_path = real_record_path + ".pt" if real_record_path is not None else None - self.__real_record_handle = None - self.__real_record_pointer = 0 - - # Recorder / emulator state section - self.__recorder_enabled = recorder_enabled # dumps black box recordings - self.__emulator_enabled = False if recorder_enabled else emulator_enabled # only one can be enabled at a time - - # Recorder / emulator initialization - if self.__recorder_enabled: - self.__record_writer_init() - elif self.__emulator_enabled: - self.__record_reader_init() - # Discrete components - self.platform = self.Platform(recorder_enabled, emulator_enabled, self.__write_record, self.__read_record) - self.datetime = self.DateTime(recorder_enabled, emulator_enabled, self.__write_record, self.__read_record) - self.file_system = self.FileSystem(recorder_enabled, emulator_enabled, self.__write_record, self.__read_record, - emulator_root_path=os.path.dirname(self.__real_record_path) if self.__real_record_path is not None else self.__real_record_path) + self.platform = self.Platform() + self.datetime = self.DateTime() + self.file_system = self.FileSystem() # Constant paths self.etc_environment_file_path = "/etc/environment" @@ -92,7 +73,7 @@ def get_package_manager(self): return ret def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): - """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is overwriten. """ + """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is over-writen. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) if environment_vars is None: @@ -128,7 +109,7 @@ def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): self.file_system.write_with_retry(self.etc_environment_file_path, environment_vars, 'w') except Exception as error: - print("Error occurred while setting environment variable [Variable={0}] [Value={1}] [Exception={2}]".format(str(var_name), str(var_value), repr(error))) + print("Error occurred while setting environment variable [Variable={0}][Value={1}][Exception={2}]".format(str(var_name), str(var_value), repr(error))) if raise_if_not_success: raise @@ -137,7 +118,7 @@ def get_env_var(self, var_name, raise_if_not_success=False): try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) if environment_vars is None: - print("Error occurred while getting environment variable: File not found. [Variable={0}] [Path={1}]".format(str(var_name), self.etc_environment_file_path)) + print("Error occurred while getting environment variable: File not found. [Variable={0}][Path={1}]".format(str(var_name), self.etc_environment_file_path)) return None # get specific environment variable value @@ -150,25 +131,15 @@ def get_env_var(self, var_name, raise_if_not_success=False): return group[group.index("=")+1:] except Exception as error: - print("Error occurred while getting environment variable [Variable={0}] [Exception={1}]".format(str(var_name), repr(error))) + print("Error occurred while getting environment variable [Variable={0}][Exception={1}]".format(str(var_name), repr(error))) if raise_if_not_success: raise - def run_command_output(self, cmd, no_output=False, chk_err=False): - operation = "RUN_CMD_OUT" - if not self.__emulator_enabled: - start = time.time() - code, output = self.__run_command_output_raw(cmd, no_output, chk_err) - self.__write_record(operation, code, output, delay=(time.time()-start)) - return code, output - else: - return self.__read_record(operation) - - def __run_command_output_raw(self, cmd, no_output, chk_err=True): + def run_command_output(self, cmd, no_output, chk_err=True): """ - Wrapper for subprocess.check_output. Execute 'cmd'. - Returns return code and STDOUT, trapping expected exceptions. - Reports exceptions to Error if chk_err parameter is True + Wrapper for subprocess.check_output. Execute 'cmd'. + Returns return code and STDOUT, trapping expected exceptions. + Reports exceptions to Error if chk_err parameter is True """ def check_output(*popenargs, **kwargs): @@ -245,23 +216,13 @@ def __convert_process_output_to_ascii(output): else: raise Exception("Unknown version of python encountered.") - def reboot_machine(self, reboot_cmd): - operation = "REBOOT_MACHINE" - if not self.__emulator_enabled: - self.__write_record(operation, 0, '', delay=0) - subprocess.Popen(reboot_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - self.__read_record(operation) # will throw if it's not the expected operation - raise Exception(Constants.EnvLayer.PRIVILEGED_OP_REBOOT) - - def exit(self, code): - operation = "EXIT_EXECUTION" - if not self.__emulator_enabled: - self.__write_record(operation, code, '', delay=0) - exit(code) - else: - self.__read_record(operation) # will throw if it's not the expected operation - raise Exception(Constants.EnvLayer.PRIVILEGED_OP_EXIT + str(code)) + @staticmethod + def reboot_machine(reboot_cmd): + subprocess.Popen(reboot_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + @staticmethod + def exit(code): + exit(code) @staticmethod def get_python_major_version(): @@ -272,94 +233,39 @@ def get_python_major_version(): # region - Platform emulation and extensions class Platform(object): - def __init__(self, recorder_enabled=True, emulator_enabled=False, write_record_delegate=None, read_record_delegate=None): - self.__recorder_enabled = recorder_enabled - self.__emulator_enabled = False if recorder_enabled else emulator_enabled - self.__write_record = write_record_delegate - self.__read_record = read_record_delegate - - def linux_distribution(self): - operation = "PLATFORM_LINUX_DISTRIBUTION" - if not self.__emulator_enabled: - major_version = EnvLayer.get_python_major_version() - - if major_version == 2: - value = platform.linux_distribution() - else: - value = distro.linux_distribution() - - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return eval(output) - - def system(self): # OS Type - operation = "PLATFORM_SYSTEM" - if not self.__emulator_enabled: - value = platform.system() - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return output - - def machine(self): # architecture - operation = "PLATFORM_MACHINE" - if not self.__emulator_enabled: - value = platform.machine() - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return output - - def node(self): # machine name - operation = "PLATFORM_NODE" - if not self.__emulator_enabled: - value = platform.node() - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return output + @staticmethod + def linux_distribution(): + return platform.linux_distribution() if (EnvLayer.get_python_major_version() == 2) else distro.linux_distribution() + + @staticmethod + def system(): # OS Type + return platform.system() + + @staticmethod + def machine(): # architecture + return platform.machine() + + @staticmethod + def node(): # machine name + return platform.node() # endregion - Platform emulation and extensions # region - File system emulation and extensions class FileSystem(object): - def __init__(self, recorder_enabled=True, emulator_enabled=False, write_record_delegate=None, read_record_delegate=None, emulator_root_path=None): - self.__recorder_enabled = recorder_enabled - self.__emulator_enabled = False if recorder_enabled else emulator_enabled - self.__write_record = write_record_delegate - self.__read_record = read_record_delegate - self.__emulator_enabled = emulator_enabled - self.__emulator_root_path = emulator_root_path - - # file-names of files that other processes may changes the contents of + def __init__(self): + # file-names of files that other processes may change the contents of self.__non_exclusive_files = [Constants.EXT_STATE_FILE] - def resolve_path(self, requested_path): - """ Resolves any paths used with desired file system paths """ - if self.__emulator_enabled and self.__emulator_root_path is not None and self.__emulator_root_path not in requested_path: - return os.path.join(self.__emulator_root_path, os.path.normpath(requested_path)) - else: - return requested_path - def open(self, file_path, mode, raise_if_not_found=True): """ Provides a file handle to the file_path requested using implicit redirection where required """ - real_path = self.resolve_path(file_path) for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): try: - return open(real_path, mode) + return open(file_path, mode) except Exception as error: if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: time.sleep(i + 1) else: - error_message = "Unable to open file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(real_path), repr(error), str(raise_if_not_found)) + error_message = "Unable to open file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path), repr(error), str(raise_if_not_found)) if raise_if_not_found: raise Exception(error_message) else: @@ -377,31 +283,23 @@ def __obtain_file_handle(self, file_path_or_handle, mode='a+', raise_if_not_foun def read_with_retry(self, file_path_or_handle, raise_if_not_found=True): """ Reads all content from a given file path in a single operation """ - operation = "FILE_READ" - - # only fully emulate non_exclusive_files from the real recording; exclusive files can be redirected and handled in emulator scenarios - if not self.__emulator_enabled or (isinstance(file_path_or_handle, str) and os.path.basename(file_path_or_handle) not in self.__non_exclusive_files): - file_handle, was_path = self.__obtain_file_handle(file_path_or_handle, 'r', raise_if_not_found) - for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): - try: - value = file_handle.read() - if was_path: # what was passed in was not a file handle, so close the handle that was init here - file_handle.close() - self.__write_record(operation, code=0, output=value, delay=0) - return value - except Exception as error: - if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: - time.sleep(i + 1) + file_handle, was_path = self.__obtain_file_handle(file_path_or_handle, 'r', raise_if_not_found) + for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): + try: + value = file_handle.read() + if was_path: # what was passed in was not a file handle, so close the handle that was init here + file_handle.close() + return value + except Exception as error: + if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: + time.sleep(i + 1) + else: + error_message = "Unable to read file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path_or_handle), repr(error), str(raise_if_not_found)) + if raise_if_not_found: + raise Exception(error_message) else: - error_message = "Unable to read file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path_or_handle), repr(error), str(raise_if_not_found)) - if raise_if_not_found: - raise Exception(error_message) - else: - print(error_message) - return None - else: - code, output = self.__read_record(operation) - return output + print(error_message) + return None def write_with_retry(self, file_path_or_handle, data, mode='a+'): """ Writes to a given real/emulated file path in a single operation """ @@ -461,45 +359,18 @@ def delete_from_dir(dir_name, identifier_list, raise_if_delete_failed=False): # region - DateTime emulation and extensions class DateTime(object): - def __init__(self, recorder_enabled=True, emulator_enabled=False, write_record_delegate=None, read_record_delegate=None): - self.__recorder_enabled = recorder_enabled - self.__emulator_enabled = False if recorder_enabled else emulator_enabled - self.__write_record = write_record_delegate - self.__read_record = read_record_delegate - + @staticmethod def time(self): - operation = "DATETIME_TIME" - if not self.__emulator_enabled: - value = time.time() - self.__write_record(operation, code=0, output=value, delay=0) - return value - else: - code, output = self.__read_record(operation) - return int(output) - - def datetime_utcnow(self): - operation = "DATETIME_UTCNOW" - if not self.__emulator_enabled: - value = datetime.datetime.utcnow() - self.__write_record(operation, code=0, output=str(value), delay=0) - return value - else: - code, output = self.__read_record(operation) - return datetime.datetime.strptime(str(output), "%Y-%m-%d %H:%M:%S.%f") - - def timestamp(self): - operation = "DATETIME_TIMESTAMP" - if not self.__emulator_enabled: - value = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - self.__write_record(operation, code=0, output=value, delay=0) - return value - else: - code, output = self.__read_record(operation) - return output + return time.time() + + @staticmethod + def datetime_utcnow(): + return datetime.datetime.utcnow() + + @staticmethod + def timestamp(): + return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - # -------------------------------------------------------------------------------------------------------------- - # Static library functions - # -------------------------------------------------------------------------------------------------------------- @staticmethod def total_minutes_from_time_delta(time_delta): return ((time_delta.microseconds + (time_delta.seconds + time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6) / 60 @@ -531,77 +402,3 @@ def standard_datetime_to_utc(std_datetime): """ Converts datetime object to string of format '"%Y-%m-%dT%H:%M:%SZ"' """ return std_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") # endregion - DateTime emulator and extensions - -# region - Core Emulator support functions - def __write_record(self, operation, code, output, delay, timestamp=None): - """ Writes a single operation record to disk if the recorder is enabled """ - if not self.__recorder_enabled or self.__real_record_handle is None: - return - - try: - record = { - "timestamp": str(timestamp) if timestamp is not None else datetime.datetime.strptime(str(datetime.datetime.utcnow()).split(".")[0], Constants.UTC_DATETIME_FORMAT), #WRONG - "operation": str(operation), - "code": int(code), - "output": base64.b64encode(str(output)), - "delay": float(delay) - } - self.__real_record_handle.write('\n{0}'.format(json.dumps(record))) - except Exception: - print("EnvLayer: Unable to write real record to disk.") - - def __record_writer_init(self): - """ Initializes the record writer handle """ - self.__real_record_handle = open(self.__real_record_path, 'a+') - - def __read_record(self, expected_operation): - """ Returns code, output for a given operation if it matches """ - if self.__real_record_handle is None: - raise Exception("Invalid real record handle.") - - # Get single record - real_record_raw = self.__real_record_handle.readline().rstrip() - real_record = json.loads(real_record_raw) - - # Load data from record - timestamp = real_record['timestamp'] - operation = real_record['operation'] - code = int(real_record['code']) - output = base64.b64decode(real_record['output']) - delay = float(real_record['delay']) - print("Real record read: {0}: {1} >> code({2}) - output.len({3} - {4})".format(timestamp, operation, str(code), str(len(output)), str(self.__real_record_pointer+1))) - - # Verify operation - if real_record['operation'] != expected_operation: - raise Exception("Execution deviation detected. Add adaptations for operation expected: {0}. Operation data found for: {1}.".format(expected_operation, real_record['operation'])) - - # Advance and persist pointer - self.__real_record_pointer += 1 - with open(self.__real_record_pointer_path, 'w') as file_handle: - file_handle.write(str(self.__real_record_pointer)) - - # Return data - time.sleep(delay) - return code, output - - def __record_reader_init(self): - """ Seeks the real record pointer to the expected location """ - # Initialize record pointer - if not os.path.exists(self.__real_record_pointer_path): - self.__real_record_pointer = 0 - else: - with open(self.__real_record_pointer_path, 'r') as file_handle: - self.__real_record_pointer = int(file_handle.read().rstrip()) # no safety checks as there's no good recovery - - # Have the handle seek to the desired position - self.__real_record_handle = open(self.__real_record_pointer_path, 'r') - for x in range(1, self.__real_record_pointer): - self.__real_record_handle.readline() -# endregion - Core Emulator support functions - -# region - Legacy mode extensions - def set_legacy_test_mode(self): - print("Switching env layer to legacy test mode...") - self.datetime = self.DateTime(False, False, self.__write_record, self.__read_record) - self.file_system = self.FileSystem(False, False, self.__write_record, self.__read_record, emulator_root_path=os.path.dirname(self.__real_record_path)) -# endregion - Legacy mode extensions diff --git a/src/core/tests/library/RuntimeCompositor.py b/src/core/tests/library/RuntimeCompositor.py index 2d755a96..80ee720e 100644 --- a/src/core/tests/library/RuntimeCompositor.py +++ b/src/core/tests/library/RuntimeCompositor.py @@ -151,7 +151,6 @@ def set_legacy_test_type(self, test_type): def reconfigure_env_layer_to_legacy_mode(self): self.env_layer.get_package_manager = self.legacy_env_layer_extensions.get_package_manager self.env_layer.platform = self.legacy_env_layer_extensions.LegacyPlatform() - self.env_layer.set_legacy_test_mode() self.env_layer.run_command_output = self.legacy_env_layer_extensions.run_command_output if os.name == 'nt': self.env_layer.etc_environment_file_path = os.getcwd() From 632051155d0781c6c3a8ea1176272bcda5ed3a03 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Wed, 9 Apr 2025 15:35:19 -0700 Subject: [PATCH 02/16] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/core/src/bootstrap/EnvLayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 453bf5bb..5f128730 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -73,7 +73,7 @@ def get_package_manager(self): return ret def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): - """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is over-writen. """ + """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is overwritten. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) if environment_vars is None: From e985403f314183134d20c8860baac0f5a01b7608 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Wed, 9 Apr 2025 15:37:34 -0700 Subject: [PATCH 03/16] Correcting a static method --- src/core/src/bootstrap/EnvLayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 5f128730..5361cc12 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -360,7 +360,7 @@ def delete_from_dir(dir_name, identifier_list, raise_if_delete_failed=False): # region - DateTime emulation and extensions class DateTime(object): @staticmethod - def time(self): + def time(): return time.time() @staticmethod From cef3bb8a0d6abf7c9009218ecf178e7f3470d9c0 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:17:40 -0700 Subject: [PATCH 04/16] Additional cleanup --- src/core/src/bootstrap/Bootstrapper.py | 19 +--- .../src/bootstrap/ConfigurationFactory.py | 16 ++-- src/core/src/bootstrap/EnvLayer.py | 92 ++++++++----------- .../core_logic/ConfigurePatchingProcessor.py | 2 +- src/core/src/core_logic/PatchAssessor.py | 2 +- src/core/src/core_logic/PatchInstaller.py | 2 +- .../src/service_interfaces/StatusHandler.py | 4 +- .../src/service_interfaces/TelemetryWriter.py | 2 +- src/core/tests/Test_EnvLayer.py | 2 +- .../tests/library/LegacyEnvLayerExtensions.py | 6 +- 10 files changed, 58 insertions(+), 89 deletions(-) diff --git a/src/core/src/bootstrap/Bootstrapper.py b/src/core/src/bootstrap/Bootstrapper.py index 50d66b03..91d7875c 100644 --- a/src/core/src/bootstrap/Bootstrapper.py +++ b/src/core/src/bootstrap/Bootstrapper.py @@ -32,12 +32,11 @@ def __init__(self, argv, capture_stdout=True): self.current_env = self.get_current_env() self.argv = argv self.auto_assessment_only = bool(self.get_value_from_argv(self.argv, Constants.ARG_AUTO_ASSESS_ONLY, "False") == "True") - self.log_file_path, self.real_record_path, self.events_folder, self.telemetry_supported = self.get_path_to_log_files_and_telemetry_dir(argv, self.auto_assessment_only) - self.recorder_enabled, self.emulator_enabled = self.get_recorder_emulator_flags(argv) + self.log_file_path, self.events_folder, self.telemetry_supported = self.get_path_to_log_files_and_telemetry_dir(argv, self.auto_assessment_only) # Container initialization print("Building bootstrap container configuration...") - self.configuration_factory = ConfigurationFactory(self.log_file_path, self.real_record_path, self.recorder_enabled, self.emulator_enabled, self.events_folder, self.telemetry_supported) + self.configuration_factory = ConfigurationFactory(self.log_file_path, self.events_folder, self.telemetry_supported) self.container = Container() self.container.build(self.configuration_factory.get_bootstrap_configuration(self.current_env)) @@ -74,10 +73,9 @@ def get_path_to_log_files_and_telemetry_dir(self, argv, auto_assessment_only): log_folder = environment_settings[Constants.EnvSettings.LOG_FOLDER] # can throw exception and that's okay (since we can't recover from this) exec_demarcator = ".aa" if auto_assessment_only else "" log_file_path = os.path.join(log_folder, str(sequence_number) + exec_demarcator + ".core.log") - real_rec_path = os.path.join(log_folder, str(sequence_number) + exec_demarcator + ".core.rec") events_folder = environment_settings[Constants.EnvSettings.EVENTS_FOLDER] # can throw exception and that's okay (since we can't recover from this) telemetry_supported = environment_settings[Constants.EnvSettings.TELEMETRY_SUPPORTED] - return log_file_path, real_rec_path, events_folder, telemetry_supported + return log_file_path, events_folder, telemetry_supported def reset_auto_assessment_log_file_if_needed(self): """ Deletes the auto assessment log file when needed to prevent excessive growth """ @@ -87,17 +85,6 @@ def reset_auto_assessment_log_file_if_needed(self): except Exception as error: print("INFO: Error while checking/removing auto-assessment log file. [Path={0}][ExistsRecheck={1}]".format(self.log_file_path, str(os.path.exists(self.log_file_path)))) - def get_recorder_emulator_flags(self, argv): - """ Determines if the recorder or emulator flags need to be changed from the defaults """ - recorder_enabled = False - emulator_enabled = False - try: - recorder_enabled = bool(self.get_value_from_argv(argv, Constants.ARG_INTERNAL_RECORDER_ENABLED)) - emulator_enabled = bool(self.get_value_from_argv(argv, Constants.ARG_INTERNAL_EMULATOR_ENABLED)) - except Exception as error: - print("INFO: Default environment layer settings loaded.") - return recorder_enabled, emulator_enabled - @staticmethod def get_value_from_argv(argv, key, default_value=Constants.DEFAULT_UNSPECIFIED_VALUE): """ Discovers the value assigned to a given key based on the core contract on arguments """ diff --git a/src/core/src/bootstrap/ConfigurationFactory.py b/src/core/src/bootstrap/ConfigurationFactory.py index 6609d64c..ad308f31 100644 --- a/src/core/src/bootstrap/ConfigurationFactory.py +++ b/src/core/src/bootstrap/ConfigurationFactory.py @@ -58,14 +58,14 @@ class ConfigurationFactory(object): """ Class for generating module definitions. Configuration is list of key value pairs. Please DON'T change key name. DI container relies on the key name to find and resolve dependencies. If you do need change it, please make sure to update the key name in all places that reference it. """ - def __init__(self, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported): + def __init__(self, log_file_path, events_folder, telemetry_supported): self.vm_cloud_type = self.get_vm_cloud_type() self.lifecycle_manager_component = self.get_lifecycle_manager_component(self.vm_cloud_type) self.bootstrap_configurations = { - 'prod_config': self.new_bootstrap_configuration(Constants.PROD, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported), - 'dev_config': self.new_bootstrap_configuration(Constants.DEV, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported), - 'test_config': self.new_bootstrap_configuration(Constants.TEST, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported) + 'prod_config': self.new_bootstrap_configuration(Constants.PROD, log_file_path, events_folder, telemetry_supported), + 'dev_config': self.new_bootstrap_configuration(Constants.DEV, log_file_path, events_folder, telemetry_supported), + 'test_config': self.new_bootstrap_configuration(Constants.TEST, log_file_path, events_folder, telemetry_supported) } self.configurations = { @@ -127,18 +127,14 @@ def get_configuration(self, env, package_manager_name): # region - Configuration Builders @staticmethod - def new_bootstrap_configuration(config_env, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported): + def new_bootstrap_configuration(config_env, log_file_path, events_folder, telemetry_supported): """ Core configuration definition. """ configuration = { 'config_env': config_env, 'env_layer': { 'component': EnvLayer, 'component_args': [], - 'component_kwargs': { - 'real_record_path': real_record_path, - 'recorder_enabled': recorder_enabled, - 'emulator_enabled': emulator_enabled - } + 'component_kwargs': {} }, 'file_logger': { 'component': FileLogger, diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 5361cc12..ae06f65f 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -32,7 +32,7 @@ class EnvLayer(object): """ Environment related functions """ - def __init__(self, real_record_path=None, recorder_enabled=False, emulator_enabled=False): + def __init__(self): # Discrete components self.platform = self.Platform() self.datetime = self.DateTime() @@ -42,47 +42,39 @@ def __init__(self, real_record_path=None, recorder_enabled=False, emulator_enabl self.etc_environment_file_path = "/etc/environment" def get_package_manager(self): + # type: () -> str """ Detects package manager type """ - ret = None - if self.platform.linux_distribution()[0] == Constants.AZURE_LINUX: code, out = self.run_command_output('which tdnf', False, False) if code == 0: - ret = Constants.TDNF + return Constants.TDNF else: - print("Error: Expected package manager tdnf not found on this Azure Linux VM") - else: - # choose default - almost surely one will match. - for b in ('apt-get', 'yum', 'zypper'): - code, out = self.run_command_output('which ' + b, False, False) - if code == 0: - ret = b - if ret == 'apt-get': - ret = Constants.APT - break - if ret == 'yum': - ret = Constants.YUM - break - if ret == 'zypper': - ret = Constants.ZYPPER - break - - if ret is None and platform.system() == 'Windows': - ret = Constants.APT - - return ret - - def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): + print("Error: Expected package manager tdnf not found on this Azure Linux VM.") + return str() + + # choose default package manager + package_manager_map = (('apt-get', Constants.APT), + ('yum', Constants.YUM), + ('zypper', Constants.ZYPPER)) + for entry in package_manager_map: + code, out = self.run_command_output('which ' + entry[0], False, False) + if code == 0: + return entry[1] + + return str() if platform.system() != 'Windows' else Constants.APT + + def set_env_var(self, var_name, var_value=str(), raise_if_not_success=False): + # type: (str, str, bool) -> None """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is overwritten. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) if environment_vars is None: - print("Error occurred while setting environment variable: File not found. [Variable={0}] [Value={1}] [Path={2}]".format(str(var_name), str(var_value), self.etc_environment_file_path)) + print("Error occurred while setting environment variable: File not found. [Variable={0}][Value={1}][Path={2}]".format(str(var_name), str(var_value), self.etc_environment_file_path)) return environment_vars_lines = environment_vars.strip().split("\n") - if var_value is None: + if var_value is str(): # remove environment variable regex = re.compile('{0}=.+'.format(var_name)) search = regex.search(environment_vars) @@ -114,6 +106,7 @@ def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): raise def get_env_var(self, var_name, raise_if_not_success=False): + # type: (str, bool) -> None """ Returns the value of an environment variable with var_name in /etc/environment. Returns None if it does not exist. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) @@ -136,16 +129,11 @@ def get_env_var(self, var_name, raise_if_not_success=False): raise def run_command_output(self, cmd, no_output, chk_err=True): - """ - Wrapper for subprocess.check_output. Execute 'cmd'. - Returns return code and STDOUT, trapping expected exceptions. - Reports exceptions to Error if chk_err parameter is True - """ + # type: (str, bool, bool) -> (int, any) + """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected exceptions. Reports exceptions to Error if chk_err parameter is True """ def check_output(*popenargs, **kwargs): - """ - Backport from subprocess module from python 2.7 - """ + """ Backport from subprocess module from python 2.7 """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') @@ -189,17 +177,13 @@ def __str__(self): output = subprocess.check_output(no_output, cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: if chk_err: - print("Error: CalledProcessError. Error Code is: " + str(e.returncode), file=sys.stdout) - print("Error: CalledProcessError. Command string was: " + e.cmd, file=sys.stdout) - print("Error: CalledProcessError. Command result was: " + self.__convert_process_output_to_ascii(e.output[:-1]), file=sys.stdout) + print("Error: CalledProcessError. [Code={0}][Command={1}][Result={2}]".format(str(e.returncode), e.cmd, self.__convert_process_output_to_ascii(e.output[:-1])), file=sys.stdout) if no_output: return e.return_code, None else: return e.return_code, self.__convert_process_output_to_ascii(e.output) except Exception as error: - message = "Exception during cmd execution. [Exception={0}][Cmd={1}]".format(repr(error),str(cmd)) - print(message) - raise message + raise "Exception during cmd execution. [Exception={0}][Cmd={1}]".format(repr(error), str(cmd)) if no_output: return 0, None @@ -226,37 +210,39 @@ def exit(code): @staticmethod def get_python_major_version(): + # type: () -> int if hasattr(sys.version_info, 'major'): return sys.version_info.major else: return sys.version_info[0] # python 2.6 doesn't have attributes like 'major' within sys.version_info -# region - Platform emulation and extensions +# region - Platform extensions class Platform(object): @staticmethod def linux_distribution(): return platform.linux_distribution() if (EnvLayer.get_python_major_version() == 2) else distro.linux_distribution() @staticmethod - def system(): # OS Type + def os_type(): # OS Type return platform.system() @staticmethod - def machine(): # architecture + def cpu_arch(): # architecture return platform.machine() @staticmethod - def node(): # machine name + def vm_name(): # machine name return platform.node() -# endregion - Platform emulation and extensions +# endregion - Platform extensions -# region - File system emulation and extensions +# region - File system extensions class FileSystem(object): def __init__(self): # file-names of files that other processes may change the contents of self.__non_exclusive_files = [Constants.EXT_STATE_FILE] - def open(self, file_path, mode, raise_if_not_found=True): + @staticmethod + def open(file_path, mode, raise_if_not_found=True): """ Provides a file handle to the file_path requested using implicit redirection where required """ for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): try: @@ -302,7 +288,7 @@ def read_with_retry(self, file_path_or_handle, raise_if_not_found=True): return None def write_with_retry(self, file_path_or_handle, data, mode='a+'): - """ Writes to a given real/emulated file path in a single operation """ + """ Writes to a file path in a single operation """ file_handle, was_path = self.__obtain_file_handle(file_path_or_handle, mode) for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): @@ -355,9 +341,9 @@ def delete_from_dir(dir_name, identifier_list, raise_if_delete_failed=False): else: print(error_message) continue -# endregion - File system emulation and extensions +# endregion - File system extensions -# region - DateTime emulation and extensions +# region - DateTime extensions class DateTime(object): @staticmethod def time(): diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index a6d70004..2db851f0 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -40,7 +40,7 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ def start_configure_patching(self): """ Start configure patching """ try: - self.composite_logger.log("\nStarting configure patching... [MachineId: " + self.env_layer.platform.node() +"][ActivityId: " + self.execution_config.activity_id +"][StartTime: " + self.execution_config.start_time +"]") + self.composite_logger.log("\nStarting configure patching... [MachineId: " + self.env_layer.platform.vm_name() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "]") self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) self.__raise_if_telemetry_unsupported() diff --git a/src/core/src/core_logic/PatchAssessor.py b/src/core/src/core_logic/PatchAssessor.py index d8220333..6c7165a3 100644 --- a/src/core/src/core_logic/PatchAssessor.py +++ b/src/core/src/core_logic/PatchAssessor.py @@ -51,7 +51,7 @@ def start_assessment(self): self.lifecycle_manager.lifecycle_status_check() return True - self.composite_logger.log("\nStarting patch assessment... [MachineId: " + self.env_layer.platform.node() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "]") + self.composite_logger.log("\nStarting patch assessment... [MachineId: " + self.env_layer.platform.vm_name() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "]") self.write_assessment_state() # success / failure does not matter, only that an attempt started self.stopwatch.start() diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index 63f8771c..37e88d1f 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -62,7 +62,7 @@ def start_installation(self, simulate=False): self.stopwatch.start() - self.composite_logger.log("\nMachine Id: " + self.env_layer.platform.node()) + self.composite_logger.log("\nMachine Id: " + self.env_layer.platform.vm_name()) self.composite_logger.log("Activity Id: " + self.execution_config.activity_id) self.composite_logger.log("Operation request time: " + self.execution_config.start_time + ", Maintenance Window Duration: " + self.execution_config.duration) diff --git a/src/core/src/service_interfaces/StatusHandler.py b/src/core/src/service_interfaces/StatusHandler.py index 94c8286d..03a56f93 100644 --- a/src/core/src/service_interfaces/StatusHandler.py +++ b/src/core/src/service_interfaces/StatusHandler.py @@ -255,8 +255,8 @@ def __get_patch_id(self, package_name, package_version): def get_os_name_and_version(self): try: - if self.env_layer.platform.system() != "Linux": - raise Exception("Unsupported OS type: {0}.".format(self.env_layer.platform.system())) + if self.env_layer.platform.os_type() != "Linux": + raise Exception("Unsupported OS type: {0}.".format(self.env_layer.platform.os_type())) platform_info = self.env_layer.platform.linux_distribution() return "{0}_{1}".format(platform_info[0], platform_info[1]) except Exception as error: diff --git a/src/core/src/service_interfaces/TelemetryWriter.py b/src/core/src/service_interfaces/TelemetryWriter.py index 026512fe..024bda76 100644 --- a/src/core/src/service_interfaces/TelemetryWriter.py +++ b/src/core/src/service_interfaces/TelemetryWriter.py @@ -83,7 +83,7 @@ def set_and_write_machine_config_info(self): # Machine info - sent only once at the start of the run self.machine_info = "[PlatformName={0}][PlatformVersion={1}][MachineCpu={2}][MachineArch={3}][DiskType={4}]".format( str(self.env_layer.platform.linux_distribution()[0]), str(self.env_layer.platform.linux_distribution()[1]), - self.get_machine_processor(), str(self.env_layer.platform.machine()), self.get_disk_type()) + self.get_machine_processor(), str(self.env_layer.platform.cpu_arch()), self.get_disk_type()) self.write_event("Machine info is: {0}".format(self.machine_info), Constants.TelemetryEventLevel.Informational) def write_execution_error(self, cmd, code, output): diff --git a/src/core/tests/Test_EnvLayer.py b/src/core/tests/Test_EnvLayer.py index 54ad3eb4..0b23c427 100644 --- a/src/core/tests/Test_EnvLayer.py +++ b/src/core/tests/Test_EnvLayer.py @@ -67,7 +67,7 @@ def test_get_package_manager(self): test_input_output_table = [ [self.mock_run_command_for_apt, self.mock_linux_distribution, Constants.APT], [self.mock_run_command_for_tdnf, self.mock_linux_distribution_to_return_azure_linux, Constants.TDNF], - [self.mock_run_command_for_yum, self.mock_linux_distribution_to_return_azure_linux, None], # check for Azure Linux machine which does not have tdnf + [self.mock_run_command_for_yum, self.mock_linux_distribution_to_return_azure_linux, str()], # check for Azure Linux machine which does not have tdnf [self.mock_run_command_for_yum, self.mock_linux_distribution, Constants.YUM], [self.mock_run_command_for_zypper, self.mock_linux_distribution, Constants.ZYPPER], ] diff --git a/src/core/tests/library/LegacyEnvLayerExtensions.py b/src/core/tests/library/LegacyEnvLayerExtensions.py index 1a1cf24f..e2d75db3 100644 --- a/src/core/tests/library/LegacyEnvLayerExtensions.py +++ b/src/core/tests/library/LegacyEnvLayerExtensions.py @@ -29,15 +29,15 @@ def linux_distribution(self): return ['Ubuntu', '16.04', 'Xenial'] @staticmethod - def system(): # OS Type + def os_type(): # OS Type return 'Linux' @staticmethod - def machine(): # architecture + def cpu_arch(): # architecture return 'x86_64' @staticmethod - def node(): # machine name + def vm_name(): # machine name return 'LegacyTestVM' def get_package_manager(self): From eae8456d191de5300bcb9d6a8dbad29b051229be Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:30:14 -0700 Subject: [PATCH 05/16] Python 3.12 support in tests --- src/core/src/bootstrap/EnvLayer.py | 2 +- src/core/tests/Test_CoreMain.py | 4 ++-- src/core/tests/Test_UbuntuProClient.py | 8 ++++++-- src/extension/tests/Test_ExtEnvHandler.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index ae06f65f..3af623ad 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -106,7 +106,7 @@ def set_env_var(self, var_name, var_value=str(), raise_if_not_success=False): raise def get_env_var(self, var_name, raise_if_not_success=False): - # type: (str, bool) -> None + # type: (str, bool) -> any """ Returns the value of an environment variable with var_name in /etc/environment. Returns None if it does not exist. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) diff --git a/src/core/tests/Test_CoreMain.py b/src/core/tests/Test_CoreMain.py index daee38d3..e25f5e99 100644 --- a/src/core/tests/Test_CoreMain.py +++ b/src/core/tests/Test_CoreMain.py @@ -1004,8 +1004,8 @@ def test_auto_assessment_resets_maintenance_run_id_and_health_store_id_to_None(s CoreMain(argument_composer.get_composed_arguments()) # check values of health_store_id and maintenance_run_id - self.assertEquals(runtime.execution_config.health_store_id, None) - self.assertEquals(runtime.execution_config.maintenance_run_id, None) + self.assertEqual(runtime.execution_config.health_store_id, None) + self.assertEqual(runtime.execution_config.maintenance_run_id, None) # check telemetry events self.__check_telemetry_events(runtime) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index ed169b31..ba376683 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -13,10 +13,11 @@ # limitations under the License. # # Requires Python 2.7+ -import imp import sys import types import unittest +if sys.version_info[0] < 3: + import imp from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer @@ -66,7 +67,10 @@ def mock_import_uaclient_version_module(self, mock_name, method_name): mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.version.v1'], mock_name, mock_method) else: - version_module = imp.new_module('version_module') + if sys.version_info[0] < 3: + version_module = imp.new_module('version_module') + else: + version_module = types.ModuleType('version_module') mock_method = getattr(self, method_name) setattr(version_module, mock_name, mock_method) self.assign_sys_modules_with_mock_module('uaclient.api.u.pro.version.v1', version_module) diff --git a/src/extension/tests/Test_ExtEnvHandler.py b/src/extension/tests/Test_ExtEnvHandler.py index d650240d..bc8c1901 100644 --- a/src/extension/tests/Test_ExtEnvHandler.py +++ b/src/extension/tests/Test_ExtEnvHandler.py @@ -233,7 +233,7 @@ def test_get_temp_folder_success(self): temp_folder_path = ext_env_handler.get_temp_folder() # validate path - self.assertEquals(temp_folder_path, ext_env_handler.temp_folder) + self.assertEqual(temp_folder_path, ext_env_handler.temp_folder) shutil.rmtree(test_dir) From a3ef8e4b048a5be415ef6340c09a7f7b880d0ff4 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:33:16 -0700 Subject: [PATCH 06/16] Bugfix: Unit tests broken in Python 3.12 --- src/core/tests/Test_CoreMain.py | 4 ++-- src/core/tests/Test_UbuntuProClient.py | 9 +++++++-- src/extension/tests/Test_ExtEnvHandler.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/core/tests/Test_CoreMain.py b/src/core/tests/Test_CoreMain.py index daee38d3..e25f5e99 100644 --- a/src/core/tests/Test_CoreMain.py +++ b/src/core/tests/Test_CoreMain.py @@ -1004,8 +1004,8 @@ def test_auto_assessment_resets_maintenance_run_id_and_health_store_id_to_None(s CoreMain(argument_composer.get_composed_arguments()) # check values of health_store_id and maintenance_run_id - self.assertEquals(runtime.execution_config.health_store_id, None) - self.assertEquals(runtime.execution_config.maintenance_run_id, None) + self.assertEqual(runtime.execution_config.health_store_id, None) + self.assertEqual(runtime.execution_config.maintenance_run_id, None) # check telemetry events self.__check_telemetry_events(runtime) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index ed169b31..af86d648 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -13,10 +13,11 @@ # limitations under the License. # # Requires Python 2.7+ -import imp import sys import types import unittest +if sys.version_info[0] < 3: + import imp from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer @@ -66,7 +67,10 @@ def mock_import_uaclient_version_module(self, mock_name, method_name): mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.version.v1'], mock_name, mock_method) else: - version_module = imp.new_module('version_module') + if sys.version_info[0] < 3: + version_module = imp.new_module('version_module') + else: + version_module = types.ModuleType('version_module') mock_method = getattr(self, method_name) setattr(version_module, mock_name, mock_method) self.assign_sys_modules_with_mock_module('uaclient.api.u.pro.version.v1', version_module) @@ -404,5 +408,6 @@ def test_get_other_updates_exception(self): package_manager.ubuntu_pro_client.get_ubuntu_pro_client_updates = backup_get_ubuntu_pro_client_updates obj.mock_unimport_uaclient_update_module() + if __name__ == '__main__': unittest.main() diff --git a/src/extension/tests/Test_ExtEnvHandler.py b/src/extension/tests/Test_ExtEnvHandler.py index d650240d..bc8c1901 100644 --- a/src/extension/tests/Test_ExtEnvHandler.py +++ b/src/extension/tests/Test_ExtEnvHandler.py @@ -233,7 +233,7 @@ def test_get_temp_folder_success(self): temp_folder_path = ext_env_handler.get_temp_folder() # validate path - self.assertEquals(temp_folder_path, ext_env_handler.temp_folder) + self.assertEqual(temp_folder_path, ext_env_handler.temp_folder) shutil.rmtree(test_dir) From 05b5ccc6040c51ed94403a7172974136d0d1a136 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:43:04 -0700 Subject: [PATCH 07/16] Corrected code coverage --- src/core/tests/Test_UbuntuProClient.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index af86d648..5289c4c0 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -16,8 +16,6 @@ import sys import types import unittest -if sys.version_info[0] < 3: - import imp from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer @@ -66,11 +64,9 @@ def mock_import_uaclient_version_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.version.v1'] = types.ModuleType('version_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.version.v1'], mock_name, mock_method) - else: - if sys.version_info[0] < 3: - version_module = imp.new_module('version_module') - else: - version_module = types.ModuleType('version_module') + else: # Python 2 only + import imp + version_module = imp.new_module('version_module') mock_method = getattr(self, method_name) setattr(version_module, mock_name, mock_method) self.assign_sys_modules_with_mock_module('uaclient.api.u.pro.version.v1', version_module) From cafa35b82865fbd81ee47ce9b325a84b191fd055 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:45:54 -0700 Subject: [PATCH 08/16] Fixed code --- src/core/tests/Test_UbuntuProClient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index 5289c4c0..430a1a8e 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -404,6 +404,5 @@ def test_get_other_updates_exception(self): package_manager.ubuntu_pro_client.get_ubuntu_pro_client_updates = backup_get_ubuntu_pro_client_updates obj.mock_unimport_uaclient_update_module() - if __name__ == '__main__': unittest.main() From 867fef57d036a37a7e41ebb8f113cbfa8c0361ba Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:49:35 -0700 Subject: [PATCH 09/16] Merge UT fix --- src/core/tests/Test_UbuntuProClient.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index ba376683..37585b9a 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -16,8 +16,6 @@ import sys import types import unittest -if sys.version_info[0] < 3: - import imp from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer @@ -66,11 +64,9 @@ def mock_import_uaclient_version_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.version.v1'] = types.ModuleType('version_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.version.v1'], mock_name, mock_method) - else: - if sys.version_info[0] < 3: - version_module = imp.new_module('version_module') - else: - version_module = types.ModuleType('version_module') + else: # Python 2 only + import imp + version_module = imp.new_module('version_module') mock_method = getattr(self, method_name) setattr(version_module, mock_name, mock_method) self.assign_sys_modules_with_mock_module('uaclient.api.u.pro.version.v1', version_module) @@ -98,6 +94,7 @@ def mock_import_uaclient_reboot_required_module(self, mock_name, method_name): mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'], mock_name, mock_method) else: + import imp reboot_module = imp.new_module('reboot_module') mock_method = getattr(self, method_name) setattr(reboot_module, mock_name, mock_method) @@ -117,7 +114,7 @@ def __init__(self, package, version, provided_by, origin): class MockUpdatesResult(MockSystemModules): - def __init__(self, updates = []): + def __init__(self, updates=[]): self.updates = updates def mock_update_list_with_all_update_types(self): From 152c1e1beef54da2d842b7ebc9fd8759aec42f65 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 11:55:05 -0700 Subject: [PATCH 10/16] More UT fixes --- src/core/tests/Test_UbuntuProClient.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index 430a1a8e..c8653dd1 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -93,7 +93,8 @@ def mock_import_uaclient_reboot_required_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'] = types.ModuleType('reboot_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'], mock_name, mock_method) - else: + else: # Python 2 only + import imp reboot_module = imp.new_module('reboot_module') mock_method = getattr(self, method_name) setattr(reboot_module, mock_name, mock_method) @@ -113,7 +114,7 @@ def __init__(self, package, version, provided_by, origin): class MockUpdatesResult(MockSystemModules): - def __init__(self, updates = []): + def __init__(self, updates=[]): self.updates = updates def mock_update_list_with_all_update_types(self): @@ -138,6 +139,7 @@ def mock_import_uaclient_update_module(self, mock_name, method_name): mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.packages.updates.v1'], mock_name, mock_method) else: + import imp update_module = imp.new_module('update_module') mock_method = getattr(self, method_name) setattr(update_module, mock_name, mock_method) From fe529ac02a95b4a7956a1799d26a4f6789269179 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 12:04:15 -0700 Subject: [PATCH 11/16] Better fix without losing code coverage --- src/core/tests/Test_UbuntuProClient.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index c8653dd1..9865df00 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -16,6 +16,8 @@ import sys import types import unittest +if sys.version_info < (3, 12): + import imp from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer @@ -64,8 +66,7 @@ def mock_import_uaclient_version_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.version.v1'] = types.ModuleType('version_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.version.v1'], mock_name, mock_method) - else: # Python 2 only - import imp + else: version_module = imp.new_module('version_module') mock_method = getattr(self, method_name) setattr(version_module, mock_name, mock_method) @@ -93,8 +94,7 @@ def mock_import_uaclient_reboot_required_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'] = types.ModuleType('reboot_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'], mock_name, mock_method) - else: # Python 2 only - import imp + else: reboot_module = imp.new_module('reboot_module') mock_method = getattr(self, method_name) setattr(reboot_module, mock_name, mock_method) @@ -139,7 +139,6 @@ def mock_import_uaclient_update_module(self, mock_name, method_name): mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.packages.updates.v1'], mock_name, mock_method) else: - import imp update_module = imp.new_module('update_module') mock_method = getattr(self, method_name) setattr(update_module, mock_name, mock_method) From 72b099bff49138844f10cc9e83c309dfd054ba3f Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 14:03:32 -0700 Subject: [PATCH 12/16] Add coverage --- src/core/src/bootstrap/EnvLayer.py | 21 +++++++++-------- src/core/src/core_logic/PatchInstaller.py | 6 +---- .../src/service_interfaces/StatusHandler.py | 2 -- src/core/tests/Test_EnvLayer.py | 23 +++++++++++++++++++ 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 3af623ad..d95f3504 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -25,6 +25,8 @@ import sys import tempfile import time +from xmlrpc.client import DateTime + from core.src.bootstrap.Constants import Constants from core.src.external_dependencies import distro @@ -44,6 +46,9 @@ def __init__(self): def get_package_manager(self): # type: () -> str """ Detects package manager type """ + if platform.system() == 'Windows': + return Constants.APT + if self.platform.linux_distribution()[0] == Constants.AZURE_LINUX: code, out = self.run_command_output('which tdnf', False, False) if code == 0: @@ -61,7 +66,7 @@ def get_package_manager(self): if code == 0: return entry[1] - return str() if platform.system() != 'Windows' else Constants.APT + return str() def set_env_var(self, var_name, var_value=str(), raise_if_not_success=False): # type: (str, str, bool) -> None @@ -345,17 +350,15 @@ def delete_from_dir(dir_name, identifier_list, raise_if_delete_failed=False): # region - DateTime extensions class DateTime(object): - @staticmethod - def time(): - return time.time() - @staticmethod def datetime_utcnow(): - return datetime.datetime.utcnow() + if sys.version_info < (3, 12): + return datetime.datetime.utcnow() + else: + return datetime.datetime.now(datetime.timezone.utc) - @staticmethod - def timestamp(): - return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + def timestamp(self): + return self.datetime_utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") @staticmethod def total_minutes_from_time_delta(time_delta): diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index 37e88d1f..5834bc95 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -58,14 +58,10 @@ def start_installation(self, simulate=False): self.raise_if_telemetry_unsupported() self.raise_if_min_python_version_not_met() - self.composite_logger.log('\nStarting patch installation...') + self.composite_logger.log("\nStarting patch installation... [MachineId: " + self.env_layer.platform.vm_name() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "][MaintenanceWindowDuration: " + self.execution_config.duration + "]") self.stopwatch.start() - self.composite_logger.log("\nMachine Id: " + self.env_layer.platform.vm_name()) - self.composite_logger.log("Activity Id: " + self.execution_config.activity_id) - self.composite_logger.log("Operation request time: " + self.execution_config.start_time + ", Maintenance Window Duration: " + self.execution_config.duration) - maintenance_window = self.maintenance_window package_manager = self.package_manager reboot_manager = self.reboot_manager diff --git a/src/core/src/service_interfaces/StatusHandler.py b/src/core/src/service_interfaces/StatusHandler.py index 03a56f93..5b34cd6a 100644 --- a/src/core/src/service_interfaces/StatusHandler.py +++ b/src/core/src/service_interfaces/StatusHandler.py @@ -255,8 +255,6 @@ def __get_patch_id(self, package_name, package_version): def get_os_name_and_version(self): try: - if self.env_layer.platform.os_type() != "Linux": - raise Exception("Unsupported OS type: {0}.".format(self.env_layer.platform.os_type())) platform_info = self.env_layer.platform.linux_distribution() return "{0}_{1}".format(platform_info[0], platform_info[1]) except Exception as error: diff --git a/src/core/tests/Test_EnvLayer.py b/src/core/tests/Test_EnvLayer.py index 0b23c427..3c6ae3c9 100644 --- a/src/core/tests/Test_EnvLayer.py +++ b/src/core/tests/Test_EnvLayer.py @@ -30,6 +30,9 @@ def tearDown(self): def mock_platform_system(self): return 'Linux' + def mock_platform_system_windows(self): + return 'Windows' + def mock_linux_distribution(self): return ['test', 'test', 'test'] @@ -70,6 +73,7 @@ def test_get_package_manager(self): [self.mock_run_command_for_yum, self.mock_linux_distribution_to_return_azure_linux, str()], # check for Azure Linux machine which does not have tdnf [self.mock_run_command_for_yum, self.mock_linux_distribution, Constants.YUM], [self.mock_run_command_for_zypper, self.mock_linux_distribution, Constants.ZYPPER], + [lambda cmd, no_output, chk_err: (-1, ''), self.mock_linux_distribution, str()], # no matches for any package manager ] for row in test_input_output_table: @@ -78,10 +82,29 @@ def test_get_package_manager(self): package_manager = self.envlayer.get_package_manager() self.assertTrue(package_manager is row[2]) + # test for Windows + platform.system = self.mock_platform_system_windows + self.assertEqual(self.envlayer.get_package_manager(), Constants.APT) + + # restore original methods self.envlayer.run_command_output = self.backup_run_command_output self.envlayer.platform.linux_distribution = self.backup_linux_distribution platform.system = self.backup_platform_system + def test_filesystem(self): + # only validates if these invocable without exceptions + backup_retry_count = Constants.MAX_FILE_OPERATION_RETRY_COUNT + Constants.MAX_FILE_OPERATION_RETRY_COUNT = 1 + self.envlayer.file_system.read_with_retry("fake.path", raise_if_not_found=False) + Constants.MAX_FILE_OPERATION_RETRY_COUNT = backup_retry_count + + def test_platform(self): + # only validates if these invocable without exceptions + self.envlayer.platform.linux_distribution() + self.envlayer.platform.os_type() + self.envlayer.platform.cpu_arch() + self.envlayer.platform.vm_name() + if __name__ == '__main__': unittest.main() \ No newline at end of file From 21a24473c19b946cf2b4f3563d09cc87d18ce461 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 15:07:30 -0700 Subject: [PATCH 13/16] Additional coverage --- src/core/src/bootstrap/EnvLayer.py | 1 - src/core/tests/Test_EnvLayer.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index d95f3504..c9cc86e4 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -25,7 +25,6 @@ import sys import tempfile import time -from xmlrpc.client import DateTime from core.src.bootstrap.Constants import Constants from core.src.external_dependencies import distro diff --git a/src/core/tests/Test_EnvLayer.py b/src/core/tests/Test_EnvLayer.py index 3c6ae3c9..25862a85 100644 --- a/src/core/tests/Test_EnvLayer.py +++ b/src/core/tests/Test_EnvLayer.py @@ -61,7 +61,7 @@ def mock_run_command_for_tdnf(self, cmd, no_output=False, chk_err=False): # endregion def test_get_package_manager(self): - self.backup_platform_system = platform.system() + self.backup_platform_system = platform.system platform.system = self.mock_platform_system self.backup_linux_distribution = self.envlayer.platform.linux_distribution self.envlayer.platform.linux_distribution = self.mock_linux_distribution @@ -94,7 +94,7 @@ def test_get_package_manager(self): def test_filesystem(self): # only validates if these invocable without exceptions backup_retry_count = Constants.MAX_FILE_OPERATION_RETRY_COUNT - Constants.MAX_FILE_OPERATION_RETRY_COUNT = 1 + Constants.MAX_FILE_OPERATION_RETRY_COUNT = 2 self.envlayer.file_system.read_with_retry("fake.path", raise_if_not_found=False) Constants.MAX_FILE_OPERATION_RETRY_COUNT = backup_retry_count From 701a68735ab28b7c6be3050dc43c7c965c8e2636 Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 15:51:15 -0700 Subject: [PATCH 14/16] Rolling back utcnow --- src/core/src/bootstrap/EnvLayer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index c9cc86e4..642a29ee 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -349,15 +349,17 @@ def delete_from_dir(dir_name, identifier_list, raise_if_delete_failed=False): # region - DateTime extensions class DateTime(object): + @staticmethod + def time(): + return time.time() + @staticmethod def datetime_utcnow(): - if sys.version_info < (3, 12): - return datetime.datetime.utcnow() - else: - return datetime.datetime.now(datetime.timezone.utc) + return datetime.datetime.utcnow() - def timestamp(self): - return self.datetime_utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + @staticmethod + def timestamp(): + return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") @staticmethod def total_minutes_from_time_delta(time_delta): From ee674a316123f8dc809de74573dce9fd6910850a Mon Sep 17 00:00:00 2001 From: Koshy John Date: Fri, 11 Apr 2025 16:12:10 -0700 Subject: [PATCH 15/16] Incorrect merge --- src/core/tests/Test_UbuntuProClient.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/core/tests/Test_UbuntuProClient.py b/src/core/tests/Test_UbuntuProClient.py index c7080047..9865df00 100644 --- a/src/core/tests/Test_UbuntuProClient.py +++ b/src/core/tests/Test_UbuntuProClient.py @@ -66,8 +66,7 @@ def mock_import_uaclient_version_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.version.v1'] = types.ModuleType('version_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.version.v1'], mock_name, mock_method) - else: # Python 2 only - import imp + else: version_module = imp.new_module('version_module') mock_method = getattr(self, method_name) setattr(version_module, mock_name, mock_method) @@ -95,8 +94,7 @@ def mock_import_uaclient_reboot_required_module(self, mock_name, method_name): sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'] = types.ModuleType('reboot_module') mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.security.status.reboot_required.v1'], mock_name, mock_method) - else: # Python 2 only - import imp + else: reboot_module = imp.new_module('reboot_module') mock_method = getattr(self, method_name) setattr(reboot_module, mock_name, mock_method) @@ -141,7 +139,6 @@ def mock_import_uaclient_update_module(self, mock_name, method_name): mock_method = getattr(self, method_name) setattr(sys.modules['uaclient.api.u.pro.packages.updates.v1'], mock_name, mock_method) else: - import imp update_module = imp.new_module('update_module') mock_method = getattr(self, method_name) setattr(update_module, mock_name, mock_method) From ab6139560430920447d1ae3a1a3d3d31c9ec017a Mon Sep 17 00:00:00 2001 From: Koshy John Date: Thu, 1 May 2025 09:00:20 -0700 Subject: [PATCH 16/16] Additional env var set check --- src/core/src/bootstrap/EnvLayer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 5aeabe6f..79b1803c 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -82,7 +82,7 @@ def set_env_var(self, var_name, var_value=str(), raise_if_not_success=False): environment_vars_lines = environment_vars.strip().split("\n") - if var_value is str(): + if var_value is None or var_value == str(): # remove environment variable regex = re.compile('{0}=.+'.format(var_name)) search = regex.search(environment_vars)