diff --git a/src/core/src/bootstrap/Bootstrapper.py b/src/core/src/bootstrap/Bootstrapper.py index 50d66b03..91d7875c 100644 --- a/src/core/src/bootstrap/Bootstrapper.py +++ b/src/core/src/bootstrap/Bootstrapper.py @@ -32,12 +32,11 @@ def __init__(self, argv, capture_stdout=True): self.current_env = self.get_current_env() self.argv = argv self.auto_assessment_only = bool(self.get_value_from_argv(self.argv, Constants.ARG_AUTO_ASSESS_ONLY, "False") == "True") - self.log_file_path, self.real_record_path, self.events_folder, self.telemetry_supported = self.get_path_to_log_files_and_telemetry_dir(argv, self.auto_assessment_only) - self.recorder_enabled, self.emulator_enabled = self.get_recorder_emulator_flags(argv) + self.log_file_path, self.events_folder, self.telemetry_supported = self.get_path_to_log_files_and_telemetry_dir(argv, self.auto_assessment_only) # Container initialization print("Building bootstrap container configuration...") - self.configuration_factory = ConfigurationFactory(self.log_file_path, self.real_record_path, self.recorder_enabled, self.emulator_enabled, self.events_folder, self.telemetry_supported) + self.configuration_factory = ConfigurationFactory(self.log_file_path, self.events_folder, self.telemetry_supported) self.container = Container() self.container.build(self.configuration_factory.get_bootstrap_configuration(self.current_env)) @@ -74,10 +73,9 @@ def get_path_to_log_files_and_telemetry_dir(self, argv, auto_assessment_only): log_folder = environment_settings[Constants.EnvSettings.LOG_FOLDER] # can throw exception and that's okay (since we can't recover from this) exec_demarcator = ".aa" if auto_assessment_only else "" log_file_path = os.path.join(log_folder, str(sequence_number) + exec_demarcator + ".core.log") - real_rec_path = os.path.join(log_folder, str(sequence_number) + exec_demarcator + ".core.rec") events_folder = environment_settings[Constants.EnvSettings.EVENTS_FOLDER] # can throw exception and that's okay (since we can't recover from this) telemetry_supported = environment_settings[Constants.EnvSettings.TELEMETRY_SUPPORTED] - return log_file_path, real_rec_path, events_folder, telemetry_supported + return log_file_path, events_folder, telemetry_supported def reset_auto_assessment_log_file_if_needed(self): """ Deletes the auto assessment log file when needed to prevent excessive growth """ @@ -87,17 +85,6 @@ def reset_auto_assessment_log_file_if_needed(self): except Exception as error: print("INFO: Error while checking/removing auto-assessment log file. [Path={0}][ExistsRecheck={1}]".format(self.log_file_path, str(os.path.exists(self.log_file_path)))) - def get_recorder_emulator_flags(self, argv): - """ Determines if the recorder or emulator flags need to be changed from the defaults """ - recorder_enabled = False - emulator_enabled = False - try: - recorder_enabled = bool(self.get_value_from_argv(argv, Constants.ARG_INTERNAL_RECORDER_ENABLED)) - emulator_enabled = bool(self.get_value_from_argv(argv, Constants.ARG_INTERNAL_EMULATOR_ENABLED)) - except Exception as error: - print("INFO: Default environment layer settings loaded.") - return recorder_enabled, emulator_enabled - @staticmethod def get_value_from_argv(argv, key, default_value=Constants.DEFAULT_UNSPECIFIED_VALUE): """ Discovers the value assigned to a given key based on the core contract on arguments """ diff --git a/src/core/src/bootstrap/ConfigurationFactory.py b/src/core/src/bootstrap/ConfigurationFactory.py index 6609d64c..ad308f31 100644 --- a/src/core/src/bootstrap/ConfigurationFactory.py +++ b/src/core/src/bootstrap/ConfigurationFactory.py @@ -58,14 +58,14 @@ class ConfigurationFactory(object): """ Class for generating module definitions. Configuration is list of key value pairs. Please DON'T change key name. DI container relies on the key name to find and resolve dependencies. If you do need change it, please make sure to update the key name in all places that reference it. """ - def __init__(self, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported): + def __init__(self, log_file_path, events_folder, telemetry_supported): self.vm_cloud_type = self.get_vm_cloud_type() self.lifecycle_manager_component = self.get_lifecycle_manager_component(self.vm_cloud_type) self.bootstrap_configurations = { - 'prod_config': self.new_bootstrap_configuration(Constants.PROD, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported), - 'dev_config': self.new_bootstrap_configuration(Constants.DEV, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported), - 'test_config': self.new_bootstrap_configuration(Constants.TEST, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported) + 'prod_config': self.new_bootstrap_configuration(Constants.PROD, log_file_path, events_folder, telemetry_supported), + 'dev_config': self.new_bootstrap_configuration(Constants.DEV, log_file_path, events_folder, telemetry_supported), + 'test_config': self.new_bootstrap_configuration(Constants.TEST, log_file_path, events_folder, telemetry_supported) } self.configurations = { @@ -127,18 +127,14 @@ def get_configuration(self, env, package_manager_name): # region - Configuration Builders @staticmethod - def new_bootstrap_configuration(config_env, log_file_path, real_record_path, recorder_enabled, emulator_enabled, events_folder, telemetry_supported): + def new_bootstrap_configuration(config_env, log_file_path, events_folder, telemetry_supported): """ Core configuration definition. """ configuration = { 'config_env': config_env, 'env_layer': { 'component': EnvLayer, 'component_args': [], - 'component_kwargs': { - 'real_record_path': real_record_path, - 'recorder_enabled': recorder_enabled, - 'emulator_enabled': emulator_enabled - } + 'component_kwargs': {} }, 'file_logger': { 'component': FileLogger, diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index 1b351d92..79b1803c 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -15,10 +15,8 @@ # Requires Python 2.7+ from __future__ import print_function -import base64 import datetime import glob -import json import os import re import platform @@ -27,6 +25,7 @@ import sys import tempfile import time + from core.src.bootstrap.Constants import Constants from core.src.external_dependencies import distro @@ -34,28 +33,11 @@ class EnvLayer(object): """ Environment related functions """ - def __init__(self, real_record_path=None, recorder_enabled=False, emulator_enabled=False): - # Recorder / emulator storage - self.__real_record_path = real_record_path - self.__real_record_pointer_path = real_record_path + ".pt" if real_record_path is not None else None - self.__real_record_handle = None - self.__real_record_pointer = 0 - - # Recorder / emulator state section - self.__recorder_enabled = recorder_enabled # dumps black box recordings - self.__emulator_enabled = False if recorder_enabled else emulator_enabled # only one can be enabled at a time - - # Recorder / emulator initialization - if self.__recorder_enabled: - self.__record_writer_init() - elif self.__emulator_enabled: - self.__record_reader_init() - + def __init__(self): # Discrete components - self.platform = self.Platform(recorder_enabled, emulator_enabled, self.__write_record, self.__read_record) - self.datetime = self.DateTime(recorder_enabled, emulator_enabled, self.__write_record, self.__read_record) - self.file_system = self.FileSystem(recorder_enabled, emulator_enabled, self.__write_record, self.__read_record, - emulator_root_path=os.path.dirname(self.__real_record_path) if self.__real_record_path is not None else self.__real_record_path) + self.platform = self.Platform() + self.datetime = self.DateTime() + self.file_system = self.FileSystem() # Constant paths self.etc_environment_file_path = "/etc/environment" @@ -65,47 +47,42 @@ def is_distro_azure_linux(distro_name): return any(x in distro_name for x in Constants.AZURE_LINUX) def get_package_manager(self): + # type: () -> str """ Detects package manager type """ - ret = None + if platform.system() == 'Windows': + return Constants.APT if self.is_distro_azure_linux(str(self.platform.linux_distribution())): code, out = self.run_command_output('which tdnf', False, False) if code == 0: - ret = Constants.TDNF + return Constants.TDNF else: - print("Error: Expected package manager tdnf not found on this Azure Linux VM") - else: - # choose default - almost surely one will match. - for b in ('apt-get', 'yum', 'zypper'): - code, out = self.run_command_output('which ' + b, False, False) - if code == 0: - ret = b - if ret == 'apt-get': - ret = Constants.APT - break - if ret == 'yum': - ret = Constants.YUM - break - if ret == 'zypper': - ret = Constants.ZYPPER - break - - if ret is None and platform.system() == 'Windows': - ret = Constants.APT - - return ret - - def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): - """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is overwriten. """ + print("Error: Expected package manager tdnf not found on this Azure Linux VM.") + return str() + + # choose default package manager + package_manager_map = (('apt-get', Constants.APT), + ('yum', Constants.YUM), + ('zypper', Constants.ZYPPER)) + for entry in package_manager_map: + code, out = self.run_command_output('which ' + entry[0], False, False) + if code == 0: + return entry[1] + + return str() + + def set_env_var(self, var_name, var_value=str(), raise_if_not_success=False): + # type: (str, str, bool) -> None + """ Sets an environment variable with var_name and var_value in /etc/environment. If it already exists, it is overwritten. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) if environment_vars is None: - print("Error occurred while setting environment variable: File not found. [Variable={0}] [Value={1}] [Path={2}]".format(str(var_name), str(var_value), self.etc_environment_file_path)) + print("Error occurred while setting environment variable: File not found. [Variable={0}][Value={1}][Path={2}]".format(str(var_name), str(var_value), self.etc_environment_file_path)) return environment_vars_lines = environment_vars.strip().split("\n") - if var_value is None: + if var_value is None or var_value == str(): # remove environment variable regex = re.compile('{0}=.+'.format(var_name)) search = regex.search(environment_vars) @@ -132,16 +109,17 @@ def set_env_var(self, var_name, var_value=None, raise_if_not_success=False): self.file_system.write_with_retry(self.etc_environment_file_path, environment_vars, 'w') except Exception as error: - print("Error occurred while setting environment variable [Variable={0}] [Value={1}] [Exception={2}]".format(str(var_name), str(var_value), repr(error))) + print("Error occurred while setting environment variable [Variable={0}][Value={1}][Exception={2}]".format(str(var_name), str(var_value), repr(error))) if raise_if_not_success: raise def get_env_var(self, var_name, raise_if_not_success=False): + # type: (str, bool) -> any """ Returns the value of an environment variable with var_name in /etc/environment. Returns None if it does not exist. """ try: environment_vars = self.file_system.read_with_retry(self.etc_environment_file_path) if environment_vars is None: - print("Error occurred while getting environment variable: File not found. [Variable={0}] [Path={1}]".format(str(var_name), self.etc_environment_file_path)) + print("Error occurred while getting environment variable: File not found. [Variable={0}][Path={1}]".format(str(var_name), self.etc_environment_file_path)) return None # get specific environment variable value @@ -154,31 +132,16 @@ def get_env_var(self, var_name, raise_if_not_success=False): return group[group.index("=")+1:] except Exception as error: - print("Error occurred while getting environment variable [Variable={0}] [Exception={1}]".format(str(var_name), repr(error))) + print("Error occurred while getting environment variable [Variable={0}][Exception={1}]".format(str(var_name), repr(error))) if raise_if_not_success: raise - def run_command_output(self, cmd, no_output=False, chk_err=False): - operation = "RUN_CMD_OUT" - if not self.__emulator_enabled: - start = time.time() - code, output = self.__run_command_output_raw(cmd, no_output, chk_err) - self.__write_record(operation, code, output, delay=(time.time()-start)) - return code, output - else: - return self.__read_record(operation) - - def __run_command_output_raw(self, cmd, no_output, chk_err=True): - """ - Wrapper for subprocess.check_output. Execute 'cmd'. - Returns return code and STDOUT, trapping expected exceptions. - Reports exceptions to Error if chk_err parameter is True - """ + def run_command_output(self, cmd, no_output, chk_err=True): + # type: (str, bool, bool) -> (int, any) + """ Wrapper for subprocess.check_output. Execute 'cmd'. Returns return code and STDOUT, trapping expected exceptions. Reports exceptions to Error if chk_err parameter is True """ def check_output(*popenargs, **kwargs): - """ - Backport from subprocess module from python 2.7 - """ + """ Backport from subprocess module from python 2.7 """ if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it will be overridden.') @@ -222,17 +185,13 @@ def __str__(self): output = subprocess.check_output(no_output, cmd, stderr=subprocess.STDOUT, shell=True) except subprocess.CalledProcessError as e: if chk_err: - print("Error: CalledProcessError. Error Code is: " + str(e.returncode), file=sys.stdout) - print("Error: CalledProcessError. Command string was: " + e.cmd, file=sys.stdout) - print("Error: CalledProcessError. Command result was: " + self.__convert_process_output_to_ascii(e.output[:-1]), file=sys.stdout) + print("Error: CalledProcessError. [Code={0}][Command={1}][Result={2}]".format(str(e.returncode), e.cmd, self.__convert_process_output_to_ascii(e.output[:-1])), file=sys.stdout) if no_output: return e.return_code, None else: return e.return_code, self.__convert_process_output_to_ascii(e.output) except Exception as error: - message = "Exception during cmd execution. [Exception={0}][Cmd={1}]".format(repr(error),str(cmd)) - print(message) - raise message + raise "Exception during cmd execution. [Exception={0}][Cmd={1}]".format(repr(error), str(cmd)) if no_output: return 0, None @@ -249,121 +208,58 @@ def __convert_process_output_to_ascii(output): else: raise Exception("Unknown version of python encountered.") - def reboot_machine(self, reboot_cmd): - operation = "REBOOT_MACHINE" - if not self.__emulator_enabled: - self.__write_record(operation, 0, '', delay=0) - subprocess.Popen(reboot_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - self.__read_record(operation) # will throw if it's not the expected operation - raise Exception(Constants.EnvLayer.PRIVILEGED_OP_REBOOT) - - def exit(self, code): - operation = "EXIT_EXECUTION" - if not self.__emulator_enabled: - self.__write_record(operation, code, '', delay=0) - exit(code) - else: - self.__read_record(operation) # will throw if it's not the expected operation - raise Exception(Constants.EnvLayer.PRIVILEGED_OP_EXIT + str(code)) + @staticmethod + def reboot_machine(reboot_cmd): + subprocess.Popen(reboot_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + @staticmethod + def exit(code): + exit(code) @staticmethod def get_python_major_version(): + # type: () -> int if hasattr(sys.version_info, 'major'): return sys.version_info.major else: return sys.version_info[0] # python 2.6 doesn't have attributes like 'major' within sys.version_info -# region - Platform emulation and extensions +# region - Platform extensions class Platform(object): - def __init__(self, recorder_enabled=True, emulator_enabled=False, write_record_delegate=None, read_record_delegate=None): - self.__recorder_enabled = recorder_enabled - self.__emulator_enabled = False if recorder_enabled else emulator_enabled - self.__write_record = write_record_delegate - self.__read_record = read_record_delegate - - def linux_distribution(self): - operation = "PLATFORM_LINUX_DISTRIBUTION" - if not self.__emulator_enabled: - major_version = EnvLayer.get_python_major_version() - - if major_version == 2: - value = platform.linux_distribution() - else: - value = distro.linux_distribution() - - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return eval(output) - - def system(self): # OS Type - operation = "PLATFORM_SYSTEM" - if not self.__emulator_enabled: - value = platform.system() - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return output - - def machine(self): # architecture - operation = "PLATFORM_MACHINE" - if not self.__emulator_enabled: - value = platform.machine() - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return output - - def node(self): # machine name - operation = "PLATFORM_NODE" - if not self.__emulator_enabled: - value = platform.node() - if self.__recorder_enabled: - self.__write_record(operation, code=0, output=str(value)) - return value - else: - code, output = self.__read_record(operation) - return output -# endregion - Platform emulation and extensions + @staticmethod + def linux_distribution(): + return platform.linux_distribution() if (EnvLayer.get_python_major_version() == 2) else distro.linux_distribution() -# region - File system emulation and extensions + @staticmethod + def os_type(): # OS Type + return platform.system() + + @staticmethod + def cpu_arch(): # architecture + return platform.machine() + + @staticmethod + def vm_name(): # machine name + return platform.node() +# endregion - Platform extensions + +# region - File system extensions class FileSystem(object): - def __init__(self, recorder_enabled=True, emulator_enabled=False, write_record_delegate=None, read_record_delegate=None, emulator_root_path=None): - self.__recorder_enabled = recorder_enabled - self.__emulator_enabled = False if recorder_enabled else emulator_enabled - self.__write_record = write_record_delegate - self.__read_record = read_record_delegate - self.__emulator_enabled = emulator_enabled - self.__emulator_root_path = emulator_root_path - - # file-names of files that other processes may changes the contents of + def __init__(self): + # file-names of files that other processes may change the contents of self.__non_exclusive_files = [Constants.EXT_STATE_FILE] - def resolve_path(self, requested_path): - """ Resolves any paths used with desired file system paths """ - if self.__emulator_enabled and self.__emulator_root_path is not None and self.__emulator_root_path not in requested_path: - return os.path.join(self.__emulator_root_path, os.path.normpath(requested_path)) - else: - return requested_path - - def open(self, file_path, mode, raise_if_not_found=True): + @staticmethod + def open(file_path, mode, raise_if_not_found=True): """ Provides a file handle to the file_path requested using implicit redirection where required """ - real_path = self.resolve_path(file_path) for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): try: - return open(real_path, mode) + return open(file_path, mode) except Exception as error: if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: time.sleep(i + 1) else: - error_message = "Unable to open file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(real_path), repr(error), str(raise_if_not_found)) + error_message = "Unable to open file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path), repr(error), str(raise_if_not_found)) if raise_if_not_found: raise Exception(error_message) else: @@ -381,34 +277,26 @@ def __obtain_file_handle(self, file_path_or_handle, mode='a+', raise_if_not_foun def read_with_retry(self, file_path_or_handle, raise_if_not_found=True): """ Reads all content from a given file path in a single operation """ - operation = "FILE_READ" - - # only fully emulate non_exclusive_files from the real recording; exclusive files can be redirected and handled in emulator scenarios - if not self.__emulator_enabled or (isinstance(file_path_or_handle, str) and os.path.basename(file_path_or_handle) not in self.__non_exclusive_files): - file_handle, was_path = self.__obtain_file_handle(file_path_or_handle, 'r', raise_if_not_found) - for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): - try: - value = file_handle.read() - if was_path: # what was passed in was not a file handle, so close the handle that was init here - file_handle.close() - self.__write_record(operation, code=0, output=value, delay=0) - return value - except Exception as error: - if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: - time.sleep(i + 1) + file_handle, was_path = self.__obtain_file_handle(file_path_or_handle, 'r', raise_if_not_found) + for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): + try: + value = file_handle.read() + if was_path: # what was passed in was not a file handle, so close the handle that was init here + file_handle.close() + return value + except Exception as error: + if i < Constants.MAX_FILE_OPERATION_RETRY_COUNT - 1: + time.sleep(i + 1) + else: + error_message = "Unable to read file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path_or_handle), repr(error), str(raise_if_not_found)) + if raise_if_not_found: + raise Exception(error_message) else: - error_message = "Unable to read file (retries exhausted). [File={0}][Error={1}][RaiseIfNotFound={2}].".format(str(file_path_or_handle), repr(error), str(raise_if_not_found)) - if raise_if_not_found: - raise Exception(error_message) - else: - print(error_message) - return None - else: - code, output = self.__read_record(operation) - return output + print(error_message) + return None def write_with_retry(self, file_path_or_handle, data, mode='a+'): - """ Writes to a given real/emulated file path in a single operation """ + """ Writes to a file path in a single operation """ file_handle, was_path = self.__obtain_file_handle(file_path_or_handle, mode) for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): @@ -461,49 +349,22 @@ def delete_from_dir(dir_name, identifier_list, raise_if_delete_failed=False): else: print(error_message) continue -# endregion - File system emulation and extensions +# endregion - File system extensions -# region - DateTime emulation and extensions +# region - DateTime extensions class DateTime(object): - def __init__(self, recorder_enabled=True, emulator_enabled=False, write_record_delegate=None, read_record_delegate=None): - self.__recorder_enabled = recorder_enabled - self.__emulator_enabled = False if recorder_enabled else emulator_enabled - self.__write_record = write_record_delegate - self.__read_record = read_record_delegate - - def time(self): - operation = "DATETIME_TIME" - if not self.__emulator_enabled: - value = time.time() - self.__write_record(operation, code=0, output=value, delay=0) - return value - else: - code, output = self.__read_record(operation) - return int(output) - - def datetime_utcnow(self): - operation = "DATETIME_UTCNOW" - if not self.__emulator_enabled: - value = datetime.datetime.utcnow() - self.__write_record(operation, code=0, output=str(value), delay=0) - return value - else: - code, output = self.__read_record(operation) - return datetime.datetime.strptime(str(output), "%Y-%m-%d %H:%M:%S.%f") - - def timestamp(self): - operation = "DATETIME_TIMESTAMP" - if not self.__emulator_enabled: - value = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - self.__write_record(operation, code=0, output=value, delay=0) - return value - else: - code, output = self.__read_record(operation) - return output + @staticmethod + def time(): + return time.time() + + @staticmethod + def datetime_utcnow(): + return datetime.datetime.utcnow() + + @staticmethod + def timestamp(): + return datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - # -------------------------------------------------------------------------------------------------------------- - # Static library functions - # -------------------------------------------------------------------------------------------------------------- @staticmethod def total_minutes_from_time_delta(time_delta): return ((time_delta.microseconds + (time_delta.seconds + time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6) / 60 @@ -535,77 +396,3 @@ def standard_datetime_to_utc(std_datetime): """ Converts datetime object to string of format '"%Y-%m-%dT%H:%M:%SZ"' """ return std_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") # endregion - DateTime emulator and extensions - -# region - Core Emulator support functions - def __write_record(self, operation, code, output, delay, timestamp=None): - """ Writes a single operation record to disk if the recorder is enabled """ - if not self.__recorder_enabled or self.__real_record_handle is None: - return - - try: - record = { - "timestamp": str(timestamp) if timestamp is not None else datetime.datetime.strptime(str(datetime.datetime.utcnow()).split(".")[0], Constants.UTC_DATETIME_FORMAT), #WRONG - "operation": str(operation), - "code": int(code), - "output": base64.b64encode(str(output)), - "delay": float(delay) - } - self.__real_record_handle.write('\n{0}'.format(json.dumps(record))) - except Exception: - print("EnvLayer: Unable to write real record to disk.") - - def __record_writer_init(self): - """ Initializes the record writer handle """ - self.__real_record_handle = open(self.__real_record_path, 'a+') - - def __read_record(self, expected_operation): - """ Returns code, output for a given operation if it matches """ - if self.__real_record_handle is None: - raise Exception("Invalid real record handle.") - - # Get single record - real_record_raw = self.__real_record_handle.readline().rstrip() - real_record = json.loads(real_record_raw) - - # Load data from record - timestamp = real_record['timestamp'] - operation = real_record['operation'] - code = int(real_record['code']) - output = base64.b64decode(real_record['output']) - delay = float(real_record['delay']) - print("Real record read: {0}: {1} >> code({2}) - output.len({3} - {4})".format(timestamp, operation, str(code), str(len(output)), str(self.__real_record_pointer+1))) - - # Verify operation - if real_record['operation'] != expected_operation: - raise Exception("Execution deviation detected. Add adaptations for operation expected: {0}. Operation data found for: {1}.".format(expected_operation, real_record['operation'])) - - # Advance and persist pointer - self.__real_record_pointer += 1 - with open(self.__real_record_pointer_path, 'w') as file_handle: - file_handle.write(str(self.__real_record_pointer)) - - # Return data - time.sleep(delay) - return code, output - - def __record_reader_init(self): - """ Seeks the real record pointer to the expected location """ - # Initialize record pointer - if not os.path.exists(self.__real_record_pointer_path): - self.__real_record_pointer = 0 - else: - with open(self.__real_record_pointer_path, 'r') as file_handle: - self.__real_record_pointer = int(file_handle.read().rstrip()) # no safety checks as there's no good recovery - - # Have the handle seek to the desired position - self.__real_record_handle = open(self.__real_record_pointer_path, 'r') - for x in range(1, self.__real_record_pointer): - self.__real_record_handle.readline() -# endregion - Core Emulator support functions - -# region - Legacy mode extensions - def set_legacy_test_mode(self): - print("Switching env layer to legacy test mode...") - self.datetime = self.DateTime(False, False, self.__write_record, self.__read_record) - self.file_system = self.FileSystem(False, False, self.__write_record, self.__read_record, emulator_root_path=os.path.dirname(self.__real_record_path)) -# endregion - Legacy mode extensions diff --git a/src/core/src/core_logic/ConfigurePatchingProcessor.py b/src/core/src/core_logic/ConfigurePatchingProcessor.py index a6d70004..2db851f0 100644 --- a/src/core/src/core_logic/ConfigurePatchingProcessor.py +++ b/src/core/src/core_logic/ConfigurePatchingProcessor.py @@ -40,7 +40,7 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ def start_configure_patching(self): """ Start configure patching """ try: - self.composite_logger.log("\nStarting configure patching... [MachineId: " + self.env_layer.platform.node() +"][ActivityId: " + self.execution_config.activity_id +"][StartTime: " + self.execution_config.start_time +"]") + self.composite_logger.log("\nStarting configure patching... [MachineId: " + self.env_layer.platform.vm_name() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "]") self.status_handler.set_current_operation(Constants.CONFIGURE_PATCHING) self.__raise_if_telemetry_unsupported() diff --git a/src/core/src/core_logic/PatchAssessor.py b/src/core/src/core_logic/PatchAssessor.py index d8220333..6c7165a3 100644 --- a/src/core/src/core_logic/PatchAssessor.py +++ b/src/core/src/core_logic/PatchAssessor.py @@ -51,7 +51,7 @@ def start_assessment(self): self.lifecycle_manager.lifecycle_status_check() return True - self.composite_logger.log("\nStarting patch assessment... [MachineId: " + self.env_layer.platform.node() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "]") + self.composite_logger.log("\nStarting patch assessment... [MachineId: " + self.env_layer.platform.vm_name() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "]") self.write_assessment_state() # success / failure does not matter, only that an attempt started self.stopwatch.start() diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index 0baa9a24..5af1b8df 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -58,14 +58,10 @@ def start_installation(self, simulate=False): self.raise_if_telemetry_unsupported() self.raise_if_min_python_version_not_met() - self.composite_logger.log('\nStarting patch installation...') + self.composite_logger.log("\nStarting patch installation... [MachineId: " + self.env_layer.platform.vm_name() + "][ActivityId: " + self.execution_config.activity_id + "][StartTime: " + self.execution_config.start_time + "][MaintenanceWindowDuration: " + self.execution_config.duration + "]") self.stopwatch.start() - self.composite_logger.log("\nMachine Id: " + self.env_layer.platform.node()) - self.composite_logger.log("Activity Id: " + self.execution_config.activity_id) - self.composite_logger.log("Operation request time: " + self.execution_config.start_time + ", Maintenance Window Duration: " + self.execution_config.duration) - maintenance_window = self.maintenance_window package_manager = self.package_manager reboot_manager = self.reboot_manager diff --git a/src/core/src/service_interfaces/StatusHandler.py b/src/core/src/service_interfaces/StatusHandler.py index 94c8286d..5b34cd6a 100644 --- a/src/core/src/service_interfaces/StatusHandler.py +++ b/src/core/src/service_interfaces/StatusHandler.py @@ -255,8 +255,6 @@ def __get_patch_id(self, package_name, package_version): def get_os_name_and_version(self): try: - if self.env_layer.platform.system() != "Linux": - raise Exception("Unsupported OS type: {0}.".format(self.env_layer.platform.system())) platform_info = self.env_layer.platform.linux_distribution() return "{0}_{1}".format(platform_info[0], platform_info[1]) except Exception as error: diff --git a/src/core/src/service_interfaces/TelemetryWriter.py b/src/core/src/service_interfaces/TelemetryWriter.py index ed44651d..7e565ebf 100644 --- a/src/core/src/service_interfaces/TelemetryWriter.py +++ b/src/core/src/service_interfaces/TelemetryWriter.py @@ -83,7 +83,7 @@ def set_and_write_machine_config_info(self): # Machine info - sent only once at the start of the run self.machine_info = "[PlatformName={0}][PlatformVersion={1}][MachineCpu={2}][MachineArch={3}][DiskType={4}]".format( str(self.env_layer.platform.linux_distribution()[0]), str(self.env_layer.platform.linux_distribution()[1]), - self.get_machine_processor(), str(self.env_layer.platform.machine()), self.get_disk_type()) + self.get_machine_processor(), str(self.env_layer.platform.cpu_arch()), self.get_disk_type()) self.write_event("Machine info is: {0}".format(self.machine_info), Constants.TelemetryEventLevel.Informational) def write_execution_error(self, cmd, code, output): diff --git a/src/core/tests/Test_EnvLayer.py b/src/core/tests/Test_EnvLayer.py index 231aee33..24bf5c73 100644 --- a/src/core/tests/Test_EnvLayer.py +++ b/src/core/tests/Test_EnvLayer.py @@ -30,6 +30,9 @@ def tearDown(self): def mock_platform_system(self): return 'Linux' + def mock_platform_system_windows(self): + return 'Windows' + def mock_linux_distribution(self): return ['test', 'test', 'test'] @@ -61,7 +64,7 @@ def mock_run_command_for_tdnf(self, cmd, no_output=False, chk_err=False): # endregion def test_get_package_manager(self): - self.backup_platform_system = platform.system() + self.backup_platform_system = platform.system platform.system = self.mock_platform_system self.backup_linux_distribution = self.envlayer.platform.linux_distribution self.envlayer.platform.linux_distribution = self.mock_linux_distribution @@ -70,10 +73,11 @@ def test_get_package_manager(self): test_input_output_table = [ [self.mock_run_command_for_apt, self.mock_linux_distribution, Constants.APT], [self.mock_run_command_for_tdnf, self.mock_linux_distribution_to_return_azure_linux_3, Constants.TDNF], - [self.mock_run_command_for_yum, self.mock_linux_distribution_to_return_azure_linux_3, None], # check for Azure Linux machine which does not have tdnf + [self.mock_run_command_for_yum, self.mock_linux_distribution_to_return_azure_linux_3, str()], # check for Azure Linux machine which does not have tdnf [self.mock_run_command_for_tdnf, self.mock_linux_distribution_to_return_azure_linux_2, Constants.TDNF], [self.mock_run_command_for_yum, self.mock_linux_distribution, Constants.YUM], [self.mock_run_command_for_zypper, self.mock_linux_distribution, Constants.ZYPPER], + [lambda cmd, no_output, chk_err: (-1, ''), self.mock_linux_distribution, str()], # no matches for any package manager ] for row in test_input_output_table: @@ -82,10 +86,29 @@ def test_get_package_manager(self): package_manager = self.envlayer.get_package_manager() self.assertTrue(package_manager is row[2]) + # test for Windows + platform.system = self.mock_platform_system_windows + self.assertEqual(self.envlayer.get_package_manager(), Constants.APT) + + # restore original methods self.envlayer.run_command_output = self.backup_run_command_output self.envlayer.platform.linux_distribution = self.backup_linux_distribution platform.system = self.backup_platform_system + def test_filesystem(self): + # only validates if these invocable without exceptions + backup_retry_count = Constants.MAX_FILE_OPERATION_RETRY_COUNT + Constants.MAX_FILE_OPERATION_RETRY_COUNT = 2 + self.envlayer.file_system.read_with_retry("fake.path", raise_if_not_found=False) + Constants.MAX_FILE_OPERATION_RETRY_COUNT = backup_retry_count + + def test_platform(self): + # only validates if these invocable without exceptions + self.envlayer.platform.linux_distribution() + self.envlayer.platform.os_type() + self.envlayer.platform.cpu_arch() + self.envlayer.platform.vm_name() + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/src/core/tests/library/LegacyEnvLayerExtensions.py b/src/core/tests/library/LegacyEnvLayerExtensions.py index 1a1cf24f..e2d75db3 100644 --- a/src/core/tests/library/LegacyEnvLayerExtensions.py +++ b/src/core/tests/library/LegacyEnvLayerExtensions.py @@ -29,15 +29,15 @@ def linux_distribution(self): return ['Ubuntu', '16.04', 'Xenial'] @staticmethod - def system(): # OS Type + def os_type(): # OS Type return 'Linux' @staticmethod - def machine(): # architecture + def cpu_arch(): # architecture return 'x86_64' @staticmethod - def node(): # machine name + def vm_name(): # machine name return 'LegacyTestVM' def get_package_manager(self): diff --git a/src/core/tests/library/RuntimeCompositor.py b/src/core/tests/library/RuntimeCompositor.py index 716d5082..5cbc6a03 100644 --- a/src/core/tests/library/RuntimeCompositor.py +++ b/src/core/tests/library/RuntimeCompositor.py @@ -151,7 +151,6 @@ def set_legacy_test_type(self, test_type): def reconfigure_env_layer_to_legacy_mode(self): self.env_layer.get_package_manager = self.legacy_env_layer_extensions.get_package_manager self.env_layer.platform = self.legacy_env_layer_extensions.LegacyPlatform() - self.env_layer.set_legacy_test_mode() self.env_layer.run_command_output = self.legacy_env_layer_extensions.run_command_output if os.name == 'nt': self.env_layer.etc_environment_file_path = os.getcwd()