diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index a59fb58d..07e908e9 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -201,6 +201,7 @@ class AutoAssessmentStates(EnumBackport): MAX_IMDS_CONNECTION_RETRY_COUNT = 5 MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5 MAX_BATCH_SIZE_FOR_PACKAGES = 3 + MAX_COMPLETE_STATUS_FILES_TO_RETAIN = 10 class PackageClassification(EnumBackport): UNCLASSIFIED = 'Unclassified' diff --git a/src/core/src/service_interfaces/StatusHandler.py b/src/core/src/service_interfaces/StatusHandler.py index 46fd4cdd..76c5c5ef 100644 --- a/src/core/src/service_interfaces/StatusHandler.py +++ b/src/core/src/service_interfaces/StatusHandler.py @@ -14,6 +14,7 @@ # # Requires Python 2.7+ import collections +import glob import json import os import re @@ -323,7 +324,7 @@ def set_assessment_substatus_json(self, status=Constants.STATUS_TRANSITIONING, c self.__assessment_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_ASSESSMENT_SUMMARY, status, code, json.dumps(self.__assessment_summary_json)) # Update complete status on disk - self.__write_complete_status_file() + self.__write_status_file() def __new_assessment_summary_json(self, assessment_packages_json, status, code): """ Called by: set_assessment_substatus_json @@ -372,7 +373,7 @@ def set_installation_substatus_json(self, status=Constants.STATUS_TRANSITIONING, self.__installation_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_INSTALLATION_SUMMARY, status, code, json.dumps(self.__installation_summary_json)) # Update complete status on disk - self.__write_complete_status_file() + self.__write_status_file() def __new_installation_summary_json(self, installation_packages_json): """ Called by: set_installation_substatus_json @@ -434,7 +435,7 @@ def set_patch_metadata_for_healthstore_substatus_json(self, status=Constants.STA self.__metadata_for_healthstore_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_METADATA_FOR_HEALTHSTORE, status, code, json.dumps(self.__metadata_for_healthstore_summary_json)) # Update complete status on disk - self.__write_complete_status_file() + self.__write_status_file() # wait period required in cases where we need to ensure HealthStore reads the status from GA if wait_after_update: @@ -467,7 +468,7 @@ def set_configure_patching_substatus_json(self, status=Constants.STATUS_TRANSITI self.__configure_patching_substatus_json = self.__new_substatus_json_for_operation(Constants.CONFIGURE_PATCHING_SUMMARY, status, code, json.dumps(self.__configure_patching_summary_json)) # Update complete status on disk - self.__write_complete_status_file() + self.__write_status_file() def __new_configure_patching_summary_json(self, automatic_os_patch_state, auto_assessment_state, status, code): """ Called by: set_configure_patching_substatus_json @@ -569,6 +570,9 @@ def load_status_file_components(self, initial_load=False): self.composite_logger.log_debug("Loading status file components [InitialLoad={0}].".format(str(initial_load))) + # Remove older complete status files + self.__removed_older_complete_status_files(self.execution_config.status_folder) + # Verify the status file exists - if not, reset status file if not os.path.exists(self.complete_status_file_path) and initial_load: self.composite_logger.log_warning("Status file not found at initial load. Resetting status file to defaults.") @@ -591,8 +595,7 @@ def load_status_file_components(self, initial_load=False): if self.execution_config.exec_auto_assess_only: self.__installation_substatus_json = complete_status_file_data['status']['substatus'][i] else: - message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message'] - self.__installation_summary_json = json.loads(message) + self.__installation_summary_json = self.__get_substatus_message(complete_status_file_data, i) self.__installation_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__installation_summary_json['patches']) self.__installation_packages = list(self.__installation_packages_map.values()) self.__maintenance_window_exceeded = bool(self.__installation_summary_json['maintenanceWindowExceeded']) @@ -602,8 +605,7 @@ def load_status_file_components(self, initial_load=False): self.__installation_errors = errors['details'] self.__installation_total_error_count = self.__get_total_error_count_from_prev_status(errors['message']) if name == Constants.PATCH_ASSESSMENT_SUMMARY: # if it exists, it must be to spec, or an exception will get thrown - message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message'] - self.__assessment_summary_json = json.loads(message) + self.__assessment_summary_json = self.__get_substatus_message(complete_status_file_data, i) self.__assessment_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__assessment_summary_json['patches']) self.__assessment_packages = list(self.__assessment_packages_map.values()) errors = self.__assessment_summary_json['errors'] @@ -614,19 +616,20 @@ def load_status_file_components(self, initial_load=False): if self.execution_config.exec_auto_assess_only: self.__metadata_for_healthstore_substatus_json = complete_status_file_data['status']['substatus'][i] else: - message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message'] - self.__metadata_for_healthstore_summary_json = json.loads(message) + self.__metadata_for_healthstore_summary_json = self.__get_substatus_message(complete_status_file_data, i) if name == Constants.CONFIGURE_PATCHING_SUMMARY: # if it exists, it must be to spec, or an exception will get thrown if self.execution_config.exec_auto_assess_only: self.__configure_patching_substatus_json = complete_status_file_data['status']['substatus'][i] else: - message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message'] - self.__configure_patching_summary_json = json.loads(message) + self.__configure_patching_summary_json = self.__get_substatus_message(complete_status_file_data, i) errors = self.__configure_patching_summary_json['errors'] if errors is not None and errors['details'] is not None: self.__configure_patching_errors = errors['details'] self.__configure_patching_top_level_error_count = self.__get_total_error_count_from_prev_status(errors['message']) + def __get_substatus_message(self, status_file_data, index): + return json.loads(status_file_data['status']['substatus'][index]['formattedMessage']['message']) + def __load_complete_status_file_data(self, file_path): # Read the status file - raise exception on persistent failure for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT): @@ -641,7 +644,7 @@ def __load_complete_status_file_data(self, file_path): raise return complete_status_file_data - def __write_complete_status_file(self): + def __write_status_file(self): """ Composes and writes the status file from **already up-to-date** in-memory data. This is usually the final call to compose and persist after an in-memory data update in a specialized method. @@ -806,3 +809,21 @@ def __set_errors_json(self, error_count_by_operation, errors_by_operation): } # endregion + def __removed_older_complete_status_files(self, status_folder): + """ Retain 10 latest status complete file and remove other .complete.status files """ + files_removed = [] + all_complete_status_files = glob.glob(os.path.join(status_folder, '*.complete.status')) # Glob return empty list if no file matched pattern + if len(all_complete_status_files) <= Constants.MAX_COMPLETE_STATUS_FILES_TO_RETAIN: + return + + all_complete_status_files.sort(key=os.path.getmtime, reverse=True) + for complete_status_file in all_complete_status_files[Constants.MAX_COMPLETE_STATUS_FILES_TO_RETAIN:]: + try: + if os.path.exists(complete_status_file): + os.remove(complete_status_file) + files_removed.append(complete_status_file) + except Exception as e: + self.composite_logger.log_debug("Error deleting complete status file. [File={0} [Exception={1}]]".format(repr(complete_status_file), repr(e))) + + self.composite_logger.log_debug("Cleaned up older complete status files: {0}".format(files_removed)) + diff --git a/src/core/tests/Test_StatusHandler.py b/src/core/tests/Test_StatusHandler.py index 75a5f7de..09e227db 100644 --- a/src/core/tests/Test_StatusHandler.py +++ b/src/core/tests/Test_StatusHandler.py @@ -14,6 +14,7 @@ # # Requires Python 2.7+ import datetime +import glob import json import os import unittest @@ -31,6 +32,9 @@ def setUp(self): def tearDown(self): self.runtime.stop() + def __mock_os_remove(self, file_to_remove): + raise Exception("File could not be deleted") + def test_set_package_assessment_status(self): # startedBy should be set to User in status for Assessment packages, package_versions = self.runtime.package_manager.get_all_updates() @@ -417,9 +421,20 @@ def test_if_status_file_resets_on_load_if_malformed(self): self.assertEqual(len(substatus_file_data["status"]["substatus"]), 0) self.runtime.env_layer.file_system.delete_files_from_dir(example_file1, "*.complete.status") - def test_if_complete_status_path_is_dir(self): + def test_if_complete_and_status_path_is_dir(self): + self.old_complete_status_path = self.runtime.execution_config.complete_status_file_path + self.runtime.execution_config.complete_status_file_path = self.runtime.execution_config.status_folder + self.runtime.status_handler.load_status_file_components(initial_load=True) + self.assertTrue(os.path.isfile(os.path.join(self.runtime.execution_config.status_folder, '1.complete.status'))) + + self.old_status_path = self.runtime.execution_config.status_file_path self.runtime.execution_config.status_file_path = self.runtime.execution_config.status_folder - self.assertRaises(Exception, self.runtime.status_handler.load_status_file_components(initial_load=True)) + self.runtime.status_handler.load_status_file_components(initial_load=True) + self.assertTrue(os.path.isfile(os.path.join(self.runtime.execution_config.status_folder, '1.status'))) + + # reset the status path + self.runtime.execution_config.complete_status_file_path = self.old_complete_status_path + self.runtime.execution_config.status_file_path = self.old_status_path def test_assessment_packages_map(self): patch_count_for_test = 5 @@ -520,6 +535,39 @@ def test_load_status_and_set_package_install_status(self): self.assertTrue('Critical' in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][2]["classifications"])) self.runtime.env_layer.file_system.delete_files_from_dir(self.runtime.status_handler.status_file_path, '*.complete.status') + def test_remove_old_complete_status_files(self): + """ Create dummy files in status folder and check if the complete_status_file_path is the latest file and delete those dummy files """ + file_path = self.runtime.execution_config.status_folder + for i in range(1, 15): + with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f: + f.write("test" + str(i)) + + packages, package_versions = self.runtime.package_manager.get_all_updates() + self.runtime.status_handler.set_package_assessment_status(packages, package_versions) + self.runtime.status_handler.load_status_file_components(initial_load=True) + + # remove 10 complete status files + count_status_files = glob.glob(os.path.join(file_path, '*.complete.status')) + self.assertEqual(10, len(count_status_files)) + self.assertTrue(os.path.isfile(self.runtime.execution_config.complete_status_file_path)) + self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status') + self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status'))) + + def test_remove_old_complete_status_files_throws_exception(self): + file_path = self.runtime.execution_config.status_folder + for i in range(1, 16): + with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f: + f.write("test" + str(i)) + + self.backup_os_remove = os.remove + os.remove = self.__mock_os_remove + self.assertRaises(Exception, self.runtime.status_handler.load_status_file_components(initial_load=True)) + + # reset os.remove() mock and remove *complete.status files + os.remove = self.backup_os_remove + self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status') + self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status'))) + # Setup functions to populate packages and versions for truncation def __set_up_packages_func(self, val): test_packages = []