User/jf/remove old complete status file #205

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

kjohn-msft merged 21 commits into master from user/jf/remove_old_complete_status_file

Aug 21, 2023

src/core/src/bootstrap/Constants.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -201,6 +201,7 @@ class AutoAssessmentStates(EnumBackport): @@
         MAX_IMDS_CONNECTION_RETRY_COUNT = 5
         MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5
         MAX_BATCH_SIZE_FOR_PACKAGES = 3
+        MAX_COMPLETE_STATUS_FILES_TO_RETAIN = 10
         class PackageClassification(EnumBackport):
             UNCLASSIFIED = 'Unclassified'
@@ Expand Down @@

src/core/src/service_interfaces/StatusHandler.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -14,6 +14,7 @@ @@
     #
     # Requires Python 2.7+
     import collections
+    import glob
     import json
     import os
     import re
@@ Expand Down Expand Up @@
             self.__assessment_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_ASSESSMENT_SUMMARY, status, code, json.dumps(self.__assessment_summary_json))
             # Update complete status on disk
-            self.__write_complete_status_file()
+            self.__write_status_file()
         def __new_assessment_summary_json(self, assessment_packages_json, status, code):
             """ Called by: set_assessment_substatus_json
@@ Expand Down Expand Up @@
             self.__installation_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_INSTALLATION_SUMMARY, status, code, json.dumps(self.__installation_summary_json))
             # Update complete status on disk
-            self.__write_complete_status_file()
+            self.__write_status_file()
         def __new_installation_summary_json(self, installation_packages_json):
             """ Called by: set_installation_substatus_json
@@ Expand Down Expand Up @@
             self.__metadata_for_healthstore_substatus_json = self.__new_substatus_json_for_operation(Constants.PATCH_METADATA_FOR_HEALTHSTORE, status, code, json.dumps(self.__metadata_for_healthstore_summary_json))
             # Update complete status on disk
-            self.__write_complete_status_file()
+            self.__write_status_file()
             # wait period required in cases where we need to ensure HealthStore reads the status from GA
             if wait_after_update:
@@ Expand Down Expand Up @@
             self.__configure_patching_substatus_json = self.__new_substatus_json_for_operation(Constants.CONFIGURE_PATCHING_SUMMARY, status, code, json.dumps(self.__configure_patching_summary_json))
             # Update complete status on disk
-            self.__write_complete_status_file()
+            self.__write_status_file()
         def __new_configure_patching_summary_json(self, automatic_os_patch_state, auto_assessment_state, status, code):
             """ Called by: set_configure_patching_substatus_json
@@ Expand Down Expand Up / @@ -569,6 +570,9 @@ def load_status_file_components(self, initial_load=False): @@
             self.composite_logger.log_debug("Loading status file components [InitialLoad={0}].".format(str(initial_load)))
+            # Remove older complete status files
+            self.__removed_older_complete_status_files(self.execution_config.status_folder)
             # Verify the status file exists - if not, reset status file
             if not os.path.exists(self.complete_status_file_path) and initial_load:
                 self.composite_logger.log_warning("Status file not found at initial load. Resetting status file to defaults.")
@@ Expand All / @@ -591,8 +595,7 @@ def load_status_file_components(self, initial_load=False): @@
                     if self.execution_config.exec_auto_assess_only:
                         self.__installation_substatus_json = complete_status_file_data['status']['substatus'][i]
                     else:
-                        message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                        self.__installation_summary_json = json.loads(message)
+                        self.__installation_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                         self.__installation_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__installation_summary_json['patches'])
                         self.__installation_packages = list(self.__installation_packages_map.values())
                         self.__maintenance_window_exceeded = bool(self.__installation_summary_json['maintenanceWindowExceeded'])
@@ Expand All / @@ -602,8 +605,7 @@ def load_status_file_components(self, initial_load=False): @@
                             self.__installation_errors = errors['details']
                             self.__installation_total_error_count = self.__get_total_error_count_from_prev_status(errors['message'])
                 if name == Constants.PATCH_ASSESSMENT_SUMMARY:     # if it exists, it must be to spec, or an exception will get thrown
-                    message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                    self.__assessment_summary_json = json.loads(message)
+                    self.__assessment_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                     self.__assessment_packages_map = collections.OrderedDict((package["patchId"], package) for package in self.__assessment_summary_json['patches'])
                     self.__assessment_packages = list(self.__assessment_packages_map.values())
                     errors = self.__assessment_summary_json['errors']
@@ Expand All @@
                     if self.execution_config.exec_auto_assess_only:
                         self.__metadata_for_healthstore_substatus_json = complete_status_file_data['status']['substatus'][i]
                     else:
-                        message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                        self.__metadata_for_healthstore_summary_json = json.loads(message)
+                        self.__metadata_for_healthstore_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                 if name == Constants.CONFIGURE_PATCHING_SUMMARY:     # if it exists, it must be to spec, or an exception will get thrown
                     if self.execution_config.exec_auto_assess_only:
                         self.__configure_patching_substatus_json = complete_status_file_data['status']['substatus'][i]
                     else:
-                        message = complete_status_file_data['status']['substatus'][i]['formattedMessage']['message']
-                        self.__configure_patching_summary_json = json.loads(message)
+                        self.__configure_patching_summary_json = self.__get_substatus_message(complete_status_file_data, i)
                         errors = self.__configure_patching_summary_json['errors']
                         if errors is not None and errors['details'] is not None:
                             self.__configure_patching_errors = errors['details']
                             self.__configure_patching_top_level_error_count = self.__get_total_error_count_from_prev_status(errors['message'])
+        def __get_substatus_message(self, status_file_data, index):
+            return json.loads(status_file_data['status']['substatus'][index]['formattedMessage']['message'])
         def __load_complete_status_file_data(self, file_path):
             # Read the status file - raise exception on persistent failure
             for i in range(0, Constants.MAX_FILE_OPERATION_RETRY_COUNT):
@@ Expand All / @@ -641,7 +644,7 @@ def __load_complete_status_file_data(self, file_path): @@
                         raise
             return complete_status_file_data
-        def __write_complete_status_file(self):
+        def __write_status_file(self):
             """ Composes and writes the status file from **already up-to-date** in-memory data.
                 This is usually the final call to compose and persist after an in-memory data update in a specialized method.
@@ Expand Down Expand Up @@
             }
         # endregion
+        def __removed_older_complete_status_files(self, status_folder):
+            """ Retain 10 latest status complete file and remove other .complete.status files """
+            files_removed = []
+            all_complete_status_files = glob.glob(os.path.join(status_folder, '*.complete.status'))    # Glob return empty list if no file matched pattern
+            if len(all_complete_status_files) <= Constants.MAX_COMPLETE_STATUS_FILES_TO_RETAIN:
+                return
+            all_complete_status_files.sort(key=os.path.getmtime, reverse=True)
+            for complete_status_file in all_complete_status_files[Constants.MAX_COMPLETE_STATUS_FILES_TO_RETAIN:]:
+                try:
+                    if os.path.exists(complete_status_file):
+                        os.remove(complete_status_file)
+                        files_removed.append(complete_status_file)
+                except Exception as e:
+                    self.composite_logger.log_debug("Error deleting complete status file. [File={0} [Exception={1}]]".format(repr(complete_status_file), repr(e)))
+            self.composite_logger.log_debug("Cleaned up older complete status files: {0}".format(files_removed))

src/core/tests/Test_StatusHandler.py

-Original file line number
+Diff line change
@@ Expand Up / @@ -14,6 +14,7 @@ @@
     #
     # Requires Python 2.7+
     import datetime
+    import glob
     import json
     import os
     import unittest
@@ Expand All / @@ -31,6 +32,9 @@ def setUp(self): @@
         def tearDown(self):
             self.runtime.stop()
+        def __mock_os_remove(self, file_to_remove):
+            raise Exception("File could not be deleted")
         def test_set_package_assessment_status(self):
             # startedBy should be set to User in status for Assessment
             packages, package_versions = self.runtime.package_manager.get_all_updates()
@@ Expand Down Expand Up / @@ -417,9 +421,20 @@ def test_if_status_file_resets_on_load_if_malformed(self): @@
             self.assertEqual(len(substatus_file_data["status"]["substatus"]), 0)
             self.runtime.env_layer.file_system.delete_files_from_dir(example_file1, "*.complete.status")
-        def test_if_complete_status_path_is_dir(self):
+        def test_if_complete_and_status_path_is_dir(self):
+            self.old_complete_status_path = self.runtime.execution_config.complete_status_file_path
+            self.runtime.execution_config.complete_status_file_path = self.runtime.execution_config.status_folder
+            self.runtime.status_handler.load_status_file_components(initial_load=True)
+            self.assertTrue(os.path.isfile(os.path.join(self.runtime.execution_config.status_folder, '1.complete.status')))
+            self.old_status_path = self.runtime.execution_config.status_file_path
             self.runtime.execution_config.status_file_path = self.runtime.execution_config.status_folder
-            self.assertRaises(Exception, self.runtime.status_handler.load_status_file_components(initial_load=True))
+            self.runtime.status_handler.load_status_file_components(initial_load=True)
+            self.assertTrue(os.path.isfile(os.path.join(self.runtime.execution_config.status_folder, '1.status')))
+            # reset the status path
+            self.runtime.execution_config.complete_status_file_path = self.old_complete_status_path
+            self.runtime.execution_config.status_file_path = self.old_status_path
         def test_assessment_packages_map(self):
             patch_count_for_test = 5
@@ Expand Down Expand Up / @@ -520,6 +535,39 @@ def test_load_status_and_set_package_install_status(self): @@
             self.assertTrue('Critical' in str(json.loads(substatus_file_data["formattedMessage"]["message"])["patches"][2]["classifications"]))
             self.runtime.env_layer.file_system.delete_files_from_dir(self.runtime.status_handler.status_file_path, '*.complete.status')
+        def test_remove_old_complete_status_files(self):
+            """ Create dummy files in status folder and check if the complete_status_file_path is the latest file and delete those dummy files """
+            file_path = self.runtime.execution_config.status_folder
+            for i in range(1, 15):
+                with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f:
+                    f.write("test" + str(i))
+            packages, package_versions = self.runtime.package_manager.get_all_updates()
+            self.runtime.status_handler.set_package_assessment_status(packages, package_versions)
+            self.runtime.status_handler.load_status_file_components(initial_load=True)
+            # remove 10 complete status files
+            count_status_files = glob.glob(os.path.join(file_path, '*.complete.status'))
+            self.assertEqual(10, len(count_status_files))
+            self.assertTrue(os.path.isfile(self.runtime.execution_config.complete_status_file_path))
+            self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status')
+            self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status')))
+        def test_remove_old_complete_status_files_throws_exception(self):
+            file_path = self.runtime.execution_config.status_folder
+            for i in range(1, 16):
+                with open(os.path.join(file_path, str(i + 100) + '.complete.status'), 'w') as f:
+                    f.write("test" + str(i))
+            self.backup_os_remove = os.remove
+            os.remove = self.__mock_os_remove
+            self.assertRaises(Exception, self.runtime.status_handler.load_status_file_components(initial_load=True))
+            # reset os.remove() mock and remove *complete.status files
+            os.remove = self.backup_os_remove
+            self.runtime.env_layer.file_system.delete_files_from_dir(file_path, '*.complete.status')
+            self.assertFalse(os.path.isfile(os.path.join(file_path, '1.complete_status')))
         # Setup functions to populate packages and versions for truncation
         def __set_up_packages_func(self, val):
             test_packages = []
@@ Expand Down @@

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

User/jf/remove old complete status file #205

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!