Implement patching in batches to improve performance.

src/core/src/bootstrap/Constants.py

@@ -200,6 +200,7 @@ class AutoAssessmentStates(EnumBackport):
     MAX_INSTALLATION_RETRY_COUNT = 3
     MAX_IMDS_CONNECTION_RETRY_COUNT = 5
     MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5
+    MAX_BATCH_SIZE_FOR_PACKAGES = 3
 
     class PackageClassification(EnumBackport):
         UNCLASSIFIED = 'Unclassified'

src/core/src/bootstrap/EnvLayer.py

@@ -501,6 +501,19 @@ def total_minutes_from_time_delta(time_delta):
         def total_seconds_from_time_delta(time_delta):
             return (time_delta.microseconds + (time_delta.seconds + time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6
 
+        @staticmethod
+        def total_seconds_from_time_delta_round_to_one_decimal_digit(time_delta):
+            """ 
+            Converts the input time in datetime.timedelta format to seconds in float format
+
+            Parameters:
+            time_delta (datetime.timedelta): time in datetime.timedelta format e.g. 0:00:00.219000
+
+            Returns:
+            time in seconds round of to one decimal digit (float): e.g. 0.2 seconds
+            """
+            return round(EnvLayer.DateTime.total_seconds_from_time_delta(time_delta), 1)
+
         @staticmethod
         def utc_to_standard_datetime(utc_datetime):
             """ Converts string of format '"%Y-%m-%dT%H:%M:%SZ"' to datetime object """

src/core/src/core_logic/MaintenanceWindow.py

@@ -59,9 +59,13 @@ def get_remaining_time_in_minutes(self, current_time=None, log_to_stdout=False):
 
         return remaining_time_in_minutes
 
-    def is_package_install_time_available(self, remaining_time_in_minutes=None):
+    def is_package_install_time_available(self, remaining_time_in_minutes=None, number_of_packages_in_batch=1):
         """Check if time still available for package installation"""
-        cutoff_time_in_minutes = Constants.REBOOT_BUFFER_IN_MINUTES + Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES
+        cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES * number_of_packages_in_batch
+
+        if Constants.REBOOT_SETTINGS[self.execution_config.reboot_setting] != Constants.REBOOT_NEVER:
+            cutoff_time_in_minutes = cutoff_time_in_minutes + Constants.REBOOT_BUFFER_IN_MINUTES
+
         if remaining_time_in_minutes is None:
             remaining_time_in_minutes = self.get_remaining_time_in_minutes()
 

src/core/src/core_logic/Stopwatch.py

@@ -23,6 +23,7 @@ class StopwatchException(Constants.EnumBackport):
         # Stopwatch exception strings
         STARTED_ALREADY = "Stopwatch is already started"
         NOT_STARTED = "Stopwatch is not started"
+        NOT_STOPPED = "Stopwatch is not stoppped"
         STOPPED_ALREADY = "Stopwatch is already stoppped"
 
     def __init__(self, env_layer, telemetry_writer, composite_logger):
@@ -50,6 +51,7 @@ def start(self):
         self.time_taken_in_secs = None
         self.task_details = None
 
+    # Stop the stopwatch and set end_time. Create new end_time even if end_time is already set
     def stop(self):
         if self.end_time is not None:
             self.composite_logger.log_debug(str(Stopwatch.StopwatchException.STOPPED_ALREADY))
@@ -58,16 +60,26 @@ def stop(self):
             self.composite_logger.log_debug(str(Stopwatch.StopwatchException.NOT_STARTED))
             self.start_time = self.end_time
 
-        self.time_taken_in_secs = self.env_layer.datetime.total_seconds_from_time_delta(self.end_time - self.start_time)
-
-        # Rounding off to one digit after decimal e.g. 14.574372666666667 will become 14.6
-        self.time_taken_in_secs = round(self.time_taken_in_secs, 1)
+        self.time_taken_in_secs = self.env_layer.datetime.total_seconds_from_time_delta_round_to_one_decimal_digit(self.end_time - self.start_time)
 
+    # Stop the stopwatch, set end_time and write details in telemetry. Create new end_time even if end_time is already set
     def stop_and_write_telemetry(self, message):
         self.stop()
         self.set_task_details(message)
         self.composite_logger.log("Stopwatch details: " + self.task_details)
 
+    # Write stopwatch details in telemetry. Use the existing end_time if it is already set otherwise set new end_time
+    def write_telemetry_for_stopwatch(self, message):
+        if self.end_time is None:
+            self.composite_logger.log_debug(str(Stopwatch.StopwatchException.NOT_STOPPED))
+            self.end_time = self.env_layer.datetime.datetime_utcnow()
+        if self.start_time is None:
+            self.composite_logger.log_debug(str(Stopwatch.StopwatchException.NOT_STARTED))
+            self.start_time = self.end_time
+        self.time_taken_in_secs = self.env_layer.datetime.total_seconds_from_time_delta_round_to_one_decimal_digit(self.end_time - self.start_time)
+        self.set_task_details(message)
+        self.composite_logger.log("Stopwatch details: " + str(self.task_details))
+
     def set_task_details(self, message):
         self.task_details = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}]".format(Constants.PerfLogTrackerParams.MESSAGE, str(message), Constants.PerfLogTrackerParams.TIME_TAKEN_IN_SECS, str(self.time_taken_in_secs), 
                              Constants.PerfLogTrackerParams.START_TIME, str(self.start_time), Constants.PerfLogTrackerParams.END_TIME, str(self.end_time))

src/core/src/package_managers/AptitudePackageManager.py

@@ -360,19 +360,27 @@ def is_package_version_installed(self, package_name, package_version):
         self.composite_logger.log_debug("   - Package version specified was determined to NOT be installed.")
         return False
 
-    def get_dependent_list(self, package_name):
-        """Returns dependent List of the package"""
-        cmd = self.single_package_dependency_resolution_template.replace('<PACKAGE-NAME>', package_name)
+    def get_dependent_list(self, packages):
+        """Returns dependent List for the list of packages"""
+        package_names = ""
+        for index, package in enumerate(packages):
+            if index != 0:
+                package_names += ' '
+            package_names += package
+
+        cmd = self.single_package_dependency_resolution_template.replace('<PACKAGE-NAME>', package_names)
 
         self.composite_logger.log_debug("\nRESOLVING DEPENDENCIES USING COMMAND: " + str(cmd))
         output = self.invoke_package_manager(cmd)
 
-        packages, package_versions = self.extract_packages_and_versions(output)
-        if package_name in packages:
-            packages.remove(package_name)
+        dependencies, dependency_versions = self.extract_packages_and_versions(output)
+
+        for package in packages:
+            if package in dependencies:
+                dependencies.remove(package)
 
-        self.composite_logger.log_debug(str(len(packages)) + " dependent updates were found for package '" + package_name + "'.")
-        return packages
+        self.composite_logger.log_debug(str(len(dependencies)) + " dependent packages were found for packages '" + str(packages) + "'.")
+        return dependencies
 
     def get_product_name(self, package_name):
         """Retrieve product name """
@@ -584,3 +592,10 @@ def __get_os_major_version(self):
     def __is_minimum_required_python_installed(self):
         """check if python version is at least 3.5"""
         return sys.version_info >= Constants.UbuntuProClientSettings.MINIMUM_PYTHON_VERSION_REQUIRED
+
+    def add_arch_dependencies(self, package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions):
+        """
+        Add the packages with same name as that of input parameter package but with different architectures from packages list to the list package_and_dependencies.
+        Only required for yum. No-op for apt and zypper.
+        """
+        return

src/core/src/package_managers/PackageManager.py

@@ -198,12 +198,19 @@ def install_updates_fail_safe(self, excluded_packages):
         pass
 
     def install_update_and_dependencies(self, package_and_dependencies, package_and_dependency_versions, simulate=False):
-        """Install a single package along with its dependencies (explicitly)"""
-        install_result = Constants.INSTALLED
-        package_no_longer_required = False
-        code_path = "| Install"
-        start_time = time.time()
-
+        """
+        Install a list of packages along with dependencies (explicitly)
+
+        Parameters:
+        package_and_dependencies (List of strings): List of packages along with dependencies to install
+        package_and_dependency_versions (List of strings): Versions of the packages in the list package_and_dependencies
+        simulate (bool): Whether this function call is from test run.
+
+        Returns:
+        code (int): Output code of the command run to install packages
+        out (string): Output string of the command run to install packages
+        exec_cmd (string): Command used to install packages
+        """
         if type(package_and_dependencies) is str:
             package_and_dependencies = [package_and_dependencies]
             package_and_dependency_versions = [package_and_dependency_versions]
@@ -216,9 +223,30 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_
 
         self.composite_logger.log_debug("UPDATING PACKAGE (WITH DEPENDENCIES) USING COMMAND: " + exec_cmd)
         out, code = self.invoke_package_manager_advanced(exec_cmd, raise_on_exception=False)
-        package_size = self.get_package_size(out)
         self.composite_logger.log_debug("\n<PackageInstallOutput>\n" + out + "\n</PackageInstallOutput>")  # wrapping multi-line for readability
 
+        return code, out, exec_cmd
+
+    def get_installation_status(self, code, out, exec_cmd, package, version, simulate=False):
+        """
+        Returns result of the package installation
+
+        Parameters:
+        code (int): Output code of the command run to install packages.
+        out (string): Output string of the command run to install packages.
+        exec_cmd (string): Command used to install packages.
+        package (string): Package name.
+        version (string): Package version.
+        simulate (bool): Whether this function call is from test run.
+
+        Returns:
+        install_result (string): Package installation result
+        """
+        install_result = Constants.INSTALLED
+        package_no_longer_required = False
+        code_path = "| Install"
+        start_time = time.time()
+
         # special case of package no longer being required (or maybe even present on the system)
         if code == 1 and self.get_package_manager_setting(Constants.PKG_MGR_SETTING_IDENTITY) == Constants.YUM:
             self.composite_logger.log_debug(" - Detecting if package is no longer required (as return code is 1):")
@@ -232,22 +260,22 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_
                 self.composite_logger.log_debug("    - Evidence of package no longer required NOT detected.")
 
         if not package_no_longer_required:
-            if not self.is_package_version_installed(package_and_dependencies[0], package_and_dependency_versions[0]):
-                if code == 0 and self.STR_ONLY_UPGRADES.replace('<PACKAGE>', package_and_dependencies[0]) in out:
+            if not self.is_package_version_installed(package, version):
+                if code == 0 and self.STR_ONLY_UPGRADES.replace('<PACKAGE>', package) in out:
                     # It is premature to fail this package. In the *unlikely* case it never gets picked up, it'll remain NotStarted.
                     # The NotStarted status must not be written again in the calling function (it's not at the time of this writing).
                     code_path += " > Package has no prior version. (no operation; return 'not started')"
                     install_result = Constants.PENDING
-                    self.composite_logger.log_warning(" |- Package " + package_and_dependencies[0] + " (" + package_and_dependency_versions[0] + ") needs to already have an older version installed in order to be upgraded. " +
+                    self.composite_logger.log_warning(" |- Package " + package + " (" + version + ") needs to already have an older version installed in order to be upgraded. " +
                                                    "\n |- Another upgradeable package requiring it as a dependency can cause it to get installed later. No action may be required.\n")
 
-                elif code == 0 and self.STR_OBSOLETED.replace('<PACKAGE>', self.get_composite_package_identifier(package_and_dependencies[0], package_and_dependency_versions[0])) in out:
+                elif code == 0 and self.STR_OBSOLETED.replace('<PACKAGE>', self.get_composite_package_identifier(package, version)) in out:
                     # Package can be obsoleted by another package installed in the run (via dependencies)
                     code_path += " > Package obsoleted. (succeeded)"
                     install_result = Constants.INSTALLED    # close approximation to obsoleted
                     self.composite_logger.log_debug(" - Package was discovered to be obsoleted.")
 
-                elif code == 0 and len(out.split(self.STR_REPLACED)) > 1 and package_and_dependencies[0] in out.split(self.STR_REPLACED)[1]:
+                elif code == 0 and len(out.split(self.STR_REPLACED)) > 1 and package in out.split(self.STR_REPLACED)[1]:
                     code_path += " > Package replaced. (succeeded)"
                     install_result = Constants.INSTALLED    # close approximation to replaced
                     self.composite_logger.log_debug(" - Package was discovered to be replaced by another during its installation.")
@@ -256,12 +284,12 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_
                     install_result = Constants.FAILED
                     if code != 0:
                         code_path += " > Package NOT installed. (failed)"
-                        self.composite_logger.log_error(" |- Package failed to install: " + package_and_dependencies[0] + " (" + package_and_dependency_versions[0] + "). " +
+                        self.composite_logger.log_error(" |- Package failed to install: " + package + " (" + version + "). " +
                                                      "\n |- Error code: " + str(code) + ". Command used: " + exec_cmd +
                                                      "\n |- Command output: " + out + "\n")
                     else:
                         code_path += " > Package NOT installed but return code: 0. (failed)"
-                        self.composite_logger.log_error(" |- Package appears to have not been installed: " + package_and_dependencies[0] + " (" + package_and_dependency_versions[0] + "). " +
+                        self.composite_logger.log_error(" |- Package appears to have not been installed: " + package + " (" + version + "). " +
                                                      "\n |- Return code: 0. Command used: " + exec_cmd + "\n" +
                                                      "\n |- Command output: " + out + "\n")
             elif code != 0:
@@ -271,15 +299,36 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_
                 code_path += " > Info, Package installed, zero return. (succeeded)"
 
         if not simulate:
+            package_size = self.get_package_size(out)
             if install_result == Constants.FAILED:
-                error = self.telemetry_writer.write_package_info(package_and_dependencies[0], package_and_dependency_versions[0], package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd, str(out))
+                error = self.telemetry_writer.write_package_info(package, version, package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd, str(out))
             else:
-                error = self.telemetry_writer.write_package_info(package_and_dependencies[0], package_and_dependency_versions[0], package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd)
+                error = self.telemetry_writer.write_package_info(package, version, package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd)
 
             if error is not None:
                 self.composite_logger.log_debug('\nEXCEPTION writing package telemetry: ' + repr(error))
 
         return install_result
+
+    def install_update_and_dependencies_and_get_status(self, package_and_dependencies, package_and_dependency_versions, simulate=False):
+        """
+        Install a single package along with its dependencies (explicitly) and return the installation status
+        Parameters:
+        package_and_dependencies (List of strings): List of packages along with dependencies to install
+        package_and_dependency_versions (List of strings): Versions of the packages in the list package_and_dependencies
+        simulate (bool): Whether this function call is from test run.
+
+        Returns:
+        install_result (string): Package installation result
+        """
+        if type(package_and_dependencies) is str:
+            package_and_dependencies = [package_and_dependencies]
+            package_and_dependency_versions = [package_and_dependency_versions]
+
+        code, out, exec_cmd = self.install_update_and_dependencies(package_and_dependencies, package_and_dependency_versions, simulate)
+        install_result = self.get_installation_status(code, out, exec_cmd, package_and_dependencies[0], package_and_dependency_versions[0], simulate)
+        return install_result
+
     # endregion
 
     # region Package Information
@@ -386,3 +435,20 @@ def do_processes_require_restart(self):
         """ Signals whether processes require a restart due to updates to files """
         pass
 
+    @abstractmethod
+    def add_arch_dependencies(self, package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions):
+        """
+        Add the packages with same name as that of input parameter package but with different architectures from packages list to the list package_and_dependencies.
+        Only required for yum. No-op for apt and zypper.
+
+        Parameters:
+        package_manager (PackageManager): Package manager used.
+        package (string): Input package for which same package name but different architecture need to be added in the list package_and_dependencies.
+        packages (List of strings): List of all packages selected by user to install.
+        package_versions (List of strings): Versions of packages in packages list.
+        package_and_dependencies (List of strings): List of packages along with dependencies. This function adds packages with same name as input parameter package 
+                                                    but different architecture in this list.
+        package_and_dependency_versions (List of strings): Versions of packages in package_and_dependencies.
+        """
+        pass
+