diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index a1e9115c..ba4f10db 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -200,6 +200,7 @@ class AutoAssessmentStates(EnumBackport): MAX_INSTALLATION_RETRY_COUNT = 3 MAX_IMDS_CONNECTION_RETRY_COUNT = 5 MAX_ZYPPER_REPO_REFRESH_RETRY_COUNT = 5 + MAX_BATCH_SIZE_FOR_PACKAGES = 3 class PackageClassification(EnumBackport): UNCLASSIFIED = 'Unclassified' diff --git a/src/core/src/bootstrap/EnvLayer.py b/src/core/src/bootstrap/EnvLayer.py index f71c95cb..ab2e1f6a 100644 --- a/src/core/src/bootstrap/EnvLayer.py +++ b/src/core/src/bootstrap/EnvLayer.py @@ -501,6 +501,19 @@ def total_minutes_from_time_delta(time_delta): def total_seconds_from_time_delta(time_delta): return (time_delta.microseconds + (time_delta.seconds + time_delta.days * 24 * 3600) * 10 ** 6) / 10.0 ** 6 + @staticmethod + def total_seconds_from_time_delta_round_to_one_decimal_digit(time_delta): + """ + Converts the input time in datetime.timedelta format to seconds in float format + + Parameters: + time_delta (datetime.timedelta): time in datetime.timedelta format e.g. 0:00:00.219000 + + Returns: + time in seconds round of to one decimal digit (float): e.g. 0.2 seconds + """ + return round(EnvLayer.DateTime.total_seconds_from_time_delta(time_delta), 1) + @staticmethod def utc_to_standard_datetime(utc_datetime): """ Converts string of format '"%Y-%m-%dT%H:%M:%SZ"' to datetime object """ diff --git a/src/core/src/core_logic/MaintenanceWindow.py b/src/core/src/core_logic/MaintenanceWindow.py index 0bc26eb9..732e4192 100644 --- a/src/core/src/core_logic/MaintenanceWindow.py +++ b/src/core/src/core_logic/MaintenanceWindow.py @@ -59,9 +59,13 @@ def get_remaining_time_in_minutes(self, current_time=None, log_to_stdout=False): return remaining_time_in_minutes - def is_package_install_time_available(self, remaining_time_in_minutes=None): + def is_package_install_time_available(self, remaining_time_in_minutes=None, number_of_packages_in_batch=1): """Check if time still available for package installation""" - cutoff_time_in_minutes = Constants.REBOOT_BUFFER_IN_MINUTES + Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES + cutoff_time_in_minutes = Constants.PACKAGE_INSTALL_EXPECTED_MAX_TIME_IN_MINUTES * number_of_packages_in_batch + + if Constants.REBOOT_SETTINGS[self.execution_config.reboot_setting] != Constants.REBOOT_NEVER: + cutoff_time_in_minutes = cutoff_time_in_minutes + Constants.REBOOT_BUFFER_IN_MINUTES + if remaining_time_in_minutes is None: remaining_time_in_minutes = self.get_remaining_time_in_minutes() diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index 8298b2c2..54cf2125 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -16,6 +16,7 @@ """ The patch install orchestrator """ import datetime +import math import os import time from core.src.bootstrap.Constants import Constants @@ -41,6 +42,10 @@ def __init__(self, env_layer, execution_config, composite_logger, telemetry_writ self.last_still_needed_packages = None # Used for 'Installed' status records self.last_still_needed_package_versions = None self.progress_template = "[Time available: {0} | A: {1}, S: {2}, F: {3} | D: {4}]\t {5}" + + self.attempted_parent_package_install_count = 0 + self.successful_parent_package_install_count = 0 + self.failed_parent_package_install_count = 0 self.stopwatch = Stopwatch(self.env_layer, self.telemetry_writer, self.composite_logger) def start_installation(self, simulate=False): @@ -162,28 +167,66 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): self.composite_logger.log("\nNote: Packages that are neither included nor excluded may still be installed if an included package has a dependency on it.") # We will see this as packages going from NotSelected --> Installed. We could remove them preemptively from not_included_packages, but we're explicitly choosing not to. - self.composite_logger.log("\n\nInstalling patches in sequence...") self.composite_logger.log("[Progress Legend: (A)ttempted, (S)ucceeded, (F)ailed, (D)ependencies est.* (Important: Dependencies are excluded in all other counts)]") - attempted_parent_update_count = 0 - successful_parent_update_count = 0 - failed_parent_update_count = 0 installed_update_count = 0 # includes dependencies patch_installation_successful = True maintenance_window_exceeded = False all_packages, all_package_versions = package_manager.get_all_updates(True) # cached is fine self.telemetry_writer.write_event("All available packages list: " + str(all_packages), Constants.TelemetryEventLevel.Verbose) - self.last_still_needed_packages = all_packages - self.last_still_needed_package_versions = all_package_versions + self.last_still_needed_packages = list(all_packages) + self.last_still_needed_package_versions = list(all_package_versions) + + stopwatch_for_batch_install_process = Stopwatch(self.env_layer, self.telemetry_writer, self.composite_logger) + stopwatch_for_batch_install_process.start() + + # Check for packages available and install them in batches. Some packages may not be installed due to: + # (a) Not enough time remaining in maintenance window + # (b) Failure during package installation + installed_update_count, patch_installation_successful, maintenance_window_batch_cutoff_reached, packages, package_versions = self.install_packages_in_batches( + all_packages, all_package_versions, packages, package_versions, maintenance_window, package_manager) + + stopwatch_for_batch_install_process.stop() + + install_update_count_in_batch_patching = installed_update_count + attempted_parent_package_install_count_in_batch_patching = self.attempted_parent_package_install_count + successful_parent_package_install_count_in_batch_patching = self.successful_parent_package_install_count + failed_parent_package_install_count_in_batch_patching = self.failed_parent_package_install_count + batch_processing_perf_log = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}][{8}={9}][{10}={11}][{12}={13}][{14}={15}]".format(Constants.PerfLogTrackerParams.TASK, "InstallPackagesInBatches", + "InstalledPackagesCountInBatchProcessing", str(install_update_count_in_batch_patching), "AttemptedParentPackageInstallCount", attempted_parent_package_install_count_in_batch_patching, + "SuccessfulParentPackageInstallCount", successful_parent_package_install_count_in_batch_patching, "FailedParentPackageInstallCount", failed_parent_package_install_count_in_batch_patching, + "RemainingPackagesToInstall", str(len(packages)), Constants.PerfLogTrackerParams.PATCH_OPERATION_SUCCESSFUL, str(patch_installation_successful), + "IsMaintenanceWindowBatchCutoffReached", str(maintenance_window_batch_cutoff_reached)) + + stopwatch_for_batch_install_process.write_telemetry_for_stopwatch(str(batch_processing_perf_log)) + + if len(packages) == 0: + self.log_final_metrics(maintenance_window, patch_installation_successful, maintenance_window_exceeded, installed_update_count) + return installed_update_count, patch_installation_successful, maintenance_window_exceeded + else: + progress_status = self.progress_template.format(str(datetime.timedelta(minutes=maintenance_window.get_remaining_time_in_minutes())), str(self.attempted_parent_package_install_count), str(self.successful_parent_package_install_count), str(self.failed_parent_package_install_count), str(installed_update_count - self.successful_parent_package_install_count), + "Following packages are not attempted or failed in batch installation: " + str(packages)) + self.composite_logger.log(progress_status) + + stopwatch_for_sequential_install_process = Stopwatch(self.env_layer, self.telemetry_writer, self.composite_logger) + stopwatch_for_sequential_install_process.start() for package, version in zip(packages, package_versions): + if package not in self.last_still_needed_packages: + self.composite_logger.log("The following package is already installed, it could have been installed as dependent package of some other package: " + package) + self.attempted_parent_package_install_count += 1 + self.successful_parent_package_install_count += 1 + continue + + single_package_install_stopwatch = Stopwatch(self.env_layer, self.telemetry_writer, self.composite_logger) + single_package_install_stopwatch.start() # Extension state check if self.lifecycle_manager is not None: self.lifecycle_manager.lifecycle_status_check() # may terminate the code abruptly, as designed # maintenance window check remaining_time = maintenance_window.get_remaining_time_in_minutes() - if maintenance_window.is_package_install_time_available(remaining_time) is False: + if maintenance_window.is_package_install_time_available(remaining_time, number_of_packages_in_batch=1) is False: error_msg = "Stopped patch installation as it is past the maintenance window cutoff time." self.composite_logger.log_error("\n" + error_msg) self.status_handler.add_error_to_status(error_msg, Constants.PatchOperationErrorCodes.DEFAULT_ERROR) @@ -192,7 +235,7 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): break # point in time status - progress_status = self.progress_template.format(str(datetime.timedelta(minutes=remaining_time)), str(attempted_parent_update_count), str(successful_parent_update_count), str(failed_parent_update_count), str(installed_update_count - successful_parent_update_count), + progress_status = self.progress_template.format(str(datetime.timedelta(minutes=remaining_time)), str(self.attempted_parent_package_install_count), str(self.successful_parent_package_install_count), str(self.failed_parent_package_install_count), str(installed_update_count - self.successful_parent_package_install_count), "Processing package: " + str(package) + " (" + str(version) + ")") if version == Constants.UA_ESM_REQUIRED: progress_status += "[Skipping - requires Ubuntu Advantage for Infrastructure with Extended Security Maintenance]" @@ -202,52 +245,43 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): self.composite_logger.log(progress_status) # include all dependencies (with specified versions) explicitly + # package_and_dependencies initially conains only one package. The dependencies are added in the list by method include_dependencies package_and_dependencies = [package] package_and_dependency_versions = [version] - dependencies = package_manager.get_dependent_list(package) - for dependency in dependencies: - if dependency not in all_packages: - continue - package_and_dependencies.append(dependency) - package_and_dependency_versions.append(package_versions[packages.index(dependency)] if dependency in packages else Constants.DEFAULT_UNSPECIFIED_VALUE) - - # multilib resolution for yum - if package_manager.get_package_manager_setting(Constants.PKG_MGR_SETTING_IDENTITY) == Constants.YUM: - package_name_without_arch = package_manager.get_product_name_without_arch(package) - for possible_arch_dependency, possible_arch_dependency_version in zip(packages, package_versions): - if package_manager.get_product_name_without_arch(possible_arch_dependency) == package_name_without_arch and possible_arch_dependency not in package_and_dependencies: - package_and_dependencies.append(possible_arch_dependency) - package_and_dependency_versions.append(possible_arch_dependency_version) - - # remove duplicates - package_and_dependencies, package_and_dependency_versions = package_manager.dedupe_update_packages(package_and_dependencies, package_and_dependency_versions) + + self.include_dependencies(package_manager, [package], all_packages, all_package_versions, packages, package_versions, package_and_dependencies, package_and_dependency_versions) # parent package install (+ dependencies) and parent package result management install_result = Constants.FAILED for i in range(0, Constants.MAX_INSTALLATION_RETRY_COUNT): - install_result = package_manager.install_update_and_dependencies(package_and_dependencies, package_and_dependency_versions, simulate) + install_result = package_manager.install_update_and_dependencies_and_get_status(package_and_dependencies, package_and_dependency_versions, simulate) + if install_result != Constants.INSTALLED: if i < Constants.MAX_INSTALLATION_RETRY_COUNT - 1: time.sleep(i + 1) self.composite_logger.log_warning("Retrying installation of package. [Package={0}]".format(package_manager.get_product_name(package_and_dependencies[0]))) + else: + break # Update reboot pending status in status_handler self.status_handler.set_reboot_pending(self.package_manager.is_reboot_pending()) if install_result == Constants.FAILED: self.status_handler.set_package_install_status(package_manager.get_product_name(str(package_and_dependencies[0])), str(package_and_dependency_versions[0]), Constants.FAILED) - failed_parent_update_count += 1 + self.failed_parent_package_install_count += 1 patch_installation_successful = False elif install_result == Constants.INSTALLED: self.status_handler.set_package_install_status(package_manager.get_product_name(str(package_and_dependencies[0])), str(package_and_dependency_versions[0]), Constants.INSTALLED) - successful_parent_update_count += 1 + self.successful_parent_package_install_count += 1 if package in self.last_still_needed_packages: index = self.last_still_needed_packages.index(package) self.last_still_needed_packages.pop(index) self.last_still_needed_package_versions.pop(index) installed_update_count += 1 - attempted_parent_update_count += 1 + self.attempted_parent_package_install_count += 1 + number_of_dependencies_installed = 0 + number_of_dependencies_failed = 0 # dependency package result management for dependency, dependency_version in zip(package_and_dependencies, package_and_dependency_versions): if dependency not in self.last_still_needed_packages or dependency == package: @@ -260,20 +294,53 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): self.last_still_needed_packages.pop(index) self.last_still_needed_package_versions.pop(index) installed_update_count += 1 + number_of_dependencies_installed += 1 else: # status is not logged by design here, in case you were wondering if that's a bug message = " - [Info] Dependency appears to have failed to install (note: it *may* be retried): " + str(dependency) + "(" + str(dependency_version) + ")" self.composite_logger.log_debug(message) + number_of_dependencies_failed += 1 # dependency package result management fallback (not reliable enough to be used as primary, and will be removed; remember to retain last_still_needed refresh when you do that) - installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, condition=(attempted_parent_update_count % Constants.PACKAGE_STATUS_REFRESH_RATE_IN_SECONDS == 0)) # reconcile status after every 10 attempted installs + installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, condition=(self.attempted_parent_package_install_count % Constants.PACKAGE_STATUS_REFRESH_RATE_IN_SECONDS == 0)) # reconcile status after every 10 attempted installs - progress_status = self.progress_template.format(str(datetime.timedelta(minutes=maintenance_window.get_remaining_time_in_minutes())), str(attempted_parent_update_count), str(successful_parent_update_count), str(failed_parent_update_count), str(installed_update_count - successful_parent_update_count), - "Completed processing packages!") - self.composite_logger.log(progress_status) + package_install_perf_log = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}][{8}={9}][{10}={11}][{12}={13}][{14}={15}]".format(Constants.PerfLogTrackerParams.TASK, "InstallPackage", + "PackageName", package, "PackageVersion", version, "PackageAndDependencies", str(package_and_dependencies),"PackageAndDependencyVersions", str(package_and_dependency_versions), + "PackageInstallResult", str(install_result), "NumberOfDependenciesInstalled", str(number_of_dependencies_installed), "NumberOfDependenciesFailed", str(number_of_dependencies_failed)) + + single_package_install_stopwatch.stop_and_write_telemetry(str(package_install_perf_log)) self.composite_logger.log_debug("\nPerforming final system state reconciliation...") - installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, True) # final reconciliation + installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, True) + self.log_final_metrics(maintenance_window, patch_installation_successful, maintenance_window_exceeded, installed_update_count) + + install_update_count_in_sequential_patching = installed_update_count - install_update_count_in_batch_patching + attempted_parent_package_install_count_in_sequential_patching = self.attempted_parent_package_install_count - attempted_parent_package_install_count_in_batch_patching + successful_parent_package_install_count_in_sequential_patching = self.successful_parent_package_install_count - successful_parent_package_install_count_in_batch_patching + failed_parent_package_install_count_in_sequential_patching = self.failed_parent_package_install_count - failed_parent_package_install_count_in_batch_patching + + sequential_processing_perf_log = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}][{8}={9}]".format(Constants.PerfLogTrackerParams.TASK, "InstallPackagesSequentially", "InstalledPackagesCountInSequentialProcessing", + install_update_count_in_sequential_patching, "AttemptedParentPackageInstallCount", attempted_parent_package_install_count_in_sequential_patching, + "SuccessfulParentPackageInstallCount", successful_parent_package_install_count_in_sequential_patching, "FailedParentPackageInstallCount", + failed_parent_package_install_count_in_sequential_patching) + + stopwatch_for_sequential_install_process.stop_and_write_telemetry(sequential_processing_perf_log) + + return installed_update_count, patch_installation_successful, maintenance_window_exceeded + + def log_final_metrics(self, maintenance_window, patch_installation_successful, maintenance_window_exceeded, installed_update_count): + """ + logs the final metrics. + + Parameters: + maintenance_window (MaintenanceWindow): Maintenance window for the job. + patch_installation_successful (bool): Whether patch installation succeeded. + maintenance_window_exceeded (bool): Whether maintenance window exceeded. + installed_update_count (int): Number of updates installed. + """ + progress_status = self.progress_template.format(str(datetime.timedelta(minutes=maintenance_window.get_remaining_time_in_minutes())), str(self.attempted_parent_package_install_count), str(self.successful_parent_package_install_count), str(self.failed_parent_package_install_count), str(installed_update_count - self.successful_parent_package_install_count), + "Completed processing packages!") + self.composite_logger.log(progress_status) if not patch_installation_successful or maintenance_window_exceeded: message = "\n\nOperation status was marked as failed because: " @@ -282,7 +349,202 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): self.status_handler.add_error_to_status(message, Constants.PatchOperationErrorCodes.OPERATION_FAILED) self.composite_logger.log_error(message) - return installed_update_count, patch_installation_successful, maintenance_window_exceeded + def include_dependencies(self, package_manager, packages_in_batch, all_packages, all_package_versions, packages, package_versions, package_and_dependencies, package_and_dependency_versions): + """ + Add dependent packages in the list of packages to install i.e. package_and_dependencies. + + Parameters: + package_manager (PackageManager): Package manager used. + packages_in_batch (List of strings): List of packages to be installed in the current batch. + all_packages (List of strings): List of all available packages to install. + all_package_versions (List of strings): Versions of packages in all_packages. + packages (List of strings): List of all packages selected by user to install. + package_versions (List of strings): Versions of packages in packages list. + package_and_dependencies (List of strings): List of packages selected by user along with packages they are dependent on. The input package_and_dependencies + does not contain dependent packages. The dependent packages are added in the list in this function. + package_and_dependency_versions (List of strings): Versions of packages in package_and_dependencies. Input list does not contain versions of the dependent packages. + The version of dependent packages are added in the list in this function. + """ + dependencies = package_manager.get_dependent_list(package_and_dependencies) + + for dependency in dependencies: + if dependency not in all_packages: + continue + package_and_dependencies.append(dependency) + version = all_package_versions[all_packages.index(dependency)] if dependency in all_packages else Constants.DEFAULT_UNSPECIFIED_VALUE + package_and_dependency_versions.append(version) + + for package in packages_in_batch: + package_manager.add_arch_dependencies(package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions) + + package_and_dependencies, package_and_dependency_versions = package_manager.dedupe_update_packages(package_and_dependencies, package_and_dependency_versions) + + self.composite_logger.log("Packages including dependencies are: " + str(package_and_dependencies)) + + def install_packages_in_batches(self, all_packages, all_package_versions, packages, package_versions, maintenance_window, package_manager, simulate=False): + """ + Install packages in batches. + + Parameters: + + all_packages (List of strings): List of all available packages to install. + all_package_versions (List of strings): Versions of the packages in the list all_packages. + packages (List of strings): List of all packages selected by user to install. + package_versions (List of strings): Versions of packages in the list packages. + maintenance_window (MaintenanceWindow): Maintenance window for the job. + package_manager (PackageManager): Package manager used. + simulate (bool): Whether this function is called from a test run. + + Returns: + installed_update_count (int): Number of packages installed through installing packages in batches. + patch_installation_successful (bool): Whether package installation succeeded for all attempted packages. + maintenance_window_batch_cutoff_reached (bool): Whether process of installing packages in batches stopped due to not enough time in maintenance window + to install packages in batch. + not_attempted_and_failed_packages (List of strings): List of packages which are (a) Not attempted due to not enough time in maintenance window to install in batch. + (b) Failed to install in batch patching. + not_attempted_and_failed_package_versions (List of strings): Versions of packages in the list not_attempted_and_failed_packages. + + """ + number_of_batches = int(math.ceil(len(packages) / float(Constants.MAX_BATCH_SIZE_FOR_PACKAGES))) + self.composite_logger.log("\nDividing package install in batches. \nNumber of packages to be installed: " + str(len(packages)) + "\nBatch Size: " + str(Constants.MAX_BATCH_SIZE_FOR_PACKAGES) + "\nNumber of batches: " + str(number_of_batches)) + installed_update_count = 0 + patch_installation_successful = True + maintenance_window_batch_cutoff_reached = False + + # remaining_packages are the packages which are not attempted to install due to there is not enough remaining time in maintenance window to install packages in batches. + # These packages will be attempted in sequential installation if there is enough time in maintenance window to install package sequentially. + remaining_packages = [] + remaining_package_versions = [] + + # failed_packages are the packages which are failed to install in batch patching. These packages will be attempted again in sequential patching if there is + # enough time remaining in maintenance window. + failed_packages = [] + failed_package_versions = [] + + for batch_index in range(0, number_of_batches): + per_batch_installation_stopwatch = Stopwatch(self.env_layer, self.telemetry_writer, self.composite_logger) + per_batch_installation_stopwatch.start() + + # Extension state check + if self.lifecycle_manager is not None: + self.lifecycle_manager.lifecycle_status_check() + + begin_index = batch_index * Constants.MAX_BATCH_SIZE_FOR_PACKAGES + end_index = begin_index + Constants.MAX_BATCH_SIZE_FOR_PACKAGES - 1 + end_index = min(end_index, len(packages) - 1) + + packages_in_batch = [] + package_versions_in_batch = [] + skip_packages = [] + already_installed_packages = [] + + for index in range(begin_index, end_index + 1): + if package_versions[index] == Constants.UA_ESM_REQUIRED: + skip_packages.append(packages[index]) + self.status_handler.set_package_install_status(package_manager.get_product_name(packages[index]), str(package_versions[index]), Constants.NOT_SELECTED) + elif packages[index] not in self.last_still_needed_packages: + # Could have got installed as dependent package of some other package. Package installation status could also have been set. + already_installed_packages.append(packages[index]) + self.attempted_parent_package_install_count += 1 + self.successful_parent_package_install_count += 1 + else: + packages_in_batch.append(packages[index]) + package_versions_in_batch.append(package_versions[index]) + + if len(already_installed_packages) > 0: + self.composite_logger.log("Following packages are already installed. Could have got installed as dependent package of some other package " + str(already_installed_packages)) + + if len(skip_packages) > 0: + self.composite_logger.log("[Skipping packages " + str(skip_packages) + " - requires Ubuntu Advantage for Infrastructure with Extended Security Maintenance]") + + if len(packages_in_batch) == 0: + continue + + remaining_time = maintenance_window.get_remaining_time_in_minutes() + + if maintenance_window.is_package_install_time_available(remaining_time, len(packages_in_batch)) is False: + self.composite_logger.log("Stopped installing packages in batches as it is past the maintenance window cutoff time for installing in batches." + + " Batch Index: {0}, remaining time: {1}, number of packages in batch: {2}".format(batch_index, remaining_time, str(len(packages_in_batch)))) + maintenance_window_batch_cutoff_reached = True + remaining_packages = packages[begin_index:] + remaining_package_versions = package_versions[begin_index:] + break + + # point in time status + progress_status = self.progress_template.format(str(datetime.timedelta(minutes=remaining_time)), str(self.attempted_parent_package_install_count), str(self.successful_parent_package_install_count), str(self.failed_parent_package_install_count), str(installed_update_count - self.successful_parent_package_install_count), + "Processing batch index: " + str(batch_index) + ", Number of packages: " + str(len(packages_in_batch)) + "\nProcessing packages: " + str(packages_in_batch)) + self.composite_logger.log(progress_status) + + # package_and_dependencies initially conains only packages in batch. The dependencies are added in the list by method include_dependencies + package_and_dependencies = list(packages_in_batch) + package_and_dependency_versions = list(package_versions_in_batch) + + self.include_dependencies(package_manager, packages_in_batch, all_packages, all_package_versions, packages, package_versions, package_and_dependencies, package_and_dependency_versions) + + parent_packages_installed_in_batch_count = 0 + parent_packages_failed_in_batch_count = 0 + number_of_dependencies_installed = 0 + number_of_dependencies_failed = 0 + + code, out, exec_cmd = package_manager.install_update_and_dependencies(package_and_dependencies, package_and_dependency_versions, simulate) + + for package,version in zip(package_and_dependencies, package_and_dependency_versions): + install_result = package_manager.get_installation_status(code, out, exec_cmd, package, version, simulate) + + if install_result == Constants.FAILED: + if package in packages_in_batch: + # parent package + self.status_handler.set_package_install_status(package_manager.get_product_name(str(package)), str(version), Constants.FAILED) + self.failed_parent_package_install_count += 1 + patch_installation_successful = False + parent_packages_failed_in_batch_count += 1 + failed_packages.append(package) + failed_package_versions.append(version) + else: + # dependent package + number_of_dependencies_failed +=1 + elif install_result == Constants.INSTALLED: + self.status_handler.set_package_install_status(package_manager.get_product_name(str(package)), str(version), Constants.INSTALLED) + if package in packages_in_batch: + # parent package + self.successful_parent_package_install_count += 1 + parent_packages_installed_in_batch_count += 1 + else: + # dependent package + number_of_dependencies_installed += 1 + + if package in self.last_still_needed_packages: + index = self.last_still_needed_packages.index(package) + self.last_still_needed_packages.pop(index) + self.last_still_needed_package_versions.pop(index) + installed_update_count += 1 + + self.attempted_parent_package_install_count += len(packages_in_batch) + + # Update reboot pending status in status_handler + self.status_handler.set_reboot_pending(self.package_manager.is_reboot_pending()) + + # dependency package result management fallback (not reliable enough to be used as primary, and will be removed; remember to retain last_still_needed refresh when you do that) + installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, condition=(self.attempted_parent_package_install_count % Constants.PACKAGE_STATUS_REFRESH_RATE_IN_SECONDS == 0)) # reconcile status after every 10 attempted installs + + per_batch_install_perf_log = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}][{8}={9}][{10}={11}][{12}={13}][{14}={15}]".format(Constants.PerfLogTrackerParams.TASK, "InstallBatchOfPackages", + "PackagesInBatch", str(packages_in_batch), "PackageAndDependencies", str(package_and_dependencies), "PackageAndDependencyVersions", str(package_and_dependency_versions), + "NumberOfParentPackagesInstalled", str(parent_packages_installed_in_batch_count), "NumberOfParentPackagesFailed", str(parent_packages_failed_in_batch_count), + "NumberOfDependenciesInstalled", str(number_of_dependencies_installed), "NumberOfDependenciesFailed", str(number_of_dependencies_failed)) + + per_batch_installation_stopwatch.stop_and_write_telemetry(str(per_batch_install_perf_log)) + + # Performing reconciliation at the end to get accurate number of installed packages through this function. + installed_update_count += self.perform_status_reconciliation_conditionally(package_manager, True) + + # not_attempted_and_failed_packages is the list of packages including two kind of packages: + # (a) Not attempted due to not enough time in maintenance window to install packages in batches. + # (b) Failed to install in batch patching. + # These packages are attempted in the sequential patching if there is enough time remaining in maintenance window. The non attempted packages are in + # the front of the list than failed packages and hence non attempated packages are attempted first in sequential patching than the failed packages. + not_attempted_and_failed_packages = remaining_packages + failed_packages + not_attempted_and_failed_package_versions = remaining_package_versions + failed_package_versions + return installed_update_count, patch_installation_successful, maintenance_window_batch_cutoff_reached, not_attempted_and_failed_packages, not_attempted_and_failed_package_versions def mark_installation_completed(self): """ Marks Installation operation as completed by updating the status of PatchInstallationSummary as success and patch metadata to be sent to healthstore. @@ -369,7 +631,7 @@ def get_excluded_updates(self, package_manager, packages, package_versions): excluded_package_versions.append(package_version) continue - dependency_list = package_manager.get_dependent_list(package) + dependency_list = package_manager.get_dependent_list([package]) if dependency_list and self.package_filter.check_for_exclusion(dependency_list): self.composite_logger.log_debug(" - Exclusion list match on dependency list for package '{0}': {1}".format(str(package), str(dependency_list))) excluded_packages.append(package) # one of the package's dependencies are excluded, so exclude the package diff --git a/src/core/src/core_logic/Stopwatch.py b/src/core/src/core_logic/Stopwatch.py index efeae8e4..0fc52452 100644 --- a/src/core/src/core_logic/Stopwatch.py +++ b/src/core/src/core_logic/Stopwatch.py @@ -23,6 +23,7 @@ class StopwatchException(Constants.EnumBackport): # Stopwatch exception strings STARTED_ALREADY = "Stopwatch is already started" NOT_STARTED = "Stopwatch is not started" + NOT_STOPPED = "Stopwatch is not stoppped" STOPPED_ALREADY = "Stopwatch is already stoppped" def __init__(self, env_layer, telemetry_writer, composite_logger): @@ -50,6 +51,7 @@ def start(self): self.time_taken_in_secs = None self.task_details = None + # Stop the stopwatch and set end_time. Create new end_time even if end_time is already set def stop(self): if self.end_time is not None: self.composite_logger.log_debug(str(Stopwatch.StopwatchException.STOPPED_ALREADY)) @@ -58,16 +60,26 @@ def stop(self): self.composite_logger.log_debug(str(Stopwatch.StopwatchException.NOT_STARTED)) self.start_time = self.end_time - self.time_taken_in_secs = self.env_layer.datetime.total_seconds_from_time_delta(self.end_time - self.start_time) - - # Rounding off to one digit after decimal e.g. 14.574372666666667 will become 14.6 - self.time_taken_in_secs = round(self.time_taken_in_secs, 1) + self.time_taken_in_secs = self.env_layer.datetime.total_seconds_from_time_delta_round_to_one_decimal_digit(self.end_time - self.start_time) + # Stop the stopwatch, set end_time and write details in telemetry. Create new end_time even if end_time is already set def stop_and_write_telemetry(self, message): self.stop() self.set_task_details(message) self.composite_logger.log("Stopwatch details: " + self.task_details) + # Write stopwatch details in telemetry. Use the existing end_time if it is already set otherwise set new end_time + def write_telemetry_for_stopwatch(self, message): + if self.end_time is None: + self.composite_logger.log_debug(str(Stopwatch.StopwatchException.NOT_STOPPED)) + self.end_time = self.env_layer.datetime.datetime_utcnow() + if self.start_time is None: + self.composite_logger.log_debug(str(Stopwatch.StopwatchException.NOT_STARTED)) + self.start_time = self.end_time + self.time_taken_in_secs = self.env_layer.datetime.total_seconds_from_time_delta_round_to_one_decimal_digit(self.end_time - self.start_time) + self.set_task_details(message) + self.composite_logger.log("Stopwatch details: " + str(self.task_details)) + def set_task_details(self, message): self.task_details = "[{0}={1}][{2}={3}][{4}={5}][{6}={7}]".format(Constants.PerfLogTrackerParams.MESSAGE, str(message), Constants.PerfLogTrackerParams.TIME_TAKEN_IN_SECS, str(self.time_taken_in_secs), Constants.PerfLogTrackerParams.START_TIME, str(self.start_time), Constants.PerfLogTrackerParams.END_TIME, str(self.end_time)) diff --git a/src/core/src/package_managers/AptitudePackageManager.py b/src/core/src/package_managers/AptitudePackageManager.py index d9cf6116..87bf34fd 100644 --- a/src/core/src/package_managers/AptitudePackageManager.py +++ b/src/core/src/package_managers/AptitudePackageManager.py @@ -360,19 +360,27 @@ def is_package_version_installed(self, package_name, package_version): self.composite_logger.log_debug(" - Package version specified was determined to NOT be installed.") return False - def get_dependent_list(self, package_name): - """Returns dependent List of the package""" - cmd = self.single_package_dependency_resolution_template.replace('', package_name) + def get_dependent_list(self, packages): + """Returns dependent List for the list of packages""" + package_names = "" + for index, package in enumerate(packages): + if index != 0: + package_names += ' ' + package_names += package + + cmd = self.single_package_dependency_resolution_template.replace('', package_names) self.composite_logger.log_debug("\nRESOLVING DEPENDENCIES USING COMMAND: " + str(cmd)) output = self.invoke_package_manager(cmd) - packages, package_versions = self.extract_packages_and_versions(output) - if package_name in packages: - packages.remove(package_name) + dependencies, dependency_versions = self.extract_packages_and_versions(output) + + for package in packages: + if package in dependencies: + dependencies.remove(package) - self.composite_logger.log_debug(str(len(packages)) + " dependent updates were found for package '" + package_name + "'.") - return packages + self.composite_logger.log_debug(str(len(dependencies)) + " dependent packages were found for packages '" + str(packages) + "'.") + return dependencies def get_product_name(self, package_name): """Retrieve product name """ @@ -584,3 +592,10 @@ def __get_os_major_version(self): def __is_minimum_required_python_installed(self): """check if python version is at least 3.5""" return sys.version_info >= Constants.UbuntuProClientSettings.MINIMUM_PYTHON_VERSION_REQUIRED + + def add_arch_dependencies(self, package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions): + """ + Add the packages with same name as that of input parameter package but with different architectures from packages list to the list package_and_dependencies. + Only required for yum. No-op for apt and zypper. + """ + return diff --git a/src/core/src/package_managers/PackageManager.py b/src/core/src/package_managers/PackageManager.py index 3b13467b..19db14da 100644 --- a/src/core/src/package_managers/PackageManager.py +++ b/src/core/src/package_managers/PackageManager.py @@ -198,12 +198,19 @@ def install_updates_fail_safe(self, excluded_packages): pass def install_update_and_dependencies(self, package_and_dependencies, package_and_dependency_versions, simulate=False): - """Install a single package along with its dependencies (explicitly)""" - install_result = Constants.INSTALLED - package_no_longer_required = False - code_path = "| Install" - start_time = time.time() - + """ + Install a list of packages along with dependencies (explicitly) + + Parameters: + package_and_dependencies (List of strings): List of packages along with dependencies to install + package_and_dependency_versions (List of strings): Versions of the packages in the list package_and_dependencies + simulate (bool): Whether this function call is from test run. + + Returns: + code (int): Output code of the command run to install packages + out (string): Output string of the command run to install packages + exec_cmd (string): Command used to install packages + """ if type(package_and_dependencies) is str: package_and_dependencies = [package_and_dependencies] package_and_dependency_versions = [package_and_dependency_versions] @@ -216,9 +223,30 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_ self.composite_logger.log_debug("UPDATING PACKAGE (WITH DEPENDENCIES) USING COMMAND: " + exec_cmd) out, code = self.invoke_package_manager_advanced(exec_cmd, raise_on_exception=False) - package_size = self.get_package_size(out) self.composite_logger.log_debug("\n\n" + out + "\n") # wrapping multi-line for readability + return code, out, exec_cmd + + def get_installation_status(self, code, out, exec_cmd, package, version, simulate=False): + """ + Returns result of the package installation + + Parameters: + code (int): Output code of the command run to install packages. + out (string): Output string of the command run to install packages. + exec_cmd (string): Command used to install packages. + package (string): Package name. + version (string): Package version. + simulate (bool): Whether this function call is from test run. + + Returns: + install_result (string): Package installation result + """ + install_result = Constants.INSTALLED + package_no_longer_required = False + code_path = "| Install" + start_time = time.time() + # special case of package no longer being required (or maybe even present on the system) if code == 1 and self.get_package_manager_setting(Constants.PKG_MGR_SETTING_IDENTITY) == Constants.YUM: self.composite_logger.log_debug(" - Detecting if package is no longer required (as return code is 1):") @@ -232,22 +260,22 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_ self.composite_logger.log_debug(" - Evidence of package no longer required NOT detected.") if not package_no_longer_required: - if not self.is_package_version_installed(package_and_dependencies[0], package_and_dependency_versions[0]): - if code == 0 and self.STR_ONLY_UPGRADES.replace('', package_and_dependencies[0]) in out: + if not self.is_package_version_installed(package, version): + if code == 0 and self.STR_ONLY_UPGRADES.replace('', package) in out: # It is premature to fail this package. In the *unlikely* case it never gets picked up, it'll remain NotStarted. # The NotStarted status must not be written again in the calling function (it's not at the time of this writing). code_path += " > Package has no prior version. (no operation; return 'not started')" install_result = Constants.PENDING - self.composite_logger.log_warning(" |- Package " + package_and_dependencies[0] + " (" + package_and_dependency_versions[0] + ") needs to already have an older version installed in order to be upgraded. " + + self.composite_logger.log_warning(" |- Package " + package + " (" + version + ") needs to already have an older version installed in order to be upgraded. " + "\n |- Another upgradeable package requiring it as a dependency can cause it to get installed later. No action may be required.\n") - elif code == 0 and self.STR_OBSOLETED.replace('', self.get_composite_package_identifier(package_and_dependencies[0], package_and_dependency_versions[0])) in out: + elif code == 0 and self.STR_OBSOLETED.replace('', self.get_composite_package_identifier(package, version)) in out: # Package can be obsoleted by another package installed in the run (via dependencies) code_path += " > Package obsoleted. (succeeded)" install_result = Constants.INSTALLED # close approximation to obsoleted self.composite_logger.log_debug(" - Package was discovered to be obsoleted.") - elif code == 0 and len(out.split(self.STR_REPLACED)) > 1 and package_and_dependencies[0] in out.split(self.STR_REPLACED)[1]: + elif code == 0 and len(out.split(self.STR_REPLACED)) > 1 and package in out.split(self.STR_REPLACED)[1]: code_path += " > Package replaced. (succeeded)" install_result = Constants.INSTALLED # close approximation to replaced self.composite_logger.log_debug(" - Package was discovered to be replaced by another during its installation.") @@ -256,12 +284,12 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_ install_result = Constants.FAILED if code != 0: code_path += " > Package NOT installed. (failed)" - self.composite_logger.log_error(" |- Package failed to install: " + package_and_dependencies[0] + " (" + package_and_dependency_versions[0] + "). " + + self.composite_logger.log_error(" |- Package failed to install: " + package + " (" + version + "). " + "\n |- Error code: " + str(code) + ". Command used: " + exec_cmd + "\n |- Command output: " + out + "\n") else: code_path += " > Package NOT installed but return code: 0. (failed)" - self.composite_logger.log_error(" |- Package appears to have not been installed: " + package_and_dependencies[0] + " (" + package_and_dependency_versions[0] + "). " + + self.composite_logger.log_error(" |- Package appears to have not been installed: " + package + " (" + version + "). " + "\n |- Return code: 0. Command used: " + exec_cmd + "\n" + "\n |- Command output: " + out + "\n") elif code != 0: @@ -271,15 +299,36 @@ def install_update_and_dependencies(self, package_and_dependencies, package_and_ code_path += " > Info, Package installed, zero return. (succeeded)" if not simulate: + package_size = self.get_package_size(out) if install_result == Constants.FAILED: - error = self.telemetry_writer.write_package_info(package_and_dependencies[0], package_and_dependency_versions[0], package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd, str(out)) + error = self.telemetry_writer.write_package_info(package, version, package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd, str(out)) else: - error = self.telemetry_writer.write_package_info(package_and_dependencies[0], package_and_dependency_versions[0], package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd) + error = self.telemetry_writer.write_package_info(package, version, package_size, round(time.time() - start_time, 2), install_result, code_path, exec_cmd) if error is not None: self.composite_logger.log_debug('\nEXCEPTION writing package telemetry: ' + repr(error)) return install_result + + def install_update_and_dependencies_and_get_status(self, package_and_dependencies, package_and_dependency_versions, simulate=False): + """ + Install a single package along with its dependencies (explicitly) and return the installation status + Parameters: + package_and_dependencies (List of strings): List of packages along with dependencies to install + package_and_dependency_versions (List of strings): Versions of the packages in the list package_and_dependencies + simulate (bool): Whether this function call is from test run. + + Returns: + install_result (string): Package installation result + """ + if type(package_and_dependencies) is str: + package_and_dependencies = [package_and_dependencies] + package_and_dependency_versions = [package_and_dependency_versions] + + code, out, exec_cmd = self.install_update_and_dependencies(package_and_dependencies, package_and_dependency_versions, simulate) + install_result = self.get_installation_status(code, out, exec_cmd, package_and_dependencies[0], package_and_dependency_versions[0], simulate) + return install_result + # endregion # region Package Information @@ -386,3 +435,20 @@ def do_processes_require_restart(self): """ Signals whether processes require a restart due to updates to files """ pass + @abstractmethod + def add_arch_dependencies(self, package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions): + """ + Add the packages with same name as that of input parameter package but with different architectures from packages list to the list package_and_dependencies. + Only required for yum. No-op for apt and zypper. + + Parameters: + package_manager (PackageManager): Package manager used. + package (string): Input package for which same package name but different architecture need to be added in the list package_and_dependencies. + packages (List of strings): List of all packages selected by user to install. + package_versions (List of strings): Versions of packages in packages list. + package_and_dependencies (List of strings): List of packages along with dependencies. This function adds packages with same name as input parameter package + but different architecture in this list. + package_and_dependency_versions (List of strings): Versions of packages in package_and_dependencies. + """ + pass + diff --git a/src/core/src/package_managers/YumPackageManager.py b/src/core/src/package_managers/YumPackageManager.py index ccb1658e..c5a8814c 100644 --- a/src/core/src/package_managers/YumPackageManager.py +++ b/src/core/src/package_managers/YumPackageManager.py @@ -282,7 +282,7 @@ def is_package_version_installed(self, package_name, package_version): return False - def get_dependent_list(self, package_name): + def extract_dependencies(self, output, packages): # Sample output for the cmd 'sudo yum update --assumeno selinux-policy.noarch' is : # # Loaded plugins: langpacks, product-id, search-disabled-repos @@ -299,10 +299,7 @@ def get_dependent_list(self, package_name): # ---> Package selinux-policy-targeted.noarch 0:3.13.1-102.el7_3.16 will be an update # --> Finished Dependency Resolution - self.composite_logger.log_debug("\nRESOLVING DEPENDENCIES USING COMMAND: " + str(self.single_package_upgrade_simulation_cmd + package_name)) - dependent_updates = [] - - output = self.invoke_package_manager(self.single_package_upgrade_simulation_cmd + package_name) + dependencies = [] lines = output.strip().split('\n') for line in lines: @@ -316,12 +313,24 @@ def get_dependent_list(self, package_name): continue dependent_package_name = self.get_product_name(updates_line[2]) - if len(dependent_package_name) != 0 and dependent_package_name != package_name: + if len(dependent_package_name) != 0 and dependent_package_name not in packages: self.composite_logger.log_debug(" - Dependency detected: " + dependent_package_name) - dependent_updates.append(dependent_package_name) + dependencies.append(dependent_package_name) + + return dependencies - self.composite_logger.log_debug(str(len(dependent_updates)) + " dependent updates were found for package '" + package_name + "'.") - return dependent_updates + def get_dependent_list(self, packages): + package_names = "" + for index, package in enumerate(packages): + if index != 0: + package_names += ' ' + package_names += package + + self.composite_logger.log_debug("\nRESOLVING DEPENDENCIES USING COMMAND: " + str(self.single_package_upgrade_simulation_cmd + package_names)) + output = self.invoke_package_manager(self.single_package_upgrade_simulation_cmd + package_names) + dependencies = self.extract_dependencies(output, packages) + self.composite_logger.log_debug(str(len(dependencies)) + " dependent packages were found for packages '" + str(packages) + "'.") + return dependencies def get_product_name(self, package_name): """Retrieve product name including arch where present""" @@ -930,3 +939,22 @@ def do_processes_require_restart(self): self.composite_logger.log(" - Processes requiring restart (" + str(process_count) + "): [" + process_list_verbose + "]") return process_count != 0 # True if there were any # endregion Reboot Management + + def add_arch_dependencies(self, package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions): + """ + Add the packages with same name as that of input parameter package but with different architectures from packages list to the list package_and_dependencies. + + Parameters: + package_manager (PackageManager): Package manager used. + package (string): Input package for which same package name but different architecture need to be added in the list package_and_dependencies. + packages (List of strings): List of all packages selected by user to install. + package_versions (List of strings): Versions of packages in packages list. + package_and_dependencies (List of strings): List of packages along with dependencies. This function adds packages with same name as input parameter package + but different architecture in this list. + package_and_dependency_versions (List of strings): Versions of packages in package_and_dependencies. + """ + package_name_without_arch = package_manager.get_product_name_without_arch(package) + for possible_arch_dependency, possible_arch_dependency_version in zip(packages, package_versions): + if package_manager.get_product_name_without_arch(possible_arch_dependency) == package_name_without_arch and possible_arch_dependency not in package_and_dependencies: + package_and_dependencies.append(possible_arch_dependency) + package_and_dependency_versions.append(possible_arch_dependency_version) diff --git a/src/core/src/package_managers/ZypperPackageManager.py b/src/core/src/package_managers/ZypperPackageManager.py index 4212c19d..e57d9f36 100644 --- a/src/core/src/package_managers/ZypperPackageManager.py +++ b/src/core/src/package_managers/ZypperPackageManager.py @@ -488,7 +488,7 @@ def get_all_available_versions_of_package_ex(self, package_name, include_install return package_versions - def get_dependent_list(self, package_name): + def extract_dependencies(self, output, packages): # Sample output for the cmd # 'sudo LANG=en_US.UTF8 zypper --non-interactive update --dry-run man' is : # @@ -510,11 +510,7 @@ def get_dependent_list(self, package_name): # Overall download size: 23.7 MiB. Already cached: 0 B. \ # After the operation, additional 85.1 MiB will be used. # Continue? [y/n/? shows all options] (y): y - - self.composite_logger.log_debug("\nRESOLVING DEPENDENCIES USING COMMAND:: " + str(self.single_package_upgrade_simulation_cmd + package_name)) - dependent_updates = [] - - output = self.invoke_package_manager(self.single_package_upgrade_simulation_cmd + package_name) + dependencies = [] lines = output.strip().split('\n') for line in lines: @@ -525,12 +521,25 @@ def get_dependent_list(self, package_name): updates_line = lines[lines.index(line) + 1] dependent_package_names = re.split(r'\s+', updates_line) for dependent_package_name in dependent_package_names: - if len(dependent_package_name) != 0 and dependent_package_name != package_name: + if len(dependent_package_name) != 0 and dependent_package_name not in packages: self.composite_logger.log_debug(" - Dependency detected: " + dependent_package_name) - dependent_updates.append(dependent_package_name) + dependencies.append(dependent_package_name) + + return dependencies + + def get_dependent_list(self, packages): + package_names = "" + for index, package in enumerate(packages): + if index != 0: + package_names += ' ' + package_names += package - self.composite_logger.log_debug(str(len(dependent_updates)) + " dependent updates were found for package '" + package_name + "'.") - return dependent_updates + self.composite_logger.log_debug("\nRESOLVING DEPENDENCIES USING COMMAND:: " + str(self.single_package_upgrade_simulation_cmd + package_names)) + + output = self.invoke_package_manager(self.single_package_upgrade_simulation_cmd + package_names) + dependencies = self.extract_dependencies(output, packages) + self.composite_logger.log_debug(str(len(dependencies)) + " dependent packages were found for packages '" + str(packages) + "'.") + return dependencies def get_product_name(self, package_name): """Retrieve product name """ @@ -796,3 +805,10 @@ def do_processes_require_restart(self): self.composite_logger.log(" - Processes requiring restart (" + str(process_count) + "): [" + process_list_verbose + "]") return process_count != 0 # True if there were any # endregion Reboot Management + + def add_arch_dependencies(self, package_manager, package, packages, package_versions, package_and_dependencies, package_and_dependency_versions): + """ + Add the packages with same name as that of input parameter package but with different architectures from packages list to the list package_and_dependencies. + Only required for yum. No-op for apt and zypper. + """ + return diff --git a/src/core/tests/Test_AptitudePackageManager.py b/src/core/tests/Test_AptitudePackageManager.py index a2cf400c..e4f77afb 100644 --- a/src/core/tests/Test_AptitudePackageManager.py +++ b/src/core/tests/Test_AptitudePackageManager.py @@ -113,7 +113,7 @@ def test_install_package_success(self): self.assertIsNotNone(package_manager) # test for successfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.INSTALLED) # needs to be fixed + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.INSTALLED) # needs to be fixed def test_is_installed_check_with_dpkg(self): self.runtime.set_legacy_test_type('SuccessInstallPath') @@ -133,7 +133,7 @@ def test_install_package_failure(self): self.assertIsNotNone(package_manager) # test for unsuccessfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.FAILED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.FAILED) self.assertRaises(Exception, lambda: package_manager.invoke_package_manager('sudo apt-get -y --only-upgrade true install force-dpkg-failure')) # ensure that error message appears in substatus properly @@ -172,7 +172,7 @@ def test_install_package_only_upgrades(self): self.assertIsNotNone(package_manager) # test for unsuccessfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('iucode-tool', '1.5.1-1ubuntu0.1', simulate=True), Constants.PENDING) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('iucode-tool', '1.5.1-1ubuntu0.1', True), Constants.PENDING) def test_disable_auto_os_update_with_two_patch_modes_enabled_success(self): package_manager = self.container.get('package_manager') diff --git a/src/core/tests/Test_MaintenanceWindow.py b/src/core/tests/Test_MaintenanceWindow.py index 793a6bb9..8051f9f0 100644 --- a/src/core/tests/Test_MaintenanceWindow.py +++ b/src/core/tests/Test_MaintenanceWindow.py @@ -19,7 +19,6 @@ from core.tests.library.ArgumentComposer import ArgumentComposer from core.tests.library.RuntimeCompositor import RuntimeCompositor - class TestMaintenanceWindow(unittest.TestCase): def setUp(self): pass @@ -107,5 +106,23 @@ def test_get_percentage_maintenance_window_used_start_time_greater_exception(sel self.assertRaises(Exception, runtime.maintenance_window.get_percentage_maintenance_window_used) runtime.stop() + def test_is_package_install_time_available(self): + argument_composer = ArgumentComposer() + + number_of_packages = 1 + remaining_time_in_minutes = 10 + + # if reboot_setting = 'Never', then cutoff time is 5 minutes. So, package install time is available. + argument_composer.reboot_setting = 'Never' + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True) + self.assertEqual(True, runtime.maintenance_window.is_package_install_time_available(remaining_time_in_minutes, number_of_packages)) + runtime.stop() + + # if reboot_setting = 'Always', then cutoff time is (5+15=20 minutes). So, package install time is not available. + argument_composer.reboot_setting = 'Always' + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True) + self.assertEqual(False, runtime.maintenance_window.is_package_install_time_available(remaining_time_in_minutes, number_of_packages)) + runtime.stop() + if __name__ == '__main__': unittest.main() diff --git a/src/core/tests/Test_PatchInstaller.py b/src/core/tests/Test_PatchInstaller.py index 10e7e1ac..2cfcd85f 100644 --- a/src/core/tests/Test_PatchInstaller.py +++ b/src/core/tests/Test_PatchInstaller.py @@ -20,7 +20,6 @@ from core.src.bootstrap.Constants import Constants from core.tests.library.ArgumentComposer import ArgumentComposer from core.tests.library.RuntimeCompositor import RuntimeCompositor -from core.src.core_logic.Stopwatch import Stopwatch class TestPatchInstaller(unittest.TestCase): def setUp(self): @@ -55,6 +54,27 @@ def test_yum_install_success(self): runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) # Path change runtime.set_legacy_test_type('SuccessInstallPath') + # As all the packages should get installed using batch patching, get_remaining_packages_to_install should return 0 packages + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(2, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_yum_install_success_not_enough_time_for_batch_patching(self): + # total packages to install is 2, reboot_setting is 'Never', so cutoff time for batch = 2*5 = 10 + # window size is 60 minutes, let time remain = 9 minutes so that not enough time to install in batch + # So td = 60-9 = 51 + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=51) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.reboot_setting = 'Never' + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + # Path change + runtime.set_legacy_test_type('SuccessInstallPath') installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) self.assertEqual(2, installed_update_count) self.assertTrue(update_run_successful) @@ -93,6 +113,25 @@ def test_zypper_install_updates_maintenance_window_exceeded(self): self.assertTrue(maintenance_window_exceeded) runtime.stop() + def test_zypper_install_success_not_enough_time_for_batch_patching(self): + # total packages to install is 2, reboot_setting is 'Never', so cutoff time for batch = 2*5 = 10 + # window size is 60 minutes, let time remain = 9 minutes so that not enough time to install in batch + # So td = 60-9 = 51 + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=51) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + # Path change + runtime.set_legacy_test_type('SuccessInstallPath') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(2, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + def test_zypper_install_success(self): current_time = datetime.datetime.utcnow() td = datetime.timedelta(hours=0, minutes=20) @@ -103,6 +142,7 @@ def test_zypper_install_success(self): runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) # Path change runtime.set_legacy_test_type('SuccessInstallPath') + # As all the packages should get installed using batch patching, get_remaining_packages_to_install should return 0 packages installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) self.assertEqual(2, installed_update_count) self.assertTrue(update_run_successful) @@ -151,12 +191,278 @@ def test_apt_install_success(self): runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) # Path change runtime.set_legacy_test_type('SuccessInstallPath') + # As all the packages should get installed using batch patching, get_remaining_packages_to_install should return 0 packages + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(3, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_apt_install_success_not_enough_time_for_batch_patching(self): + # total packages to install is 3, reboot_setting is 'Never', so cutoff time for batch = 3*5 = 15 + # window size is 60 minutes, let time remain = 14 minutes so that not enough time to install in batch + # So td = 60-14 = 46 + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=46) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('SuccessInstallPath') installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) self.assertEqual(3, installed_update_count) self.assertTrue(update_run_successful) self.assertFalse(maintenance_window_exceeded) runtime.stop() + def test_dependency_installed_successfully(self): + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=20) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallSuccessfully') + # As all the packages should get installed using batch patching, get_remaining_packages_to_install should return 0 packages + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(4, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_dependency_install_failed(self): + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=20) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallFailed') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(2, installed_update_count) + self.assertFalse(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_not_enough_time_for_batch_patching_dependency_installed_successfully(self): + # total packages to install is 3, reboot_setting is 'Never', so cutoff time for batch = 3*5 = 15 + # window size is 60 minutes, let time remain = 14 minutes so that not enough time to install in batch + # So td = 60-14 = 46 + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=46) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.reboot_setting = 'Never' + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallSuccessfully') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(4, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_not_enough_time_for_batch_patching_dependency_install_failed(self): + # total packages to install is 3, reboot_setting is 'Never', so cutoff time for batch = 3*5 = 15 + # window size is 60 minutes, let time remain = 14 minutes so that not enough time to install in batch + # So td = 60-14 = 46 + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=46) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.reboot_setting = 'Never' + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallFailed') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(2, installed_update_count) + self.assertFalse(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_include_dependency_apt(self): + # all_packages contains: git-man, git, grub-efi-amd64-signed and grub-efi-amd64-bin + # All the classifications selected and hence all packages to install + # Batch contains packages git-man, git and grub-efi-amd64-signed + # grub-efi-amd64-signed is dependent on grub-efi-amd64-bin so include_dependencies should add grub-efi-amd64-bin in package_and_dependencies + argument_composer = ArgumentComposer() + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallFailed') + all_packages, all_packages_version = runtime.package_manager.get_available_updates(runtime.package_filter) + packages = list(all_packages) + package_versions = list(all_packages_version) + packages_in_batch = packages[0:3] + package_versions_in_batch = package_versions[0:3] + package_and_dependencies = list(packages_in_batch) + package_and_dependency_versions = list(package_versions_in_batch) + self.assertEqual(3, len(package_and_dependencies)) + self.assertEqual(3, len(package_and_dependency_versions)) + self.assertTrue("git-man" in package_and_dependencies) + self.assertTrue("git" in package_and_dependencies) + self.assertTrue("grub-efi-amd64-signed" in package_and_dependencies) + self.assertTrue("grub-efi-amd64-bin" not in package_and_dependencies) + runtime.patch_installer.include_dependencies(runtime.package_manager, packages_in_batch, all_packages, all_packages_version, packages, package_versions, package_and_dependencies, package_and_dependency_versions) + self.assertEqual(4, len(package_and_dependencies)) + self.assertEqual(4, len(package_and_dependency_versions)) + self.assertTrue("git-man" in package_and_dependencies) + self.assertTrue("git" in package_and_dependencies) + self.assertTrue("grub-efi-amd64-signed" in package_and_dependencies) + self.assertTrue("grub-efi-amd64-bin" in package_and_dependencies) + runtime.stop() + + def test_include_dependency_yum(self): + # all_packages contains: selinux-policy.noarch, selinux-policy-targeted.noarch, libgcc.i686, tar.x86_64 and tcpdump.x86_64 + # All the classifications selected and hence all packages to install + # Batch contains the package selinux-policy.noarch + # selinux-policy.noarch is dependent on selinux-policy-targeted.noarch so include_dependencies should add selinux-policy-targeted.noarch in package_and_dependencies + argument_composer = ArgumentComposer() + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + # Path change + runtime.set_legacy_test_type('HappyPath') + all_packages, all_packages_version = runtime.package_manager.get_available_updates(runtime.package_filter) + packages = list(all_packages) + package_versions = list(all_packages_version) + packages_in_batch = ["selinux-policy.noarch"] + package_versions_in_batch = ["3.13.1-102.el7_3.16"] + package_and_dependencies = list(packages_in_batch) + package_and_dependency_versions = list(package_versions_in_batch) + runtime.patch_installer.include_dependencies(runtime.package_manager, packages_in_batch, all_packages, all_packages_version, packages, package_versions, package_and_dependencies, package_and_dependency_versions) + self.assertEqual(2, len(package_and_dependencies)) + self.assertEqual(2, len(package_and_dependency_versions)) + self.assertTrue(package_and_dependencies[0] == "selinux-policy.noarch") + self.assertTrue(package_and_dependency_versions[0] == "3.13.1-102.el7_3.16") + self.assertTrue(package_and_dependencies[1] == "selinux-policy-targeted.noarch") + self.assertTrue(package_and_dependency_versions[1] == "3.13.1-102.el7_3.16") + runtime.stop() + + def test_include_dependency_zypper(self): + # all_packages contains: kernel-default, libgcc and libgoa-1_0-0 + # All the classifications selected and hence all packages to install + # Batch contains the package libgcc + # libgcc is not dependent on any package so include_dependencies should not add any package in package_and_dependencies + argument_composer = ArgumentComposer() + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.ZYPPER) + # Path change + runtime.set_legacy_test_type('HappyPath') + all_packages, all_packages_version = runtime.package_manager.get_available_updates(runtime.package_filter) + packages = list(all_packages) + package_versions = list(all_packages_version) + packages_in_batch = ["libgcc"] + package_versions_in_batch = ["5.60.7-8.1"] + package_and_dependencies = list(packages_in_batch) + package_and_dependency_versions = list(package_versions_in_batch) + runtime.patch_installer.include_dependencies(runtime.package_manager, packages_in_batch, all_packages, all_packages_version, packages, package_versions, package_and_dependencies, package_and_dependency_versions) + self.assertEqual(1, len(package_and_dependencies)) + self.assertEqual(1, len(package_and_dependency_versions)) + self.assertTrue(package_and_dependencies[0] == "libgcc") + self.assertTrue(package_and_dependency_versions[0] == "5.60.7-8.1") + runtime.stop() + + def test_skip_package_version_UA_ESM_REQUIRED(self): + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=20) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('UA_ESM_Required') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(0, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_dependent_package_excluded(self): + # exclusion list contains grub-efi-amd64-bin + # grub-efi-amd64-signed is dependent on grub-efi-amd64-bin, so grub-efi-amd64-signed should also get excluded + # so, out of 4 packages, only 2 packages are installed and 2 are excluded + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=20) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.patches_to_exclude = ["grub-efi-amd64-bin"] + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallSuccessfully') + # As all the packages should get installed using batch patching, get_remaining_packages_to_install should return 0 packages + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(2, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_dependent_package_excluded_and_not_enough_time_for_batch_patching(self): + # exclusion list contains grub-efi-amd64-bin + # grub-efi-amd64-signed is dependent on grub-efi-amd64-bin, so grub-efi-amd64-signed should also get excluded + # so, out of 4 packages, only 2 packages are installed and 2 are excluded. + # total packages to install is 2, reboot_setting is 'Never', so cutoff time for batch = 2*5 = 10 + # window size is 60 minutes, let time remain = 9 minutes so that not enough time to install in batch + # So td = 60-9 = 51 + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=51) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.patches_to_exclude = ["grub-efi-amd64-bin"] + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.APT) + # Path change + runtime.set_legacy_test_type('DependencyInstallSuccessfully') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(2, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_arch_dependency_install_success(self): + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=20) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + # Path change + runtime.set_legacy_test_type('ArchDependency') + # As all the packages should get installed using batch patching, get_remaining_packages_to_install should return 0 packages + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(4, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + + def test_no_updates_to_install(self): + # Verify that if there are no updates available then also install_updates method runs successfully + current_time = datetime.datetime.utcnow() + td = datetime.timedelta(hours=0, minutes=20) + job_start_time = (current_time - td).strftime("%Y-%m-%dT%H:%M:%S.9999Z") + argument_composer = ArgumentComposer() + argument_composer.maximum_duration = 'PT1H' + argument_composer.start_time = job_start_time + runtime = RuntimeCompositor(argument_composer.get_composed_arguments(), True, Constants.YUM) + # Path change. There is no path as NoUpdatesToInstall so the command to get available updates will return empty string + runtime.set_legacy_test_type('NoUpdatesToInstall') + installed_update_count, update_run_successful, maintenance_window_exceeded = runtime.patch_installer.install_updates(runtime.maintenance_window, runtime.package_manager, simulate=True) + self.assertEqual(0, installed_update_count) + self.assertTrue(update_run_successful) + self.assertFalse(maintenance_window_exceeded) + runtime.stop() + def test_healthstore_writes(self): self.healthstore_writes_helper("HealthStoreId", None, False, expected_patch_version="HealthStoreId") self.healthstore_writes_helper("HealthStoreId", "MaintenanceRunId", False, expected_patch_version="HealthStoreId") diff --git a/src/core/tests/Test_Stopwatch.py b/src/core/tests/Test_Stopwatch.py index b85d9c08..83ea3540 100644 --- a/src/core/tests/Test_Stopwatch.py +++ b/src/core/tests/Test_Stopwatch.py @@ -130,8 +130,16 @@ def test_stopped_already(self): self.assertTrue(start_time1 == start_time2) self.assertTrue(end_time1 <= end_time2) self.assertTrue(time_taken1 <= time_taken2) - + def test_write_telemetry_for_stopwatch(self): + stopwatch = Stopwatch(self.runtime.env_layer, self.runtime.telemetry_writer, self.runtime.composite_logger) + stopwatch.write_telemetry_for_stopwatch("test") + self.assertTrue(stopwatch.start_time is not None) + self.assertTrue(stopwatch.end_time is not None) + self.assertTrue(stopwatch.time_taken_in_secs is not None) + self.assertTrue(stopwatch.task_details is not None) + self.assertTrue(stopwatch.start_time <= stopwatch.end_time) + self.assertTrue(stopwatch.time_taken_in_secs >= 0) if __name__ == '__main__': diff --git a/src/core/tests/Test_YumPackageManager.py b/src/core/tests/Test_YumPackageManager.py index 2fd53ad9..6a185261 100644 --- a/src/core/tests/Test_YumPackageManager.py +++ b/src/core/tests/Test_YumPackageManager.py @@ -131,13 +131,13 @@ def test_package_manager(self): # test for get_dependent_list # legacy_test_type ='HappyPath' - dependent_list = package_manager.get_dependent_list("selinux-policy.noarch") + dependent_list = package_manager.get_dependent_list(["selinux-policy.noarch"]) self.assertIsNotNone(dependent_list) self.assertEqual(len(dependent_list), 1) self.assertEqual(dependent_list[0], "selinux-policy-targeted.noarch") # test for get_dependent_list with 'install' instead of update - dependent_list = package_manager.get_dependent_list("kmod-kvdo.x86_64") + dependent_list = package_manager.get_dependent_list(["kmod-kvdo.x86_64"]) self.assertIsNotNone(dependent_list) self.assertEqual(len(dependent_list), 1) self.assertEqual(dependent_list[0], "kernel.x86_64") @@ -181,7 +181,7 @@ def test_package_manager(self): # test for get_dependent_list # legacy_test_type ='Exception Path' try: - package_manager.get_dependent_list("man") + package_manager.get_dependent_list(["man"]) except Exception as exception: self.assertTrue(str(exception)) else: @@ -197,7 +197,7 @@ def test_install_package_success(self): self.assertIsNotNone(package_filter) # test for successfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.INSTALLED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.INSTALLED) def test_install_package_failure(self): """Unit test for install package failure""" @@ -209,7 +209,7 @@ def test_install_package_failure(self): self.assertIsNotNone(package_filter) # test for unsuccessfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('selinux-policy', '3.13.1-102.el7_3.16', simulate=True), Constants.FAILED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('selinux-policy', '3.13.1-102.el7_3.16', simulate=True), Constants.FAILED) def test_install_package_obsoleted(self): """Unit test for install package failure""" @@ -221,7 +221,7 @@ def test_install_package_obsoleted(self): self.assertIsNotNone(package_filter) # test for unsuccessfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('rdma.noarch', '7.3_4.7_rc2-6.el7_3', simulate=True), Constants.INSTALLED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('rdma.noarch', '7.3_4.7_rc2-6.el7_3', simulate=True), Constants.INSTALLED) def test_install_package_replaced(self): """Unit test for install package failure""" @@ -233,7 +233,7 @@ def test_install_package_replaced(self): self.assertIsNotNone(package_filter) # test for unsuccessfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('python-rhsm.x86_64', '1.19.10-1.el7_4', simulate=True), Constants.INSTALLED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('python-rhsm.x86_64', '1.19.10-1.el7_4', simulate=True), Constants.INSTALLED) def test_get_product_name(self): """Unit test for retrieving product Name""" diff --git a/src/core/tests/Test_ZypperPackageManager.py b/src/core/tests/Test_ZypperPackageManager.py index 67f51a27..ffadb910 100644 --- a/src/core/tests/Test_ZypperPackageManager.py +++ b/src/core/tests/Test_ZypperPackageManager.py @@ -154,7 +154,7 @@ def test_package_manager(self): # test for get_dependent_list # legacy_test_type ='Happy Path' - dependent_list = package_manager.get_dependent_list("man") + dependent_list = package_manager.get_dependent_list(["man"]) self.assertIsNotNone(dependent_list) self.assertEqual(len(dependent_list), 16) @@ -180,7 +180,7 @@ def test_package_manager(self): # test for get_dependent_list # legacy_test_type ='Exception Path' try: - package_manager.get_dependent_list("man") + package_manager.get_dependent_list(["man"]) except Exception as exception: self.assertTrue(str(exception)) else: @@ -227,7 +227,7 @@ def test_install_package_success(self): self.assertIsNotNone(package_manager) # test for successfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('selinux-policy', '3.13.1-102.el7_3.16', simulate=True), Constants.INSTALLED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('selinux-policy', '3.13.1-102.el7_3.16', simulate=True), Constants.INSTALLED) def test_install_package_failure(self): self.runtime.set_legacy_test_type('FailInstallPath') @@ -236,7 +236,7 @@ def test_install_package_failure(self): self.assertIsNotNone(package_manager) # test for unsuccessfully installing a package - self.assertEqual(package_manager.install_update_and_dependencies('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.FAILED) + self.assertEqual(package_manager.install_update_and_dependencies_and_get_status('selinux-policy.noarch', '3.13.1-102.el7_3.16', simulate=True), Constants.FAILED) def test_get_process_tree_from_package_manager_output_success(self): self.runtime.set_legacy_test_type('HappyPath') diff --git a/src/core/tests/library/LegacyEnvLayerExtensions.py b/src/core/tests/library/LegacyEnvLayerExtensions.py index bcc6ccb6..4eea7757 100644 --- a/src/core/tests/library/LegacyEnvLayerExtensions.py +++ b/src/core/tests/library/LegacyEnvLayerExtensions.py @@ -963,6 +963,146 @@ def run_command_output(self, cmd, no_output=False, chk_err=True): else: code = 0 output = "Error: Cannot retrieve repository metadata (repomd.xml) for repository: addons. Please verify its path and try again" + elif self.legacy_test_type == 'DependencyInstallSuccessfully': + if self.legacy_package_manager_name is Constants.APT: + # Total 4 packages: git-man, git, grub-efi-amd64-signed and grub-efi-amd64-bin + # grub-efi-amd64-signed is dependent on grub-efi-amd64-bin + # All packages installs successfully + if cmd.find("dist-upgrade") > -1: + code = 0 + output = "Inst git-man [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [all])" \ + "Inst git [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [amd64])" \ + "Inst grub-efi-amd64-signed [1.187.2~18.04.1+2.06-2ubuntu14] " \ + "(1.187.3~18.04.1+2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64]) []" \ + "Inst grub-efi-amd64-bin [2.06-2ubuntu14] " \ + "(2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64])" + elif cmd.find("apt-get -y --only-upgrade true -s install git-man git grub-efi-amd64-signed") > -1: + code = 0 + output = "Inst git-man [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [all])" \ + "Inst git [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [amd64])" \ + "Inst grub-efi-amd64-signed [1.187.2~18.04.1+2.06-2ubuntu14] " \ + "(1.187.3~18.04.1+2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64]) []" \ + "Inst grub-efi-amd64-bin [2.06-2ubuntu14] " \ + "(2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64])" + elif cmd.find("apt-get -y --only-upgrade true -s install grub-efi-amd64-signed") > -1: + code = 0 + output = "Inst grub-efi-amd64-signed [1.187.2~18.04.1+2.06-2ubuntu14] " \ + "(1.187.3~18.04.1+2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64]) []" \ + "Inst grub-efi-amd64-bin [2.06-2ubuntu14] " \ + "(2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64])" + elif cmd.find("sudo apt list --installed git-man") > -1: + code = 0 + output = "Listing... Done\n" + \ + "git-man/bionic-updates,bionic-security,now 1:2.17.1-1ubuntu0.16 all [installed,automatic]" + elif cmd.find("sudo apt list --installed git") > -1: + code = 0 + output = "Listing... Done\n" + \ + "git/bionic-updates,bionic-security,now 1:2.17.1-1ubuntu0.16 amd64 [installed,automatic]" + elif cmd.find("sudo apt list --installed grub-efi-amd64-signed") > -1: + code = 0 + output = "Listing... Done\n" + \ + "grub-efi-amd64-signed/bionic-updates,now 1.187.3~18.04.1+2.06-2ubuntu14.1 amd64 [installed]" + elif cmd.find("sudo apt list --installed grub-efi-amd64-bin") > -1: + code = 0 + output = "Listing... Done\n" + \ + "grub-efi-amd64-bin/bionic-updates,now 2.06-2ubuntu14.1 amd64 [installed]" + elif cmd.find("simulate-install") > -1 or cmd.find( + "apt-get -y --only-upgrade true -s install") > -1 or cmd.find( + "LANG=en_US.UTF8 sudo yum install --assumeno") > -1 or cmd.find( + "sudo LANG=en_US.UTF8 zypper --non-interactive update --dry-run") > -1: + code = 0 + output = "Package sucessfully installed!" + elif self.legacy_test_type == 'DependencyInstallFailed': + if self.legacy_package_manager_name is Constants.APT: + # Total 4 packages: git-man, git, grub-efi-amd64-signed and grub-efi-amd64-bin + # grub-efi-amd64-signed is dependent on grub-efi-amd64-bin + # Installation of grub-efi-amd64-bin fails and as grub-efi-amd64-signed is dependent, it also failed + # Rest all packages install successfully + if cmd.find("dist-upgrade") > -1: + code = 0 + output = "Inst git-man [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [all])" \ + "Inst git [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [amd64])" \ + "Inst grub-efi-amd64-signed [1.187.2~18.04.1+2.06-2ubuntu14] " \ + "(1.187.3~18.04.1+2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64]) []" \ + "Inst grub-efi-amd64-bin [2.06-2ubuntu14] " \ + "(2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64])" + elif cmd.find("apt-get -y --only-upgrade true -s install git-man git grub-efi-amd64-signed") > -1: + code = 0 + output = "Inst git-man [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [all])" \ + "Inst git [1:2.17.1-1ubuntu0.15] (1:2.17.1-1ubuntu0.16 Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [amd64])" \ + "Inst grub-efi-amd64-signed [1.187.2~18.04.1+2.06-2ubuntu14] " \ + "(1.187.3~18.04.1+2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64]) []" \ + "Inst grub-efi-amd64-bin [2.06-2ubuntu14] " \ + "(2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64])" + elif cmd.find("apt-get -y --only-upgrade true -s install grub-efi-amd64-signed") > -1: + code = 0 + output = "Inst grub-efi-amd64-signed [1.187.2~18.04.1+2.06-2ubuntu14] " \ + "(1.187.3~18.04.1+2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64]) []" \ + "Inst grub-efi-amd64-bin [2.06-2ubuntu14] " \ + "(2.06-2ubuntu14.1 Ubuntu:18.04/bionic-updates [amd64])" + elif cmd.find("sudo apt list --installed git-man") > -1: + code = 0 + output = "Listing... Done\n" + \ + "git-man/bionic-updates,bionic-security,now 1:2.17.1-1ubuntu0.16 all [installed,automatic]" + elif cmd.find("sudo apt list --installed git") > -1: + code = 0 + output = "Listing... Done\n" + \ + "git/bionic-updates,bionic-security,now 1:2.17.1-1ubuntu0.16 amd64 [installed,automatic]" + elif cmd.find("sudo apt list --installed grub-efi-amd64-signed") > -1: + code = 0 + output = "Listing... Done\n" + elif cmd.find("sudo apt list --installed grub-efi-amd64-bin") > -1: + code = 0 + output = "Listing... Done\n" + elif cmd.find("simulate-install") > -1 or cmd.find( + "apt-get -y --only-upgrade true -s install") > -1 or cmd.find( + "LANG=en_US.UTF8 sudo yum install --assumeno") > -1 or cmd.find( + "sudo LANG=en_US.UTF8 zypper --non-interactive update --dry-run") > -1: + code = 0 + output = "Package sucessfully installed!" + elif self.legacy_test_type == 'UA_ESM_Required': + if self.legacy_package_manager_name is Constants.APT: + if cmd.find("dist-upgrade") > -1: + code = 0 + output = "Inst git-man [1:2.17.1-1ubuntu0.15] (UA_ESM_Required Ubuntu:18.04/bionic-updates, " \ + "Ubuntu:18.04/bionic-security [all])" + elif self.legacy_test_type == 'ArchDependency': + if self.legacy_package_manager_name is Constants.YUM: + if cmd.find("check-update") > -1: + code = 100 + output = "\n" + \ + "selinux-policy.noarch " + \ + "3.13.1-102.el7_3.16 " + \ + "rhui-rhel-7-server-rhui-rpms\n" + \ + "selinux-policy-targeted.noarch " + \ + "3.13.1-102.el7_3.16 " + \ + "rhui-rhel-7-server-rhui-rpms\n" + \ + "libgcc.i686 " + \ + "4.8.5-28.el7 " + \ + "rhui-rhel-7-server-rhui-rpms\n" + \ + "libgcc.x86_64 " + \ + "4.8.5-28.el7 " + \ + "rhui-rhel-7-server-rhui-rpms\n" + elif cmd.find("list installed") > -1: + code = 0 + package = cmd.replace('sudo yum list installed ', '') + whitelisted_versions = [ + '3.13.1-102.el7_3.16', '4.8.5-28.el7'] # any list of versions you want to work for *any* package + output = "Loaded plugins: product-id, search-disabled-repos, subscription-manager\n" + \ + "Installed Packages\n" + template = " @anaconda/7.3\n" + for version in whitelisted_versions: + entry = template.replace('', package) + entry = entry.replace('', version) + output += entry major_version = self.get_python_major_version() if major_version == 2: