From 6270937fbac46896daf13e20b8be2d74c7d9eecc Mon Sep 17 00:00:00 2001 From: SathishMSFT Date: Thu, 29 Jun 2023 14:37:08 +0530 Subject: [PATCH 1/4] telemetry change to get esm packages selected count --- src/core/src/package_managers/AptitudePackageManager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/src/package_managers/AptitudePackageManager.py b/src/core/src/package_managers/AptitudePackageManager.py index 9f35895b..99c84715 100644 --- a/src/core/src/package_managers/AptitudePackageManager.py +++ b/src/core/src/package_managers/AptitudePackageManager.py @@ -684,6 +684,7 @@ def filter_out_esm_packages(self, packages, package_versions): esm_packages = [] esm_package_versions = [] esm_packages_found = False + esm_packages_selected_to_install = [] for pkg, version in zip(packages, package_versions): if version != Constants.UA_ESM_REQUIRED: @@ -700,6 +701,11 @@ def filter_out_esm_packages(self, packages, package_versions): self.status_handler.add_error_to_status("{0} patches requires Ubuntu Pro for Infrastructure with Extended Security Maintenance".format(len(esm_packages)), Constants.PatchOperationErrorCodes.UA_ESM_REQUIRED) # Set the error status with the esm_package details. Will be used in portal. self.telemetry_writer.write_event("Filter esm packages [EsmPackagesCount={0}]".format(len(esm_packages)), Constants.TelemetryEventLevel.Informational) - self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}]".format(len(packages), len(esm_packages))) + # Log the Esm packages that will be patched in case of an Ubuntu Pro attached VM. + security_esm_update_query_success, security_esm_updates, security_esm_updates_versions = self.get_security_esm_updates() + if security_esm_update_query_success: + esm_packages_selected_to_install = [package for package in non_esm_packages if package in security_esm_updates] + + self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}][SelectedEsmPackagesCount={2}]".format(len(packages), len(esm_packages), len(esm_packages_selected_to_install))) return non_esm_packages, non_esm_package_versions, esm_packages, esm_package_versions, esm_packages_found From 4ba85d4798713cba242fb9beebbda4131e8c458e Mon Sep 17 00:00:00 2001 From: SathishMSFT Date: Tue, 4 Jul 2023 12:26:20 +0530 Subject: [PATCH 2/4] telemetry change to get esm packages selected count --- src/core/src/core_logic/PatchAssessor.py | 3 +- src/core/src/core_logic/PatchInstaller.py | 2 +- .../AptitudePackageManager.py | 42 ++++++++++--------- .../src/package_managers/PackageManager.py | 2 +- .../src/package_managers/YumPackageManager.py | 2 +- .../package_managers/ZypperPackageManager.py | 2 +- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/core/src/core_logic/PatchAssessor.py b/src/core/src/core_logic/PatchAssessor.py index 9e079fcf..7eb9127a 100644 --- a/src/core/src/core_logic/PatchAssessor.py +++ b/src/core/src/core_logic/PatchAssessor.py @@ -81,7 +81,8 @@ def start_assessment(self): self.status_handler.set_package_assessment_status(sec_packages, sec_package_versions, Constants.PackageClassification.SECURITY) # Set the security-esm packages in status. - self.package_manager.set_security_esm_package_status(Constants.ASSESSMENT) + packages = [] # Passing empty packages. + self.package_manager.set_security_esm_package_status(Constants.ASSESSMENT, packages) # ensure reboot status is set reboot_pending = self.package_manager.is_reboot_pending() diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index 1f69a82f..2e780e51 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -177,7 +177,7 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): self.status_handler.set_package_install_status_classification(sec_packages, sec_package_versions, classification="Security") # Set the security-esm package status. - package_manager.set_security_esm_package_status(Constants.INSTALLATION) + package_manager.set_security_esm_package_status(Constants.INSTALLATION, packages) self.composite_logger.log("\nNote: Packages that are neither included nor excluded may still be installed if an included package has a dependency on it.") # We will see this as packages going from NotSelected --> Installed. We could remove them preemptively from not_included_packages, but we're explicitly choosing not to. diff --git a/src/core/src/package_managers/AptitudePackageManager.py b/src/core/src/package_managers/AptitudePackageManager.py index 99c84715..653d2dd1 100644 --- a/src/core/src/package_managers/AptitudePackageManager.py +++ b/src/core/src/package_managers/AptitudePackageManager.py @@ -643,7 +643,7 @@ def check_pro_client_prerequisites(self): self.composite_logger.log_debug("Ubuntu Pro Client pre-requisite checks:[IsFeatureEnabled={0}][IsOSVersionCompatible={1}][IsPythonCompatible={2}][Error={3}]".format(Constants.UbuntuProClientSettings.FEATURE_ENABLED, self.__get_os_major_version() <= Constants.UbuntuProClientSettings.MAX_OS_MAJOR_VERSION_SUPPORTED, self.__is_minimum_required_python_installed(), exception_error)) return self.__pro_client_prereq_met - def set_security_esm_package_status(self, operation): + def set_security_esm_package_status(self, operation, packages): """Set the security-ESM classification for the esm packages.""" security_esm_update_query_success, security_esm_updates, security_esm_updates_versions = self.get_security_esm_updates() if self.__pro_client_prereq_met and security_esm_update_query_success and len(security_esm_updates) > 0: @@ -654,6 +654,9 @@ def set_security_esm_package_status(self, operation): if not self.ubuntu_pro_client.is_ubuntu_pro_client_attached: self.status_handler.add_error_to_status("{0} patches requires Ubuntu Pro for Infrastructure with Extended Security Maintenance".format(len(security_esm_updates)), Constants.PatchOperationErrorCodes.UA_ESM_REQUIRED) elif operation == Constants.INSTALLATION: + if security_esm_update_query_success: + esm_packages_selected_to_install = [package for package in packages if package in security_esm_updates] + self.composite_logger.log_debug("[SelectedEsmPackagesCount={0}]".format(len(esm_packages_selected_to_install))) self.status_handler.set_package_install_status_classification(security_esm_updates, security_esm_updates_versions, Constants.PackageClassification.SECURITY_ESM) def __get_os_major_version(self): @@ -679,33 +682,34 @@ def filter_out_esm_packages(self, packages, package_versions): """ Filter out packages from the list where the version matches the UA_ESM_REQUIRED string. """ - non_esm_packages = [] - non_esm_package_versions = [] - esm_packages = [] - esm_package_versions = [] - esm_packages_found = False + selected_packages = [] + selected_packages_versions = [] + ua_esm_required_packages = [] + ua_esm_required_package_versions = [] + ua_esm_required_packages_found = False esm_packages_selected_to_install = [] for pkg, version in zip(packages, package_versions): if version != Constants.UA_ESM_REQUIRED: - non_esm_packages.append(pkg) - non_esm_package_versions.append(version) + selected_packages.append(pkg) + selected_packages_versions.append(version) continue # version is UA_ESM_REQUIRED. - esm_packages.append(pkg) - esm_package_versions.append(version) + ua_esm_required_packages.append(pkg) + ua_esm_required_package_versions.append(version) - esm_packages_found = len(esm_packages) > 0 - if esm_packages_found: - self.status_handler.add_error_to_status("{0} patches requires Ubuntu Pro for Infrastructure with Extended Security Maintenance".format(len(esm_packages)), Constants.PatchOperationErrorCodes.UA_ESM_REQUIRED) # Set the error status with the esm_package details. Will be used in portal. - self.telemetry_writer.write_event("Filter esm packages [EsmPackagesCount={0}]".format(len(esm_packages)), Constants.TelemetryEventLevel.Informational) + ua_esm_required_packages_found = len(ua_esm_required_packages) > 0 + if ua_esm_required_packages_found: + self.status_handler.add_error_to_status("{0} patches requires Ubuntu Pro for Infrastructure with Extended Security Maintenance".format(len(ua_esm_required_packages)), Constants.PatchOperationErrorCodes.UA_ESM_REQUIRED) # Set the error status with the esm_package details. Will be used in portal. + self.telemetry_writer.write_event("Filter esm packages [EsmPackagesCount={0}]".format(len(ua_esm_required_packages)), Constants.TelemetryEventLevel.Informational) # Log the Esm packages that will be patched in case of an Ubuntu Pro attached VM. - security_esm_update_query_success, security_esm_updates, security_esm_updates_versions = self.get_security_esm_updates() - if security_esm_update_query_success: - esm_packages_selected_to_install = [package for package in non_esm_packages if package in security_esm_updates] + # security_esm_update_query_success, security_esm_updates, security_esm_updates_versions = self.get_security_esm_updates() + # if security_esm_update_query_success: + # esm_packages_selected_to_install = [package for package in selected_packages if package in security_esm_updates] - self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}][SelectedEsmPackagesCount={2}]".format(len(packages), len(esm_packages), len(esm_packages_selected_to_install))) - return non_esm_packages, non_esm_package_versions, esm_packages, esm_package_versions, esm_packages_found + # self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}][SelectedEsmPackagesCount={2}]".format(len(packages), len(ua_esm_required_packages), len(esm_packages_selected_to_install))) + self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}]".format( len(packages), len(ua_esm_required_packages))) + return selected_packages, selected_packages_versions, ua_esm_required_packages, ua_esm_required_package_versions, ua_esm_required_packages_found diff --git a/src/core/src/package_managers/PackageManager.py b/src/core/src/package_managers/PackageManager.py index 13ba3eff..d78b4ae1 100644 --- a/src/core/src/package_managers/PackageManager.py +++ b/src/core/src/package_managers/PackageManager.py @@ -453,7 +453,7 @@ def add_arch_dependencies(self, package_manager, package, packages, package_vers pass @abstractmethod - def set_security_esm_package_status(self, operation): + def set_security_esm_package_status(self, operation, packages): pass @abstractmethod diff --git a/src/core/src/package_managers/YumPackageManager.py b/src/core/src/package_managers/YumPackageManager.py index 7c1f9268..b66bfc4f 100644 --- a/src/core/src/package_managers/YumPackageManager.py +++ b/src/core/src/package_managers/YumPackageManager.py @@ -958,7 +958,7 @@ def add_arch_dependencies(self, package_manager, package, packages, package_vers package_and_dependencies.append(possible_arch_dependency) package_and_dependency_versions.append(possible_arch_dependency_version) - def set_security_esm_package_status(self, operation): + def set_security_esm_package_status(self, operation, packages): """ Set the security-ESM classification for the esm packages. Only needed for apt. No-op for yum and zypper. """ diff --git a/src/core/src/package_managers/ZypperPackageManager.py b/src/core/src/package_managers/ZypperPackageManager.py index 5e866b4e..8ff9e0a2 100644 --- a/src/core/src/package_managers/ZypperPackageManager.py +++ b/src/core/src/package_managers/ZypperPackageManager.py @@ -813,7 +813,7 @@ def add_arch_dependencies(self, package_manager, package, packages, package_vers """ return - def set_security_esm_package_status(self, operation): + def set_security_esm_package_status(self, operation, packages): """ Set the security-ESM classification for the esm packages. Only needed for apt. No-op for yum and zypper. """ From 1b1d9d2370b6c1ec353f9a71bcbb43a7c78e82ef Mon Sep 17 00:00:00 2001 From: SathishMSFT Date: Tue, 4 Jul 2023 12:32:44 +0530 Subject: [PATCH 3/4] Refactoring code --- src/core/src/package_managers/AptitudePackageManager.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/core/src/package_managers/AptitudePackageManager.py b/src/core/src/package_managers/AptitudePackageManager.py index 653d2dd1..af5f5cc6 100644 --- a/src/core/src/package_managers/AptitudePackageManager.py +++ b/src/core/src/package_managers/AptitudePackageManager.py @@ -687,7 +687,6 @@ def filter_out_esm_packages(self, packages, package_versions): ua_esm_required_packages = [] ua_esm_required_package_versions = [] ua_esm_required_packages_found = False - esm_packages_selected_to_install = [] for pkg, version in zip(packages, package_versions): if version != Constants.UA_ESM_REQUIRED: @@ -704,12 +703,6 @@ def filter_out_esm_packages(self, packages, package_versions): self.status_handler.add_error_to_status("{0} patches requires Ubuntu Pro for Infrastructure with Extended Security Maintenance".format(len(ua_esm_required_packages)), Constants.PatchOperationErrorCodes.UA_ESM_REQUIRED) # Set the error status with the esm_package details. Will be used in portal. self.telemetry_writer.write_event("Filter esm packages [EsmPackagesCount={0}]".format(len(ua_esm_required_packages)), Constants.TelemetryEventLevel.Informational) - # Log the Esm packages that will be patched in case of an Ubuntu Pro attached VM. - # security_esm_update_query_success, security_esm_updates, security_esm_updates_versions = self.get_security_esm_updates() - # if security_esm_update_query_success: - # esm_packages_selected_to_install = [package for package in selected_packages if package in security_esm_updates] - - # self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}][SelectedEsmPackagesCount={2}]".format(len(packages), len(ua_esm_required_packages), len(esm_packages_selected_to_install))) - self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}]".format( len(packages), len(ua_esm_required_packages))) + self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}]".format(len(packages), len(ua_esm_required_packages))) return selected_packages, selected_packages_versions, ua_esm_required_packages, ua_esm_required_package_versions, ua_esm_required_packages_found From 9f7efb4d669d580560bf6f5ccaf4c54434a42412 Mon Sep 17 00:00:00 2001 From: SathishMSFT Date: Fri, 7 Jul 2023 12:10:48 +0530 Subject: [PATCH 4/4] Addressing review comments. --- src/core/src/core_logic/PatchAssessor.py | 3 +-- src/core/src/core_logic/PatchInstaller.py | 2 +- .../package_managers/AptitudePackageManager.py | 15 +++++++-------- src/core/src/package_managers/PackageManager.py | 2 +- .../src/package_managers/YumPackageManager.py | 2 +- .../src/package_managers/ZypperPackageManager.py | 2 +- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/core/src/core_logic/PatchAssessor.py b/src/core/src/core_logic/PatchAssessor.py index 7eb9127a..3ee0c991 100644 --- a/src/core/src/core_logic/PatchAssessor.py +++ b/src/core/src/core_logic/PatchAssessor.py @@ -81,8 +81,7 @@ def start_assessment(self): self.status_handler.set_package_assessment_status(sec_packages, sec_package_versions, Constants.PackageClassification.SECURITY) # Set the security-esm packages in status. - packages = [] # Passing empty packages. - self.package_manager.set_security_esm_package_status(Constants.ASSESSMENT, packages) + self.package_manager.set_security_esm_package_status(Constants.ASSESSMENT, packages=[]) # ensure reboot status is set reboot_pending = self.package_manager.is_reboot_pending() diff --git a/src/core/src/core_logic/PatchInstaller.py b/src/core/src/core_logic/PatchInstaller.py index 2e780e51..a97523ca 100644 --- a/src/core/src/core_logic/PatchInstaller.py +++ b/src/core/src/core_logic/PatchInstaller.py @@ -160,7 +160,7 @@ def install_updates(self, maintenance_window, package_manager, simulate=False): # These packages will already be marked with version as 'UA_ESM_REQUIRED'. # Esm packages will not be dependent packages to non-esm packages. This is confirmed by Canonical. So, once these are removed from processing, we need not worry about handling it in our batch / sequential patch processing logic. # Adding this after filtering excluded packages, so we don`t un-intentionally mark excluded esm-package status as failed. - packages, package_versions, self.skipped_esm_packages, self.skipped_esm_package_versions, self.esm_packages_found_without_attach = package_manager.filter_out_esm_packages(packages, package_versions) + packages, package_versions, self.skipped_esm_packages, self.skipped_esm_package_versions, self.esm_packages_found_without_attach = package_manager.separate_out_esm_packages(packages, package_versions) self.telemetry_writer.write_event("Final package list: " + str(packages), Constants.TelemetryEventLevel.Verbose) diff --git a/src/core/src/package_managers/AptitudePackageManager.py b/src/core/src/package_managers/AptitudePackageManager.py index af5f5cc6..a0d47280 100644 --- a/src/core/src/package_managers/AptitudePackageManager.py +++ b/src/core/src/package_managers/AptitudePackageManager.py @@ -656,7 +656,7 @@ def set_security_esm_package_status(self, operation, packages): elif operation == Constants.INSTALLATION: if security_esm_update_query_success: esm_packages_selected_to_install = [package for package in packages if package in security_esm_updates] - self.composite_logger.log_debug("[SelectedEsmPackagesCount={0}]".format(len(esm_packages_selected_to_install))) + self.composite_logger.log_debug("Setting security ESM package status. [SelectedEsmPackagesCount={0}]".format(len(esm_packages_selected_to_install))) self.status_handler.set_package_install_status_classification(security_esm_updates, security_esm_updates_versions, Constants.PackageClassification.SECURITY_ESM) def __get_os_major_version(self): @@ -678,20 +678,20 @@ def add_arch_dependencies(self, package_manager, package, packages, package_vers """ return - def filter_out_esm_packages(self, packages, package_versions): + def separate_out_esm_packages(self, packages, package_versions): """ Filter out packages from the list where the version matches the UA_ESM_REQUIRED string. """ - selected_packages = [] - selected_packages_versions = [] + non_esm_packages = [] + non_esm_package_versions = [] ua_esm_required_packages = [] ua_esm_required_package_versions = [] ua_esm_required_packages_found = False for pkg, version in zip(packages, package_versions): if version != Constants.UA_ESM_REQUIRED: - selected_packages.append(pkg) - selected_packages_versions.append(version) + non_esm_packages.append(pkg) + non_esm_package_versions.append(version) continue # version is UA_ESM_REQUIRED. @@ -701,8 +701,7 @@ def filter_out_esm_packages(self, packages, package_versions): ua_esm_required_packages_found = len(ua_esm_required_packages) > 0 if ua_esm_required_packages_found: self.status_handler.add_error_to_status("{0} patches requires Ubuntu Pro for Infrastructure with Extended Security Maintenance".format(len(ua_esm_required_packages)), Constants.PatchOperationErrorCodes.UA_ESM_REQUIRED) # Set the error status with the esm_package details. Will be used in portal. - self.telemetry_writer.write_event("Filter esm packages [EsmPackagesCount={0}]".format(len(ua_esm_required_packages)), Constants.TelemetryEventLevel.Informational) self.composite_logger.log_debug("Filter esm packages : [TotalPackagesCount={0}][EsmPackagesCount={1}]".format(len(packages), len(ua_esm_required_packages))) - return selected_packages, selected_packages_versions, ua_esm_required_packages, ua_esm_required_package_versions, ua_esm_required_packages_found + return non_esm_packages, non_esm_package_versions, ua_esm_required_packages, ua_esm_required_package_versions, ua_esm_required_packages_found diff --git a/src/core/src/package_managers/PackageManager.py b/src/core/src/package_managers/PackageManager.py index d78b4ae1..824fa999 100644 --- a/src/core/src/package_managers/PackageManager.py +++ b/src/core/src/package_managers/PackageManager.py @@ -457,6 +457,6 @@ def set_security_esm_package_status(self, operation, packages): pass @abstractmethod - def filter_out_esm_packages(self, packages, package_versions): + def separate_out_esm_packages(self, packages, package_versions): pass diff --git a/src/core/src/package_managers/YumPackageManager.py b/src/core/src/package_managers/YumPackageManager.py index b66bfc4f..bfddfafc 100644 --- a/src/core/src/package_managers/YumPackageManager.py +++ b/src/core/src/package_managers/YumPackageManager.py @@ -964,7 +964,7 @@ def set_security_esm_package_status(self, operation, packages): """ pass - def filter_out_esm_packages(self, packages, package_versions): + def separate_out_esm_packages(self, packages, package_versions): """ Filter out packages from the list where the version matches the UA_ESM_REQUIRED string. Only needed for apt. No-op for yum and zypper diff --git a/src/core/src/package_managers/ZypperPackageManager.py b/src/core/src/package_managers/ZypperPackageManager.py index 8ff9e0a2..5f172d40 100644 --- a/src/core/src/package_managers/ZypperPackageManager.py +++ b/src/core/src/package_managers/ZypperPackageManager.py @@ -819,7 +819,7 @@ def set_security_esm_package_status(self, operation, packages): """ pass - def filter_out_esm_packages(self, packages, package_versions): + def separate_out_esm_packages(self, packages, package_versions): """ Filter out packages from the list where the version matches the UA_ESM_REQUIRED string. Only needed for apt. No-op for yum and zypper