diff --git a/src/core/src/bootstrap/Constants.py b/src/core/src/bootstrap/Constants.py index 99d58939..3c228de0 100644 --- a/src/core/src/bootstrap/Constants.py +++ b/src/core/src/bootstrap/Constants.py @@ -346,6 +346,9 @@ class EnvLayer(EnumBackport): PRIVILEGED_OP_REBOOT = PRIVILEGED_OP_MARKER + "Reboot_Exception" PRIVILEGED_OP_EXIT = PRIVILEGED_OP_MARKER + "Exit_" + # Supported Package Architectures - if this is changed, review YumPackageManage + SUPPORTED_PACKAGE_ARCH = ['.x86_64', '.noarch', '.i686', '.aarch64'] + # Package / Patch State Ordering Constants # This ordering ensures that the most important information is preserved in the case of patch object truncation PackageClassificationOrderInStatusReporting = { diff --git a/src/core/src/core_logic/PackageFilter.py b/src/core/src/core_logic/PackageFilter.py index efed752a..c8277c5e 100644 --- a/src/core/src/core_logic/PackageFilter.py +++ b/src/core/src/core_logic/PackageFilter.py @@ -117,8 +117,8 @@ def single_package_check_for_match(self, package, matching_list, package_version @staticmethod def get_product_name_without_arch(package_name): - """Splits out product name without architecture - if this is changed, review YumPackageManager""" - architectures = ['.x86_64', '.noarch', '.i686'] + """Splits out product name without architecture""" + architectures = Constants.SUPPORTED_PACKAGE_ARCH for arch in architectures: if package_name.endswith(arch): return package_name.replace(arch, '') diff --git a/src/core/src/package_managers/YumPackageManager.py b/src/core/src/package_managers/YumPackageManager.py index 1568906e..ea6d00de 100644 --- a/src/core/src/package_managers/YumPackageManager.py +++ b/src/core/src/package_managers/YumPackageManager.py @@ -212,7 +212,7 @@ def extract_packages_and_versions_including_duplicates(self, output): self.composite_logger.log_debug("\nExtracting package and version data...") packages = [] versions = [] - package_extensions = ['.x86_64', '.noarch', '.i686'] + package_extensions = Constants.SUPPORTED_PACKAGE_ARCH def is_package(chunk): # Using a list comprehension to determine if chunk is a package @@ -309,42 +309,47 @@ def is_package_version_installed(self, package_name, package_version): return False def extract_dependencies(self, output, packages): - # Sample output for the cmd 'sudo yum update --assumeno selinux-policy.noarch' is : - # - # Loaded plugins: langpacks, product-id, search-disabled-repos - # Resolving Dependencies - # --> Running transaction check - # ---> Package selinux-policy.noarch 0:3.13.1-102.el7_3.15 will be updated - # --> Processing Dependency: selinux-policy = 3.13.1-102.el7_3.15 for \ - # package: selinux-policy-targeted-3.13.1-102.el7_3.15.noarch - # --> Processing Dependency: selinux-policy = 3.13.1-102.el7_3.15 for \ - # package: selinux-policy-targeted-3.13.1-102.el7_3.15.noarch - # ---> Package selinux-policy.noarch 0:3.13.1-102.el7_3.16 will be an update - # --> Running transaction check - # ---> Package selinux-policy-targeted.noarch 0:3.13.1-102.el7_3.15 will be updated - # ---> Package selinux-policy-targeted.noarch 0:3.13.1-102.el7_3.16 will be an update - # --> Finished Dependency Resolution + # Extracts dependent packages from output. Refer yum_update_output_expected_formats.txt for examples of supported output formats. dependencies = [] - lines = output.strip().split('\n') + package_arch_to_look_for = ["x86_64", "noarch", "i686", "aarch64"] # if this is changed, review Constants - for line in lines: - if line.find(" will be updated") < 0 and line.find(" will be an update") < 0 and line.find(" will be installed") < 0: - self.composite_logger.log_debug(" - Inapplicable line: " + str(line)) - continue + lines = output.strip().splitlines() + + for line_index in range(0, len(lines)): + line = re.split(r'\s+', (lines[line_index].replace("--->", "")).strip()) + next_line = [] + dependent_package_name = "" - updates_line = re.split(r'\s+', line.strip()) - if len(updates_line) != 7: + if line_index < len(lines) - 1: + next_line = re.split(r'\s+', (lines[line_index + 1].replace("--->", "")).strip()) + + if self.is_valid_update(line, package_arch_to_look_for): + dependent_package_name = self.get_product_name_with_arch(line, package_arch_to_look_for) + elif self.is_valid_update(line+next_line, package_arch_to_look_for): + dependent_package_name = self.get_product_name_with_arch(line+next_line, package_arch_to_look_for) + else: self.composite_logger.log_debug(" - Inapplicable line: " + str(line)) continue - dependent_package_name = self.get_product_name(updates_line[2]) - if len(dependent_package_name) != 0 and dependent_package_name not in packages: + if len(dependent_package_name) != 0 and dependent_package_name not in packages and dependent_package_name not in dependencies: self.composite_logger.log_debug(" - Dependency detected: " + dependent_package_name) dependencies.append(dependent_package_name) return dependencies + def is_valid_update(self, package_details_in_output, package_arch_to_look_for): + # Verifies whether the line under consideration (i.e. package_details_in_output) contains relevant package details. + # package_details_in_output will be of the following format if it is valid + # In Yum 3: Package selinux-policy.noarch 0:3.13.1-102.el7_3.15 will be updated + # In Yum 4: kernel-tools x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 8.4 M + return len(package_details_in_output) == 6 and self.is_arch_in_package_details(package_details_in_output[1], package_arch_to_look_for) + + @staticmethod + def is_arch_in_package_details(package_detail, package_arch_to_look_for): + # Using a list comprehension to determine if chunk is a package + return len([p for p in package_arch_to_look_for if p in package_detail]) == 1 + def get_dependent_list(self, packages): package_names = "" for index, package in enumerate(packages): @@ -364,7 +369,7 @@ def get_product_name(self, package_name): def get_product_name_and_arch(self, package_name): """Splits out product name and architecture - if this is changed, modify in PackageFilter also""" - architectures = ['.x86_64', '.noarch', '.i686'] + architectures = Constants.SUPPORTED_PACKAGE_ARCH for arch in architectures: if package_name.endswith(arch): return package_name[:-len(arch)], arch @@ -380,6 +385,10 @@ def get_product_arch(self, package_name): product_name, arch = self.get_product_name_and_arch(package_name) return arch + def get_product_name_with_arch(self, package_detail, package_arch_to_look_for): + """Retrieve product name with arch separated by '.'. Note: This format is default in yum3. Refer samples noted within func extract_dependencies() for more clarity""" + return package_detail[0] + "." + package_detail[1] if package_detail[1] in package_arch_to_look_for else package_detail[1] + def get_package_version_without_epoch(self, package_version): """Returns the package version stripped of any epoch""" package_version_split = str(package_version).split(':', 1) diff --git a/src/core/tests/Test_YumPackageManager.py b/src/core/tests/Test_YumPackageManager.py index db9be641..5e4f486f 100644 --- a/src/core/tests/Test_YumPackageManager.py +++ b/src/core/tests/Test_YumPackageManager.py @@ -676,5 +676,45 @@ def __assert_test_rhel8_image(self): self.assertEqual(available_updates[1], "tcpdump.x86_64") self.assertEqual(package_versions[1], "14:4.9.2-3.el7") + def test_get_dependent_list_yum_version_4(self): + # Creating new RuntimeCompositor with test_type YumVersion4Dependency because there are some command runs in constructor of YumPackageManager + # for which the sample output is in the test_type YumVersion4Dependency. + self.runtime.stop() # First stopping the existing runtime + self.runtime = RuntimeCompositor(ArgumentComposer().get_composed_arguments(), True, Constants.YUM, test_type="YumVersion4Dependency") + + self.container = self.runtime.container + package_manager = self.container.get('package_manager') + dependent_list = package_manager.get_dependent_list(["iptables.x86_64"]) + self.assertEqual(len(dependent_list), 2) + self.assertEqual(dependent_list[0], "iptables-ebtables.x86_64") + self.assertEqual(dependent_list[1], "iptables-libs.x86_64") + + def test_get_dependent_list_yum_version_4_update_in_two_lines(self): + # Creating new RuntimeCompositor with test_type YumVersion4Dependency because there are some command runs in constructor of YumPackageManager + # for which the sample output is in the test_type YumVersion4Dependency. + self.runtime.stop() # First stopping the existing runtime + self.runtime = RuntimeCompositor(ArgumentComposer().get_composed_arguments(), True, Constants.YUM, test_type="YumVersion4DependencyInTwoLines") + + self.container = self.runtime.container + package_manager = self.container.get('package_manager') + dependent_list = package_manager.get_dependent_list(["polkit.x86_64"]) + self.assertEqual(len(dependent_list), 1) + self.assertEqual(dependent_list[0], "polkit-libs.x86_64") + + def test_get_dependent_list_yum_version_4_update_in_two_lines_with_unexpected_output(self): + # This test is for adding code coverage for code handling unexpected output in the command for get dependencies. + # There are two packages in the output and for the second package i.e. polkit-libs, the package name is not present. Only other package details are present. + # So, there are 0 dependencies expected. + + # Creating new RuntimeCompositor with test_type YumVersion4Dependency because there are some command runs in constructor of YumPackageManager + # for which the sample output is in the test_type YumVersion4Dependency. + self.runtime.stop() # First stopping the existing runtime + self.runtime = RuntimeCompositor(ArgumentComposer().get_composed_arguments(), True, Constants.YUM, test_type="YumVersion4DependencyInTwoLinesWithUnexpectedOutput") + + self.container = self.runtime.container + package_manager = self.container.get('package_manager') + dependent_list = package_manager.get_dependent_list(["polkit.x86_64"]) + self.assertEqual(len(dependent_list), 0) + if __name__ == '__main__': unittest.main() diff --git a/src/core/tests/library/LegacyEnvLayerExtensions.py b/src/core/tests/library/LegacyEnvLayerExtensions.py index d5267ac9..b2448898 100644 --- a/src/core/tests/library/LegacyEnvLayerExtensions.py +++ b/src/core/tests/library/LegacyEnvLayerExtensions.py @@ -19,9 +19,9 @@ class LegacyEnvLayerExtensions(): - def __init__(self, package_manager_name): + def __init__(self, package_manager_name, test_type="HappyPath"): self.legacy_package_manager_name = package_manager_name - self.legacy_test_type = "HappyPath" + self.legacy_test_type = test_type self.temp_folder_path = "" class LegacyPlatform(object): @@ -1214,6 +1214,99 @@ def run_command_output(self, cmd, no_output=False, chk_err=True): "1:2.02-123.el8 " + \ "@System\n" + elif self.legacy_test_type == 'YumVersion4Dependency': + if self.legacy_package_manager_name is Constants.YUM: + if cmd.find("--version") > -1: + code = 0 + output = "4.7.0\n" + \ + " Installed: dnf-0:4.7.0-8.el8.noarch at Mon 29 May 2023 02:58:31 PM GMT\n" + \ + " Built : Red Hat, Inc. at Fri 18 Mar 2022 03:21:28 PM GMT" + \ + "\n" + \ + " Installed: rpm-0:4.14.3-24.el8_6.x86_64 at Mon 29 May 2023 05:07:36 PM GMT" + \ + " Built : Red Hat, Inc. at Wed 14 Sep 2022 09:12:50 AM GMT" + if cmd.find("assumeno --skip-broken iptables") > -1: + code = 0 + output = "Updating Subscription Management repositories.\n" + \ + "Unable to read consumer identity\n" + \ + "\n" + \ + "This system is not registered to Red Hat Subscription Management. You can use subscription-manager to register.\n" + \ + "\n" + \ + "Last metadata expiration check: 0:12:39 ago on Mon 27 Feb 2023 12:10:02 PM UTC.\n" + \ + "Package iptables-libs-1.8.4-17.el8_4.1.x86_64 is already installed.\n" + \ + "Dependencies resolved.\n" + \ + "===========================================================================================\n" + \ + " Package Arch Version Repository Size\n" + \ + "===========================================================================================\n" + \ + "Upgrading:\n" + \ + " iptables x86_64 1.8.4-17.el8_4.2 rhel-8-for-x86_64-baseos-eus-rhui-rpms 584 k\n" + \ + " iptables-ebtables x86_64 1.8.4-17.el8_4.2 rhel-8-for-x86_64-baseos-eus-rhui-rpms 72 k\n" + \ + " iptables-libs x86_64 1.8.4-17.el8_4.2 rhel-8-for-x86_64-baseos-eus-rhui-rpms 108 k\n" + \ + "\n" + \ + "Transaction Summary\n" + \ + "===========================================================================================\n" + \ + "Upgrade 3 Packages\n" + \ + "\n" + \ + "Total download size: 764 k\n" + \ + "Operation aborted." + elif self.legacy_test_type == 'YumVersion4DependencyInTwoLines': + if self.legacy_package_manager_name is Constants.YUM: + if cmd.find("--version") > -1: + code = 0 + output = "4.7.0\n" + \ + " Installed: dnf-0:4.7.0-8.el8.noarch at Mon 29 May 2023 02:58:31 PM GMT\n" + \ + " Built : Red Hat, Inc. at Fri 18 Mar 2022 03:21:28 PM GMT" + \ + "\n" + \ + " Installed: rpm-0:4.14.3-24.el8_6.x86_64 at Mon 29 May 2023 05:07:36 PM GMT" + \ + " Built : Red Hat, Inc. at Wed 14 Sep 2022 09:12:50 AM GMT" + if cmd.find("assumeno --skip-broken polkit") > -1: + code = 0 + output = "Last metadata expiration check: 0:08:47 ago on Tue 25 Jul 2023 02:14:28 PM UTC.\n" + \ + "Package polkit-0.115-13.el8_5.2.x86_64 is already installed.\n" + \ + "Dependencies resolved.\n" + \ + "================================================================================\n" + \ + " Package Arch Version Repository Size \n" + \ + "================================================================================\n" + \ + "Upgrading:\n" + \ + " polkit x86_64 0.115-14.el8_6.1 rhel-8-for-x86_64-baseos-eus-rhui-rpms 154 k\n" + \ + " polkit-libs\n" + \ + " x86_64 0.115-14.el8_6.1 rhel-8-for-x86_64-baseos-eus-rhui-rpms 77 k\n" + \ + "\n" + \ + "Transaction Summary\n" + \ + "================================================================================\n" + \ + "Upgrade 2 Packages\n" + \ + "\n" + \ + "Total download size: 231 k\n" + \ + "Operation aborted.\n" + elif self.legacy_test_type == 'YumVersion4DependencyInTwoLinesWithUnexpectedOutput': + if self.legacy_package_manager_name is Constants.YUM: + if cmd.find("--version") > -1: + code = 0 + output = "4.7.0\n" + \ + " Installed: dnf-0:4.7.0-8.el8.noarch at Mon 29 May 2023 02:58:31 PM GMT\n" + \ + " Built : Red Hat, Inc. at Fri 18 Mar 2022 03:21:28 PM GMT" + \ + "\n" + \ + " Installed: rpm-0:4.14.3-24.el8_6.x86_64 at Mon 29 May 2023 05:07:36 PM GMT" + \ + " Built : Red Hat, Inc. at Wed 14 Sep 2022 09:12:50 AM GMT" + if cmd.find("assumeno --skip-broken polkit") > -1: + # The below is example for unexpected output. For the second package i.e. polkit-libs, the package name is not present. + # Only other package details are present. This scenario is added for code coverage. + code = 0 + output = "Last metadata expiration check: 0:08:47 ago on Tue 25 Jul 2023 02:14:28 PM UTC.\n" + \ + "Package polkit-0.115-13.el8_5.2.x86_64 is already installed.\n" + \ + "Dependencies resolved.\n" + \ + "================================================================================\n" + \ + " Package Arch Version Repository Size \n" + \ + "================================================================================\n" + \ + "Upgrading:\n" + \ + " polkit x86_64 0.115-14.el8_6.1 rhel-8-for-x86_64-baseos-eus-rhui-rpms 154 k\n" + \ + " x86_64 0.115-14.el8_6.1 rhel-8-for-x86_64-baseos-eus-rhui-rpms 77 k\n" + \ + "\n" + \ + "Transaction Summary\n" + \ + "================================================================================\n" + \ + "Upgrade 2 Packages\n" + \ + "\n" + \ + "Total download size: 231 k\n" + \ + "Operation aborted.\n" major_version = self.get_python_major_version() if major_version == 2: return code, output.decode('utf8', 'ignore').encode('ascii', 'ignore') diff --git a/src/core/tests/library/RuntimeCompositor.py b/src/core/tests/library/RuntimeCompositor.py index 86dae841..eaee7b20 100644 --- a/src/core/tests/library/RuntimeCompositor.py +++ b/src/core/tests/library/RuntimeCompositor.py @@ -41,7 +41,7 @@ class RuntimeCompositor(object): - def __init__(self, argv=Constants.DEFAULT_UNSPECIFIED_VALUE, legacy_mode=False, package_manager_name=Constants.APT, vm_cloud_type=Constants.VMCloudType.AZURE): + def __init__(self, argv=Constants.DEFAULT_UNSPECIFIED_VALUE, legacy_mode=False, package_manager_name=Constants.APT, vm_cloud_type=Constants.VMCloudType.AZURE, test_type="HappyPath"): # Init data self.original_rm_start_reboot = None self.current_env = Constants.DEV @@ -78,7 +78,7 @@ def mkdtemp_runner(): # Reconfigure env layer for legacy mode tests self.env_layer = bootstrapper.env_layer if legacy_mode: - self.legacy_env_layer_extensions = LegacyEnvLayerExtensions(package_manager_name) + self.legacy_env_layer_extensions = LegacyEnvLayerExtensions(package_manager_name, test_type) self.reconfigure_env_layer_to_legacy_mode() # Core components diff --git a/src/tools/references/cmd_output_references/yum_update_output_expected_formats b/src/tools/references/cmd_output_references/yum_update_output_expected_formats new file mode 100644 index 00000000..fb8afc74 --- /dev/null +++ b/src/tools/references/cmd_output_references/yum_update_output_expected_formats @@ -0,0 +1,100 @@ +Example of different output formats for 'sudo yum update ____ ' depending yum version and details within the output + +In Yum 3: Sample output for the cmd 'sudo yum update --assumeno selinux-policy.noarch' is : + Loaded plugins: langpacks, product-id, search-disabled-repos + Resolving Dependencies + --> Running transaction check + ---> Package selinux-policy.noarch 0:3.13.1-102.el7_3.15 will be updated + --> Processing Dependency: selinux-policy = 3.13.1-102.el7_3.15 for \ + package: selinux-policy-targeted-3.13.1-102.el7_3.15.noarch + --> Processing Dependency: selinux-policy = 3.13.1-102.el7_3.15 for \ + package: selinux-policy-targeted-3.13.1-102.el7_3.15.noarch + ---> Package selinux-policy.noarch 0:3.13.1-102.el7_3.16 will be an update + --> Running transaction check + ---> Package selinux-policy-targeted.noarch 0:3.13.1-102.el7_3.15 will be updated + ---> Package selinux-policy-targeted.noarch 0:3.13.1-102.el7_3.16 will be an update + --> Finished Dependency Resolution + +In Yum 4: Sample 1 (Upgrades are available, no installation required): +Sample output for the cmd 'sudo yum update --assumeno selinux-policy.noarch' is : + Last metadata expiration check: 0:08:56 ago on Tue 25 Jul 2023 02:14:28 PM UTC. + Package selinux-policy-3.14.3-95.el8_6.6.noarch is already installed. + Dependencies resolved. + ================================================================================================== + Package Arch Version Repository Size + ================================================================================================== + Upgrading: + selinux-policy noarch 3.14.3-95.el8_6.8 rhel-8-for-x86_64-baseos-eus-rhui-rpms 651 k + selinux-policy-targeted noarch 3.14.3-95.el8_6.8 rhel-8-for-x86_64-baseos-eus-rhui-rpms 15 M + + Transaction Summary + ================================================================================================== + Upgrade 2 Packages + + Total download size: 16 M + Operation aborted.. + +In Yum 4: Sample 2 (No upgrades available, installations required): +Sample output for the cmd 'sudo yum update --assumeno kernel-modules.x86_64' is : + + Last metadata expiration check: 0:09:14 ago on Tue 25 Jul 2023 02:14:28 PM UTC. + Package kernel-modules-4.18.0-372.9.1.el8.x86_64 is already installed. + Package kernel-modules-4.18.0-372.52.1.el8_6.x86_64 is already installed. + Dependencies resolved. + ============================================================================================= + Package Arch Version Repository Size + ============================================================================================= + Installing dependencies: + kernel-core x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 40 M + kernel-modules x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 32 M + + Transaction Summary + ============================================================================================= + Install 2 Packages + + Total download size: 72 M + Installed size: 93 M + Operation aborted. + +In Yum 4: Sample 3 (Both upgrades and installations required): +Sample output for the cmd 'sudo yum update --assumeno kernel-modules.x86_64' is : + + Last metadata expiration check: 0:01:56 ago on Tue 25 Jul 2023 12:01:47 PM UTC. + Dependencies resolved. + ================================================================================================ + Package Arch Version Repository Size + ================================================================================================ + Upgrading: + kernel-tools x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 8.4 M + kernel-tools-libs x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 8.2 M + openssl x86_64 1:1.1.1k-9.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 710 k + openssl-libs x86_64 1:1.1.1k-9.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 1.5 M + Installing dependencies: + kernel-core x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 40 M + kernel-modules x86_64 4.18.0-372.64.1.el8_6 rhel-8-for-x86_64-baseos-eus-rhui-rpms 32 M + + Transaction Summary + ================================================================================================ + Install 2 Packages + Upgrade 4 Packages + +In Yum 4: Sample 4 (dependent patch detail split over 2 lines): +Sample output for the cmd 'sudo yum update --assumeno polkit.x86_64' is : + + Last metadata expiration check: 0:08:47 ago on Tue 25 Jul 2023 02:14:28 PM UTC. + Package polkit-0.115-13.el8_5.2.x86_64 is already installed. + Dependencies resolved. + ================================================================================ + Package Arch Version Repository Size + ================================================================================ + Upgrading: + polkit x86_64 0.115-14.el8_6.1 rhel-8-for-x86_64-baseos-eus-rhui-rpms 154 k + polkit-libs + x86_64 0.115-14.el8_6.1 rhel-8-for-x86_64-baseos-eus-rhui-rpms 77 k + + Transaction Summary + ================================================================================ + Upgrade 2 Packages + + Total download size: 231 k + Operation aborted. \ No newline at end of file