Fix check_checksums when using templates, patch-dicts or checksums.json

easybuild/framework/easyblock.py

@@ -42,8 +42,10 @@
 * Caspar van Leeuwen (SURF)
 * Jan Andre Reuter (Juelich Supercomputing Centre)
 * Jasper Grimm (UoY)
+* Alexander Grund (TU Dresden)
 """
 import concurrent
+import contextlib
 import copy
 import functools
 import glob
@@ -593,7 +595,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
             if not isinstance(patch_specs, tuple) or len(patch_specs) != 2:
                 raise EasyBuildError('Patch specs must be a tuple of (patches, post-install patches) or a list')
             post_install_patches = patch_specs[1]
-            patch_specs = itertools.chain(*patch_specs)
+            patch_specs = itertools.chain.from_iterable(patch_specs)
 
         patches = []
         for index, patch_spec in enumerate(patch_specs):
@@ -683,6 +685,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
 
                     source_urls = resolve_template(ext_options.get('source_urls', []), template_values)
                     checksums = resolve_template(ext_options.get('checksums', []), template_values)
+                    ext_src['checksums'] = checksums
 
                     download_instructions = resolve_template(ext_options.get('download_instructions'), template_values)
 
@@ -725,6 +728,11 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                                 # copy 'path' entry to 'src' for use with extensions
                                 'src': src['path'],
                             })
+                            filename = src['name']
+                        else:
+                            filename = source.get('filename')
+                        if filename is not None:
+                            ext_src['sources'] = [filename]
 
                     else:
 
@@ -738,6 +746,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                             raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
 
                         src_fn = resolve_template(src_fn, template_values)
+                        ext_src['sources'] = [src_fn]
 
                         if fetch_files:
                             src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
@@ -781,7 +790,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                     if fetch_files:
                         ext_patches = self.fetch_patches(patch_specs=ext_patch_specs, extension=True)
                     else:
-                        ext_patches = [create_patch_info(p) for p in itertools.chain(*ext_patch_specs)]
+                        ext_patches = [create_patch_info(p) for p in itertools.chain.from_iterable(ext_patch_specs)]
 
                     if ext_patches:
                         self.log.debug('Found patches for extension %s: %s', ext_name, ext_patches)
@@ -2748,10 +2757,11 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
             checksums = ent.get('checksums', [])
         except EasyBuildError:
             if isinstance(ent, EasyConfig):
-                sources = ent.get_ref('sources')
-                data_sources = ent.get_ref('data_sources')
-                patches = ent.get_ref('patches') + ent.get_ref('postinstallpatches')
-                checksums = ent.get_ref('checksums')
+                with ent.disable_templating():
+                    sources = ent['sources']
+                    data_sources = ent['data_sources']
+                    patches = ent['patches'] + ent['postinstallpatches']
+                    checksums = ent['checksums']
 
         # Single source should be re-wrapped as a list, and checksums with it
         if isinstance(sources, dict):
@@ -2761,25 +2771,30 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
         if isinstance(checksums, str):
             checksums = [checksums]
 
-        sources = sources + data_sources
+        def get_name(fn, key):
+            # if the filename is a tuple, the actual source file name is the first element
+            if isinstance(fn, tuple):
+                fn = fn[0]
+            # if the filename is a dict, the actual source file name is inside
+            if isinstance(fn, dict):
+                fn = fn[key]
+            return fn
+
+        sources = [get_name(src, 'filename') for src in itertools.chain(sources, data_sources)]
+        patches = [get_name(patch, 'name') for patch in patches]
+
+        if source_cnt is None:
+            source_cnt = len(sources)
+        patch_cnt = len(patches)
 
-        if not checksums:
+        if not checksums and (source_cnt + patch_cnt) > 0:
             checksums_from_json = self.get_checksums_from_json()
             # recreate a list of checksums. If each filename is found, the generated list of checksums should match
             # what is expected in list format
-            for fn in sources + patches:
-                # if the filename is a tuple, the actual source file name is the first element
-                if isinstance(fn, tuple):
-                    fn = fn[0]
-                # if the filename is a dict, the actual source file name is the "filename" element
-                if isinstance(fn, dict):
-                    fn = fn["filename"]
-                if fn in checksums_from_json.keys():
-                    checksums += [checksums_from_json[fn]]
+            with contextlib.suppress(KeyError):
+                checksums.extend(checksums_from_json[fn] for fn in sources + patches)
 
-        if source_cnt is None:
-            source_cnt = len(sources)
-        patch_cnt, checksum_cnt = len(patches), len(checksums)
+        checksum_cnt = len(checksums)
 
         if (source_cnt + patch_cnt) != checksum_cnt:
             if sub:
@@ -2835,17 +2850,9 @@ def check_checksums(self):
         checksum_issues.extend(self.check_checksums_for(self.cfg))
 
         # also check checksums for extensions
-        for ext in self.cfg.get_ref('exts_list'):
-            # just skip extensions for which only a name is specified
-            # those are just there to check for things that are in the "standard library"
-            if not isinstance(ext, str):
-                ext_name = ext[0]
-                # take into account that extension may be a 2-tuple with just name/version
-                ext_opts = ext[2] if len(ext) == 3 else {}
-                # only a single source per extension is supported (see source_tmpl)
-                source_cnt = 1 if not ext_opts.get('nosource') else 0
-                res = self.check_checksums_for(ext_opts, sub="of extension %s" % ext_name, source_cnt=source_cnt)
-                checksum_issues.extend(res)
+        for ext in self.collect_exts_file_info(fetch_files=False, verify_checksums=False):
+            res = self.check_checksums_for(ext, sub=f"of extension {ext['name']}")
+            checksum_issues.extend(res)
 
         return checksum_issues
 

test/framework/easyblock.py

@@ -29,14 +29,17 @@
 @author: Kenneth Hoste (Ghent University)
 @author: Maxime Boissonneault (Compute Canada)
 @author: Jan Andre Reuter (Juelich Supercomputing Centre)
+@author: Alexander Grund (TU Dresden)
 """
 import copy
 import fileinput
+import itertools
 import os
 import re
 import shutil
 import sys
 import tempfile
+import textwrap
 from inspect import cleandoc
 from test.framework.github import requires_github_access
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
@@ -2350,56 +2353,94 @@ def test_collect_exts_file_info(self):
         toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy')
         toy_ext_sources = os.path.join(toy_sources, 'extensions')
         toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
-        toy_ec = process_easyconfig(toy_ec_file)[0]
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        new_ext_txt = "('baz', '0.0', {'nosource': True}),"  # With nosource option
+        new_ext_txt += "('barbar', '0.0', {'sources': [SOURCE_TAR_GZ]}),"  # With sources containing a list
+        test_ectxt = re.sub(r'\(name, version', new_ext_txt+r"\g<0>", read_file(toy_ec_file))
+        write_file(test_ec, test_ectxt)
+
+        toy_ec = process_easyconfig(test_ec)[0]
         toy_eb = EasyBlock(toy_ec['ec'])
 
         exts_file_info = toy_eb.collect_exts_file_info()
 
         self.assertIsInstance(exts_file_info, list)
-        self.assertEqual(len(exts_file_info), 4)
+        self.assertEqual(len(exts_file_info), 6)
 
         self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
+        self.assertEqual(exts_file_info[1]['sources'], ['bar-0.0.tar.gz'])
         self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
         bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch'
         self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
         self.assertEqual(exts_file_info[1]['patches'][0]['path'], os.path.join(toy_ext_sources, bar_patch1))
         bar_patch2 = 'bar-0.0_fix-very-silly-typo-in-printf-statement.patch'
         self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
         self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))
+        self.assertEqual(len(exts_file_info[1]['checksums']), 1)
 
         self.assertEqual(exts_file_info[2]['name'], 'barbar')
+        self.assertEqual(exts_file_info[2]['sources'], ['barbar-1.2.tar.gz'])
         self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-1.2.tar.gz'))
         self.assertNotIn('patches', exts_file_info[2])
+        self.assertEqual(len(exts_file_info[2]['checksums']), 0)
 
-        self.assertEqual(exts_file_info[3]['name'], 'toy')
-        self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz'))
+        self.assertEqual(exts_file_info[3]['name'], 'baz')
+        self.assertNotIn('sources', exts_file_info[3])
+        self.assertNotIn('sources', exts_file_info[3]['options'])
+        self.assertNotIn('src', exts_file_info[3])
         self.assertNotIn('patches', exts_file_info[3])
+        self.assertEqual(len(exts_file_info[3]['checksums']), 0)
+
+        self.assertEqual(exts_file_info[4]['name'], 'barbar')
+        self.assertEqual(exts_file_info[4]['sources'], ['barbar-0.0.tar.gz'])
+        self.assertEqual(exts_file_info[4]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
+        self.assertNotIn('patches', exts_file_info[4])
+
+        self.assertEqual(exts_file_info[5]['name'], 'toy')
+        self.assertEqual(exts_file_info[5]['sources'], ['toy-0.0.tar.gz'])
+        self.assertEqual(exts_file_info[5]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz'))
+        self.assertNotIn('patches', exts_file_info[5])
 
         # location of files is missing when fetch_files is set to False
         exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False)
 
         self.assertIsInstance(exts_file_info, list)
-        self.assertEqual(len(exts_file_info), 4)
+        self.assertEqual(len(exts_file_info), 6)
 
         self.assertEqual(exts_file_info[0], {'name': 'ulimit'})
 
         self.assertEqual(exts_file_info[1]['name'], 'bar')
+        self.assertEqual(exts_file_info[1]['sources'], ['bar-0.0.tar.gz'])
         self.assertNotIn('src', exts_file_info[1])
         self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
         self.assertNotIn('path', exts_file_info[1]['patches'][0])
         self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
         self.assertNotIn('path', exts_file_info[1]['patches'][1])
 
         self.assertEqual(exts_file_info[2]['name'], 'barbar')
+        self.assertEqual(exts_file_info[2]['sources'], ['barbar-1.2.tar.gz'])
         self.assertNotIn('src', exts_file_info[2])
         self.assertNotIn('patches', exts_file_info[2])
 
-        self.assertEqual(exts_file_info[3]['name'], 'toy')
+        self.assertEqual(exts_file_info[3]['name'], 'baz')
+        self.assertNotIn('sources', exts_file_info[3])
+        self.assertNotIn('sources', exts_file_info[3]['options'])
         self.assertNotIn('src', exts_file_info[3])
         self.assertNotIn('patches', exts_file_info[3])
 
+        self.assertEqual(exts_file_info[4]['name'], 'barbar')
+        self.assertEqual(exts_file_info[4]['sources'], ['barbar-0.0.tar.gz'])
+        self.assertNotIn('src', exts_file_info[4])
+        self.assertNotIn('patches', exts_file_info[4])
+
+        self.assertEqual(exts_file_info[5]['name'], 'toy')
+        self.assertEqual(exts_file_info[5]['sources'], ['toy-0.0.tar.gz'])
+        self.assertNotIn('src', exts_file_info[5])
+        self.assertNotIn('patches', exts_file_info[5])
+
         error_msg = "Can't verify checksums for extension files if they are not being fetched"
         self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False)
 
@@ -3337,21 +3378,21 @@ def run_checks():
             self.assertEqual(res[0], expected)
             self.assertTrue(res[1].startswith("Non-SHA256 checksum(s) found for toy-0.0.tar.gz:"))
 
+        ext_error_tmpl = "Checksums missing for one or more sources/patches of extension %s in "
+
         # check for main sources/patches should reveal two issues with checksums
         res = eb.check_checksums_for(eb.cfg)
         self.assertEqual(len(res), 2)
         run_checks()
 
         # full check also catches checksum issues with extensions
+        eb.json_checksums = {}  # Avoid picking up checksums from JSON file
         res = eb.check_checksums()
         self.assertEqual(len(res), 4)
         run_checks()
 
-        idx = 2
-        for ext in ['bar', 'barbar']:
-            expected = "Checksums missing for one or more sources/patches of extension %s in " % ext
-            self.assertTrue(res[idx].startswith(expected))
-            idx += 1
+        for ext, line in zip(('bar', 'barbar'), res[2:]):
+            self.assertIn(ext_error_tmpl % ext, line)
 
         # check whether tuple of alternative SHA256 checksums is correctly recognized
         toy_ec = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
@@ -3414,6 +3455,24 @@ def run_checks():
         )]
         self.assertEqual(eb.check_checksums(), [])
 
+        self.contents = textwrap.dedent("""
+            easyblock = "ConfigureMake"
+            name = "Uniq_1"
+            version = "3.14"
+            homepage = "http://example.com"
+            description = "test"
+            toolchain = SYSTEM
+            # Templates of parent used in extensions
+            exts_list = [
+                ('%(namelower)s', version),
+            ]
+        """)
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+        res = eb.check_checksums()
+        self.assertEqual(len(res), 1)
+        self.assertIn(ext_error_tmpl % 'uniq_1', res[0])
+
         # no checksums in easyconfig, then picked up from checksums.json next to easyconfig file
         test_ec = os.path.join(self.test_prefix, 'test.eb')
         copy_file(toy_ec, test_ec)
@@ -3426,14 +3485,84 @@ def run_checks():
         expected += "found 1 sources + 2 patches vs 0 checksums"
         self.assertEqual(res[0], expected)
 
-        # all is fine is checksums.json is also copied
+        # all is fine if checksums.json is also copied
         copy_file(os.path.join(os.path.dirname(toy_ec), 'checksums.json'), self.test_prefix)
         eb.json_checksums = None
         self.assertEqual(eb.check_checksums(), [])
 
+        self.contents = textwrap.dedent("""
+            easyblock = "ConfigureMake"
+            name = "Uniq_1"
+            version = "3"
+            homepage = "http://example.com"
+            description = "test"
+            toolchain = SYSTEM
+            # Different ways of specifying sources, patches, template usages to make sure they are resolved correctly
+            exts_list = [
+                'ulimit',  # extension that is part of "standard library"
+                ('ext1', '0.0'), # Default source filename
+                ('ext2', '1.2', {
+                    'source_tmpl': "%(name)s.zip",
+                    'patches': ['%(name)s.patch'],
+                }),
+                ('ext3', '1.2', {
+                    'sources': "%(name)s.zip",
+                    'postinstallpatches': ['%(name)s.patch'],
+                }),
+                ('ext-%(namelower)s', version + '.14', {
+                    'sources': {'filename': '%(name)s-%(version)s.zip', 'download_filename': 'foo.tar'},
+                    'patches': [{'name': '%(name)s.patch'}],
+                }),
+                ('ext-ok1', version, {
+                    'checksums': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'
+                }),
+                ('ext-ok2', version, {
+                    'data_sources': {'filename': '%(name)s-%(version)s.zip', 'download_filename': 'bar.tar'},
+                    'checksums': '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'
+                }),
+                ('ext-ok3', version, {
+                    'nosource': True
+                }),
+                ('ext-ok1', version, {
+                    'checksums': ['44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc']
+                }),
+                ('ext-ok2', version, {
+                    'nosource': True
+                }),
+            ]
+        """)
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+        res = eb.check_checksums()
+        self.assertEqual(len(res), 4)
+        extensions = ['ext1', 'ext2', 'ext3', 'ext-uniq_1']
+        for ext, line in zip(extensions, res):
+            self.assertIn(ext_error_tmpl % ext, line)
+
+        # Gradually add checksums to JSON dict and test that the associated extension checksums are now fine
+        sha256_cs = '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'  # Valid format only
+        eb.json_checksums = {'ext1-0.0.tar.gz': sha256_cs}
+        for ext, line in itertools.zip_longest(extensions[1:], eb.check_checksums(), fillvalue=''):
+            self.assertIn(ext_error_tmpl % ext, line)
+        eb.json_checksums['ext2.zip'] = sha256_cs
+        for ext, line in itertools.zip_longest(extensions[1:], eb.check_checksums(), fillvalue=''):
+            self.assertIn(ext_error_tmpl % ext, line)
+        eb.json_checksums['ext2.patch'] = sha256_cs
+        for ext, line in itertools.zip_longest(extensions[2:], eb.check_checksums(), fillvalue=''):
+            self.assertIn(ext_error_tmpl % ext, line)
+        eb.json_checksums['ext3.zip'] = sha256_cs
+        for ext, line in itertools.zip_longest(extensions[2:], eb.check_checksums(), fillvalue=''):
+            self.assertIn(ext_error_tmpl % ext, line)
+        eb.json_checksums['ext3.patch'] = sha256_cs
+        for ext, line in itertools.zip_longest(extensions[3:], eb.check_checksums(), fillvalue=''):
+            self.assertIn(ext_error_tmpl % ext, line)
+        eb.json_checksums['ext-uniq_1-3.14.zip'] = sha256_cs
+        eb.json_checksums['ext-uniq_1.patch'] = sha256_cs
+        self.assertEqual(eb.check_checksums(), [])
+
         # more checks for check_checksums_for method, which also takes regular dict as input
         self.assertEqual(eb.check_checksums_for({}), [])
-        expected = "Checksums missing for one or more sources/patches in test.eb: "
+        expected = f"Checksums missing for one or more sources/patches in {os.path.basename(self.eb_file)}: "
         expected += "found 1 sources + 0 patches vs 0 checksums"
         self.assertEqual(eb.check_checksums_for({'sources': ['test.tar.gz']}), [expected])