Merge pull request #3383 from easybuilders/4.2.x

release EasyBuild v4.2.2

RELEASE_NOTES

@@ -3,6 +3,26 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.2.2 (July 8th 2020)
+----------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for using 'sources' and 'git_config' for extensions in 'exts_list' (#3294)
+  - add support for software minver template (like %(pyminver)s, etc.) (#3344, #3345)
+  - add support for updating dictionary or tuple easyconfig parameters with self.cfg.update (#3356)
+- various bug fixes, including:
+  - fix crash in --avail-easyconfig-constants when using --output-format=rst + ensure sorted output (#3341)
+  - always take into account builddependencies when generating template values, also when we're not iterating over builddependencies (#3346)
+  - fix running command as 'easybuild' user in generated Singularity definition file (#3347)
+  - allow ignoring versionsuffix in --try-update-deps (#3350, #3353)
+  - retain order of paths when generating prepend_path statements for module file (don't sort them alphabetically) (#3367)
+  - also put easyblocks used by extensions in 'reprod' directory (#3375)
+  - also copy template values in EasyConfig.copy method to ensure they are defined when installing extensions (#3377)
+  - skip lines that start with 'module-version' when determining whether a module exists in ModulesTool.exist (#3379)
+
+
 v4.2.1 (May 20th 2020)
 ----------------------
 

easybuild/framework/easyconfig/easyconfig.py

@@ -570,34 +570,57 @@ def copy(self, validate=None):
         if self.path:
             ec.path = self.path
 
+        # also copy template values, since re-generating them may not give the same set of template values straight away
+        ec.template_values = copy.deepcopy(self.template_values)
+
         return ec
 
     def update(self, key, value, allow_duplicate=True):
         """
-        Update a string configuration value with a value (i.e. append to it).
+        Update an easyconfig parameter with the specified value (i.e. append to it).
+        Note: For dictionary easyconfig parameters, 'allow_duplicate' is ignored (since it's meaningless).
         """
         if isinstance(value, string_type):
-            lval = [value]
-        elif isinstance(value, list):
-            lval = value
+            inval = [value]
+        elif isinstance(value, (list, dict, tuple)):
+            inval = value
         else:
-            msg = "Can't update configuration value for %s, because the "
-            msg += "attempted update value, '%s', is not a string or list."
+            msg = "Can't update configuration value for %s, because the attempted"
+            msg += " update value, '%s', is not a string, list, tuple or dictionary."
             raise EasyBuildError(msg, key, value)
 
-        param_value = self[key]
+        # For easyconfig parameters that are dictionaries, input value must also be a dictionary
+        if isinstance(self[key], dict) and not isinstance(value, dict):
+            msg = "Can't update configuration value for %s, because the attempted"
+            msg += "update value (%s), is not a dictionary (type: %s)."
+            raise EasyBuildError(msg, key, value, type(value))
+
+        # Grab current parameter value so we can modify it
+        param_value = copy.deepcopy(self[key])
+
         if isinstance(param_value, string_type):
-            for item in lval:
+            for item in inval:
                 # re.search: only add value to string if it's not there yet (surrounded by whitespace)
                 if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % re.escape(item), param_value)):
                     param_value = param_value + ' %s ' % item
-        elif isinstance(param_value, list):
-            for item in lval:
+
+        elif isinstance(param_value, (list, tuple)):
+            # make sure we have a list value so we can just append to it
+            param_value = list(param_value)
+            for item in inval:
                 if allow_duplicate or item not in param_value:
-                    param_value = param_value + [item]
+                    param_value.append(item)
+            # cast back to tuple if original value was a tuple
+            if isinstance(self[key], tuple):
+                param_value = tuple(param_value)
+
+        elif isinstance(param_value, dict):
+            param_value.update(inval)
         else:
-            raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key)
+            msg = "Can't update configuration value for %s, because it's not a string, list, tuple or dictionary."
+            raise EasyBuildError(msg, key)
 
+        # Overwrite easyconfig parameter value with updated value, preserving type
         self[key] = param_value
 
     def set_keys(self, params):
@@ -1573,6 +1596,7 @@ def _finalize_dependencies(self):
     def generate_template_values(self):
         """Try to generate all template values."""
 
+        self.log.info("Generating template values...")
         self._generate_template_values()
 
         # recursive call, until there are no more changes to template values;
@@ -1591,6 +1615,8 @@ def generate_template_values(self):
                     # KeyError's may occur when not all templates are defined yet, but these are safe to ignore
                     pass
 
+        self.log.info("Template values: %s", ', '.join("%s='%s'" % x for x in sorted(self.template_values.items())))
+
     def _generate_template_values(self, ignore=None):
         """Actual code to generate the template values"""
 

easybuild/framework/easyconfig/templates.py

@@ -216,9 +216,25 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
         # copy to avoid changing original list below
         deps = copy.copy(config.get('dependencies', []))
 
-        # only consider build dependencies for defining *ver and *shortver templates if we're in iterative mode
-        if hasattr(config, 'iterating') and config.iterating:
-            deps += config.get('builddependencies', [])
+        # also consider build dependencies for *ver and *shortver templates;
+        # we need to be a bit careful here, because for iterative installations
+        # (when multi_deps is used for example) the builddependencies value may be a list of lists
+
+        # first, determine if we have an EasyConfig instance
+        # (indirectly by checking for 'iterating' and 'iterate_options' attributes,
+        #  because we can't import the EasyConfig class here without introducing
+        #  a cyclic import...);
+        # we need to know to determine whether we're iterating over a list of build dependencies
+        is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options')
+
+        if is_easyconfig:
+            # if we're iterating over different lists of build dependencies,
+            # only consider build dependencies when we're actually in iterative mode!
+            if 'builddependencies' in config.iterate_options:
+                if config.iterating:
+                    deps += config.get('builddependencies', [])
+            else:
+                deps += config.get('builddependencies', [])
 
         for dep in deps:
             if isinstance(dep, dict):
@@ -245,6 +261,8 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
                 template_values['%sver' % pref] = dep_version
                 dep_version_parts = dep_version.split('.')
                 template_values['%smajver' % pref] = dep_version_parts[0]
+                if len(dep_version_parts) > 1:
+                    template_values['%sminver' % pref] = dep_version_parts[1]
                 template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2])
                 break
 

easybuild/framework/easyconfig/tweak.py

@@ -64,10 +64,8 @@
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES
 from easybuild.tools.utilities import flatten, nub, quote_str
 
-
 _log = fancylogger.getLogger('easyconfig.tweak', fname=False)
 
-
 EASYCONFIG_TEMPLATE = "TEMPLATE"
 
 
@@ -126,6 +124,10 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
             pruned_build_specs = copy.copy(build_specs)
 
             update_dependencies = pruned_build_specs.pop('update_deps', None)
+            ignore_versionsuffixes = pruned_build_specs.pop('ignore_versionsuffixes', None)
+            if ignore_versionsuffixes and not update_dependencies:
+                print_warning("--try-ignore-versionsuffixes is ignored if --try-update-deps is not True")
+                ignore_versionsuffixes = False
             if 'toolchain' in pruned_build_specs:
                 target_toolchain = pruned_build_specs.pop('toolchain')
                 pruned_build_specs.pop('toolchain_name', '')
@@ -197,7 +199,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                     new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
                                                                         targetdir=tweaked_ecs_path,
                                                                         update_build_specs=pruned_build_specs,
-                                                                        update_dep_versions=update_dependencies)
+                                                                        update_dep_versions=update_dependencies,
+                                                                        ignore_versionsuffixes=ignore_versionsuffixes)
                     # Need to update the toolchain in the build_specs to match the toolchain mapping
                     keys = verification_build_specs.keys()
                     if 'toolchain_name' in keys:
@@ -219,7 +222,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None):
                     # Note pruned_build_specs are not passed down for dependencies
                     map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping,
                                                           targetdir=tweaked_ecs_deps_path,
-                                                          update_dep_versions=update_dependencies)
+                                                          update_dep_versions=update_dependencies,
+                                                          ignore_versionsuffixes=ignore_versionsuffixes)
             else:
                 tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path)
 
@@ -277,6 +281,7 @@ def tweak_one(orig_ec, tweaked_ec, tweaks, targetdir=None):
 
         class TcDict(dict):
             """A special dict class that represents trivial toolchains properly."""
+
             def __repr__(self):
                 return "{'name': '%(name)s', 'version': '%(version)s'}" % self
 
@@ -899,7 +904,7 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp
                 'versionsuffix': versionsuffix or '',
             }
             # See what this dep would be mapped to
-            version_matches = find_potential_version_mappings(software_as_dep, toolchain_mapping)
+            version_matches = find_potential_version_mappings(software_as_dep, toolchain_mapping, quiet=True)
             if version_matches:
                 target_version = version_matches[0]['version']
                 if LooseVersion(target_version) > LooseVersion(version):
@@ -940,7 +945,7 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain):
 
 
 def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_build_specs=None,
-                                          update_dep_versions=False):
+                                          update_dep_versions=False, ignore_versionsuffixes=False):
     """
     Take an easyconfig spec, parse it, map it to a target toolchain and dump it out
 
@@ -961,6 +966,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
     if update_dep_versions and (list_deps_versionsuffixes(ec_spec) or parsed_ec['versionsuffix']):
         # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8`
         versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['toolchain'], toolchain_mapping)
+        versonsuffix_mapping.update(map_common_versionsuffixes('Perl', parsed_ec['toolchain'], toolchain_mapping))
 
     if update_build_specs is not None:
         if 'version' in update_build_specs:
@@ -1033,21 +1039,25 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
             elif update_dep_versions:
                 # search for available updates for this dependency:
                 # first get highest version candidate paths for this (include search through subtoolchains)
-                potential_version_mappings = find_potential_version_mappings(dep, toolchain_mapping,
-                                                                             versionsuffix_mapping=versonsuffix_mapping)
+                potential_version_mappings = find_potential_version_mappings(
+                    dep,
+                    toolchain_mapping,
+                    versionsuffix_mapping=versonsuffix_mapping,
+                    ignore_versionsuffixes=ignore_versionsuffixes
+                )
                 # only highest version match is retained by default in potential_version_mappings,
                 # compare that version to the original version and replace if appropriate (upgrades only).
                 if potential_version_mappings:
                     highest_version_match = potential_version_mappings[0]['version']
+                    highest_versionsuffix_match = potential_version_mappings[0]['versionsuffix']
                     if LooseVersion(highest_version_match) > LooseVersion(dep['version']):
                         _log.info("Updating version of %s dependency from %s to %s", dep['name'], dep['version'],
                                   highest_version_match)
                         _log.info("Depending on your configuration, this will be resolved with one of the following "
                                   "easyconfigs: \n%s", '\n'.join(cand['path'] for cand in potential_version_mappings))
                         orig_dep['version'] = highest_version_match
-                        if orig_dep['versionsuffix'] in versonsuffix_mapping:
-                            dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']]
-                            orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']]
+                        dep['versionsuffix'] = highest_versionsuffix_match
+                        orig_dep['versionsuffix'] = highest_versionsuffix_match
                         dep_changed = True
 
             if dep_changed:
@@ -1090,7 +1100,8 @@ def list_deps_versionsuffixes(ec_spec):
     return list(set(versionsuffix_list))
 
 
-def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True):
+def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True,
+                                    ignore_versionsuffixes=False, quiet=False):
     """
     Find potential version mapping for a dependency in a new hierarchy
 
@@ -1139,7 +1150,9 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
     if len(version_components) > 1:  # Have at least major.minor
         candidate_ver_list.append(r'%s\..*' % major_version)
     candidate_ver_list.append(r'.*')  # Include a major version search
-    potential_version_mappings, highest_version = [], None
+    potential_version_mappings = []
+    highest_version = None
+    highest_version_ignoring_versionsuffix = None
 
     for candidate_ver in candidate_ver_list:
 
@@ -1152,7 +1165,8 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
                     toolchain_suffix = ''
                 else:
                     toolchain_suffix = '-%s-%s' % (toolchain['name'], toolchain['version'])
-                full_versionsuffix = toolchain_suffix + versionsuffix + EB_FORMAT_EXTENSION
+                # Search for any version suffix but only use what we are allowed to
+                full_versionsuffix = toolchain_suffix + r'.*' + EB_FORMAT_EXTENSION
                 depver = '^' + prefix_to_version + candidate_ver + full_versionsuffix
                 cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False,
                                                 case_sensitive=True)
@@ -1178,14 +1192,46 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin
 
                 # add what is left to the possibilities
                 for path in cand_paths:
-                    version = fetch_parameters_from_easyconfig(read_file(path), ['version'])[0]
+                    version, newversionsuffix = fetch_parameters_from_easyconfig(read_file(path), ['version',
+                                                                                                   'versionsuffix'])
+                    if not newversionsuffix:
+                        newversionsuffix = ''
                     if version:
-                        if highest_version is None or LooseVersion(version) > LooseVersion(highest_version):
-                            highest_version = version
+                        if versionsuffix == newversionsuffix:
+                            if highest_version is None or LooseVersion(version) > LooseVersion(highest_version):
+                                highest_version = version
+                        else:
+                            if highest_version_ignoring_versionsuffix is None or \
+                                    LooseVersion(version) > LooseVersion(highest_version_ignoring_versionsuffix):
+                                highest_version_ignoring_versionsuffix = version
                     else:
                         raise EasyBuildError("Failed to determine version from contents of %s", path)
 
-                    potential_version_mappings.append({'path': path, 'toolchain': toolchain, 'version': version})
+                    potential_version_mappings.append({'path': path, 'toolchain': toolchain, 'version': version,
+                                                       'versionsuffix': newversionsuffix})
+
+    ignored_versionsuffix_greater = \
+        highest_version_ignoring_versionsuffix is not None and highest_version is None or \
+        (highest_version_ignoring_versionsuffix is not None and highest_version is not None and
+         LooseVersion(highest_version_ignoring_versionsuffix) > LooseVersion(highest_version))
+
+    exclude_alternate_versionsuffixes = False
+    if ignored_versionsuffix_greater:
+        if ignore_versionsuffixes:
+            highest_version = highest_version_ignoring_versionsuffix
+        else:
+            if not quiet:
+                print_warning(
+                    "There may be newer version(s) of dep '%s' available with a different versionsuffix to '%s': %s",
+                    dep['name'], versionsuffix, [d['path'] for d in potential_version_mappings if
+                                                 d['version'] == highest_version_ignoring_versionsuffix])
+            # exclude candidates with a different versionsuffix
+            exclude_alternate_versionsuffixes = True
+    else:
+        # If the other version suffixes are not greater, then just ignore them
+        exclude_alternate_versionsuffixes = True
+    if exclude_alternate_versionsuffixes:
+        potential_version_mappings = [d for d in potential_version_mappings if d['versionsuffix'] == versionsuffix]
 
     if highest_versions_only and highest_version is not None:
         potential_version_mappings = [d for d in potential_version_mappings if d['version'] == highest_version]