Merge pull request #3764 from easybuilders/4.4.x

release EasyBuild v4.4.1

.github/workflows/bootstrap_script.yml

@@ -107,12 +107,14 @@ jobs:
           EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g')
           EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ')
           EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM"
-          EB_BOOTSTRAP_EXPECTED="20210106.01 c2d93de0dd91123eb4f51cfc16d1f5efb80f1d238b3d6cd100994086887a1ae0"
+          EB_BOOTSTRAP_EXPECTED="20210618.01 e5d477d717c6d3648ba2027ab735713ba5804fbf52f4b4adcca0bc1379b44618"
           test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1)
 
           # test bootstrap script
           export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap
+          export EASYBUILD_BOOTSTRAP_DEPRECATED=1
           python easybuild/scripts/bootstrap_eb.py $PREFIX
+          unset EASYBUILD_BOOTSTRAP_DEPRECATED
           # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module
           unset PYTHONPATH
           # simple sanity check on bootstrapped EasyBuild module

RELEASE_NOTES

@@ -4,6 +4,37 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.4.1 (July 6th 2021)
+----------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - enhance detection of patch files supplied to 'eb' command with better error messages (#3709)
+  - add per-step timing information (#3716)
+  - add module-write hook (#3728)
+  - add ignore-test-failure configuration option to ignore failing test step (#3732)
+  - add toolchain definition for nvompic (NVHPC + OpenMPI) (#3735)
+  - warn about generic milestone in --review-pr and --merge-pr (#3751)
+- various bug fixes, including:
+  - don't override COMPILER_MODULE_NAME, inherited from Ffmpi, in Fujitsu toolchain definition (#3721)
+  - avoid overwritting pr_nr in post_pr_test_report for reports with --include-easyblocks-from-pr (#3724, #3726)
+  - fix crash in get_config_dict when copying modules that were imported in easyconfig file (like 'import os') (#3729)
+  - parse C standard flags in CFLAGS for Fujitsu compiler (#3731)
+  - fix error message for --use-ccache (#3733)
+  - error out when passing a list to run_cmd* to avoid running wrong command unintended, unless shell=True is used (#3737)
+  - minor fixes to output of test reports when using multiple PRs (#3741)
+  - fix location for modules installed with intel-compilers toolchain in HierarchicalMNS by always checking toolchain compiler name against template map (#3745)
+  - avoid checking msg attribute of GitCommandError (#3756)
+  - consider sources provided as dict in EasyBlock.check_checksums_for (#3758)
+  - don't make changes to software installation directory when using --sanity-check-only (#3761)
+  - honor specified easyblock for extensions (#3762)
+- other changes:
+  - make sure that tests requiring a github token have 'github' in the test name so that they can be easily filtered (#3730)
+  - deprecate EasyBuild bootstrap script (#3742)
+  - use temporary download folder in test_http_header_fields_urlpat (#3755)
+
+
 v4.4.0 (June 2nd 2021)
 ----------------------
 

easybuild/base/testing.py

@@ -35,6 +35,7 @@
 import pprint
 import re
 import sys
+from contextlib import contextmanager
 
 try:
     from cStringIO import StringIO  # Python 2
@@ -185,6 +186,26 @@ def get_stderr(self):
         """Return output captured from stderr until now."""
         return sys.stderr.getvalue()
 
+    @contextmanager
+    def mocked_stdout_stderr(self, mock_stdout=True, mock_stderr=True):
+        """Context manager to mock stdout and stderr"""
+        if mock_stdout:
+            self.mock_stdout(True)
+        if mock_stderr:
+            self.mock_stderr(True)
+        try:
+            if mock_stdout and mock_stderr:
+                yield sys.stdout, sys.stderr
+            elif mock_stdout:
+                yield sys.stdout
+            else:
+                yield sys.stderr
+        finally:
+            if mock_stdout:
+                self.mock_stdout(False)
+            if mock_stderr:
+                self.mock_stderr(False)
+
     def tearDown(self):
         """Cleanup after running a test."""
         self.mock_stdout(False)

easybuild/framework/easyconfig/format/one.py

@@ -135,8 +135,14 @@ def get_config_dict(self):
         # we can't use copy.deepcopy() directly because in Python 2 copying the (irrelevant) __builtins__ key fails...
         cfg_copy = {}
         for key in cfg:
-            if key != '__builtins__':
-                cfg_copy[key] = copy.deepcopy(cfg[key])
+            # skip special variables like __builtins__, and imported modules (like 'os')
+            if key != '__builtins__' and "'module'" not in str(type(cfg[key])):
+                try:
+                    cfg_copy[key] = copy.deepcopy(cfg[key])
+                except Exception as err:
+                    raise EasyBuildError("Failed to copy '%s' easyconfig parameter: %s" % (key, err))
+            else:
+                self.log.debug("Not copying '%s' variable from parsed easyconfig", key)
 
         return cfg_copy
 

easybuild/framework/easyconfig/tools.py

@@ -334,7 +334,7 @@ def det_easyconfig_paths(orig_paths):
     :return: list of paths to easyconfig files
     """
     try:
-        from_prs = [int(pr_nr) for pr_nr in build_option('from_pr')]
+        from_prs = [int(x) for x in build_option('from_pr')]
     except ValueError:
         raise EasyBuildError("Argument to --from-pr must be a comma separated list of PR #s.")
 
@@ -539,6 +539,12 @@ def review_pr(paths=None, pr=None, colored=True, branch='develop', testing=False
         if missing_labels:
             lines.extend(['', "This PR should be labelled with %s" % ', '.join(["'%s'" % ml for ml in missing_labels])])
 
+        if not pr_data['milestone']:
+            lines.extend(['', "This PR should be associated with a milestone"])
+        elif '.x' in pr_data['milestone']['title']:
+            lines.extend(['', "This PR is associated with a generic '.x' milestone, "
+                              "it should be associated to the next release milestone once merged"])
+
     return '\n'.join(lines)
 
 
@@ -623,6 +629,14 @@ def categorize_files_by_type(paths):
         # file must exist in order to check whether it's a patch file
         elif os.path.isfile(path) and is_patch_file(path):
             res['patch_files'].append(path)
+        elif path.endswith('.patch'):
+            if not os.path.exists(path):
+                raise EasyBuildError('File %s does not exist, did you mistype the path?', path)
+            elif not os.path.isfile(path):
+                raise EasyBuildError('File %s is expected to be a regular file, but is a folder instead', path)
+            else:
+                raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!',
+                                     path)
         else:
             # anything else is considered to be an easyconfig file
             res['easyconfigs'].append(path)

easybuild/main.py

@@ -55,6 +55,7 @@
 from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for
 from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available
 from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
+from easybuild.tools.build_log import print_warning
 from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
 from easybuild.tools.containers.common import containerize
 from easybuild.tools.docs import list_software
@@ -304,6 +305,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
     init_session_state.update({'module_list': modlist})
     _log.debug("Initial session state: %s" % init_session_state)
 
+    if options.skip_test_step:
+        if options.ignore_test_failure:
+            raise EasyBuildError("Found both ignore-test-failure and skip-test-step enabled. "
+                                 "Please use only one of them.")
+        else:
+            print_warning("Will not run the test step as requested via skip-test-step. "
+                          "Consider using ignore-test-failure instead and verify the results afterwards")
+
     # determine easybuild-easyconfigs package install path
     easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
     if not easyconfigs_pkg_paths:

easybuild/scripts/bootstrap_eb.py

@@ -62,7 +62,7 @@
     import urllib.request as std_urllib
 
 
-EB_BOOTSTRAP_VERSION = '20210106.01'
+EB_BOOTSTRAP_VERSION = '20210618.01'
 
 # argparse preferrred, optparse deprecated >=2.7
 HAVE_ARGPARSE = False
@@ -82,6 +82,9 @@
 
 STAGE1_SUBDIR = 'eb_stage1'
 
+# the EasyBuild bootstrap script is deprecated, and will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined
+EASYBUILD_BOOTSTRAP_DEPRECATED = os.environ.pop('EASYBUILD_BOOTSTRAP_DEPRECATED', None)
+
 # set print_debug to True for detailed progress info
 print_debug = os.environ.pop('EASYBUILD_BOOTSTRAP_DEBUG', False)
 
@@ -854,6 +857,17 @@ def main():
     self_txt = open(__file__).read()
     if IS_PY3:
         self_txt = self_txt.encode('utf-8')
+
+    url = 'https://docs.easybuild.io/en/latest/Installation.html'
+    info("Use of the EasyBuild boostrap script is DEPRECATED (since June 2021).")
+    info("It is strongly recommended to use one of the installation methods outlined at %s instead!\n" % url)
+    if not EASYBUILD_BOOTSTRAP_DEPRECATED:
+        error("The EasyBuild bootstrap script will only run if $EASYBUILD_BOOTSTRAP_DEPRECATED is defined.")
+    else:
+        msg = "You have opted to continue with the EasyBuild bootstrap script by defining "
+        msg += "$EASYBUILD_BOOTSTRAP_DEPRECATED. Good luck!\n"
+        info(msg)
+
     info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest()))
     info("Found Python %s\n" % '; '.join(sys.version.split('\n')))
 

easybuild/toolchains/compiler/fujitsu.py

@@ -30,6 +30,7 @@
 :author: Miguel Dias Costa (National University of Singapore)
 """
 import os
+import re
 
 import easybuild.tools.environment as env
 import easybuild.tools.systemtools as systemtools
@@ -88,6 +89,13 @@ class FujitsuCompiler(Compiler):
     def prepare(self, *args, **kwargs):
         super(FujitsuCompiler, self).prepare(*args, **kwargs)
 
+        # fcc doesn't accept e.g. -std=c++11 or -std=gnu++11, only -std=c11 or -std=gnu11
+        pattern = r'-std=(gnu|c)\+\+(\d+)'
+        if re.search(pattern, self.vars['CFLAGS']):
+            self.log.debug("Found '-std=(gnu|c)++' in CFLAGS, fcc doesn't accept '++' here, removing it")
+            self.vars['CFLAGS'] = re.sub(pattern, r'-std=\1\2', self.vars['CFLAGS'])
+            self._setenv_variables()
+
         # make sure the fujitsu module libraries are found (and added to rpath by wrapper)
         library_path = os.getenv('LIBRARY_PATH', '')
         libdir = os.path.join(os.getenv(TC_CONSTANT_MODULE_VAR), 'lib64')

easybuild/toolchains/fujitsu.py

@@ -36,4 +36,3 @@ class Fujitsu(Ffmpi, FujitsuFFTW, FujitsuSSL):
     """Compiler toolchain for Fujitsu."""
     NAME = 'Fujitsu'
     SUBTOOLCHAIN = Ffmpi.NAME
-    COMPILER_MODULE_NAME = []

easybuild/toolchains/nvompic.py

@@ -0,0 +1,43 @@
+##
+# Copyright 2016-2021 Ghent University
+# Copyright 2016-2021 Forschungszentrum Juelich
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for nvompic compiler toolchain (includes NVHPC and OpenMPI, and CUDA as dependency).
+
+:author: Damian Alvarez (Forschungszentrum Juelich)
+:author: Sebastian Achilles (Forschungszentrum Juelich)
+"""
+
+from easybuild.toolchains.nvhpc import NVHPCToolchain
+# We pull in MPI and CUDA at once so this maps nicely to HMNS
+from easybuild.toolchains.mpi.openmpi import OpenMPI
+from easybuild.toolchains.compiler.cuda import Cuda
+
+
+# Order matters!
+class NVompic(NVHPCToolchain, Cuda, OpenMPI):
+    """Compiler toolchain with NVHPC and OpenMPI, with CUDA as dependency."""
+    NAME = 'nvompic'
+    SUBTOOLCHAIN = NVHPCToolchain.NAME

easybuild/tools/config.py

@@ -248,6 +248,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'ignore_checksums',
         'ignore_index',
         'ignore_locks',
+        'ignore_test_failure',
         'install_latest_eb_release',
         'logtostdout',
         'minimal_toolchains',

easybuild/tools/github.py

@@ -1183,7 +1183,10 @@ def not_eligible(msg):
     # check whether a milestone is set
     msg_tmpl = "* milestone is set: %s"
     if pr_data['milestone']:
-        print_msg(msg_tmpl % "OK (%s)" % pr_data['milestone']['title'], prefix=False)
+        milestone = pr_data['milestone']['title']
+        if '.x' in milestone:
+            milestone += ", please change to the next release milestone once the PR is merged"
+        print_msg(msg_tmpl % "OK (%s)" % milestone, prefix=False)
     else:
         res = not_eligible(msg_tmpl % 'no milestone found')
 
@@ -1232,7 +1235,7 @@ def reasons_for_closing(pr_data):
 
     robot_paths = build_option('robot_path')
 
-    pr_files = [path for path in fetch_easyconfigs_from_pr(pr_data['number']) if path.endswith('.eb')]
+    pr_files = [p for p in fetch_easyconfigs_from_pr(pr_data['number']) if p.endswith('.eb')]
 
     obsoleted = []
     uses_archived_tc = []
@@ -1491,17 +1494,17 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH):
 
     download_repo_path = download_repo(branch=branch, path=tmpdir)
 
-    pr_files = [path for path in fetch_easyconfigs_from_pr(pr) if path.endswith('.eb')]
+    pr_files = [p for p in fetch_easyconfigs_from_pr(pr) if p.endswith('.eb')]
 
     file_info = det_file_info(pr_files, download_repo_path)
 
     pr_target_account = build_option('pr_target_account')
     github_user = build_option('github_user')
     pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user)
-    pr_labels = [label['name'] for label in pr_data['labels']]
+    pr_labels = [x['name'] for x in pr_data['labels']]
 
     expected_labels = det_pr_labels(file_info, pr_target_repo)
-    missing_labels = [label for label in expected_labels if label not in pr_labels]
+    missing_labels = [x for x in expected_labels if x not in pr_labels]
 
     dry_run = build_option('dry_run') or build_option('extended_dry_run')
 
@@ -2025,7 +2028,7 @@ def check_github():
             ver, req_ver = git.__version__, '1.0'
             if LooseVersion(ver) < LooseVersion(req_ver):
                 check_res = "FAIL (GitPython version %s is too old, should be version %s or newer)" % (ver, req_ver)
-            elif "Could not read from remote repository" in push_err.msg:
+            elif "Could not read from remote repository" in str(push_err):
                 check_res = "FAIL (GitHub SSH key missing? %s)" % push_err
             else:
                 check_res = "FAIL (unexpected exception: %s)" % push_err