Merge pull request #3526 from easybuilders/4.3.x

release EasyBuild v4.3.2

.github/workflows/unit_tests.yml

@@ -43,6 +43,14 @@ jobs:
             python: 3.8
           - modules_tool: modules-4.1.4
             python: 3.9
+          - modules_tool: Lmod-7.8.22
+            python: 3.5
+          - modules_tool: Lmod-7.8.22
+            python: 3.7
+          - modules_tool: Lmod-7.8.22
+            python: 3.8
+          - modules_tool: Lmod-7.8.22
+            python: 3.9
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
@@ -122,6 +130,9 @@ jobs:
         EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
         TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
       run: |
+          # run tests *outside* of checked out easybuild-framework directory,
+          # to ensure we're testing installed version (see previous step)
+          cd $HOME
           # initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
           source $(cat $HOME/mod_init); type module
@@ -153,15 +164,12 @@ jobs:
           # run test suite
           python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
           # try and make sure output of running tests is clean (no printed messages/warnings)
-          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend"
+          IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2"
           # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
           PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
           test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1)
 
     - name: test bootstrap script
-      # skip testing of bootstrap script with Python 3.9,
-      # until an EasyBuild release that is compatible with Python 3.9 is available
-      if: ${{ matrix.python != 3.9 }}
       run: |
           # (re)initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi

MANIFEST.in

@@ -4,7 +4,7 @@ include optcomplete.bash
 recursive-include etc *
 recursive-include easybuild *py
 recursive-include easybuild/scripts *
-recursive-include test *py *eb
+recursive-include test *py *eb *yaml
 recursive-include test/framework/modules *
 recursive-include test/framework/sandbox/sources *
 include CONTRIBUTING.md

RELEASE_NOTES

@@ -3,6 +3,41 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
+v4.3.2 (December 10th 2020)
+---------------------------
+
+update/bugfix release
+
+- add (experimental) support for specifying easyconfig files via an "easystack" file (#3479, #3511, #3515, #3517, #3520, #3521)
+  - see also https://easybuild.readthedocs.io/en/latest/Easystack-files.html
+- add definition for new 'gobff' toolchain using BLIS and LibFLAME (#3505)
+- various enhancements, including:
+  - add support for toolchain options like 'extra_cxxflags' to specify extra compiler options (#2193)
+  - fix combination of --copy-ec and --from-pr (#3482)
+  - enhance copy_files function: support single file target, error on empty input list, support verbose mode (#3483)
+  - cache result of fetch_files_from_pr function (mainly to speed up framework test suite) (#3484)
+  - add locate_files function to filetools module (#3485)
+  - add support for %(module_name)s template value (#3497)
+  - clarify input format for --cuda-compute-capabilities in 'eb --help' output (#3509)
+  - add support for skiping unit tests (test step) via --skip-test-step (#3524)
+- various bug fixes, including:
+  - also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace (#3491)
+  - don't pass username in github_api_get_request when no GitHub token is available (#3494)
+  - also inject -rpath options for all entries in $LIBRARY_PATH in RPATH wrappers (#3495)
+  - avoid TypeError being raised by list_toolchains (#3499)
+  - check if PR is already merged in --merge-pr (#3502)
+  - graciously handle wrong PR id in fetch_pr_data (#3503)
+  - fix regression in apply_regex_substitutions: also accept list of paths to patch (#3507)
+  - update installation procedure for EasyBuild in generated Singularity container recipes (#3510)
+  - fix GitHub Actions workflow for test suite: run outside of repo checkout + also test bootstrap script with Python 3.9 (#3518)
+  - bump cryptography from 2.9.2 to 3.2 for Python 2 in requirements.txt (#3519)
+  - fix 'eb --help=rst' when running with Python 3 (#3525)
+- other changes:
+  - exclude test configurations with Lmod 7 and Python 3, except for Python 3.6 (#3496)
+  - significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed (#3498)
+  - don't include file/ldd/readelf commands run during RPATH sanity check in --trace output (#3508)
+
+
 v4.3.1 (October 29th 2020)
 --------------------------
 

easybuild/framework/easyblock.py

@@ -2139,10 +2139,11 @@ def build_step(self):
 
     def test_step(self):
         """Run unit tests provided by software (if any)."""
-        if self.cfg['runtest']:
+        unit_test_cmd = self.cfg['runtest']
+        if unit_test_cmd:
 
-            self.log.debug("Trying to execute %s as a command for running unit tests...")
-            (out, _) = run_cmd(self.cfg['runtest'], log_all=True, simple=False)
+            self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd)
+            (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False)
 
             return out
 
@@ -2460,7 +2461,7 @@ def sanity_check_rpath(self, rpath_dirs=None):
                 for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]:
                     self.log.debug("Sanity checking RPATH for %s", path)
 
-                    out, ec = run_cmd("file %s" % path, simple=False)
+                    out, ec = run_cmd("file %s" % path, simple=False, trace=False)
                     if ec:
                         fails.append("Failed to run 'file %s': %s" % (path, out))
 
@@ -2470,7 +2471,7 @@ def sanity_check_rpath(self, rpath_dirs=None):
                     # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped
                     if "dynamically linked" in out:
                         # check whether all required libraries are found via 'ldd'
-                        out, ec = run_cmd("ldd %s" % path, simple=False)
+                        out, ec = run_cmd("ldd %s" % path, simple=False, trace=False)
                         if ec:
                             fail_msg = "Failed to run 'ldd %s': %s" % (path, out)
                             self.log.warning(fail_msg)
@@ -2483,7 +2484,7 @@ def sanity_check_rpath(self, rpath_dirs=None):
                             self.log.debug("Output of 'ldd %s' checked, looks OK", path)
 
                         # check whether RPATH section in 'readelf -d' output is there
-                        out, ec = run_cmd("readelf -d %s" % path, simple=False)
+                        out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False)
                         if ec:
                             fail_msg = "Failed to run 'readelf %s': %s" % (path, out)
                             self.log.warning(fail_msg)
@@ -3310,6 +3311,10 @@ def build_and_install_one(ecdict, init_env):
         _log.debug("Skip set to %s" % skip)
         app.cfg['skip'] = skip
 
+    if build_option('skip_test_step'):
+        _log.debug('Adding test_step to skipped steps')
+        app.cfg.update('skipsteps', TEST_STEP, allow_duplicate=False)
+
     # build easyconfig
     errormsg = '(no error)'
     # timing info

easybuild/framework/easyconfig/format/format.py

@@ -612,12 +612,17 @@ def __init__(self):
             raise EasyBuildError('Invalid version number %s (incorrect length)', self.VERSION)
 
         self.rawtext = None  # text version of the easyconfig
-        self.comments = {}  # comments in easyconfig file
+        self._comments = {}  # comments in easyconfig file
         self.header = None  # easyconfig header (e.g., format version, license, ...)
         self.docstring = None  # easyconfig docstring (e.g., author, maintainer, ...)
 
         self.specs = {}
 
+    @property
+    def comments(self):
+        """Return comments in easyconfig file"""
+        return self._comments
+
     def set_specifications(self, specs):
         """Set specifications."""
         self.log.debug('Set copy of specs %s' % specs)

easybuild/framework/easyconfig/format/one.py

@@ -135,7 +135,8 @@ def parse(self, txt):
         """
         Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced.
         """
-        super(FormatOneZero, self).parse(txt, strict_section_markers=True)
+        self.rawcontent = txt
+        super(FormatOneZero, self).parse(self.rawcontent, strict_section_markers=True)
 
     def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
         """
@@ -356,14 +357,24 @@ def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy
 
         return '\n'.join(dump)
 
+    @property
+    def comments(self):
+        """
+        Return comments (and extract them first if needed).
+        """
+        if not self._comments:
+            self.extract_comments(self.rawcontent)
+
+        return self._comments
+
     def extract_comments(self, rawtxt):
         """
         Extract comments from raw content.
 
         Discriminates between comment header, comments above a line (parameter definition), and inline comments.
         Inline comments on items of iterable values are also extracted.
         """
-        self.comments = {
+        self._comments = {
             'above': {},  # comments above a parameter definition
             'header': [],  # header comment lines
             'inline': {},  # inline comments

easybuild/framework/easyconfig/parser.py

@@ -110,8 +110,6 @@ def __init__(self, filename=None, format_version=None, rawcontent=None,
         else:
             raise EasyBuildError("Neither filename nor rawcontent provided to EasyConfigParser")
 
-        self._formatter.extract_comments(self.rawcontent)
-
     def process(self, filename=None):
         """Create an instance"""
         self._read(filename=filename)

easybuild/framework/easyconfig/templates.py

@@ -47,6 +47,7 @@
 # derived from easyconfig, but not from ._config directly
 TEMPLATE_NAMES_EASYCONFIG = [
     ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"),
+    ('module_name', "Module name"),
     ('nameletter', "First letter of software name"),
     ('toolchain_name', "Toolchain name"),
     ('toolchain_version', "Toolchain version"),
@@ -72,8 +73,8 @@
 ]
 # values taken from the EasyBlock before each step
 TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [
-    ('installdir', "Installation directory"),
     ('builddir', "Build directory"),
+    ('installdir', "Installation directory"),
 ]
 # software names for which to define <pref>ver and <pref>shortver templates
 TEMPLATE_SOFTWARE_VERSIONS = [
@@ -208,6 +209,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
                 softname = config['name']
                 if softname is not None:
                     template_values['nameletter'] = softname[0]
+
+        elif name[0] == 'module_name':
+            template_values['module_name'] = getattr(config, 'short_mod_name', None)
+
         else:
             raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name)
 

easybuild/framework/easyconfig/tools.py

@@ -53,8 +53,9 @@
 from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.environment import restore_env
-from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file
-from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo
+from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files
+from easybuild.tools.filetools import read_file, resolve_path, which, write_file
+from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_files_from_pr, download_repo
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import OrderedDict
 from easybuild.tools.toolchain.toolchain import is_system_toolchain
@@ -348,44 +349,13 @@ def det_easyconfig_paths(orig_paths):
             ec_files = [path for path in pr_files if path.endswith('.eb')]
 
     if ec_files and robot_path:
-        # look for easyconfigs with relative paths in robot search path,
-        # unless they were found at the given relative paths
-
-        # determine which easyconfigs files need to be found, if any
-        ecs_to_find = []
-        for idx, ec_file in enumerate(ec_files):
-            if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file):
-                ecs_to_find.append((idx, ec_file))
-        _log.debug("List of easyconfig files to find: %s" % ecs_to_find)
-
-        # find missing easyconfigs by walking paths in robot search path
-        for path in robot_path:
-            _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
-            for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
-                for idx, orig_path in ecs_to_find[:]:
-                    if orig_path in filenames:
-                        full_path = os.path.join(subpath, orig_path)
-                        _log.info("Found %s in %s: %s" % (orig_path, path, full_path))
-                        ec_files[idx] = full_path
-                        # if file was found, stop looking for it (first hit wins)
-                        ecs_to_find.remove((idx, orig_path))
-
-                # stop os.walk insanity as soon as we have all we need (os.walk loop)
-                if not ecs_to_find:
-                    break
-
-                # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
-                dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')]
-
-                # ignore archived easyconfigs, unless specified otherwise
-                if not build_option('consider_archived_easyconfigs'):
-                    dirnames[:] = [d for d in dirnames if d != EASYCONFIGS_ARCHIVE_DIR]
-
-            # stop os.walk insanity as soon as we have all we need (outer loop)
-            if not ecs_to_find:
-                break
-
-    return [os.path.abspath(ec_file) for ec_file in ec_files]
+        ignore_subdirs = build_option('ignore_dirs')
+        if not build_option('consider_archived_easyconfigs'):
+            ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR)
+
+        ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs)
+
+    return ec_files
 
 
 def parse_easyconfigs(paths, validate=True):
@@ -728,3 +698,65 @@ def avail_easyblocks():
                                        easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path)
 
     return easyblocks
+
+
+def det_copy_ec_specs(orig_paths, from_pr):
+    """Determine list of paths + target directory for --copy-ec."""
+
+    target_path, paths = None, []
+
+    # if only one argument is specified, use current directory as target directory
+    if len(orig_paths) == 1:
+        target_path = os.getcwd()
+        paths = orig_paths[:]
+
+    # if multiple arguments are specified, assume that last argument is target location,
+    # and remove that from list of paths to copy
+    elif orig_paths:
+        target_path = orig_paths[-1]
+        paths = orig_paths[:-1]
+
+    # if --from-pr was used in combination with --copy-ec, some extra care must be taken
+    if from_pr:
+        # pull in the paths to all the changed files in the PR,
+        # which includes easyconfigs but also patch files (& maybe more);
+        # do this in a dedicated subdirectory of the working tmpdir,
+        # to avoid potential trouble with already existing files in the working tmpdir
+        # (note: we use a fixed subdirectory in the working tmpdir here rather than a unique random subdirectory,
+        #  to ensure that the caching for fetch_files_from_pr works across calls for the same PR)
+        tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % from_pr)
+        pr_paths = fetch_files_from_pr(pr=from_pr, path=tmpdir)
+
+        # assume that files need to be copied to current working directory for now
+        target_path = os.getcwd()
+
+        if orig_paths:
+            last_path = orig_paths[-1]
+
+            # check files touched by PR and see if the target directory for --copy-ec
+            # corresponds to the name of one of these files;
+            # if so we should copy the specified file(s) to the current working directory,
+            # since interpreting the last argument as target location is very unlikely to be correct in this case
+            pr_filenames = [os.path.basename(p) for p in pr_paths]
+            if last_path in pr_filenames:
+                paths = orig_paths[:]
+            else:
+                target_path = last_path
+                # exclude last argument that is used as target location
+                paths = orig_paths[:-1]
+
+        # if list of files to copy is empty at this point,
+        # we simply copy *all* files touched by the PR
+        if not paths:
+            paths = pr_paths
+
+        # replace path for files touched by PR (no need to worry about others)
+        for idx, path in enumerate(paths):
+            filename = os.path.basename(path)
+            pr_matches = [x for x in pr_paths if os.path.basename(x) == filename]
+            if len(pr_matches) == 1:
+                paths[idx] = pr_matches[0]
+            elif pr_matches:
+                raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches)
+
+    return paths, target_path