From 9413335872b5c1c10caaa1d25e94627bc2321cf3 Mon Sep 17 00:00:00 2001 From: Caylo Date: Tue, 26 Jul 2016 11:46:54 +0200 Subject: [PATCH 001/344] WIP --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/github.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dbd2acd512..b7536285a1 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -668,7 +668,7 @@ def obtain_file(self, filename, extension=False, urls=None): raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) - # + # a # GETTER/SETTER UTILITY FUNCTIONS # @property diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index a599b77a2b..1669c57821 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -81,6 +81,7 @@ GITHUB_API_URL = 'https://api.github.com' GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'hpcugent' +GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks' GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs' GITHUB_FILE_TYPE = u'file' GITHUB_MAX_PER_PAGE = 100 @@ -584,9 +585,11 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') git_repo = init_repo(git_working_dir, pr_target_repo) - if pr_target_repo != GITHUB_EASYCONFIGS_REPO: + if pr_target_repo != GITHUB_EASYCONFIGS_REPO and pr_target_repo != GITHUB_EASYBLOCKS_REPO: raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo) + easyblocks = pr_target_repo == GITHUB_EASYBLOCKS_REPO + if start_branch is None: start_branch = build_option('pr_target_branch') @@ -596,7 +599,10 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco _log.debug("git status: %s", git_repo.git.status()) # copy files to right place - file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) + if easyblocks: + copy_easyblocks(paths, os.path.join(git_working_dir, pr_target_repo)) + else: + file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) # checkout target branch if pr_branch is None: @@ -659,6 +665,12 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco return file_info, git_repo, pr_branch, diff_stat +def copy_easyblocks(paths, targetdir): + # circular dependencies ugh + from easybuild.framework.easyblock import EasyBlock + print "new pr for easyblock" + + @only_if_module_is_available('git', pkgname='GitPython') def new_pr(paths, title=None, descr=None, commit_msg=None): """Open new pull request using specified files.""" @@ -1059,3 +1071,4 @@ def validate_github_token(token, github_user): _log.info("GitHub token can be used for authenticated GitHub access, validation passed") return sanity_check and token_test + From 874b53a4df018352307fc2258eeb0934fe7362b5 Mon Sep 17 00:00:00 2001 From: Caylo Date: Tue, 26 Jul 2016 15:53:19 +0200 Subject: [PATCH 002/344] --new-pr for easyblocks repo --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/github.py | 106 +++++++++++++++++++++++-------- 2 files changed, 82 insertions(+), 26 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b7536285a1..dbd2acd512 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -668,7 +668,7 @@ def obtain_file(self, filename, extension=False, urls=None): raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) - # a + # # GETTER/SETTER UTILITY FUNCTIONS # @property diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 1669c57821..709d39f592 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -31,6 +31,8 @@ """ import base64 import getpass +import imp +import inspect import os import random import re @@ -47,8 +49,8 @@ from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option -from easybuild.tools.filetools import det_patched_files, download_file, extract_file, mkdir, read_file -from easybuild.tools.filetools import which, write_file +from easybuild.tools.filetools import det_patched_files, decode_class_name, download_file, extract_file, mkdir +from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.systemtools import UNKNOWN, get_tool_version from easybuild.tools.utilities import only_if_module_is_available @@ -77,6 +79,8 @@ _log.warning("Failed to import 'git' Python module: %s", err) +EB_PREFIX = 'EB_' +GENERIC_EB = 'generic' GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' GITHUB_DIR_TYPE = u'dir' @@ -91,6 +95,7 @@ HTTP_STATUS_OK = 200 HTTP_STATUS_CREATED = 201 KEYRING_GITHUB_TOKEN = 'github_token' +PYTHON_EXTENSION = 'py' URL_SEPARATOR = '/' @@ -600,13 +605,14 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco # copy files to right place if easyblocks: - copy_easyblocks(paths, os.path.join(git_working_dir, pr_target_repo)) + name_version, file_info = copy_easyblocks(paths, os.path.join(git_working_dir, pr_target_repo)) + else: file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) + name_version = file_info['ecs'][0].name + string.translate(file_info['ecs'][0].version, None, '-.') # checkout target branch if pr_branch is None: - name_version = file_info['ecs'][0].name + string.translate(file_info['ecs'][0].version, None, '-.') pr_branch = '%s_new_pr_%s' % (time.strftime("%Y%m%d%H%M%S"), name_version) # create branch to commit to and push; @@ -665,10 +671,49 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco return file_info, git_repo, pr_branch, diff_stat -def copy_easyblocks(paths, targetdir): - # circular dependencies ugh - from easybuild.framework.easyblock import EasyBlock - print "new pr for easyblock" +def copy_easyblocks(paths, target_dir): + file_info = { + 'paths_in_repo': [], + 'new': [], + 'ebs' : [], + } + + subdir = os.path.join('easybuild', 'easyblocks') + if os.path.exists(os.path.join(target_dir, subdir)): + for path in paths: + fn = os.path.basename(path).split('.')[0] + + mod = imp.load_source(fn, path) + clsmembers = inspect.getmembers(mod, inspect.isclass) + classnames = [cl[1].__name__ for cl in clsmembers if cl[1].__module__ == mod.__name__] + + if len(classnames) > 1: + raise EasyBuildError("Invalid EB file") + + cn = classnames[0] + eb_name = decode_class_name(cn).lower() # TODO not fully right yet. - to _ (and others??) + if cn.startswith(EB_PREFIX): + # regular eb file + letter = fn.lower()[0] + target_path = os.path.join(subdir, letter, "%s.%s" % (eb_name, PYTHON_EXTENSION)) + else: + # generic + target_path = os.path.join(subdir, GENERIC_EB, "%s.%s" % (eb_name.lower(), PYTHON_EXTENSION)) + + full_target_path = os.path.join(target_dir, target_path) + file_info['paths_in_repo'].append(full_target_path) + file_info['ebs'].append(eb_name) + try: + file_info['new'].append(not os.path.exists(full_target_path)) + + mkdir(os.path.dirname(full_target_path), parents=True) + shutil.copy2(path, full_target_path) + _log.info("%s copied to %s", path, full_target_path) + + except OSError as err: + raise EasyBuildError("Failed to copy %s to %s: %s", path, target_path, err) + + return eb_name, file_info @only_if_module_is_available('git', pkgname='GitPython') @@ -698,24 +743,36 @@ def new_pr(paths, title=None, descr=None, commit_msg=None): commit_msg=commit_msg) # only use most common toolchain(s) in toolchain label of PR title - toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] - toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) - toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]]) - - # only use most common module class(es) in moduleclass label of PR title - classes = [ec['moduleclass'] for ec in file_info['ecs']] - classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) - class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) - if title is None: - # mention software name/version in PR title (only first 3) - names_and_versions = ["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']] - if len(names_and_versions) <= 3: - main_title = ', '.join(names_and_versions) - else: - main_title = ', '.join(names_and_versions[:3] + ['...']) + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + + toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] + toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) + toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]]) + + # only use most common module class(es) in moduleclass label of PR title + classes = [ec['moduleclass'] for ec in file_info['ecs']] + classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) + class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) + + if title is None: + # mention software name/version in PR title (only first 3) + names_and_versions = ["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']] + if len(names_and_versions) <= 3: + main_title = ', '.join(names_and_versions) + else: + main_title = ', '.join(names_and_versions[:3] + ['...']) + + title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) + + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: + names = file_info['ebs'] + if len(names) <= 3: + main_title = ', '.join(names) + else: + main_title = ', '.join(names_and_versions[:3] + ['...']) + title = "EasyBlock for %s" % main_title - title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) full_descr = "(created using `eb --new-pr`)\n" if descr is not None: @@ -1071,4 +1128,3 @@ def validate_github_token(token, github_user): _log.info("GitHub token can be used for authenticated GitHub access, validation passed") return sanity_check and token_test - From a484f9ed8961646ec5017007b34503d2dd928977 Mon Sep 17 00:00:00 2001 From: Caylo Date: Wed, 27 Jul 2016 10:07:33 +0200 Subject: [PATCH 003/344] small fix --- easybuild/tools/github.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 709d39f592..170ea78ba4 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -605,7 +605,8 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco # copy files to right place if easyblocks: - name_version, file_info = copy_easyblocks(paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = copy_easyblocks(paths, os.path.join(git_working_dir, pr_target_repo)) + name_version = file_info['ebs'][0] else: file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) @@ -713,7 +714,7 @@ def copy_easyblocks(paths, target_dir): except OSError as err: raise EasyBuildError("Failed to copy %s to %s: %s", path, target_path, err) - return eb_name, file_info + return file_info @only_if_module_is_available('git', pkgname='GitPython') @@ -755,15 +756,14 @@ def new_pr(paths, title=None, descr=None, commit_msg=None): classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) - if title is None: - # mention software name/version in PR title (only first 3) - names_and_versions = ["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']] - if len(names_and_versions) <= 3: - main_title = ', '.join(names_and_versions) - else: - main_title = ', '.join(names_and_versions[:3] + ['...']) + # mention software name/version in PR title (only first 3) + names_and_versions = ["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']] + if len(names_and_versions) <= 3: + main_title = ', '.join(names_and_versions) + else: + main_title = ', '.join(names_and_versions[:3] + ['...']) - title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) + title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: names = file_info['ebs'] From b889c8ca782ae7ad8b58216e9f7ae0b948a8c4cf Mon Sep 17 00:00:00 2001 From: Caylo Date: Tue, 26 Jul 2016 13:08:11 +0200 Subject: [PATCH 004/344] --new-pr for framework repo --- easybuild/tools/github.py | 98 ++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 18 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index a599b77a2b..0d361086d6 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -82,6 +82,7 @@ GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'hpcugent' GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs' +GITHUB_FRAMEWORK_REPO = 'easybuild-framework' GITHUB_FILE_TYPE = u'file' GITHUB_MAX_PER_PAGE = 100 GITHUB_MERGEABLE_STATE_CLEAN = 'clean' @@ -584,9 +585,11 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') git_repo = init_repo(git_working_dir, pr_target_repo) - if pr_target_repo != GITHUB_EASYCONFIGS_REPO: + if not pr_target_repo in [GITHUB_EASYCONFIGS_REPO, GITHUB_FRAMEWORK_REPO,]: raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo) + framework = pr_target_repo == GITHUB_FRAMEWORK_REPO + if start_branch is None: start_branch = build_option('pr_target_branch') @@ -596,11 +599,16 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco _log.debug("git status: %s", git_repo.git.status()) # copy files to right place - file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) + if framework: + file_info = copy_framework_files(paths, os.path.join(git_working_dir, pr_target_repo)) + name_version = file_info['files'][0] + + else: + file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) + name_version = file_info['ecs'][0].name + string.translate(file_info['ecs'][0].version, None, '-.') # checkout target branch if pr_branch is None: - name_version = file_info['ecs'][0].name + string.translate(file_info['ecs'][0].version, None, '-.') pr_branch = '%s_new_pr_%s' % (time.strftime("%Y%m%d%H%M%S"), name_version) # create branch to commit to and push; @@ -659,6 +667,50 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco return file_info, git_repo, pr_branch, diff_stat +def copy_framework_files(paths, target_dir): + file_info = { + 'paths_in_repo': [], + 'new': [], + 'files' : [], + } + + + dirs = [x[0] for x in os.walk(target_dir)] + paths = [os.path.abspath(path) for path in paths] + + target_path = None + for path in paths: + fn = os.path.basename(path) + dirnames = os.path.dirname(path).split(os.path.sep) + + if 'easybuild-framework' in dirnames: + ind = dirnames.index('easybuild-framework') + 1 + parent_dir = os.path.join(*dirnames[ind:]) + + if os.path.exists(os.path.join(target_dir, parent_dir)): + target_path = os.path.join(target_dir, parent_dir) + + if target_path is None: + raise EasyBuildError("Couldn't find parent folder of updated file: %s" % path) + + full_target_path = os.path.join(target_path, os.path.basename(path)) + + file_info['paths_in_repo'].append(full_target_path) + file_info['files'].append(os.path.basename(path).split('.')[0]) + + try: + file_info['new'].append(not os.path.exists(full_target_path)) + + mkdir(os.path.dirname(full_target_path), parents=True) + shutil.copy2(path, full_target_path) + _log.info("%s copied to %s", path, full_target_path) + + except OSError as err: + raise EasyBuildError("Failed to copy %s to %s: %s", path, full_target_path, err) + + return file_info + + @only_if_module_is_available('git', pkgname='GitPython') def new_pr(paths, title=None, descr=None, commit_msg=None): """Open new pull request using specified files.""" @@ -686,24 +738,34 @@ def new_pr(paths, title=None, descr=None, commit_msg=None): commit_msg=commit_msg) # only use most common toolchain(s) in toolchain label of PR title - toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] - toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) - toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]]) + if title is None: + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: - # only use most common module class(es) in moduleclass label of PR title - classes = [ec['moduleclass'] for ec in file_info['ecs']] - classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) - class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) + toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] + toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) + toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]]) - if title is None: - # mention software name/version in PR title (only first 3) - names_and_versions = ["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']] - if len(names_and_versions) <= 3: - main_title = ', '.join(names_and_versions) - else: - main_title = ', '.join(names_and_versions[:3] + ['...']) + # only use most common module class(es) in moduleclass label of PR title + classes = [ec['moduleclass'] for ec in file_info['ecs']] + classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) + class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) - title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) + # mention software name/version in PR title (only first 3) + names_and_versions = ["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']] + if len(names_and_versions) <= 3: + main_title = ', '.join(names_and_versions) + else: + main_title = ', '.join(names_and_versions[:3] + ['...']) + + title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) + + elif pr_target_repo == GITHUB_FRAMEWORK_REPO: + names = file_info['files'] + if len(names) <= 3: + main_title = ', '.join(names) + else: + main_title = ', '.join(names_and_versions[:3] + ['...']) + title = "Changes to %s" % main_title full_descr = "(created using `eb --new-pr`)\n" if descr is not None: From 6c79086dd61435b24a2618978a0dcc840c2ab8f2 Mon Sep 17 00:00:00 2001 From: Caylo Date: Wed, 10 Aug 2016 16:45:39 +0200 Subject: [PATCH 005/344] fix remark --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 170ea78ba4..c78718bde1 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -49,6 +49,7 @@ from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option +from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX from easybuild.tools.filetools import det_patched_files, decode_class_name, download_file, extract_file, mkdir from easybuild.tools.filetools import read_file, which, write_file from easybuild.tools.systemtools import UNKNOWN, get_tool_version @@ -79,7 +80,6 @@ _log.warning("Failed to import 'git' Python module: %s", err) -EB_PREFIX = 'EB_' GENERIC_EB = 'generic' GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' @@ -693,7 +693,7 @@ def copy_easyblocks(paths, target_dir): cn = classnames[0] eb_name = decode_class_name(cn).lower() # TODO not fully right yet. - to _ (and others??) - if cn.startswith(EB_PREFIX): + if cn.startswith(EASYBLOCK_CLASS_PREFIX): # regular eb file letter = fn.lower()[0] target_path = os.path.join(subdir, letter, "%s.%s" % (eb_name, PYTHON_EXTENSION)) From a046c9e8b50914c2f30c1f45441c08755d171cce Mon Sep 17 00:00:00 2001 From: Caylo Date: Fri, 12 Aug 2016 12:49:57 +0200 Subject: [PATCH 006/344] fix variable name --- easybuild/tools/github.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ee03ecd537..941e50f12a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -645,16 +645,16 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco # copy easyconfig files to right place target_dir = os.path.join(git_working_dir, pr_target_repo) - print_msg("copying easyconfigs to %s..." % target_dir) + print_msg("copying files to %s..." % target_dir) if easyblocks: - file_info = copy_easyblocks(paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = copy_easyblocks(ec_paths, os.path.join(git_working_dir, pr_target_repo)) elif framework: - file_info = copy_framework_files(paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = copy_framework_files(ec_paths, os.path.join(git_working_dir, pr_target_repo)) else: - file_info = copy_easyconfigs(paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = copy_easyconfigs(ec_paths, os.path.join(git_working_dir, pr_target_repo)) # figure out to which software name patches relate, and copy them to the right place if patch_paths: From 36bfbfe60e07b3347e705577dbba827aab63d6e3 Mon Sep 17 00:00:00 2001 From: Caylo Date: Fri, 12 Aug 2016 12:57:35 +0200 Subject: [PATCH 007/344] copy functions --- easybuild/tools/github.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 941e50f12a..f0caf2d197 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -628,9 +628,6 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco if pr_target_repo not in [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO,]: raise EasyBuildError("Don't know how to create/update a pull request to the %s repository", pr_target_repo) - easyblocks = pr_target_repo == GITHUB_EASYBLOCKS_REPO - framework = pr_target_repo == GITHUB_FRAMEWORK_REPO - if start_branch is None: start_branch = build_option('pr_target_branch') @@ -646,15 +643,7 @@ def _easyconfigs_pr_common(paths, start_branch=None, pr_branch=None, target_acco # copy easyconfig files to right place target_dir = os.path.join(git_working_dir, pr_target_repo) print_msg("copying files to %s..." % target_dir) - - if easyblocks: - file_info = copy_easyblocks(ec_paths, os.path.join(git_working_dir, pr_target_repo)) - - elif framework: - file_info = copy_framework_files(ec_paths, os.path.join(git_working_dir, pr_target_repo)) - - else: - file_info = copy_easyconfigs(ec_paths, os.path.join(git_working_dir, pr_target_repo)) + file_info = COPY_FUNCTIONS[pr_target_repo](ec_paths, os.path.join(git_working_dir, pr_target_repo)) # figure out to which software name patches relate, and copy them to the right place if patch_paths: @@ -1319,3 +1308,10 @@ def validate_github_token(token, github_user): _log.info("GitHub token can be used for authenticated GitHub access, validation passed") return sanity_check and token_test + +# copy fucntions for --new-pr +COPY_FUNCTIONS = { + GITHUB_EASYCONFIGS_REPO: copy_easyconfigs, + GITHUB_EASYBLOCKS_REPO: copy_easyblocks, + GITHUB_FRAMEWORK_REPO: copy_framework_files, +} From d56f4fa696d6b6e6840bb473f950ddcc6533dd28 Mon Sep 17 00:00:00 2001 From: victorusu Date: Mon, 10 Jul 2017 16:00:12 +0200 Subject: [PATCH 008/344] Adding the lowercase module name scheme We got a request from one of our customers to enable the module names always as lowercases. This is the implementation of this feature. We would like to contribute this feature back. --- .../lowercase_module_naming_scheme.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py diff --git a/easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py b/easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py new file mode 100644 index 0000000000..8bf090272d --- /dev/null +++ b/easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py @@ -0,0 +1,62 @@ +## +# Copyright 2013-2017 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of (default) EasyBuild module naming scheme. +:author: Kenneth Hoste (Ghent University) +:author: Guilherme Peretti-Pezzi (CSCS) +""" + +import os +import re + +from easybuild.tools.module_naming_scheme import ModuleNamingScheme +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version + +class LowercaseModuleNamingScheme(ModuleNamingScheme): + """Class implementing a lowercase module naming scheme.""" + + REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain'] + + def det_full_module_name(self, ec): + """ + Determine full module name from given easyconfig, according to the EasyBuild module naming scheme. + :param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.) + :return: string with full module name /, e.g.: 'gzip/1.5-goolf-1.4.10' + """ + return os.path.join(ec['name'], det_full_ec_version(ec)).lower() + + def is_short_modname_for(self, short_modname, name): + """ + Determine whether the specified (short) module name is a module for software with the specified name. + Default implementation checks via a strict regex pattern, and assumes short module names are of the form: + /[-] + """ + modname_regex = re.compile('^%s(/\S+)?$' % re.escape(name.lower())) + res = bool(modname_regex.match(short_modname)) + + self.log.debug("Checking whether '%s' is a module name for software with name '%s' via regex %s: %s", + short_modname, name, modname_regex.pattern, res) + + return res From 882b3ecb50f58b6c5e417703bd6b3309036593ec Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 29 Sep 2018 16:17:35 +0200 Subject: [PATCH 009/344] Add ability to upgrade dependencies based on available easyconfigs --- easybuild/framework/easyconfig/easyconfig.py | 36 +++++++++++----- easybuild/framework/easyconfig/tweak.py | 44 +++++++++++++++++++- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 47e761ce18..a7a7223404 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1428,9 +1428,10 @@ def create_paths(path, name, version): return ['%s.eb' % os.path.join(path, *cand_path) for cand_path in cand_paths] -def robot_find_easyconfig(name, version): +def robot_find_easyconfig(name, version, all_paths=False): """ - Find an easyconfig for module in path, returns (absolute) path to easyconfig file (or None, if none is found). + Find an easyconfig (or easyconfigs) in available paths, returns (absolute) path to easyconfig file or list of all + matches (since version may contain a glob value), returns None if none is found. """ key = (name, version) if key in _easyconfig_files_cache: @@ -1447,18 +1448,33 @@ def robot_find_easyconfig(name, version): if build_option('consider_archived_easyconfigs'): paths = paths + [os.path.join(p, EASYCONFIGS_ARCHIVE_DIR) for p in paths] + res = None + candidate_paths = [] for path in paths: easyconfigs_paths = create_paths(path, name, version) - for easyconfig_path in easyconfigs_paths: - _log.debug("Checking easyconfig path %s" % easyconfig_path) - if os.path.isfile(easyconfig_path): - _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) - _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) - res = _easyconfig_files_cache[key] + if all_paths: + candidate_paths += easyconfigs_paths + else: + for easyconfig_path in easyconfigs_paths: + log.debug("Checking easyconfig path %s" % easyconfig_path) + if os.path.isfile(easyconfig_path): + _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) + _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) + res = _easyconfig_files_cache[key] + break + if res: break - if res: - break + if all_paths: + full_cand_paths = [] + for easyconfig_path in candidate_paths: + log.debug("Checking easyconfig path %s" % easyconfig_path) + if os.path.isfile(easyconfig_path): + _log.debug("Found easyconfig file for name %s at %s" % (name, easyconfig_path)) + full_cand_paths += [os.path.abspath(easyconfig_path)] + _easyconfig_files_cache[key] = full_cand_paths + if _easyconfig_files_cache[key]: + res = _easyconfig_files_cache[key] return res diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 0beda4b3f7..6c24367ad2 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -795,7 +795,7 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): return tc_mapping -def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None): +def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_dep_versions=False): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out @@ -833,6 +833,48 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # set module names orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) + elif update_dep_versions: + # Search for available updates for this dependency: + # First get all candidate paths for this (include search through subtoolchains) + toolchain_hierarchy = get_toolchain_hierarchy(orig_dep['toolchain']) + potential_version_matches = [] + for toolchain in toolchain_hierarchy: + candidate_ver= '*' + # determine main install version based on toolchain + if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + # prepend/append version prefix/suffix + ecver = ''.join([x for x in [parsed_ec.get('versionprefix', ''), candidate_ver, toolchain_suffix, + ec.get('versionsuffix', '')] if x]) + cand_paths = robot_find_easyconfig(orig_dep['name'], ecver, all_paths=True) + # Add them to the possibilities + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + try: + version = re.search('%s(.+?)%s' % (orig_dep['name'] + '-', toolchain_suffix), text).group(1) + except AttributeError: + raise EasyBuildError("Somethings wrong, could not extract version from %s", filename) + potential_version_matches += [{'version': version, 'path': path, 'toolchain': toolchain}] + _log.info("Found possible dependency upgrades: %s\n",potential_version_matches) + + # Compare this version to the original versions and replace if appropriate (upgrades only) + highest_version = orig_dep['version'] + highest_version_paths = [] + for candidate in potential_version_matches: + if LooseVersion(candidate['version']) >= LooseVersion(highest_version): + highest_version = candidate['version'] + highest_version_paths += [candidate_ver['path']] + if highest_version_paths: + _log.info("Increasing version to %s for dependency %s.", highest_version, orig_dep['name']) + _log.info("Depending on your configuration, this will be resolved with one of the following " + "easyconfigs: %s", highest_version_paths) + orig_dep['version'] = highest_version + orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) + orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) + + # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) From 8509d77a7c4723204e02ea12b35d9f74ec7ef6b9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 29 Sep 2018 16:23:08 +0200 Subject: [PATCH 010/344] MAke hound happy --- easybuild/framework/easyconfig/tweak.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 6c24367ad2..c206c339da 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -839,7 +839,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= toolchain_hierarchy = get_toolchain_hierarchy(orig_dep['toolchain']) potential_version_matches = [] for toolchain in toolchain_hierarchy: - candidate_ver= '*' + candidate_ver = '*' # determine main install version based on toolchain if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) @@ -857,7 +857,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s", filename) potential_version_matches += [{'version': version, 'path': path, 'toolchain': toolchain}] - _log.info("Found possible dependency upgrades: %s\n",potential_version_matches) + _log.info("Found possible dependency upgrades: %s\n", potential_version_matches) # Compare this version to the original versions and replace if appropriate (upgrades only) highest_version = orig_dep['version'] From e56671b9e65e9cb639485f5c651a2863578660bf Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Sat, 29 Sep 2018 17:08:14 +0200 Subject: [PATCH 011/344] Fix typo --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index a7a7223404..f363a3946c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1457,7 +1457,7 @@ def robot_find_easyconfig(name, version, all_paths=False): candidate_paths += easyconfigs_paths else: for easyconfig_path in easyconfigs_paths: - log.debug("Checking easyconfig path %s" % easyconfig_path) + _log.debug("Checking easyconfig path %s" % easyconfig_path) if os.path.isfile(easyconfig_path): _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) From ca8911c7825606081d9a65a7e252d9bccea415bb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 09:15:52 +0200 Subject: [PATCH 012/344] Change to use search_easyconfigs --- easybuild/framework/easyconfig/easyconfig.py | 8 +++-- easybuild/framework/easyconfig/format/yeb.py | 2 +- easybuild/framework/easyconfig/tweak.py | 35 ++++++++++++-------- easybuild/tools/robot.py | 11 ++++-- test/framework/options.py | 6 ++++ 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f363a3946c..fcdef2d2f1 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1454,7 +1454,7 @@ def robot_find_easyconfig(name, version, all_paths=False): for path in paths: easyconfigs_paths = create_paths(path, name, version) if all_paths: - candidate_paths += easyconfigs_paths + candidate_paths.append(easyconfigs_paths) else: for easyconfig_path in easyconfigs_paths: _log.debug("Checking easyconfig path %s" % easyconfig_path) @@ -1465,13 +1465,15 @@ def robot_find_easyconfig(name, version, all_paths=False): break if res: break + if all_paths: full_cand_paths = [] + print candidate_paths for easyconfig_path in candidate_paths: - log.debug("Checking easyconfig path %s" % easyconfig_path) + _log.debug("Checking easyconfig path %s" % easyconfig_path) if os.path.isfile(easyconfig_path): _log.debug("Found easyconfig file for name %s at %s" % (name, easyconfig_path)) - full_cand_paths += [os.path.abspath(easyconfig_path)] + full_cand_paths.append(os.path.abspath(easyconfig_path)) _easyconfig_files_cache[key] = full_cand_paths if _easyconfig_files_cache[key]: res = _easyconfig_files_cache[key] diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 54f987ccad..4189052019 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -129,7 +129,7 @@ def extract_comments(self, txt): def is_yeb_format(filename, rawcontent): """ - Determine whether easyconfig is in .yeb format. + Determine whether easyconfig is in .dep['name'] format. If filename is None, rawcontent will be used to check the format. """ isyeb = False diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c206c339da..c344199730 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -46,13 +46,14 @@ from easybuild.framework.easyconfig.default import get_easyconfig_parameter_default from easybuild.framework.easyconfig.easyconfig import EasyConfig, create_paths, process_easyconfig from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS +from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import read_file, write_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version -from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig +from easybuild.tools.robot import resolve_dependencies, robot_find_easyconfig, search_easyconfigs from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES from easybuild.tools.utilities import quote_str @@ -841,40 +842,46 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= for toolchain in toolchain_hierarchy: candidate_ver = '*' # determine main install version based on toolchain - if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: + if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: + version_suffix = ''.join([x for x in [parsed_ec.get('versionsuffix', '')] if x]) + else: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + version_suffix = ''.join([x for x in [toolchain_suffix, + parsed_ec.get('versionsuffix', '')] if x]) # prepend/append version prefix/suffix - ecver = ''.join([x for x in [parsed_ec.get('versionprefix', ''), candidate_ver, toolchain_suffix, - ec.get('versionsuffix', '')] if x]) - cand_paths = robot_find_easyconfig(orig_dep['name'], ecver, all_paths=True) + ecver = ''.join([x for x in [parsed_ec.get('versionprefix', ''), + candidate_ver, version_suffix] if x]) + cand_paths = search_easyconfigs(dep['name'] + '-' + ecver, return_list=True) # Add them to the possibilities for path in cand_paths: # Get the version from the path filename = os.path.basename(path) # Find the version sandwiched between our known values try: - version = re.search('%s(.+?)%s' % (orig_dep['name'] + '-', toolchain_suffix), text).group(1) + version_prefix = ''.join([x for x in [dep['name'] + '-', + parsed_ec.get('versionprefix', '')] if x]) + if not version_suffix: + version_suffix = EB_FORMAT_EXTENSION + version = re.search('%s(.+?)%s' % (version_prefix, version_suffix), filename).group(1) except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s", filename) - potential_version_matches += [{'version': version, 'path': path, 'toolchain': toolchain}] - _log.info("Found possible dependency upgrades: %s\n", potential_version_matches) + potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) + _log.debug("Found possible dependency upgrades: %s\n", potential_version_matches) # Compare this version to the original versions and replace if appropriate (upgrades only) - highest_version = orig_dep['version'] - highest_version_paths = [] + highest_version = dep['version'] for candidate in potential_version_matches: if LooseVersion(candidate['version']) >= LooseVersion(highest_version): highest_version = candidate['version'] - highest_version_paths += [candidate_ver['path']] - if highest_version_paths: + if highest_version != dep['version']: _log.info("Increasing version to %s for dependency %s.", highest_version, orig_dep['name']) _log.info("Depending on your configuration, this will be resolved with one of the following " - "easyconfigs: %s", highest_version_paths) + "easyconfigs: %s", '\n'.join(cand['path'] for cand in potential_version_matches + if cand['version'] == highest_version)) orig_dep['version'] = highest_version orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) - # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 8e40328b60..1c46d6056f 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -379,7 +379,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): return ordered_ecs -def search_easyconfigs(query, short=False, filename_only=False, terse=False): +def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_list=False): """Search for easyconfigs, if a query is provided.""" search_path = build_option('robot_path') if not search_path: @@ -394,7 +394,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse, silent=True, filename_only=False) - # filter out archived easyconfigs, these are handled separately + # filter out archived easyconfigs, these are handled separately hits, archived_hits = [], [] for hit in _hits: if EASYCONFIGS_ARCHIVE_DIR in hit.split(os.path.sep): @@ -402,6 +402,13 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False): else: hits.append(hit) + # if requested return the matches as a list + if return_list: + if build_option('consider_archived_easyconfigs'): + return hits + archived_hits + else: + return hits + # check whether only filenames should be printed if filename_only: hits = [os.path.basename(hit) for hit in hits] diff --git a/test/framework/options.py b/test/framework/options.py index aed8e8b8b5..9f1da7928a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2340,6 +2340,12 @@ def test_minimal_toolchains(self): self.assertTrue(os.path.exists(robot_find_easyconfig('hwloc', '1.6.2-GCC-4.7.2'))) self.assertTrue(os.path.exists(robot_find_easyconfig('SQLite', '3.8.10.2-gompi-1.4.10'))) self.assertTrue(os.path.exists(robot_find_easyconfig('SQLite', '3.8.10.2-GCC-4.7.2'))) + # test when we include glob values that we get a list back + paths = robot_find_easyconfig('binutils', '*-GCCcore-4.9.3', all_paths=True) + print paths + self.assertTrue(paths) + for path in paths: + self.assertTrue(os.path.exists(path)) args = [ ec_file, From 2fdbf51ce1757d73fcc819fe39c67a680af1cc51 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 09:27:50 +0200 Subject: [PATCH 013/344] Revert unneeded modifications --- easybuild/framework/easyconfig/easyconfig.py | 36 +++++--------------- easybuild/framework/easyconfig/format/yeb.py | 2 +- test/framework/options.py | 6 ---- 3 files changed, 10 insertions(+), 34 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index fcdef2d2f1..47e761ce18 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1428,10 +1428,9 @@ def create_paths(path, name, version): return ['%s.eb' % os.path.join(path, *cand_path) for cand_path in cand_paths] -def robot_find_easyconfig(name, version, all_paths=False): +def robot_find_easyconfig(name, version): """ - Find an easyconfig (or easyconfigs) in available paths, returns (absolute) path to easyconfig file or list of all - matches (since version may contain a glob value), returns None if none is found. + Find an easyconfig for module in path, returns (absolute) path to easyconfig file (or None, if none is found). """ key = (name, version) if key in _easyconfig_files_cache: @@ -1448,35 +1447,18 @@ def robot_find_easyconfig(name, version, all_paths=False): if build_option('consider_archived_easyconfigs'): paths = paths + [os.path.join(p, EASYCONFIGS_ARCHIVE_DIR) for p in paths] - res = None - candidate_paths = [] for path in paths: easyconfigs_paths = create_paths(path, name, version) - if all_paths: - candidate_paths.append(easyconfigs_paths) - else: - for easyconfig_path in easyconfigs_paths: - _log.debug("Checking easyconfig path %s" % easyconfig_path) - if os.path.isfile(easyconfig_path): - _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) - _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) - res = _easyconfig_files_cache[key] - break - if res: - break - - if all_paths: - full_cand_paths = [] - print candidate_paths - for easyconfig_path in candidate_paths: + for easyconfig_path in easyconfigs_paths: _log.debug("Checking easyconfig path %s" % easyconfig_path) if os.path.isfile(easyconfig_path): - _log.debug("Found easyconfig file for name %s at %s" % (name, easyconfig_path)) - full_cand_paths.append(os.path.abspath(easyconfig_path)) - _easyconfig_files_cache[key] = full_cand_paths - if _easyconfig_files_cache[key]: - res = _easyconfig_files_cache[key] + _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) + _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) + res = _easyconfig_files_cache[key] + break + if res: + break return res diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 4189052019..54f987ccad 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -129,7 +129,7 @@ def extract_comments(self, txt): def is_yeb_format(filename, rawcontent): """ - Determine whether easyconfig is in .dep['name'] format. + Determine whether easyconfig is in .yeb format. If filename is None, rawcontent will be used to check the format. """ isyeb = False diff --git a/test/framework/options.py b/test/framework/options.py index 9f1da7928a..aed8e8b8b5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2340,12 +2340,6 @@ def test_minimal_toolchains(self): self.assertTrue(os.path.exists(robot_find_easyconfig('hwloc', '1.6.2-GCC-4.7.2'))) self.assertTrue(os.path.exists(robot_find_easyconfig('SQLite', '3.8.10.2-gompi-1.4.10'))) self.assertTrue(os.path.exists(robot_find_easyconfig('SQLite', '3.8.10.2-GCC-4.7.2'))) - # test when we include glob values that we get a list back - paths = robot_find_easyconfig('binutils', '*-GCCcore-4.9.3', all_paths=True) - print paths - self.assertTrue(paths) - for path in paths: - self.assertTrue(os.path.exists(path)) args = [ ec_file, From 5f7c2972bbdb6dbaec463599aa04566c1508ecdb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 09:41:30 +0200 Subject: [PATCH 014/344] make the option name more explicit --- easybuild/tools/robot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 1c46d6056f..a351138317 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -379,13 +379,14 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): return ordered_ecs -def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_list=False): +def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_robot_list=False): """Search for easyconfigs, if a query is provided.""" search_path = build_option('robot_path') if not search_path: search_path = [os.getcwd()] extra_search_paths = build_option('search_paths') - if extra_search_paths: + # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths + if extra_search_paths and not return_robot_list: search_path.extend(extra_search_paths) ignore_dirs = build_option('ignore_dirs') @@ -403,7 +404,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, ret hits.append(hit) # if requested return the matches as a list - if return_list: + if return_robot_list: if build_option('consider_archived_easyconfigs'): return hits + archived_hits else: From a913bd35c59058093db230e7977ed66ff03f7d70 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 09:42:57 +0200 Subject: [PATCH 015/344] Should have refactored! --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c344199730..2a846fc815 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -851,7 +851,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # prepend/append version prefix/suffix ecver = ''.join([x for x in [parsed_ec.get('versionprefix', ''), candidate_ver, version_suffix] if x]) - cand_paths = search_easyconfigs(dep['name'] + '-' + ecver, return_list=True) + cand_paths = search_easyconfigs(dep['name'] + '-' + ecver, return_robot_list=True) # Add them to the possibilities for path in cand_paths: # Get the version from the path From cd4dd1bb2eabb044093df4fe5c05d61081c868ff Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 10:35:32 +0200 Subject: [PATCH 016/344] Start writing a test --- test/framework/robot.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/framework/robot.py b/test/framework/robot.py index 368174284b..8cee0f6f1b 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -55,7 +55,7 @@ from easybuild.tools.github import fetch_github_token from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import invalidate_module_caches_for -from easybuild.tools.robot import check_conflicts, det_robot_path, resolve_dependencies +from easybuild.tools.robot import check_conflicts, det_robot_path, resolve_dependencies, search_easyconfigs from test.framework.utilities import find_full_path @@ -1250,6 +1250,15 @@ def test_robot_archived_easyconfigs(self): expected = os.path.join(test_ecs, '__archive__', 'i', 'ictce', 'ictce-3.2.2.u3.eb') self.assertTrue(os.path.samefile(res[0]['spec'], expected)) + def test_search_easyconfigs(self): + """Test search_easyconfigs function.""" + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'robot_path': [test_ecs], + }) + paths = search_easyconfigs('binutils-*-GCCcore-4.9.3', return_robot_list=True) + print paths + def suite(): """ returns all the testcases in this module """ From 8281e58b477ffb8df62cc4da0ae9c5e849a50bf4 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 10:51:26 +0200 Subject: [PATCH 017/344] Correct test --- easybuild/framework/easyconfig/tweak.py | 2 +- test/framework/robot.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 2a846fc815..226a5a627d 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -840,7 +840,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= toolchain_hierarchy = get_toolchain_hierarchy(orig_dep['toolchain']) potential_version_matches = [] for toolchain in toolchain_hierarchy: - candidate_ver = '*' + candidate_ver = '.*' # using regex for * # determine main install version based on toolchain if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: version_suffix = ''.join([x for x in [parsed_ec.get('versionsuffix', '')] if x]) diff --git a/test/framework/robot.py b/test/framework/robot.py index 8cee0f6f1b..853dc75f84 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1256,8 +1256,10 @@ def test_search_easyconfigs(self): init_config(build_options={ 'robot_path': [test_ecs], }) - paths = search_easyconfigs('binutils-*-GCCcore-4.9.3', return_robot_list=True) - print paths + paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', return_robot_list=True) + ref_paths = [os.path.join(test_ecs, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', + 'binutils-2.26-GCCcore-4.9.3.eb']] + self.assertEqual(paths, ref_paths) def suite(): From b51f87d8c43c81252f9cc8363e5bb9509083ab67 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 10:52:36 +0200 Subject: [PATCH 018/344] Appease hound --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 226a5a627d..d6ed7f3a9c 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -840,7 +840,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= toolchain_hierarchy = get_toolchain_hierarchy(orig_dep['toolchain']) potential_version_matches = [] for toolchain in toolchain_hierarchy: - candidate_ver = '.*' # using regex for * + candidate_ver = '.*' # using regex for * # determine main install version based on toolchain if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: version_suffix = ''.join([x for x in [parsed_ec.get('versionsuffix', '')] if x]) From 75fef2ae183897cdf65abdec236c5a31268e0706 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 10:59:41 +0200 Subject: [PATCH 019/344] Add proper docstring for search_easyconfig --- easybuild/tools/robot.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index a351138317..d6352c3ea1 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -380,7 +380,17 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_robot_list=False): - """Search for easyconfigs, if a query is provided.""" + """ + Search for easyconfigs, if a query is provided. + + :param query: regex query string + :param short: figure out common prefix of hits, use variable to factor it out + :param filename_only: only print filenames, not paths + :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing + :param return_robot_list: return the list rather than print it out + + :return: return a list of paths for the query iff return_robot_list else no return value + """ search_path = build_option('robot_path') if not search_path: search_path = [os.getcwd()] From b902f835098756231735e9e4a31032c9ee452a3b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 12:09:11 +0200 Subject: [PATCH 020/344] Add test for dependency upgrading --- .../h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb | 3 +++ test/framework/tweak.py | 23 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb index c445c7d4cb..6765b00412 100644 --- a/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb +++ b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb @@ -18,4 +18,7 @@ sources = [SOURCE_TAR_GZ] builddependencies = [('binutils', '2.26')] +# introduce fake dependency for testing dep upgrades +dependencies = [('gzip', '1.4')] + moduleclass = 'system' diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 427fdf2484..93f77cee25 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -281,7 +281,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), - 'robot_path': test_easyconfigs, + 'robot_path': [test_easyconfigs], }) get_toolchain_hierarchy.clear() @@ -301,6 +301,27 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): self.assertTrue(key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key]) + # Now test the case where we try to update the dependencies + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': [test_easyconfigs], + }) + get_toolchain_hierarchy.clear() + tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping, update_dep_versions=True) + tweaked_ec = process_easyconfig(tweaked_spec)[0] + tweaked_dict = tweaked_ec['ec'].asdict() + # First check the mapped toolchain + key, value = 'toolchain', iccifort_binutils_tc + self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + # Also check that binutils has been mapped + for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): + self.assertTrue(key in tweaked_dict['builddependencies'][0] and + value == tweaked_dict['builddependencies'][0][key]) + # Also check that the gzip dependency was upgraded + for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): + self.assertTrue(key in tweaked_dict['dependencies'][0] and + value == tweaked_dict['dependencies'][0][key]) + def suite(): """ return all the tests in this file """ return TestLoaderFiltered().loadTestsFromTestCase(TweakTest, sys.argv[1:]) From 601010c04a91d6d2d19899f4ad6f3669a5c22947 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 12:35:14 +0200 Subject: [PATCH 021/344] Add build option for dependency upgrading --- easybuild/framework/easyconfig/tweak.py | 17 +++++++++++------ easybuild/tools/options.py | 3 +++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index d6ed7f3a9c..a63e0c4f7e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -89,12 +89,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): toolchains) # Toolchain is unique, let's store it source_toolchain = easyconfigs[-1]['ec']['toolchain'] - modifying_toolchains = False + modifying_toolchains_or_deps = False target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False + if build_specs['upgrade_dependencies']: + upgrade_dependencies = build_specs['upgrade_dependencies'] + else: + upgrade_dependencies = False - if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs: + if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs or upgrade_dependencies: keys = build_specs.keys() # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not @@ -108,7 +112,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # so build specifications should be applied to whole dependency graph; # obtain full dependency graph for specified easyconfigs; # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) - modifying_toolchains = True + modifying_toolchains_or_deps = True if 'toolchain_name' in keys: target_toolchain['name'] = build_specs['toolchain_name'] @@ -162,10 +166,11 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): new_ec_file = None verification_build_specs = copy.copy(build_specs) if orig_ec['spec'] in listed_ec_paths: - if modifying_toolchains: + if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - tweaked_ecs_path) + tweaked_ecs_path, + update_dep_versions=upgrade_dependencies) # Need to update the toolchain in the build_specs to match the toolchain mapping keys = verification_build_specs.keys() if 'toolchain_name' in keys: @@ -182,7 +187,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): tweaked_easyconfigs.extend(new_ecs) else: # Place all tweaked dependency easyconfigs in the directory appended to the robot path - if modifying_toolchains: + if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, targetdir=tweaked_ecs_deps_path) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 85cdab8a32..b806fcc98c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -307,6 +307,9 @@ def software_options(self): None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), + 'upgrade-dependencies': ("Try to upgrade versions of the dependencies of an easyconfig based on what is " + "available in the robot path", + None, 'store_true', False), }) longopts = opts.keys() From ca190f20edf2c0f675352f291c2ba5ed9a2d8da0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 12:40:24 +0200 Subject: [PATCH 022/344] Add new files to test dependency upgrades --- .../g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb | 40 +++++++++++++++++++ ...-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb | 40 +++++++++++++++++++ ...-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb | 40 +++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb create mode 100644 test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb create mode 100644 test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb new file mode 100644 index 0000000000..406f4e3342 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.9.3-2.26.eb @@ -0,0 +1,40 @@ +## +# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild +# +# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC +# Authors:: Thekla Loizou +# License:: MIT/GPL +# $Id$ +# +# This work implements a part of the HPCBIOS project and is a component of the policy: +# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html +## +easyblock = 'ConfigureMake' + +name = 'gzip' +version = '1.4' + +homepage = "http://www.gzip.org/" +description = "gzip (GNU zip) is a popular data compression program as a replacement for compress" + +# test toolchain specification +toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} + +# source tarball filename +sources = ['%(name)s-%(version)s.tar.gz'] + +# download location for source files +source_urls = [GNU_SOURCE] + +# make sure the gzip and gunzip binaries are available after installation +sanity_check_paths = { + 'files': ["bin/gunzip", "bin/gzip"], + 'dirs': [], +} + +# run 'gzip -h' and 'gzip --version' after installation +sanity_check_commands = [True, ('gzip', '--version')] + +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb new file mode 100644 index 0000000000..d0f441a526 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb @@ -0,0 +1,40 @@ +## +# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild +# +# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC +# Authors:: Thekla Loizou +# License:: MIT/GPL +# $Id$ +# +# This work implements a part of the HPCBIOS project and is a component of the policy: +# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html +## +easyblock = 'ConfigureMake' + +name = 'gzip' +version = '1.4' + +homepage = "http://www.gzip.org/" +description = "gzip (GNU zip) is a popular data compression program as a replacement for compress" + +# test toolchain specification +toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + +# source tarball filename +sources = ['%(name)s-%(version)s.tar.gz'] + +# download location for source files +source_urls = [GNU_SOURCE] + +# make sure the gzip and gunzip binaries are available after installation +sanity_check_paths = { + 'files': ["bin/gunzip", "bin/gzip"], + 'dirs': [], +} + +# run 'gzip -h' and 'gzip --version' after installation +sanity_check_commands = [True, ('gzip', '--version')] + +software_license = GPLv3 + +moduleclass = 'tools' diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb new file mode 100644 index 0000000000..d0f441a526 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb @@ -0,0 +1,40 @@ +## +# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild +# +# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC +# Authors:: Thekla Loizou +# License:: MIT/GPL +# $Id$ +# +# This work implements a part of the HPCBIOS project and is a component of the policy: +# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html +## +easyblock = 'ConfigureMake' + +name = 'gzip' +version = '1.4' + +homepage = "http://www.gzip.org/" +description = "gzip (GNU zip) is a popular data compression program as a replacement for compress" + +# test toolchain specification +toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + +# source tarball filename +sources = ['%(name)s-%(version)s.tar.gz'] + +# download location for source files +source_urls = [GNU_SOURCE] + +# make sure the gzip and gunzip binaries are available after installation +sanity_check_paths = { + 'files': ["bin/gunzip", "bin/gzip"], + 'dirs': [], +} + +# run 'gzip -h' and 'gzip --version' after installation +sanity_check_commands = [True, ('gzip', '--version')] + +software_license = GPLv3 + +moduleclass = 'tools' From dbd450dd792e82d7e3e17bbf958fe522626b15ad Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 13:00:29 +0200 Subject: [PATCH 023/344] Correctly add build option --- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/tools/options.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a63e0c4f7e..a4ae4e3be1 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -103,7 +103,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not # supported - if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain'] for key in keys): + if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'upgrade_dependencies'] for key in keys): print_warning("Combining --try-toolchain* with other build options is not fully supported: using regex") revert_to_regex = True diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b806fcc98c..4d3e744368 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1175,6 +1175,7 @@ def process_software_build_specs(options): 'version': options.try_software_version, 'toolchain_name': options.try_toolchain_name, 'toolchain_version': options.try_toolchain_version, + 'upgrade_dependencies': options.try_upgrade_dependencies } # process easy options From 366989da1241d43aebe2726ed1f1dff5826ce6df Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 13:10:44 +0200 Subject: [PATCH 024/344] Correct logic for build option --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a4ae4e3be1..79ca1f818d 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -93,7 +93,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False - if build_specs['upgrade_dependencies']: + if 'upgrade_dependencies' in build_specs: upgrade_dependencies = build_specs['upgrade_dependencies'] else: upgrade_dependencies = False From a457179f8a25321396536da6f6e54212cae83ca2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 13:14:17 +0200 Subject: [PATCH 025/344] Shorten opt name --- easybuild/framework/easyconfig/tweak.py | 4 ++-- easybuild/tools/options.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 79ca1f818d..185907fb0e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -93,8 +93,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False - if 'upgrade_dependencies' in build_specs: - upgrade_dependencies = build_specs['upgrade_dependencies'] + if 'upgrade_deps' in build_specs: + upgrade_dependencies = build_specs['upgrade_deps'] else: upgrade_dependencies = False diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4d3e744368..4d3c08b474 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -307,7 +307,7 @@ def software_options(self): None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), - 'upgrade-dependencies': ("Try to upgrade versions of the dependencies of an easyconfig based on what is " + 'upgrade-deps': ("Try to upgrade versions of the dependencies of an easyconfig based on what is " "available in the robot path", None, 'store_true', False), }) @@ -1175,7 +1175,7 @@ def process_software_build_specs(options): 'version': options.try_software_version, 'toolchain_name': options.try_toolchain_name, 'toolchain_version': options.try_toolchain_version, - 'upgrade_dependencies': options.try_upgrade_dependencies + 'upgrade_deps': options.try_upgrade_dependencies } # process easy options From 5ea9fa5c260d0af92cf8295199e5a463a2a1ba78 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 13:16:07 +0200 Subject: [PATCH 026/344] Correct omissions --- easybuild/framework/easyconfig/tweak.py | 2 +- easybuild/tools/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 185907fb0e..ffcda965af 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -103,7 +103,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not # supported - if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'upgrade_dependencies'] for key in keys): + if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'upgrade_deps'] for key in keys): print_warning("Combining --try-toolchain* with other build options is not fully supported: using regex") revert_to_regex = True diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 4d3c08b474..7e6be23a9b 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1175,7 +1175,7 @@ def process_software_build_specs(options): 'version': options.try_software_version, 'toolchain_name': options.try_toolchain_name, 'toolchain_version': options.try_toolchain_version, - 'upgrade_deps': options.try_upgrade_dependencies + 'upgrade_deps': options.try_upgrade_deps } # process easy options From 2b3aa870facc185137b3021dd63b903629a06df3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 13:17:18 +0200 Subject: [PATCH 027/344] Appease the hound --- easybuild/tools/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7e6be23a9b..586e1bbc1c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -307,9 +307,9 @@ def software_options(self): None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), - 'upgrade-deps': ("Try to upgrade versions of the dependencies of an easyconfig based on what is " - "available in the robot path", - None, 'store_true', False), + 'upgrade-deps': ("Try to upgrade versions of the dependencies of an easyconfig based on what is available " + "in the robot path", + None, 'store_true', False), }) longopts = opts.keys() From 03dfbb557873268d239ee3c167b920afc14d6dbf Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 1 Oct 2018 15:44:58 +0200 Subject: [PATCH 028/344] Delete gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb Not needed since this will be created if required --- ...-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb deleted file mode 100644 index d0f441a526..0000000000 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-iccifort-2016.1.150-GCC-4.9.3-2.25.eb +++ /dev/null @@ -1,40 +0,0 @@ -## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild -# -# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC -# Authors:: Thekla Loizou -# License:: MIT/GPL -# $Id$ -# -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html -## -easyblock = 'ConfigureMake' - -name = 'gzip' -version = '1.4' - -homepage = "http://www.gzip.org/" -description = "gzip (GNU zip) is a popular data compression program as a replacement for compress" - -# test toolchain specification -toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} - -# source tarball filename -sources = ['%(name)s-%(version)s.tar.gz'] - -# download location for source files -source_urls = [GNU_SOURCE] - -# make sure the gzip and gunzip binaries are available after installation -sanity_check_paths = { - 'files': ["bin/gunzip", "bin/gzip"], - 'dirs': [], -} - -# run 'gzip -h' and 'gzip --version' after installation -sanity_check_commands = [True, ('gzip', '--version')] - -software_license = GPLv3 - -moduleclass = 'tools' From 8c87d39593fa815d4320da432d18e2eee67e2a25 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 17:10:11 +0200 Subject: [PATCH 029/344] Correct stuff related to versionprefix --- easybuild/framework/easyconfig/tweak.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ffcda965af..f7f22e8bfb 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -846,28 +846,27 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= potential_version_matches = [] for toolchain in toolchain_hierarchy: candidate_ver = '.*' # using regex for * + if 'versionsuffix' in dep: + versionsuffix = dep['versionsuffix'] + if 'versionprefix' in dep: + versionprefix = dep['versionprefix'] # determine main install version based on toolchain - if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: - version_suffix = ''.join([x for x in [parsed_ec.get('versionsuffix', '')] if x]) - else: + if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - version_suffix = ''.join([x for x in [toolchain_suffix, - parsed_ec.get('versionsuffix', '')] if x]) + versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix] if x]) # prepend/append version prefix/suffix - ecver = ''.join([x for x in [parsed_ec.get('versionprefix', ''), - candidate_ver, version_suffix] if x]) - cand_paths = search_easyconfigs(dep['name'] + '-' + ecver, return_robot_list=True) + depver = ''.join([x for x in [versionprefix, candidate_ver, versionsuffix] if x]) + cand_paths = search_easyconfigs(dep['name'] + '-' + depver, return_robot_list=True) # Add them to the possibilities for path in cand_paths: # Get the version from the path filename = os.path.basename(path) # Find the version sandwiched between our known values try: - version_prefix = ''.join([x for x in [dep['name'] + '-', - parsed_ec.get('versionprefix', '')] if x]) - if not version_suffix: - version_suffix = EB_FORMAT_EXTENSION - version = re.search('%s(.+?)%s' % (version_prefix, version_suffix), filename).group(1) + versionprefix = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) + if not versionsuffix: + versionsuffix = EB_FORMAT_EXTENSION + version = re.search('%s(.+?)%s' % (versionprefix, versionsuffix), filename).group(1) except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s", filename) potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) From 8fbb9273fd69a347c681b51c3f4e4e805ab5f44d Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 17:10:49 +0200 Subject: [PATCH 030/344] Correct stuff related to versionprefix --- easybuild/framework/easyconfig/tweak.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index f7f22e8bfb..6975d186e2 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -846,10 +846,10 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= potential_version_matches = [] for toolchain in toolchain_hierarchy: candidate_ver = '.*' # using regex for * - if 'versionsuffix' in dep: - versionsuffix = dep['versionsuffix'] if 'versionprefix' in dep: versionprefix = dep['versionprefix'] + if 'versionsuffix' in dep: + versionsuffix = dep['versionsuffix'] # determine main install version based on toolchain if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) From e2f77687a7cdc8612d4833fc2551d84ab12c8642 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 1 Oct 2018 17:48:53 +0200 Subject: [PATCH 031/344] Correct stuff related to versionprefix --- easybuild/framework/easyconfig/tweak.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 6975d186e2..c9cf8d09f4 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -848,8 +848,12 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= candidate_ver = '.*' # using regex for * if 'versionprefix' in dep: versionprefix = dep['versionprefix'] + else: + versionprefix = '' if 'versionsuffix' in dep: versionsuffix = dep['versionsuffix'] + else: + versionsuffix = '' # determine main install version based on toolchain if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) From 72ad2b9fef249e12f4162520bb62514e4c2e2dd2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 2 Oct 2018 14:15:25 +0200 Subject: [PATCH 032/344] Handle common versionsuffixes --- easybuild/framework/easyconfig/tweak.py | 123 +++++++++++++++++++++--- 1 file changed, 107 insertions(+), 16 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c9cf8d09f4..99a4104580 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -800,6 +800,76 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): return tc_mapping +# This really needs a cache +def map_common_versionsuffixes(original_toolchain, toolchain_mapping): + """ + Create a mapping of common versionssuffixes (like `-Python-%(pyvers)`) between toolchains + + :param toolchain_mapping: + :return: dictionary of possible mappings + """ + orig_toolchain_hierarchy = get_toolchain_hierarchy(original_toolchain) + target_toolchain_hierarchy = get_toolchain_hierarchy(toolchain_mapping[original_toolchain['name']]) + + versionsuffix_mappings = {} + for required_mappings in [['Python', '2'], ['Python', '3'], ['Java', '']]: + software = required_mappings[0] + versionstub = required_mappings[1] + # Find highest value in the target + target_version = '0' + for toolchain in target_toolchain_hierarchy: + if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = EB_FORMAT_EXTENSION + else: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + regex_search_query = '^%s-%s.*' % (software, versionstub) + toolchain_suffix + versionprefix_name = '%s-' % software + + cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + try: + regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) + version = re.search(regex, filename).group(1) + if LooseVersion(version) > LooseVersion(target_version): + target_version = version + except AttributeError: + raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, + regex) + + # Now map all matching versions in the source toolchain to this target + if target_version > 0: + source_versions = set() + for toolchain in orig_toolchain_hierarchy: + if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = EB_FORMAT_EXTENSION + else: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + regex_search_query = '^%s-%s.*' % (software, versionstub) + toolchain_suffix + versionprefix_name = '%s-' % software + + cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + try: + regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) + version = re.search(regex, filename).group(1) + source_versions.add(version) + except AttributeError: + raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, + regex) + + # Finally we add to the mapping + for source_version in source_versions: + versionsuffix_mappings['-%s-%s' % (software, source_version)] = '-%s-%s' % (software, target_version) + + _log.info("Identified version suffix mappings: %s", versionsuffix_mappings) + return versionsuffix_mappings + def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_dep_versions=False): """ @@ -813,6 +883,14 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= """ # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0] + + # There are some common versionsuffixes (like '-Python-(%pyver)s') that also need dynamic searching/updating + versonsuffix_mapping = map_common_versionsuffixes(parsed_ec['ec']['toolchain'], toolchain_mapping) + if update_dep_versions: + # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8` + if parsed_ec['ec']['versionsuffix'] in versonsuffix_mapping: + parsed_ec['ec']['versionsuffix'] = versonsuffix_mapping[parsed_ec['ec']['versionsuffix']] + # Replace the toolchain if the mapping exists tc_name = parsed_ec['ec']['toolchain']['name'] if tc_name in toolchain_mapping: @@ -844,35 +922,46 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # First get all candidate paths for this (include search through subtoolchains) toolchain_hierarchy = get_toolchain_hierarchy(orig_dep['toolchain']) potential_version_matches = [] - for toolchain in toolchain_hierarchy: - candidate_ver = '.*' # using regex for * - if 'versionprefix' in dep: - versionprefix = dep['versionprefix'] + + # Figure out what precedes the version + if 'versionprefix' in dep: + versionprefix = dep['versionprefix'] + else: + versionprefix = '' + versionprefix = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) + # Figure out the main versionsuffix (altered depending on toolchain in the loop below) + if 'versionsuffix' in dep: + if dep['versionsuffix'] in versonsuffix_mapping: + versionsuffix = versonsuffix_mapping[dep['versionsuffix']] else: - versionprefix = '' - if 'versionsuffix' in dep: versionsuffix = dep['versionsuffix'] - else: - versionsuffix = '' + else: + versionsuffix = '' + + for toolchain in toolchain_hierarchy: + candidate_ver = '.*' # using regex for * + # determine main install version based on toolchain if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix] if x]) + else: + toolchain_suffix = '' + full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] if x]) + # prepend/append version prefix/suffix - depver = ''.join([x for x in [versionprefix, candidate_ver, versionsuffix] if x]) - cand_paths = search_easyconfigs(dep['name'] + '-' + depver, return_robot_list=True) + depver = ''.join([x for x in ['^', versionprefix, candidate_ver, full_versionsuffix] if x]) + cand_paths = search_easyconfigs(depver, return_robot_list=True) # Add them to the possibilities for path in cand_paths: # Get the version from the path filename = os.path.basename(path) # Find the version sandwiched between our known values try: - versionprefix = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) - if not versionsuffix: - versionsuffix = EB_FORMAT_EXTENSION - version = re.search('%s(.+?)%s' % (versionprefix, versionsuffix), filename).group(1) + regex = '^%s(.+?)%s' % (versionprefix, full_versionsuffix) + version = re.search(regex, filename).group(1) except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s", filename) + raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", + filename, regex) potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) _log.debug("Found possible dependency upgrades: %s\n", potential_version_matches) @@ -887,6 +976,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= "easyconfigs: %s", '\n'.join(cand['path'] for cand in potential_version_matches if cand['version'] == highest_version)) orig_dep['version'] = highest_version + if orig_dep['versionsuffix'] in versonsuffix_mapping: + orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) From 915ced7a1b8b0b0efa7231029d95a2077b4bd440 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 2 Oct 2018 14:16:52 +0200 Subject: [PATCH 033/344] Handle common versionsuffixes --- easybuild/framework/easyconfig/tweak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 99a4104580..1fd0cbac27 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -800,6 +800,7 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): return tc_mapping + # This really needs a cache def map_common_versionsuffixes(original_toolchain, toolchain_mapping): """ From 78ae3f95d5d299f95a22e1d517cd20f5adcb35a6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 2 Oct 2018 14:17:31 +0200 Subject: [PATCH 034/344] Handle common versionsuffixes --- easybuild/framework/easyconfig/tweak.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 1fd0cbac27..a19bc725d0 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -947,7 +947,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) else: toolchain_suffix = '' - full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] if x]) + full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] + if x]) # prepend/append version prefix/suffix depver = ''.join([x for x in ['^', versionprefix, candidate_ver, full_versionsuffix] if x]) From 90b75b9ca243183f269ea5f4da9bc1fd171372aa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 2 Oct 2018 14:28:11 +0200 Subject: [PATCH 035/344] Add a cache to versionsuffix mapping --- easybuild/framework/easyconfig/tweak.py | 29 ++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a19bc725d0..ecca42bff8 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -35,6 +35,7 @@ :author: Alan O'Cais (Juelich Supercomputing Centre) """ import copy +import functools import glob import os import re @@ -801,7 +802,33 @@ def map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool): return tc_mapping -# This really needs a cache +def map_versionsuffixes_cache(func): + """Function decorator to cache (and retrieve cached) versionsuffixes mapping between toolchains.""" + cache = {} + + @functools.wraps(func) + def cache_aware_func(original_toolchain, toolchain_mapping): + """Look up original_toolchain in cache first, determine and cache it if not available yet.""" + # No need for toolchain_mapping to change to be part of the key, it is unique in this context + cache_key = (original_toolchain['name'], original_toolchain['version']) + + # fetch from cache if available, cache it if it's not + if cache_key in cache: + _log.debug("Using cache to return version suffix mapping for toolchain %s: %s", str(original_toolchain), + cache[cache_key]) + return cache[cache_key] + else: + versionsuffix_mappings = func(original_toolchain, toolchain_mapping) + cache[cache_key] = versionsuffix_mappings + return cache[cache_key] + + # Expose clear method of cache to wrapped function + cache_aware_func.clear = cache.clear + + return cache_aware_func + + +@map_versionsuffixes_cache def map_common_versionsuffixes(original_toolchain, toolchain_mapping): """ Create a mapping of common versionssuffixes (like `-Python-%(pyvers)`) between toolchains From a7b00f3e3588fefcf9e30a032df709ccfa635b70 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 3 Oct 2018 16:01:49 +0200 Subject: [PATCH 036/344] Add a test for versionsuffix mapping --- easybuild/framework/easyconfig/tweak.py | 92 +++++++++++++------------ test/framework/tweak.py | 18 ++++- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ecca42bff8..081b00aa78 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -807,18 +807,18 @@ def map_versionsuffixes_cache(func): cache = {} @functools.wraps(func) - def cache_aware_func(original_toolchain, toolchain_mapping): + def cache_aware_func(software_name, software_version_stub, original_toolchain, toolchain_mapping): """Look up original_toolchain in cache first, determine and cache it if not available yet.""" # No need for toolchain_mapping to change to be part of the key, it is unique in this context - cache_key = (original_toolchain['name'], original_toolchain['version']) + cache_key = (software_name, software_version_stub, original_toolchain['name'], original_toolchain['version']) # fetch from cache if available, cache it if it's not if cache_key in cache: - _log.debug("Using cache to return version suffix mapping for toolchain %s: %s", str(original_toolchain), + _log.debug("Using cache to return version suffix mapping for toolchain %s: %s", str(cache_key), cache[cache_key]) return cache[cache_key] else: - versionsuffix_mappings = func(original_toolchain, toolchain_mapping) + versionsuffix_mappings = func(software_name, software_version_stub, original_toolchain, toolchain_mapping) cache[cache_key] = versionsuffix_mappings return cache[cache_key] @@ -829,29 +829,55 @@ def cache_aware_func(original_toolchain, toolchain_mapping): @map_versionsuffixes_cache -def map_common_versionsuffixes(original_toolchain, toolchain_mapping): +def map_common_versionsuffixes(software_name, software_version_stub, original_toolchain, toolchain_mapping): """ Create a mapping of common versionssuffixes (like `-Python-%(pyvers)`) between toolchains - :param toolchain_mapping: + :param software_name: Name of software + :param software_version_stub: initial characters of version + :param original_toolchain: original toolchain + :param toolchain_mapping: toolchain mapping from that containing original to target :return: dictionary of possible mappings """ orig_toolchain_hierarchy = get_toolchain_hierarchy(original_toolchain) target_toolchain_hierarchy = get_toolchain_hierarchy(toolchain_mapping[original_toolchain['name']]) versionsuffix_mappings = {} - for required_mappings in [['Python', '2'], ['Python', '3'], ['Java', '']]: - software = required_mappings[0] - versionstub = required_mappings[1] - # Find highest value in the target - target_version = '0' - for toolchain in target_toolchain_hierarchy: + + # Find highest value in the target + target_version = '0' + for toolchain in target_toolchain_hierarchy: + if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = EB_FORMAT_EXTENSION + else: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + regex_search_query = '^%s-%s.*' % (software_name, software_version_stub) + toolchain_suffix + versionprefix_name = '%s-' % software_name + + cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + try: + regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) + version = re.search(regex, filename).group(1) + if LooseVersion(version) > LooseVersion(target_version): + target_version = version + except AttributeError: + raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, + regex) + + # Now map all matching versions in the source toolchain to this target + if target_version > 0: + source_versions = set() + for toolchain in orig_toolchain_hierarchy: if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: toolchain_suffix = EB_FORMAT_EXTENSION else: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - regex_search_query = '^%s-%s.*' % (software, versionstub) + toolchain_suffix - versionprefix_name = '%s-' % software + regex_search_query = '^%s-%s.*' % (software_name, software_version_stub) + toolchain_suffix + versionprefix_name = '%s-' % software_name cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) for path in cand_paths: @@ -861,39 +887,15 @@ def map_common_versionsuffixes(original_toolchain, toolchain_mapping): try: regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) version = re.search(regex, filename).group(1) - if LooseVersion(version) > LooseVersion(target_version): - target_version = version + source_versions.add(version) except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, regex) - # Now map all matching versions in the source toolchain to this target - if target_version > 0: - source_versions = set() - for toolchain in orig_toolchain_hierarchy: - if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: - toolchain_suffix = EB_FORMAT_EXTENSION - else: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - regex_search_query = '^%s-%s.*' % (software, versionstub) + toolchain_suffix - versionprefix_name = '%s-' % software - - cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) - for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - try: - regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) - version = re.search(regex, filename).group(1) - source_versions.add(version) - except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, - regex) - - # Finally we add to the mapping - for source_version in source_versions: - versionsuffix_mappings['-%s-%s' % (software, source_version)] = '-%s-%s' % (software, target_version) + # Finally we add to the mapping + for source_version in source_versions: + versionsuffix_mappings['-%s-%s' % (software_name, source_version)] = '-%s-%s' % (software_name, + target_version) _log.info("Identified version suffix mappings: %s", versionsuffix_mappings) return versionsuffix_mappings @@ -913,7 +915,9 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= parsed_ec = process_easyconfig(ec_spec, validate=False)[0] # There are some common versionsuffixes (like '-Python-(%pyver)s') that also need dynamic searching/updating - versonsuffix_mapping = map_common_versionsuffixes(parsed_ec['ec']['toolchain'], toolchain_mapping) + versonsuffix_mapping = map_common_versionsuffixes('Python', '2', parsed_ec['ec']['toolchain'], toolchain_mapping) + versonsuffix_mapping.update(map_common_versionsuffixes('Python', '3', parsed_ec['ec']['toolchain'], + toolchain_mapping)) if update_dep_versions: # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8` if parsed_ec['ec']['versionsuffix'] in versonsuffix_mapping: diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 93f77cee25..553cc9548b 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -36,7 +36,7 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one from easybuild.framework.easyconfig.tweak import check_capability_mapping, match_minimum_tc_specs -from easybuild.framework.easyconfig.tweak import get_dep_tree_of_toolchain +from easybuild.framework.easyconfig.tweak import get_dep_tree_of_toolchain, map_common_versionsuffixes from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies, map_easyconfig_to_target_tc_hierarchy from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes @@ -276,6 +276,22 @@ def test_map_toolchain_hierarchies(self): } self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected) + def test_map_common_versionsuffixes(self): + """Test mapping between two toolchain hierarchies""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': [test_easyconfigs], + }) + get_toolchain_hierarchy.clear() + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} + iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + toolchain_mapping = map_toolchain_hierarchies(iccifort_binutils_tc, gcc_binutils_tc, self.modtool) + possible_mappings = map_common_versionsuffixes('binutils', '2', iccifort_binutils_tc, toolchain_mapping) + expected_mappings = {'-binutils-2.25': '-binutils-2.26', + '-binutils-2.26': '-binutils-2.26'} + self.assertEqual(possible_mappings, expected_mappings) + def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From e7be7cee61f260ea926df26718475c0376194146 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 3 Oct 2018 16:15:34 +0200 Subject: [PATCH 037/344] Improve test for versionsuffix mapping --- easybuild/framework/easyconfig/tweak.py | 3 ++- test/framework/tweak.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 081b00aa78..ab766d3f0b 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -887,7 +887,8 @@ def map_common_versionsuffixes(software_name, software_version_stub, original_to try: regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) version = re.search(regex, filename).group(1) - source_versions.add(version) + if LooseVersion(version) < LooseVersion(target_version): + source_versions.add(version) except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, regex) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 553cc9548b..448e60fc1a 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -288,8 +288,16 @@ def test_map_common_versionsuffixes(self): iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} toolchain_mapping = map_toolchain_hierarchies(iccifort_binutils_tc, gcc_binutils_tc, self.modtool) possible_mappings = map_common_versionsuffixes('binutils', '2', iccifort_binutils_tc, toolchain_mapping) - expected_mappings = {'-binutils-2.25': '-binutils-2.26', - '-binutils-2.26': '-binutils-2.26'} + expected_mappings = {'-binutils-2.25': '-binutils-2.26'} + self.assertEqual(possible_mappings, expected_mappings) + + # Make sure we only map upwards, here it's gzip 1.4 in gcc and 1.6 in iccifort + possible_mappings = map_common_versionsuffixes('gzip', '', iccifort_binutils_tc, toolchain_mapping) + expected_mappings = {} + self.assertEqual(possible_mappings, expected_mappings) + toolchain_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) + possible_mappings = map_common_versionsuffixes('gzip', '', gcc_binutils_tc, toolchain_mapping) + expected_mappings = {'-gzip-1.4': '-gzip-1.6'} self.assertEqual(possible_mappings, expected_mappings) def test_map_easyconfig_to_target_tc_hierarchy(self): From ec633a87ce3890fc5a1e12e1fb4375a641180988 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 16:05:25 +0100 Subject: [PATCH 038/344] Address some of the comments --- easybuild/framework/easyconfig/tweak.py | 123 +++++++++--------- easybuild/tools/robot.py | 78 +++++------ ...-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb | 2 +- test/framework/tweak.py | 23 +++- 4 files changed, 125 insertions(+), 101 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 75f13eb483..f811b93586 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -95,18 +95,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False - if 'upgrade_deps' in build_specs: - upgrade_dependencies = build_specs['upgrade_deps'] - else: - upgrade_dependencies = False + update_dependencies = build_specs.get('update_deps', None) - if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs or upgrade_dependencies: + if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs or update_dependencies: keys = build_specs.keys() # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not # supported - if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'upgrade_deps'] for key in keys): - print_warning("Combining --try-toolchain* with other build options is not fully supported: using regex") + if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'update_deps'] for key in keys): + print_warning("Combining --try-toolchain* or --try-update-deps with other build options is not fully " + + "supported: using regex") revert_to_regex = True if not revert_to_regex: @@ -192,7 +190,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, tweaked_ecs_path, - update_dep_versions=upgrade_dependencies) + update_dep_versions=update_dependencies) # Need to update the toolchain in the build_specs to match the toolchain mapping keys = verification_build_specs.keys() if 'toolchain_name' in keys: @@ -821,20 +819,19 @@ def map_versionsuffixes_cache(func): cache = {} @functools.wraps(func) - def cache_aware_func(software_name, software_version_stub, original_toolchain, toolchain_mapping): + def cache_aware_func(software_name, original_toolchain, toolchain_mapping): """Look up original_toolchain in cache first, determine and cache it if not available yet.""" # No need for toolchain_mapping to change to be part of the key, it is unique in this context - cache_key = (software_name, software_version_stub, original_toolchain['name'], original_toolchain['version']) + cache_key = (software_name, original_toolchain['name'], original_toolchain['version']) # fetch from cache if available, cache it if it's not if cache_key in cache: _log.debug("Using cache to return version suffix mapping for toolchain %s: %s", str(cache_key), cache[cache_key]) - return cache[cache_key] else: - versionsuffix_mappings = func(software_name, software_version_stub, original_toolchain, toolchain_mapping) + versionsuffix_mappings = func(software_name, original_toolchain, toolchain_mapping) cache[cache_key] = versionsuffix_mappings - return cache[cache_key] + return cache[cache_key] # Expose clear method of cache to wrapped function cache_aware_func.clear = cache.clear @@ -843,12 +840,11 @@ def cache_aware_func(software_name, software_version_stub, original_toolchain, t @map_versionsuffixes_cache -def map_common_versionsuffixes(software_name, software_version_stub, original_toolchain, toolchain_mapping): +def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapping): """ Create a mapping of common versionssuffixes (like `-Python-%(pyvers)`) between toolchains :param software_name: Name of software - :param software_version_stub: initial characters of version :param original_toolchain: original toolchain :param toolchain_mapping: toolchain mapping from that containing original to target :return: dictionary of possible mappings @@ -858,64 +854,74 @@ def map_common_versionsuffixes(software_name, software_version_stub, original_to versionsuffix_mappings = {} - # Find highest value in the target - target_version = '0' + # Find highest value in the target (for each major version) + target_versions = {} for toolchain in target_toolchain_hierarchy: - if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: - toolchain_suffix = EB_FORMAT_EXTENSION - else: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - regex_search_query = '^%s-%s.*' % (software_name, software_version_stub) + toolchain_suffix - versionprefix_name = '%s-' % software_name - - cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) + prefix_stub = '%s-' % software_name + cand_paths, toolchain_suffix = get_matching_easyconfig_candidates(prefix_stub, toolchain) for path in cand_paths: # Get the version from the path filename = os.path.basename(path) # Find the version sandwiched between our known values try: - regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) + regex = '%s(.*)%s' % (prefix_stub, toolchain_suffix) version = re.search(regex, filename).group(1) - if LooseVersion(version) > LooseVersion(target_version): - target_version = version + major_version = version.split('.')[0] + # We make a list for all major values, make sure the major value in the list is initialised to zero + if major_version not in target_versions: + target_versions[major_version] = version + elif LooseVersion(version) > LooseVersion(target_versions[major_version]): + target_versions[major_version] = version except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, regex) # Now map all matching versions in the source toolchain to this target - if target_version > 0: - source_versions = set() - for toolchain in orig_toolchain_hierarchy: - if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: - toolchain_suffix = EB_FORMAT_EXTENSION - else: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - regex_search_query = '^%s-%s.*' % (software_name, software_version_stub) + toolchain_suffix - versionprefix_name = '%s-' % software_name - - cand_paths = search_easyconfigs(regex_search_query, return_robot_list=True) - for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - try: - regex = '%s(.*)%s' % (versionprefix_name, toolchain_suffix) - version = re.search(regex, filename).group(1) - if LooseVersion(version) < LooseVersion(target_version): - source_versions.add(version) - except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, - regex) - - # Finally we add to the mapping - for source_version in source_versions: - versionsuffix_mappings['-%s-%s' % (software_name, source_version)] = '-%s-%s' % (software_name, - target_version) + for major_version, target_version in target_versions.iteritems(): + if target_version > 0: + source_versions = set() + for toolchain in orig_toolchain_hierarchy: + prefix_stub = '%s-%s' % (software_name, major_version) + cand_paths, toolchain_suffix = get_matching_easyconfig_candidates(prefix_stub, toolchain) + + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + try: + regex = '%s-(.*)%s' % (software_name, toolchain_suffix) + version = re.search(regex, filename).group(1) + if LooseVersion(version) < LooseVersion(target_version): + source_versions.add(version) + except AttributeError: + raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, + regex) + + # Finally we add to the mapping + for source_version in source_versions: + versionsuffix_mappings['-%s-%s' % (software_name, source_version)] = '-%s-%s' % (software_name, + target_version) _log.info("Identified version suffix mappings: %s", versionsuffix_mappings) return versionsuffix_mappings +def get_matching_easyconfig_candidates(prefix_stub, toolchain): + """ + + :param prefix_stub: stub used in regex (e.g., 'Python-' or 'Python-2') + :param toolchain: the toolchain to use with the search + :return: list of candidate paths, toolchain_suffix of candidates + """ + if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = EB_FORMAT_EXTENSION + else: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + regex_search_query = '^%s.*' % prefix_stub + toolchain_suffix + cand_paths = search_easyconfigs(regex_search_query, consider_extra_paths=False, print_result=False) + return cand_paths, toolchain_suffix + + def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_dep_versions=False): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out @@ -930,9 +936,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= parsed_ec = process_easyconfig(ec_spec, validate=False)[0] # There are some common versionsuffixes (like '-Python-(%pyver)s') that also need dynamic searching/updating - versonsuffix_mapping = map_common_versionsuffixes('Python', '2', parsed_ec['ec']['toolchain'], toolchain_mapping) - versonsuffix_mapping.update(map_common_versionsuffixes('Python', '3', parsed_ec['ec']['toolchain'], - toolchain_mapping)) + versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['ec']['toolchain'], toolchain_mapping) + if update_dep_versions: # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8` if parsed_ec['ec']['versionsuffix'] in versonsuffix_mapping: diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index d6352c3ea1..8bffa903d1 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -379,7 +379,8 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): return ordered_ecs -def search_easyconfigs(query, short=False, filename_only=False, terse=False, return_robot_list=False): +def search_easyconfigs(query, short=False, filename_only=False, terse=False, consider_extra_paths=True, + print_result=True): """ Search for easyconfigs, if a query is provided. @@ -387,16 +388,17 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, ret :param short: figure out common prefix of hits, use variable to factor it out :param filename_only: only print filenames, not paths :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing - :param return_robot_list: return the list rather than print it out + :param consider_extra_paths: consider all paths when searching + :param print_result: print the list of easyconfigs - :return: return a list of paths for the query iff return_robot_list else no return value + :return: return a list of paths for the query """ search_path = build_option('robot_path') if not search_path: search_path = [os.getcwd()] extra_search_paths = build_option('search_paths') # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths - if extra_search_paths and not return_robot_list: + if extra_search_paths and consider_extra_paths: search_path.extend(extra_search_paths) ignore_dirs = build_option('ignore_dirs') @@ -413,39 +415,41 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, ret else: hits.append(hit) - # if requested return the matches as a list - if return_robot_list: - if build_option('consider_archived_easyconfigs'): - return hits + archived_hits - else: - return hits + if print_result: + # check whether only filenames should be printed + if filename_only: + hits = [os.path.basename(hit) for hit in hits] + archived_hits = [os.path.basename(hit) for hit in archived_hits] - # check whether only filenames should be printed - if filename_only: - hits = [os.path.basename(hit) for hit in hits] - archived_hits = [os.path.basename(hit) for hit in archived_hits] + # prepare output format + if terse: + lines, tmpl = [], '%s' + else: + lines = ['%s=%s' % var_def for var_def in var_defs] + tmpl = ' * %s' + + # non-archived hits are shown first + lines.extend(tmpl % hit for hit in hits) + + # also take into account archived hits + if archived_hits: + if build_option('consider_archived_easyconfigs'): + if not terse: + lines.extend(['', "Matching archived easyconfigs:", '']) + lines.extend(tmpl % hit for hit in archived_hits) + elif not terse: + cnt = len(archived_hits) + lines.extend([ + '', + "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt, + ]) + + print '\n'.join(lines) - # prepare output format - if terse: - lines, tmpl = [], '%s' + # if requested return the matches as a list + if build_option('consider_archived_easyconfigs'): + final_hits = hits + archived_hits else: - lines = ['%s=%s' % var_def for var_def in var_defs] - tmpl = ' * %s' - - # non-archived hits are shown first - lines.extend(tmpl % hit for hit in hits) - - # also take into account archived hits - if archived_hits: - if build_option('consider_archived_easyconfigs'): - if not terse: - lines.extend(['', "Matching archived easyconfigs:", '']) - lines.extend(tmpl % hit for hit in archived_hits) - elif not terse: - cnt = len(archived_hits) - lines.extend([ - '', - "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt, - ]) - - print '\n'.join(lines) + final_hits = hits + + return final_hits \ No newline at end of file diff --git a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb index d0f441a526..974fbad0c0 100644 --- a/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb +++ b/test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb @@ -12,7 +12,7 @@ easyblock = 'ConfigureMake' name = 'gzip' -version = '1.4' +version = '1.6' homepage = "http://www.gzip.org/" description = "gzip (GNU zip) is a popular data compression program as a replacement for compress" diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 4fc6ef6979..2c880623f8 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -37,7 +37,8 @@ from easybuild.framework.easyconfig.tweak import find_matching_easyconfigs, obtain_ec_for, pick_version, tweak_one from easybuild.framework.easyconfig.tweak import check_capability_mapping, match_minimum_tc_specs from easybuild.framework.easyconfig.tweak import get_dep_tree_of_toolchain, map_common_versionsuffixes -from easybuild.framework.easyconfig.tweak import map_toolchain_hierarchies, map_easyconfig_to_target_tc_hierarchy +from easybuild.framework.easyconfig.tweak import get_matching_easyconfig_candidates, map_toolchain_hierarchies +from easybuild.framework.easyconfig.tweak import map_easyconfig_to_target_tc_hierarchy from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes from easybuild.tools.filetools import write_file @@ -277,6 +278,20 @@ def test_map_toolchain_hierarchies(self): } self.assertEqual(map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool), expected) + def test_get_matching_easyconfig_candidates(self): + """Test searching for easyconfig candidates based on a stub and toolchain""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': [test_easyconfigs], + }) + toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} + expected_toolchain_stub = '-GCC-4.9.3-2.26' + expected_paths = [os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_stub + '.eb')] + paths, toolchain_stub = get_matching_easyconfig_candidates('gzip', toolchain) + self.assertEqual(toolchain_stub, expected_toolchain_stub) + self.assertEqual(paths, expected_paths) + def test_map_common_versionsuffixes(self): """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -288,16 +303,16 @@ def test_map_common_versionsuffixes(self): gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} toolchain_mapping = map_toolchain_hierarchies(iccifort_binutils_tc, gcc_binutils_tc, self.modtool) - possible_mappings = map_common_versionsuffixes('binutils', '2', iccifort_binutils_tc, toolchain_mapping) + possible_mappings = map_common_versionsuffixes('binutils', iccifort_binutils_tc, toolchain_mapping) expected_mappings = {'-binutils-2.25': '-binutils-2.26'} self.assertEqual(possible_mappings, expected_mappings) # Make sure we only map upwards, here it's gzip 1.4 in gcc and 1.6 in iccifort - possible_mappings = map_common_versionsuffixes('gzip', '', iccifort_binutils_tc, toolchain_mapping) + possible_mappings = map_common_versionsuffixes('gzip', iccifort_binutils_tc, toolchain_mapping) expected_mappings = {} self.assertEqual(possible_mappings, expected_mappings) toolchain_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) - possible_mappings = map_common_versionsuffixes('gzip', '', gcc_binutils_tc, toolchain_mapping) + possible_mappings = map_common_versionsuffixes('gzip', gcc_binutils_tc, toolchain_mapping) expected_mappings = {'-gzip-1.4': '-gzip-1.6'} self.assertEqual(possible_mappings, expected_mappings) From ae9ae94ce07864c32d71c59a81c8bd0a4fcf33d5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 16:07:08 +0100 Subject: [PATCH 039/344] Make sure option name is updated --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index dea3b394f8..9600a0806a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -317,7 +317,7 @@ def software_options(self): None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), - 'upgrade-deps': ("Try to upgrade versions of the dependencies of an easyconfig based on what is available " + 'update-deps': ("Try to updade versions of the dependencies of an easyconfig based on what is available " "in the robot path", None, 'store_true', False), }) @@ -1306,7 +1306,7 @@ def process_software_build_specs(options): 'version': options.try_software_version, 'toolchain_name': options.try_toolchain_name, 'toolchain_version': options.try_toolchain_version, - 'upgrade_deps': options.try_upgrade_deps + 'update_deps': options.try_update_deps } # process easy options From 17319eae869577b8e65cc5ff954292cb7fe35858 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 16:13:26 +0100 Subject: [PATCH 040/344] Fix broken test --- easybuild/tools/robot.py | 3 ++- test/framework/robot.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 8bffa903d1..f5d8078dbd 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -441,7 +441,8 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con cnt = len(archived_hits) lines.extend([ '', - "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt, + "Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" + % cnt, ]) print '\n'.join(lines) diff --git a/test/framework/robot.py b/test/framework/robot.py index 38b9376a58..fca341b793 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1280,7 +1280,7 @@ def test_search_easyconfigs(self): init_config(build_options={ 'robot_path': [test_ecs], }) - paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', return_robot_list=True) + paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', consider_extra_paths=False, print_result=False) ref_paths = [os.path.join(test_ecs, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', 'binutils-2.26-GCCcore-4.9.3.eb']] self.assertEqual(paths, ref_paths) From 00f882221a83bc3d1e4372e3c6514565664dd6c9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 16:15:53 +0100 Subject: [PATCH 041/344] Appease hound --- easybuild/tools/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index f5d8078dbd..15ddea7747 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -453,4 +453,4 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con else: final_hits = hits - return final_hits \ No newline at end of file + return final_hits From eab766fa1b77a95dcbfb16356e80d42c78727890 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 16:39:27 +0100 Subject: [PATCH 042/344] Fix broken test --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index f811b93586..e9aa341838 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1003,7 +1003,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # prepend/append version prefix/suffix depver = ''.join([x for x in ['^', versionprefix, candidate_ver, full_versionsuffix] if x]) - cand_paths = search_easyconfigs(depver, return_robot_list=True) + cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False) # Add them to the possibilities for path in cand_paths: # Get the version from the path From ff71a68bc4579b2d0a062bd4a247b4fe76e350f0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 17:50:13 +0100 Subject: [PATCH 043/344] Extract additional method and add test --- easybuild/framework/easyconfig/tweak.py | 109 ++++++++++++++---------- test/framework/tweak.py | 23 +++++ 2 files changed, 85 insertions(+), 47 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index e9aa341838..1ca338702a 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -972,56 +972,13 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= elif update_dep_versions: # Search for available updates for this dependency: # First get all candidate paths for this (include search through subtoolchains) - toolchain_hierarchy = get_toolchain_hierarchy(orig_dep['toolchain']) - potential_version_matches = [] + potential_version_matches = find_potential_version_mappings(dep, toolchain_mapping, + versonsuffix_mapping) - # Figure out what precedes the version - if 'versionprefix' in dep: - versionprefix = dep['versionprefix'] - else: - versionprefix = '' - versionprefix = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) - # Figure out the main versionsuffix (altered depending on toolchain in the loop below) - if 'versionsuffix' in dep: - if dep['versionsuffix'] in versonsuffix_mapping: - versionsuffix = versonsuffix_mapping[dep['versionsuffix']] - else: - versionsuffix = dep['versionsuffix'] - else: - versionsuffix = '' - - for toolchain in toolchain_hierarchy: - candidate_ver = '.*' # using regex for * - - # determine main install version based on toolchain - if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - else: - toolchain_suffix = '' - full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] - if x]) - - # prepend/append version prefix/suffix - depver = ''.join([x for x in ['^', versionprefix, candidate_ver, full_versionsuffix] if x]) - cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False) - # Add them to the possibilities - for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - try: - regex = '^%s(.+?)%s' % (versionprefix, full_versionsuffix) - version = re.search(regex, filename).group(1) - except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", - filename, regex) - potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) - _log.debug("Found possible dependency upgrades: %s\n", potential_version_matches) - - # Compare this version to the original versions and replace if appropriate (upgrades only) + # Compare these versions to the original version and replace if appropriate (upgrades only) highest_version = dep['version'] for candidate in potential_version_matches: - if LooseVersion(candidate['version']) >= LooseVersion(highest_version): + if LooseVersion(candidate['version']) > LooseVersion(highest_version): highest_version = candidate['version'] if highest_version != dep['version']: _log.info("Increasing version to %s for dependency %s.", highest_version, orig_dep['name']) @@ -1042,3 +999,61 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) return tweaked_spec + + +def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping): + """ + Find potential version mapping for a dependency in a new hierarchy + :param dep: dependency + :param toolchain_mapping: toolchain mapping used for search + :param versonsuffix_mapping: mapping of version suffixes (required by software with a special version suffix, such + as python packages) + :return: list of dependencies that match + """ + + potential_version_matches = [] + + # Find the target toolchain and create the hierarchy to search within + dep_tc_name = dep['toolchain']['name'] + if dep_tc_name in toolchain_mapping: + search_toolchain = toolchain_mapping[dep_tc_name] + toolchain_hierarchy = get_toolchain_hierarchy(search_toolchain) + # Figure out what precedes the version + versionprefix = dep.get('versionprefix', '') + prefix_to_version = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) + # Figure out the main versionsuffix (altered depending on toolchain in the loop below) + if 'versionsuffix' in dep: + if dep['versionsuffix'] in versonsuffix_mapping: + versionsuffix = versonsuffix_mapping[dep['versionsuffix']] + else: + versionsuffix = dep['versionsuffix'] + else: + versionsuffix = '' + for toolchain in toolchain_hierarchy: + candidate_ver = '.*' # using regex for * + + # determine main install version based on toolchain + if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + else: + toolchain_suffix = '' + full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] + if x]) + + # prepend/append version prefix/suffix + depver = ''.join([x for x in ['^', prefix_to_version, candidate_ver, full_versionsuffix] if x]) + cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False) + # Add them to the possibilities + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + try: + regex = '^%s(.+?)%s' % (prefix_to_version, full_versionsuffix) + version = re.search(regex, filename).group(1) + except AttributeError: + raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", + filename, regex) + potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) + _log.debug("Found possible dependency upgrades: %s\n", potential_version_matches) + return potential_version_matches diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 2c880623f8..d452974880 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -38,6 +38,7 @@ from easybuild.framework.easyconfig.tweak import check_capability_mapping, match_minimum_tc_specs from easybuild.framework.easyconfig.tweak import get_dep_tree_of_toolchain, map_common_versionsuffixes from easybuild.framework.easyconfig.tweak import get_matching_easyconfig_candidates, map_toolchain_hierarchies +from easybuild.framework.easyconfig.tweak import find_potential_version_mappings from easybuild.framework.easyconfig.tweak import map_easyconfig_to_target_tc_hierarchy from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes @@ -316,6 +317,28 @@ def test_map_common_versionsuffixes(self): expected_mappings = {'-gzip-1.4': '-gzip-1.6'} self.assertEqual(possible_mappings, expected_mappings) + def test_find_potential_version_mappings(self): + """Test ability to find potential version mappings of a dependency for a given toolchain mapping""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': [test_easyconfigs], + }) + get_toolchain_hierarchy.clear() + + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} + iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + # The below mapping includes a binutils mapping (2.26 to 2.25) + tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) + ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') + parsed_ec = process_easyconfig(ec_spec)[0] + gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'] + potential_versions = find_potential_version_mappings(gzip_dep[0], tc_mapping, []) + # Should see version 1.6 of gzip with iccifort toolchain + expected_dep_path = os.path.join(test_easyconfigs, 'g', 'gzip', + 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb') + self.assertEqual(potential_versions[0]['path'], expected_dep_path) + def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 86a80e4eefc3dd6a9ce688561e693ea3e43461fd Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 17:57:44 +0100 Subject: [PATCH 044/344] Address another comment --- easybuild/framework/easyconfig/tweak.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 1ca338702a..29abfb7086 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -966,9 +966,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # Replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): orig_dep.update(toolchain_mapping['binutils']) - # set module names - orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) - orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) + dep_changed = True elif update_dep_versions: # Search for available updates for this dependency: # First get all candidate paths for this (include search through subtoolchains) @@ -981,15 +979,19 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= if LooseVersion(candidate['version']) > LooseVersion(highest_version): highest_version = candidate['version'] if highest_version != dep['version']: - _log.info("Increasing version to %s for dependency %s.", highest_version, orig_dep['name']) + _log.info("Upgrading version of %s dependency from %s to %s", dep['name'], dep['version'], + highest_version) _log.info("Depending on your configuration, this will be resolved with one of the following " "easyconfigs: %s", '\n'.join(cand['path'] for cand in potential_version_matches if cand['version'] == highest_version)) orig_dep['version'] = highest_version if orig_dep['versionsuffix'] in versonsuffix_mapping: orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] - orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) - orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) + dep_changed = True + + if dep_changed: + orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) + orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) From 42153b7940f9930c635daf78b9a07b95479535d8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 18:07:41 +0100 Subject: [PATCH 045/344] Address more comments --- easybuild/framework/easyconfig/tweak.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 29abfb7086..702a257816 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1024,13 +1024,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping versionprefix = dep.get('versionprefix', '') prefix_to_version = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) # Figure out the main versionsuffix (altered depending on toolchain in the loop below) - if 'versionsuffix' in dep: - if dep['versionsuffix'] in versonsuffix_mapping: - versionsuffix = versonsuffix_mapping[dep['versionsuffix']] - else: - versionsuffix = dep['versionsuffix'] - else: - versionsuffix = '' + versionsuffix = versonsuffix_mapping.get(dep.get('versionsuffix', None), '') for toolchain in toolchain_hierarchy: candidate_ver = '.*' # using regex for * @@ -1050,12 +1044,13 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping # Get the version from the path filename = os.path.basename(path) # Find the version sandwiched between our known values - try: - regex = '^%s(.+?)%s' % (prefix_to_version, full_versionsuffix) - version = re.search(regex, filename).group(1) - except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", - filename, regex) + regex = re.compile('^%s(.+?)%s' % (versionprefix, full_versionsuffix)) + res = regex.search(filename) + if res: + version = res.group(1) + else: + raise EasyBuildError("Failed to determine version from '%s' using regex pattern '%s'", filename, + regex.pattern) potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) - _log.debug("Found possible dependency upgrades: %s\n", potential_version_matches) + _log.debug("Found possible dependency upgrades: %s", potential_version_matches) return potential_version_matches From 174cb6cb3093d8a23381b264e86238e18e5b2789 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 18:08:40 +0100 Subject: [PATCH 046/344] Address more comments --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 702a257816..ec8b28047d 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1024,7 +1024,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping versionprefix = dep.get('versionprefix', '') prefix_to_version = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) # Figure out the main versionsuffix (altered depending on toolchain in the loop below) - versionsuffix = versonsuffix_mapping.get(dep.get('versionsuffix', None), '') + versionsuffix = dep.get('versionsuffix', '') for toolchain in toolchain_hierarchy: candidate_ver = '.*' # using regex for * From 3637ea85fc463560213d97ef285800b458f0fbde Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 18:11:50 +0100 Subject: [PATCH 047/344] Fix broken test --- easybuild/framework/easyconfig/tweak.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ec8b28047d..406a97a7b2 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -964,6 +964,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= if dep_tc_name in toolchain_mapping: orig_dep['toolchain'] = toolchain_mapping[dep_tc_name] # Replace the binutils version (if necessary) + dep_changed = False if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): orig_dep.update(toolchain_mapping['binutils']) dep_changed = True From 1952fc73d229bb3b06f1bfa7a85414a79dbf7113 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 27 Nov 2018 20:33:54 +0100 Subject: [PATCH 048/344] Fix broken test --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 406a97a7b2..860eaddf7b 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1045,7 +1045,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping # Get the version from the path filename = os.path.basename(path) # Find the version sandwiched between our known values - regex = re.compile('^%s(.+?)%s' % (versionprefix, full_versionsuffix)) + regex = re.compile('^%s(.+?)%s' % (prefix_to_version, full_versionsuffix)) res = regex.search(filename) if res: version = res.group(1) From f5e6d0ac2fea6577074719c74789c9e8cccfd488 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 09:28:23 +0100 Subject: [PATCH 049/344] Be more careful when using search since it doesn't care about case, also make sure we extract an integer version --- easybuild/framework/easyconfig/tweak.py | 26 ++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 860eaddf7b..1e477f2cbc 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -867,11 +867,17 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp regex = '%s(.*)%s' % (prefix_stub, toolchain_suffix) version = re.search(regex, filename).group(1) major_version = version.split('.')[0] - # We make a list for all major values, make sure the major value in the list is initialised to zero - if major_version not in target_versions: - target_versions[major_version] = version - elif LooseVersion(version) > LooseVersion(target_versions[major_version]): - target_versions[major_version] = version + try: + # make sure we have a have an integer value for the major version + int(major_version) + # We make a list for all major values, make sure the major value in the list is initialised to zero + if major_version not in target_versions: + target_versions[major_version] = version + elif LooseVersion(version) > LooseVersion(target_versions[major_version]): + target_versions[major_version] = version + except ValueError: + _log.info("Cannot extract major version for %s from %s, probably a name clash", prefix_stub, + filename) except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, regex) @@ -919,6 +925,8 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain): toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) regex_search_query = '^%s.*' % prefix_stub + toolchain_suffix cand_paths = search_easyconfigs(regex_search_query, consider_extra_paths=False, print_result=False) + # The stubs have to be an exact match + cand_paths = [path for path in cand_paths if path.startswith(prefix_stub)] return cand_paths, toolchain_suffix @@ -935,11 +943,11 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # Fully parse the original easyconfig parsed_ec = process_easyconfig(ec_spec, validate=False)[0] - # There are some common versionsuffixes (like '-Python-(%pyver)s') that also need dynamic searching/updating - versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['ec']['toolchain'], toolchain_mapping) + versonsuffix_mapping = {} if update_dep_versions: # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8` + versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['ec']['toolchain'], toolchain_mapping) if parsed_ec['ec']['versionsuffix'] in versonsuffix_mapping: parsed_ec['ec']['versionsuffix'] = versonsuffix_mapping[parsed_ec['ec']['versionsuffix']] @@ -1026,6 +1034,10 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping prefix_to_version = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) # Figure out the main versionsuffix (altered depending on toolchain in the loop below) versionsuffix = dep.get('versionsuffix', '') + # If versionsuffix is in our mapping then we expect it to be updated + if versionsuffix in versonsuffix_mapping: + versionsuffix = versonsuffix_mapping[versionsuffix] + for toolchain in toolchain_hierarchy: candidate_ver = '.*' # using regex for * From 297f767f0e10a5e95e5c4d15d910469ad4bd0bab Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 12:00:00 +0100 Subject: [PATCH 050/344] Fix complex use case where we have recursion, make sure that we omit all easyconfigs that form part of a toolchain, unless they were explicitly listed on the command line --- easybuild/framework/easyconfig/tweak.py | 34 +++++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 1e477f2cbc..31539601a8 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -81,6 +81,9 @@ def ec_filename_for(path): def tweak(easyconfigs, build_specs, modtool, targetdirs=None): """Tweak list of easyconfigs according to provided build specifications.""" + # keep track of originally listed easyconfigs (via their path) + listed_ec_paths = [ec['spec'] for ec in easyconfigs] + tweaked_ecs_path, tweaked_ecs_deps_path = None, None if targetdirs is not None: tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs @@ -151,12 +154,15 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) # We can leave any dependencies they may have as they will only be used if required (or originally listed) - _log.debug("Filtering out toolchain hierarchy for %s", source_toolchain) - + _log.debug("Filtering out toolchain hierarchy and dependencies for %s", source_toolchain) + path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) + toolchain_ec = process_easyconfig(path) + toolchain_deps = resolve_dependencies(toolchain_ec, modtool, retain_all_deps=True) + toolchain_dep_names = [dep['ec']['name'] for dep in toolchain_deps if + dep['spec'] not in listed_ec_paths] i = 0 while i < len(orig_ecs): - tc_names = [tc['name'] for tc in get_toolchain_hierarchy(source_toolchain)] - if orig_ecs[i]['ec']['name'] in tc_names: + if orig_ecs[i]['ec']['name'] in toolchain_dep_names: # drop elements in toolchain hierarchy del orig_ecs[i] else: @@ -170,9 +176,6 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): orig_ecs = easyconfigs _log.debug("Software name/version found, so not applying build specifications recursively: %s" % build_specs) - # keep track of originally listed easyconfigs (via their path) - listed_ec_paths = [ec['spec'] for ec in easyconfigs] - # generate tweaked easyconfigs, and continue with those instead tweaked_easyconfigs = [] for orig_ec in orig_ecs: @@ -210,7 +213,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - targetdir=tweaked_ecs_deps_path) + targetdir=tweaked_ecs_deps_path, + update_dep_versions=update_dependencies) else: new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) @@ -907,7 +911,6 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp for source_version in source_versions: versionsuffix_mappings['-%s-%s' % (software_name, source_version)] = '-%s-%s' % (software_name, target_version) - _log.info("Identified version suffix mappings: %s", versionsuffix_mappings) return versionsuffix_mappings @@ -926,7 +929,7 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain): regex_search_query = '^%s.*' % prefix_stub + toolchain_suffix cand_paths = search_easyconfigs(regex_search_query, consider_extra_paths=False, print_result=False) # The stubs have to be an exact match - cand_paths = [path for path in cand_paths if path.startswith(prefix_stub)] + cand_paths = [path for path in cand_paths if prefix_stub in path] return cand_paths, toolchain_suffix @@ -948,8 +951,6 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= if update_dep_versions: # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8` versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['ec']['toolchain'], toolchain_mapping) - if parsed_ec['ec']['versionsuffix'] in versonsuffix_mapping: - parsed_ec['ec']['versionsuffix'] = versonsuffix_mapping[parsed_ec['ec']['versionsuffix']] # Replace the toolchain if the mapping exists tc_name = parsed_ec['ec']['toolchain']['name'] @@ -988,13 +989,14 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= if LooseVersion(candidate['version']) > LooseVersion(highest_version): highest_version = candidate['version'] if highest_version != dep['version']: - _log.info("Upgrading version of %s dependency from %s to %s", dep['name'], dep['version'], + _log.info("Updating version of %s dependency from %s to %s", dep['name'], dep['version'], highest_version) _log.info("Depending on your configuration, this will be resolved with one of the following " "easyconfigs: %s", '\n'.join(cand['path'] for cand in potential_version_matches if cand['version'] == highest_version)) orig_dep['version'] = highest_version if orig_dep['versionsuffix'] in versonsuffix_mapping: + dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] dep_changed = True @@ -1003,6 +1005,8 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) # Determine the name of the modified easyconfig and dump it to target_dir + if parsed_ec['ec']['versionsuffix'] in versonsuffix_mapping: + parsed_ec['ec']['versionsuffix'] = versonsuffix_mapping[parsed_ec['ec']['versionsuffix']] ec_filename = '%s-%s.eb' % (parsed_ec['ec']['name'], det_full_ec_version(parsed_ec['ec'])) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) @@ -1028,6 +1032,9 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping dep_tc_name = dep['toolchain']['name'] if dep_tc_name in toolchain_mapping: search_toolchain = toolchain_mapping[dep_tc_name] + else: + # dummy + search_toolchain = dep['toolchain'] toolchain_hierarchy = get_toolchain_hierarchy(search_toolchain) # Figure out what precedes the version versionprefix = dep.get('versionprefix', '') @@ -1037,7 +1044,6 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping # If versionsuffix is in our mapping then we expect it to be updated if versionsuffix in versonsuffix_mapping: versionsuffix = versonsuffix_mapping[versionsuffix] - for toolchain in toolchain_hierarchy: candidate_ver = '.*' # using regex for * From 89ec2306c2b8a4b6ecd006667b7b7238f2d4b5b7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 12:44:41 +0100 Subject: [PATCH 051/344] Make search_easyconfig able to do a case senstive search --- easybuild/framework/easyconfig/tweak.py | 7 +++---- easybuild/tools/filetools.py | 9 ++++++--- easybuild/tools/robot.py | 5 +++-- test/framework/robot.py | 9 +++++++++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 31539601a8..57ccf2e394 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -927,9 +927,8 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain): else: toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) regex_search_query = '^%s.*' % prefix_stub + toolchain_suffix - cand_paths = search_easyconfigs(regex_search_query, consider_extra_paths=False, print_result=False) - # The stubs have to be an exact match - cand_paths = [path for path in cand_paths if prefix_stub in path] + cand_paths = search_easyconfigs(regex_search_query, consider_extra_paths=False, print_result=False, + case_sensitive=True) return cand_paths, toolchain_suffix @@ -1057,7 +1056,7 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping # prepend/append version prefix/suffix depver = ''.join([x for x in ['^', prefix_to_version, candidate_ver, full_versionsuffix] if x]) - cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False) + cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, case_sensitive=True) # Add them to the possibilities for path in cand_paths: # Get the version from the path diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 937d4e33db..4a4920b4e8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -595,7 +595,8 @@ def find_easyconfigs(path, ignore_dirs=None): return files -def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False): +def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False, + case_sensitive=False): """ Search for files using in specified paths using specified search query (regular expression) @@ -613,8 +614,10 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen raise EasyBuildError("search_file: ignore_dirs (%s) should be of type list, not %s", ignore_dirs, type(ignore_dirs)) - # compile regex, case-insensitive - query = re.compile(query, re.I) + if case_sensitive: + query = re.compile(query) + else:# compile regex, case-insensitive + query = re.compile(query, re.I) var_defs = [] hits = [] diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 15ddea7747..b8fc68eba3 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -380,7 +380,7 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False): def search_easyconfigs(query, short=False, filename_only=False, terse=False, consider_extra_paths=True, - print_result=True): + print_result=True, case_sensitive=False): """ Search for easyconfigs, if a query is provided. @@ -390,6 +390,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con :param terse: stick to terse (machine-readable) output, as opposed to pretty-printing :param consider_extra_paths: consider all paths when searching :param print_result: print the list of easyconfigs + :param case_sensitive: boolean to decide whether search is case sensitive :return: return a list of paths for the query """ @@ -405,7 +406,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con # note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse, - silent=True, filename_only=False) + silent=True, filename_only=False, case_sensitive=case_sensitive) # filter out archived easyconfigs, these are handled separately hits, archived_hits = [], [] diff --git a/test/framework/robot.py b/test/framework/robot.py index fca341b793..94c88028b9 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1285,6 +1285,15 @@ def test_search_easyconfigs(self): 'binutils-2.26-GCCcore-4.9.3.eb']] self.assertEqual(paths, ref_paths) + paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) + ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2.6.10.eb'), + os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb')] + self.assertEqual(paths, ref_paths) + + # Now do a case sensitive search + paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False, case_sensitive=True) + ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2.6.10.eb')] + self.assertEqual(paths, ref_paths) def suite(): """ returns all the testcases in this module """ From 2ad48f6bd7774f14f9d441e9671d01c45aa393fd Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 12:46:09 +0100 Subject: [PATCH 052/344] Appease hound --- easybuild/tools/filetools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4a4920b4e8..7d9680f833 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -616,7 +616,8 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if case_sensitive: query = re.compile(query) - else:# compile regex, case-insensitive + else: + # compile regex, case-insensitive query = re.compile(query, re.I) var_defs = [] From 5f318cbfc93b21ba101f89fea7a6ea7d891800b2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 13:08:57 +0100 Subject: [PATCH 053/344] Fix for bug found by test --- easybuild/framework/easyconfig/tweak.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 57ccf2e394..688d5dd55e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -155,11 +155,14 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) # We can leave any dependencies they may have as they will only be used if required (or originally listed) _log.debug("Filtering out toolchain hierarchy and dependencies for %s", source_toolchain) - path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) - toolchain_ec = process_easyconfig(path) - toolchain_deps = resolve_dependencies(toolchain_ec, modtool, retain_all_deps=True) - toolchain_dep_names = [dep['ec']['name'] for dep in toolchain_deps if - dep['spec'] not in listed_ec_paths] + if source_toolchain['name'] != DUMMY_TOOLCHAIN_NAME: + path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) + toolchain_ec = process_easyconfig(path) + toolchain_deps = resolve_dependencies(toolchain_ec, modtool, retain_all_deps=True) + toolchain_dep_names = [dep['ec']['name'] for dep in toolchain_deps if + dep['spec'] not in listed_ec_paths] + else: + toolchain_dep_names = [] i = 0 while i < len(orig_ecs): if orig_ecs[i]['ec']['name'] in toolchain_dep_names: From fa85c6116f799c63df6f7d8499d4b78630464361 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 14:01:15 +0100 Subject: [PATCH 054/344] Only do minor version updates if possible, if that's not possible only then do a major version update --- easybuild/framework/easyconfig/tweak.py | 57 ++++++++++++++----------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 688d5dd55e..8fc59b7526 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1046,32 +1046,37 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping # If versionsuffix is in our mapping then we expect it to be updated if versionsuffix in versonsuffix_mapping: versionsuffix = versonsuffix_mapping[versionsuffix] - for toolchain in toolchain_hierarchy: - candidate_ver = '.*' # using regex for * - # determine main install version based on toolchain - if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - else: - toolchain_suffix = '' - full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] - if x]) - - # prepend/append version prefix/suffix - depver = ''.join([x for x in ['^', prefix_to_version, candidate_ver, full_versionsuffix] if x]) - cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, case_sensitive=True) - # Add them to the possibilities - for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - regex = re.compile('^%s(.+?)%s' % (prefix_to_version, full_versionsuffix)) - res = regex.search(filename) - if res: - version = res.group(1) - else: - raise EasyBuildError("Failed to determine version from '%s' using regex pattern '%s'", filename, - regex.pattern) - potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) + # the candidate version is a regex string, let's be conservative and search for a minor version upgrade first + # only if that fails will we try a global search, i.e, a major version upgrade (assumes major.minor.XXX versioning) + major_version = dep['version'].split('.')[0] + for candidate_ver in ['%s.*' % major_version, '.*']: + if not potential_version_matches: + for toolchain in toolchain_hierarchy: + # determine main install version based on toolchain + if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + else: + toolchain_suffix = '' + full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] + if x]) + + # prepend/append version prefix/suffix + depver = ''.join([x for x in ['^', prefix_to_version, candidate_ver, full_versionsuffix] if x]) + cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, + case_sensitive=True) + # Add them to the possibilities + for path in cand_paths: + # Get the version from the path + filename = os.path.basename(path) + # Find the version sandwiched between our known values + regex = re.compile('^%s(.+?)%s' % (prefix_to_version, full_versionsuffix)) + res = regex.search(filename) + if res: + version = res.group(1) + else: + raise EasyBuildError("Failed to determine version from '%s' using regex pattern '%s'", filename, + regex.pattern) + potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) _log.debug("Found possible dependency upgrades: %s", potential_version_matches) return potential_version_matches From 44d2b8b7e1927480c6f063c29e2e63bb26ee8986 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 28 Nov 2018 17:00:17 +0100 Subject: [PATCH 055/344] Add flexibility --- easybuild/framework/easyconfig/tweak.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 8fc59b7526..0f4afba694 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1047,10 +1047,20 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping if versionsuffix in versonsuffix_mapping: versionsuffix = versonsuffix_mapping[versionsuffix] - # the candidate version is a regex string, let's be conservative and search for a minor version upgrade first - # only if that fails will we try a global search, i.e, a major version upgrade (assumes major.minor.XXX versioning) - major_version = dep['version'].split('.')[0] - for candidate_ver in ['%s.*' % major_version, '.*']: + # the candidate version is a regex string, let's be conservative and search for patch upgrade first, if that doesn't + # work look for a minor version upgrade and if that fails will we try a global search, i.e, a major version upgrade + # (assumes major.minor.XXX versioning) + candidate_ver_list = [] + version_components = dep['version'].split('.') + major_version = version_components[0] + # This doesn't work well with the versionsuffix mapping which isn't clever enough for this so ignoring + # if len(version_components) > 1: + # minor_version = version_components[1] + # candidate_ver_list.append('%s\.%s.*' % (major_version, minor_version)) + candidate_ver_list.append('%s.*' % major_version) + candidate_ver_list.append('.*') + + for candidate_ver in candidate_ver_list: if not potential_version_matches: for toolchain in toolchain_hierarchy: # determine main install version based on toolchain From fe6b09f7cdef7fd848d9ef69558e7cca88576bb0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Dec 2018 12:13:33 +0100 Subject: [PATCH 056/344] Make versionsuffix mapping more robust Make version upgrading have an order of preference (if a patch upgrade exists, use that first, only then a minor version upgrade and only after that a major version upgrade) --- easybuild/framework/easyconfig/tweak.py | 121 ++++++++++++------------ test/framework/robot.py | 11 ++- 2 files changed, 67 insertions(+), 65 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 005c04aed7..3f677d01e6 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -857,13 +857,11 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp :return: dictionary of possible mappings """ orig_toolchain_hierarchy = get_toolchain_hierarchy(original_toolchain) - target_toolchain_hierarchy = get_toolchain_hierarchy(toolchain_mapping[original_toolchain['name']]) versionsuffix_mappings = {} - # Find highest value in the target (for each major version) - target_versions = {} - for toolchain in target_toolchain_hierarchy: + # Find all Python versions in the original toolchain hierarchy and register what they would be mapped to + for toolchain in orig_toolchain_hierarchy: prefix_stub = '%s-' % software_name cand_paths, toolchain_suffix = get_matching_easyconfig_candidates(prefix_stub, toolchain) for path in cand_paths: @@ -877,43 +875,34 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp try: # make sure we have a have an integer value for the major version int(major_version) - # We make a list for all major values, make sure the major value in the list is initialised to zero - if major_version not in target_versions: - target_versions[major_version] = version - elif LooseVersion(version) > LooseVersion(target_versions[major_version]): - target_versions[major_version] = version except ValueError: _log.info("Cannot extract major version for %s from %s, probably a name clash", prefix_stub, filename) except AttributeError: raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, regex) + # Use these values to construct a dependency + software_as_dep={ + 'name': software_name, + 'version': version, + 'toolchain': toolchain + } + # See what this dep would be mapped to + version_matches = find_potential_version_mappings(software_as_dep, toolchain_mapping) + if version_matches: + target_version = version_matches[0]['version'] + if LooseVersion(target_version) > LooseVersion(version): + original_suffix = '-%s-%s' % (software_name, version) + mapped_suffix = '-%s-%s' % (software_name, target_version) + # Make sure mapping is unique + if original_suffix in versionsuffix_mappings: + if mapped_suffix != versionsuffix_mappings[original_suffix]: + raise EasyBuildError("No unique versionsuffix mapping for %s in %s toolchain " + "hierarchy to %s toolchain hierarchy", original_suffix, + original_toolchain, toolchain_mapping[original_toolchain['name']]) + else: + versionsuffix_mappings[original_suffix] = mapped_suffix - # Now map all matching versions in the source toolchain to this target - for major_version, target_version in target_versions.iteritems(): - if target_version > 0: - source_versions = set() - for toolchain in orig_toolchain_hierarchy: - prefix_stub = '%s-%s' % (software_name, major_version) - cand_paths, toolchain_suffix = get_matching_easyconfig_candidates(prefix_stub, toolchain) - - for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - try: - regex = '%s-(.*)%s' % (software_name, toolchain_suffix) - version = re.search(regex, filename).group(1) - if LooseVersion(version) < LooseVersion(target_version): - source_versions.add(version) - except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, - regex) - - # Finally we add to the mapping - for source_version in source_versions: - versionsuffix_mappings['-%s-%s' % (software_name, source_version)] = '-%s-%s' % (software_name, - target_version) _log.info("Identified version suffix mappings: %s", versionsuffix_mappings) return versionsuffix_mappings @@ -981,26 +970,23 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= dep_changed = True elif update_dep_versions: # Search for available updates for this dependency: - # First get all candidate paths for this (include search through subtoolchains) + # First get highest version candidate paths for this (include search through subtoolchains) potential_version_matches = find_potential_version_mappings(dep, toolchain_mapping, - versonsuffix_mapping) - - # Compare these versions to the original version and replace if appropriate (upgrades only) - highest_version = dep['version'] - for candidate in potential_version_matches: - if LooseVersion(candidate['version']) > LooseVersion(highest_version): - highest_version = candidate['version'] - if highest_version != dep['version']: - _log.info("Updating version of %s dependency from %s to %s", dep['name'], dep['version'], - highest_version) - _log.info("Depending on your configuration, this will be resolved with one of the following " - "easyconfigs: %s", '\n'.join(cand['path'] for cand in potential_version_matches - if cand['version'] == highest_version)) - orig_dep['version'] = highest_version - if orig_dep['versionsuffix'] in versonsuffix_mapping: - dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] - orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] - dep_changed = True + versonsuffix_mapping=versonsuffix_mapping) + # Only highest version match is retained by default in potential_version_matches, compare that version + # to the original version and replace if appropriate (upgrades only). + if potential_version_matches: + highest_version_match = potential_version_matches[0]['version'] + if LooseVersion(highest_version_match) > LooseVersion(dep['version']): + _log.info("Updating version of %s dependency from %s to %s", dep['name'], dep['version'], + highest_version_match) + _log.info("Depending on your configuration, this will be resolved with one of the following " + "easyconfigs: \n%s", '\n'.join(cand['path'] for cand in potential_version_matches)) + orig_dep['version'] = highest_version_match + if orig_dep['versionsuffix'] in versonsuffix_mapping: + dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] + orig_dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] + dep_changed = True if dep_changed: orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) @@ -1018,7 +1004,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= return tweaked_spec -def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping): +def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping={}, highest_versions_only=True): """ Find potential version mapping for a dependency in a new hierarchy :param dep: dependency @@ -1053,13 +1039,14 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping candidate_ver_list = [] version_components = dep['version'].split('.') major_version = version_components[0] - # This doesn't work well with the versionsuffix mapping which isn't clever enough for this so ignoring - # if len(version_components) > 1: - # minor_version = version_components[1] - # candidate_ver_list.append('%s\.%s.*' % (major_version, minor_version)) - candidate_ver_list.append('%s.*' % major_version) - candidate_ver_list.append('.*') - + if len(version_components) > 2: # Have something like major.minor.XXX + minor_version = version_components[1] + candidate_ver_list.append('%s\.%s\..*' % (major_version, minor_version)) + if len(version_components) > 1: # Have at least major.minor + candidate_ver_list.append('%s\..*' % major_version) + candidate_ver_list.append('.*') # Include a major version search + + highest_version = None for candidate_ver in candidate_ver_list: if not potential_version_matches: for toolchain in toolchain_hierarchy: @@ -1075,7 +1062,11 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping depver = ''.join([x for x in ['^', prefix_to_version, candidate_ver, full_versionsuffix] if x]) cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, case_sensitive=True) - # Add them to the possibilities + # Filter out easyconfigs that have been tweaked in this instance (they sit in the tempdir in a subdir + # that starts with 'tweaked_*') + tweaked_ec_stub = os.path.join(tempfile.gettempdir(), 'tweaked_') + cand_paths = [path for path in cand_paths if not path.startswith(tweaked_ec_stub)] + # Add what is left to the possibilities for path in cand_paths: # Get the version from the path filename = os.path.basename(path) @@ -1084,9 +1075,15 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping res = regex.search(filename) if res: version = res.group(1) + if highest_version is None or LooseVersion(version) > LooseVersion(highest_version): + highest_version = version else: raise EasyBuildError("Failed to determine version from '%s' using regex pattern '%s'", filename, regex.pattern) potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) + + if highest_versions_only and highest_version is not None: + potential_version_matches = [d for d in potential_version_matches if d['version'] == highest_version] + _log.debug("Found possible dependency upgrades: %s", potential_version_matches) return potential_version_matches diff --git a/test/framework/robot.py b/test/framework/robot.py index a7d4bf039d..f7b5624948 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1310,13 +1310,18 @@ def test_search_easyconfigs(self): self.assertEqual(paths, ref_paths) paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) - ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2.6.10.eb'), - os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb')] + ref_paths = [ + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-6.4.0-2.28.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-7.3.0-2.30.eb'), + os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb'), + os.path.join(test_ecs, 'o', 'OpenBLAS', 'OpenBLAS-0.2.8-GCC-4.8.2-LAPACK-3.4.2.eb') + ] self.assertEqual(paths, ref_paths) # Now do a case sensitive search paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False, case_sensitive=True) - ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2.6.10.eb')] + ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] self.assertEqual(paths, ref_paths) def suite(): From 6b43bb3a5668a645918c9dc2d6026380bd24e4eb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Dec 2018 12:15:05 +0100 Subject: [PATCH 057/344] Appease the hound --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3f677d01e6..522a320435 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -882,7 +882,7 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, regex) # Use these values to construct a dependency - software_as_dep={ + software_as_dep = { 'name': software_name, 'version': version, 'toolchain': toolchain From dba1b5b0b2cf08a0aa37e23872992c620e52ded9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Dec 2018 12:49:46 +0100 Subject: [PATCH 058/344] Style fix --- easybuild/tools/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a212191929..df0dd9ef3c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -317,9 +317,9 @@ def software_options(self): None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), - 'update-deps': ("Try to updade versions of the dependencies of an easyconfig based on what is available " - "in the robot path", - None, 'store_true', False), + 'update-deps': ("Try to update versions of the dependencies of an easyconfig based on what is available in " + "the robot path", + None, 'store_true', False), }) longopts = opts.keys() From 4290d0de1140e4d5daa445bf7b1e25ef9be44189 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Dec 2018 13:15:29 +0100 Subject: [PATCH 059/344] Modify logic order to pass failing test --- easybuild/framework/easyconfig/tweak.py | 41 ++++++++++++------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 522a320435..95908b1595 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -105,7 +105,13 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not # supported - if any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'update_deps'] for key in keys): + if not build_option('map_toolchains'): + msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, " + msg += "disabling recursion and not changing (sub)toolchains for dependencies" + _log.info(msg) + revert_to_regex = True + modifying_toolchains = False + elif any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'update_deps'] for key in keys): print_warning("Combining --try-toolchain* or --try-update-deps with other build options is not fully " + "supported: using regex") revert_to_regex = True @@ -127,26 +133,19 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): else: target_toolchain['version'] = source_toolchain['version'] - if build_option('map_toolchains'): - try: - src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) - except EasyBuildError as err: - # make sure exception was raised by match_minimum_tc_specs because toolchain mapping didn't work - if "No possible mapping from source toolchain" in err.msg: - error_msg = err.msg + '\n' - error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. " - error_msg += "(If you know what you are doing, " - error_msg += "you can use --disable-map-toolchains to proceed anyway.)" - raise EasyBuildError(error_msg, target_toolchain['name'], source_toolchain['name']) - else: - # simply re-raise the exception if something else went wrong - raise err - else: - msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, " - msg += "disabling recursion and not changing (sub)toolchains for dependencies" - _log.info(msg) - revert_to_regex = True - modifying_toolchains = False + try: + src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) + except EasyBuildError as err: + # make sure exception was raised by match_minimum_tc_specs because toolchain mapping didn't work + if "No possible mapping from source toolchain" in err.msg: + error_msg = err.msg + '\n' + error_msg += "Toolchain %s is not equivalent to toolchain %s in terms of capabilities. " + error_msg += "(If you know what you are doing, " + error_msg += "you can use --disable-map-toolchains to proceed anyway.)" + raise EasyBuildError(error_msg, target_toolchain['name'], source_toolchain['name']) + else: + # simply re-raise the exception if something else went wrong + raise err if not revert_to_regex: _log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs) From f932af420a05b2be9c9a40872c8f5838b918f171 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Dec 2018 13:18:30 +0100 Subject: [PATCH 060/344] Careful of renamed variable --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 95908b1595..2bf91de25e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -110,7 +110,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): msg += "disabling recursion and not changing (sub)toolchains for dependencies" _log.info(msg) revert_to_regex = True - modifying_toolchains = False + modifying_toolchains_or_deps = False elif any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'update_deps'] for key in keys): print_warning("Combining --try-toolchain* or --try-update-deps with other build options is not fully " + "supported: using regex") From 611a2edf4deebd674b78adf82587480052208e3c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 11 Dec 2018 13:19:50 +0100 Subject: [PATCH 061/344] Only include assignment if we are making a change --- easybuild/framework/easyconfig/tweak.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 2bf91de25e..b464644082 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -110,7 +110,6 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): msg += "disabling recursion and not changing (sub)toolchains for dependencies" _log.info(msg) revert_to_regex = True - modifying_toolchains_or_deps = False elif any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'update_deps'] for key in keys): print_warning("Combining --try-toolchain* or --try-update-deps with other build options is not fully " + "supported: using regex") From ac0ebaa7c2377ef82556f2657e867701d4694a9f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 14 Dec 2018 10:55:59 +0100 Subject: [PATCH 062/344] Use spec to filter out toolchain components --- easybuild/framework/easyconfig/tweak.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index b464644082..ec00a77552 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -157,13 +157,13 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) toolchain_ec = process_easyconfig(path) toolchain_deps = resolve_dependencies(toolchain_ec, modtool, retain_all_deps=True) - toolchain_dep_names = [dep['ec']['name'] for dep in toolchain_deps if + toolchain_dep_specs = [dep['spec'] for dep in toolchain_deps if dep['spec'] not in listed_ec_paths] else: - toolchain_dep_names = [] + toolchain_dep_specs = [] i = 0 while i < len(orig_ecs): - if orig_ecs[i]['ec']['name'] in toolchain_dep_names: + if orig_ecs[i]['spec'] in toolchain_dep_specs: # drop elements in toolchain hierarchy del orig_ecs[i] else: From ed8cb328d1ef14b0288320dbec17c360f72c996e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 14 Dec 2018 12:03:57 +0100 Subject: [PATCH 063/344] Fix typo --- easybuild/framework/easyconfig/format/one.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 7e505deb4a..6795bd205a 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -67,7 +67,7 @@ def dump_dependency(dep, toolchain): if dep['external_module']: res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name']) else: - # mininal spec: (name, version) + # minimal spec: (name, version) tup = (dep['name'], dep['version']) if dep['toolchain'] != toolchain: if dep['dummy']: From afe01295c87def83b18a9ce42f05254b7447279c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 16 Jan 2019 09:29:51 +0100 Subject: [PATCH 064/344] Fix print statement --- easybuild/tools/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index eee2d88874..b2d6242166 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -487,7 +487,7 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con % cnt, ]) - print '\n'.join(lines) + print('\n'.join(lines)) # if requested return the matches as a list if build_option('consider_archived_easyconfigs'): From 0d2f9767a21091cd3ad4c747ccc7ec103eaf954a Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:03:23 +0100 Subject: [PATCH 065/344] Add a contrib/hooks dir with some examples of hooks used. --- contrib/hooks/README.rst | 20 +++ contrib/hooks/add_delete_configopt.py | 29 ++++ contrib/hooks/hpc2n_hooks.py | 206 ++++++++++++++++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 contrib/hooks/README.rst create mode 100644 contrib/hooks/add_delete_configopt.py create mode 100644 contrib/hooks/hpc2n_hooks.py diff --git a/contrib/hooks/README.rst b/contrib/hooks/README.rst new file mode 100644 index 0000000000..42e699443b --- /dev/null +++ b/contrib/hooks/README.rst @@ -0,0 +1,20 @@ +.. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png + :align: center + +`EasyBuild `_ is a software build +and installation framework that allows you to manage (scientific) software +on High Performance Computing (HPC) systems in an efficient way. + +The **easybuild-framework** package is the core of EasyBuild. It +supports the implementation and use of so-called easyblocks which +implement the software install procedure for a particular (group of) software +package(s). + +The EasyBuild documentation is available at http://easybuild.readthedocs.org/. + +The EasyBuild framework source code is hosted on GitHub, along +with an issue tracker for bug reports and feature requests, see +https://github.com/easybuilders/easybuild-framework. + +This directory, contrib/hooks, contain examples of hooks used at various +sites and also a couple of small examples with explanations. diff --git a/contrib/hooks/add_delete_configopt.py b/contrib/hooks/add_delete_configopt.py new file mode 100644 index 0000000000..054d08b958 --- /dev/null +++ b/contrib/hooks/add_delete_configopt.py @@ -0,0 +1,29 @@ +# Small example of how to add/delete a configure option. + +# We need to be able to distinguish between versions of OpenMPI +from distutils.version import LooseVersion + +def pre_configure_hook(self, *args, **kwargs): + # Check that we're dealing with the correct easyconfig file + if self.name == 'OpenMPI': + extra_opts = "" + # Enable using pmi from slurm + extra_opts += "--with-pmi=/lap/slurm " + + # And enable munge for OpenMPI versions that knows about it + if LooseVersion(self.version) >= LooseVersion('2'): + extra_opts += "--with-munge " + + # Now add the options + self.log.info("[pre-configure hook] Adding %s" % extra_opts) + self.cfg.update('configopts', extra_opts) + + # Now we delete some options + # For newer versions of OpenMPI we can re-enable ucx, i.e. delete the --without-ucx flag + if LooseVersion(self.version) >= LooseVersion('2.1'): + self.log.info("[pre-configure hook] Re-enabling ucx") + self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ') + + # And we can remove the --disable-dlopen option from the easyconfig file + self.log.info("[pre-configure hook] Re-enabling dlopen") + self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py new file mode 100644 index 0000000000..da78c480f9 --- /dev/null +++ b/contrib/hooks/hpc2n_hooks.py @@ -0,0 +1,206 @@ +# Hooks for HPC2N site changes. +# +# Author: Ake Sandgren, HPC2N + +import os + +from distutils.version import LooseVersion +from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS +from easybuild.tools.filetools import apply_regex_substitutions +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import get_software_root +from easybuild.tools.systemtools import get_shared_lib_ext + +# Add/remove dependencies and/or patches +# Access to the raw values before templating and such. +def parse_hook(ec): + + # Internal helper function + def add_extra_dependencies(ec, dep_type, extra_deps): + """dep_type: must be in DEPENDENCY_PARAMETERS or 'osdependencies'""" + ec.log.info("[parse hook] Adding %s: %s" % (dep_type, extra_deps)) + + if dep_type in DEPENDENCY_PARAMETERS: + for dep in extra_deps: + ec[dep_type].append(dep) + elif dep_type == 'osdependencies': + if isinstance(extra_deps, tuple): + ec[dep_type].append(extra_deps) + else: + raise EasyBuildError("parse_hook: Type of extra_deps argument (%s), for 'osdependencies' must be tuple, found %s" % (extra_deps, type(extra_deps))) + else: + raise EasyBuildError("parse_hook: Incorrect dependency type in add_extra_dependencies: %s" % dep_type) + + extra_deps = [] + + if ec.name == 'OpenMPI': + if LooseVersion(ec.version) >= LooseVersion('2') and LooseVersion(ec.version) < LooseVersion('2.1.2'): + ec.log.info("[parse hook] Adding pmi and lustre patches") + if LooseVersion(ec.version) < LooseVersion('2.1.1'): + ec['patches'].append('OpenMPI-2.0.0_fix_bad-include_of_pmi_h.patch') + + if LooseVersion(ec.version) < LooseVersion('2.0.2'): + ec['patches'].append('OpenMPI-2.0.1_fix_lustre.patch') + elif LooseVersion(ec.version) < LooseVersion('2.1'): + ec['patches'].append('OpenMPI-2.0.2_fix_lustre.patch') + elif LooseVersion(ec.version) < LooseVersion('2.1.1'): + ec['patches'].append('OpenMPI-2.1.0_fix_lustre.patch') + else: + ec['patches'].append('OpenMPI-2.1.1_fix_lustre.patch') + + if LooseVersion(ec.version) == LooseVersion('4.0.0'): + ec['patches'].append('OpenMPI-4.0.0_fix_configure_bug.patch') + + if LooseVersion(ec.version) >= LooseVersion('2.1'): + pmix_version = '1.2.5' + ucx_version = '1.4.0' + if LooseVersion(ec.version) >= LooseVersion('3'): + pmix_version = '2.2.1' + if LooseVersion(ec.version) >= LooseVersion('4'): + pmix_version = '3.0.2' # OpenMPI 4.0.0 is not compatible with PMIx 3.1.x + + extra_deps.append(('PMIx', pmix_version)) + # Use of external PMIx requires external libevent + # But PMIx already has it as a dependency so we don't need + # to explicitly set it. + + extra_deps.append(('UCX', ucx_version)) + + if ec.name == 'impi': + pmix_version = '3.1.1' + extra_deps.append(('PMIx', pmix_version)) + + if extra_deps: + add_extra_dependencies(ec, 'dependencies', extra_deps) + + + +def pre_configure_hook(self, *args, **kwargs): + if self.name == 'GROMACS': + # HPC2N always uses -DGMX_USE_NVML=ON on GPU builds + if get_software_root('CUDA'): + self.log.info("[pre-configure hook] Adding -DGMX_USE_NVML=ON") + self.cfg.update('configopts', "-DGMX_USE_NVML=ON ") + + if self.name == 'OpenMPI': + extra_opts = "" + # Old versions don't work with PMIx, use slurms PMI1 + if LooseVersion(self.version) < LooseVersion('2.1'): + extra_opts += "--with-pmi=/lap/slurm " + if LooseVersion(self.version) >= LooseVersion('2'): + extra_opts += "--with-munge " + + # Using PMIx dependency in easyconfig, see above + if LooseVersion(self.version) >= LooseVersion('2.1'): + if get_software_root('PMIx'): + extra_opts += "--with-pmix=$EBROOTPMIX " + # Use of external PMIx requires external libevent + # We're using the libevent that comes from the PMIx dependency + if get_software_root('libevent'): + extra_opts += "--with-libevent=$EBROOTLIBEVENT " + else: + raise EasyBuildError("Error in pre_configure_hook for OpenMPI: External use of PMIx requires external libevent, which was not found. Check parse_hook for dependency settings.") + else: + raise EasyBuildError("Error in pre_configure_hook for OpenMPI: PMIx not defined in dependencies. Check parse_hook for dependency settings.") + + if get_software_root('UCX'): + extra_opts += "--with-ucx=$EBROOTUCX " + + if LooseVersion(self.version) >= LooseVersion('2'): + extra_opts += "--with-cma " + extra_opts += "--with-lustre " + + # We still need to fix the knem package to install its + # pkg-config .pc file correctly, and we need a more generic + # install dir. + # extra_opts += "--with-knem=/opt/knem-1.1.2.90mlnx1 " + + self.log.info("[pre-configure hook] Adding %s" % extra_opts) + self.cfg.update('configopts', extra_opts) + + if LooseVersion(self.version) >= LooseVersion('2.1'): + self.log.info("[pre-configure hook] Re-enabling ucx") + self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ') + + self.log.info("[pre-configure hook] Re-enabling dlopen") + self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') + + if self.name == 'PMIx': + self.log.info("[pre-configure hook] Adding --with-munge") + self.cfg.update('configopts', "--with-munge ") + if LooseVersion(self.version) >= LooseVersion('2'): + self.log.info("[pre-configure hook] Adding --with-tests-examples") + self.cfg.update('configopts', "--with-tests-examples ") + self.log.info("[pre-configure hook] Adding --disable-per-user-config-files") + self.cfg.update('configopts', "--disable-per-user-config-files") + + +def pre_build_hook(self, *args, **kwargs): + if self.name == 'pyslurm': + self.log.info("[pre-build hook] Adding --slurm=/lap/slurm") + self.cfg.update('buildopts', "--slurm=/lap/slurm ") + + +def post_install_hook(self, *args, **kwargs): + if self.name == 'impi': + # Fix mpirun from IntelMPI to explicitly unset I_MPI_PMI_LIBRARY + # it can only be used with srun. + self.log.info("[post-install hook] Unset I_MPI_PMI_LIBRARY in mpirun") + apply_regex_substitutions(os.path.join(self.installdir, "intel64", "bin", "mpirun"), [ + (r'^(#!/bin/sh.*)$', r'\1\nunset I_MPI_PMI_LIBRARY'), + ]) + + +def pre_module_hook(self, *args, **kwargs): + if self.name == 'impi': + # Add I_MPI_PMI_LIBRARY to module for IntelMPI so it works with + # srun. + self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module") + # Must be done this way, updating self.cfg['modextravars'] + # directly doesn't work due to templating. + en_templ = self.cfg.enable_templating + self.cfg.enable_templating = False + shlib_ext = get_shared_lib_ext() + pmix_root = get_software_root('PMIx') + if pmix_root: + mpi_type = 'pmix_v3' + self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext)}) + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) + else: + self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) + self.cfg.enable_templating = en_templ + + if self.name == 'OpenBLAS': + self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module") + self.cfg.update('modluafooter', 'if ((mode() == "load" and os.getenv("OMP_NUM_THREADS") == nil) or (mode() == "unload" and os.getenv("__OpenBLAS_set_OMP_NUM_THREADS") == "1")) then setenv("OMP_NUM_THREADS","1"); setenv("__OpenBLAS_set_OMP_NUM_THREADS", "1") end') + + if self.name == 'OpenMPI': + if LooseVersion(self.version) < LooseVersion('2.1'): + mpi_type = 'openmpi' + elif LooseVersion(self.version) < LooseVersion('3'): + mpi_type = 'pmix_v1' + elif LooseVersion(self.version) < LooseVersion('4'): + mpi_type = 'pmix_v2' + else: + mpi_type = 'pmix_v3' + + self.log.info("[pre-module hook] Set SLURM_MPI_TYPE=%s in OpenMPI module" % mpi_type) + # Must be done this way, updating self.cfg['modextravars'] + # directly doesn't work due to templating. + en_templ = self.cfg.enable_templating + self.cfg.enable_templating = False + self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) + # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. + self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) + self.cfg.enable_templating = en_templ + + if self.name == 'PMIx': + # This is a, hopefully, temporary workaround for https://github.com/pmix/pmix/issues/1114 + if LooseVersion(self.version) > LooseVersion('2') and LooseVersion(self.version) < LooseVersion('3'): + self.log.info("[pre-module hook] Set PMIX_MCA_gds=^ds21 in PMIx module") + en_templ = self.cfg.enable_templating + self.cfg.enable_templating = False + self.cfg['modextravars'].update({'PMIX_MCA_gds': '^ds21'}) + self.cfg.enable_templating = en_templ From 82d09c771801c153c419bc323714051330dcc82e Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:09:18 +0100 Subject: [PATCH 066/344] Fix leftover tabs. --- contrib/hooks/add_delete_configopt.py | 4 +-- contrib/hooks/hpc2n_hooks.py | 38 +++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/contrib/hooks/add_delete_configopt.py b/contrib/hooks/add_delete_configopt.py index 054d08b958..b187ef25c3 100644 --- a/contrib/hooks/add_delete_configopt.py +++ b/contrib/hooks/add_delete_configopt.py @@ -20,10 +20,10 @@ def pre_configure_hook(self, *args, **kwargs): # Now we delete some options # For newer versions of OpenMPI we can re-enable ucx, i.e. delete the --without-ucx flag - if LooseVersion(self.version) >= LooseVersion('2.1'): + if LooseVersion(self.version) >= LooseVersion('2.1'): self.log.info("[pre-configure hook] Re-enabling ucx") self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ') # And we can remove the --disable-dlopen option from the easyconfig file self.log.info("[pre-configure hook] Re-enabling dlopen") - self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') + self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index da78c480f9..aa99228a80 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -34,8 +34,8 @@ def add_extra_dependencies(ec, dep_type, extra_deps): extra_deps = [] if ec.name == 'OpenMPI': - if LooseVersion(ec.version) >= LooseVersion('2') and LooseVersion(ec.version) < LooseVersion('2.1.2'): - ec.log.info("[parse hook] Adding pmi and lustre patches") + if LooseVersion(ec.version) >= LooseVersion('2') and LooseVersion(ec.version) < LooseVersion('2.1.2'): + ec.log.info("[parse hook] Adding pmi and lustre patches") if LooseVersion(ec.version) < LooseVersion('2.1.1'): ec['patches'].append('OpenMPI-2.0.0_fix_bad-include_of_pmi_h.patch') @@ -48,10 +48,10 @@ def add_extra_dependencies(ec, dep_type, extra_deps): else: ec['patches'].append('OpenMPI-2.1.1_fix_lustre.patch') - if LooseVersion(ec.version) == LooseVersion('4.0.0'): + if LooseVersion(ec.version) == LooseVersion('4.0.0'): ec['patches'].append('OpenMPI-4.0.0_fix_configure_bug.patch') - if LooseVersion(ec.version) >= LooseVersion('2.1'): + if LooseVersion(ec.version) >= LooseVersion('2.1'): pmix_version = '1.2.5' ucx_version = '1.4.0' if LooseVersion(ec.version) >= LooseVersion('3'): @@ -78,20 +78,20 @@ def add_extra_dependencies(ec, dep_type, extra_deps): def pre_configure_hook(self, *args, **kwargs): if self.name == 'GROMACS': # HPC2N always uses -DGMX_USE_NVML=ON on GPU builds - if get_software_root('CUDA'): - self.log.info("[pre-configure hook] Adding -DGMX_USE_NVML=ON") - self.cfg.update('configopts', "-DGMX_USE_NVML=ON ") + if get_software_root('CUDA'): + self.log.info("[pre-configure hook] Adding -DGMX_USE_NVML=ON") + self.cfg.update('configopts', "-DGMX_USE_NVML=ON ") if self.name == 'OpenMPI': extra_opts = "" # Old versions don't work with PMIx, use slurms PMI1 - if LooseVersion(self.version) < LooseVersion('2.1'): + if LooseVersion(self.version) < LooseVersion('2.1'): extra_opts += "--with-pmi=/lap/slurm " if LooseVersion(self.version) >= LooseVersion('2'): extra_opts += "--with-munge " # Using PMIx dependency in easyconfig, see above - if LooseVersion(self.version) >= LooseVersion('2.1'): + if LooseVersion(self.version) >= LooseVersion('2.1'): if get_software_root('PMIx'): extra_opts += "--with-pmix=$EBROOTPMIX " # Use of external PMIx requires external libevent @@ -106,7 +106,7 @@ def pre_configure_hook(self, *args, **kwargs): if get_software_root('UCX'): extra_opts += "--with-ucx=$EBROOTUCX " - if LooseVersion(self.version) >= LooseVersion('2'): + if LooseVersion(self.version) >= LooseVersion('2'): extra_opts += "--with-cma " extra_opts += "--with-lustre " @@ -118,17 +118,17 @@ def pre_configure_hook(self, *args, **kwargs): self.log.info("[pre-configure hook] Adding %s" % extra_opts) self.cfg.update('configopts', extra_opts) - if LooseVersion(self.version) >= LooseVersion('2.1'): + if LooseVersion(self.version) >= LooseVersion('2.1'): self.log.info("[pre-configure hook] Re-enabling ucx") self.cfg['configopts'] = self.cfg['configopts'].replace('--without-ucx', ' ') self.log.info("[pre-configure hook] Re-enabling dlopen") - self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') + self.cfg['configopts'] = self.cfg['configopts'].replace('--disable-dlopen', ' ') if self.name == 'PMIx': self.log.info("[pre-configure hook] Adding --with-munge") self.cfg.update('configopts', "--with-munge ") - if LooseVersion(self.version) >= LooseVersion('2'): + if LooseVersion(self.version) >= LooseVersion('2'): self.log.info("[pre-configure hook] Adding --with-tests-examples") self.cfg.update('configopts', "--with-tests-examples ") self.log.info("[pre-configure hook] Adding --disable-per-user-config-files") @@ -156,10 +156,10 @@ def pre_module_hook(self, *args, **kwargs): # Add I_MPI_PMI_LIBRARY to module for IntelMPI so it works with # srun. self.log.info("[pre-module hook] Set I_MPI_PMI_LIBRARY in impi module") - # Must be done this way, updating self.cfg['modextravars'] - # directly doesn't work due to templating. - en_templ = self.cfg.enable_templating - self.cfg.enable_templating = False + # Must be done this way, updating self.cfg['modextravars'] + # directly doesn't work due to templating. + en_templ = self.cfg.enable_templating + self.cfg.enable_templating = False shlib_ext = get_shared_lib_ext() pmix_root = get_software_root('PMIx') if pmix_root: @@ -170,14 +170,14 @@ def pre_module_hook(self, *args, **kwargs): self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) else: self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': "/lap/slurm/lib/libpmi.so"}) - self.cfg.enable_templating = en_templ + self.cfg.enable_templating = en_templ if self.name == 'OpenBLAS': self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module") self.cfg.update('modluafooter', 'if ((mode() == "load" and os.getenv("OMP_NUM_THREADS") == nil) or (mode() == "unload" and os.getenv("__OpenBLAS_set_OMP_NUM_THREADS") == "1")) then setenv("OMP_NUM_THREADS","1"); setenv("__OpenBLAS_set_OMP_NUM_THREADS", "1") end') if self.name == 'OpenMPI': - if LooseVersion(self.version) < LooseVersion('2.1'): + if LooseVersion(self.version) < LooseVersion('2.1'): mpi_type = 'openmpi' elif LooseVersion(self.version) < LooseVersion('3'): mpi_type = 'pmix_v1' From 740125c834f1c13c3d1ceab80036b3db59d12b0d Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:10:40 +0100 Subject: [PATCH 067/344] Delete blank line --- contrib/hooks/hpc2n_hooks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index aa99228a80..f6b96d45ab 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -74,7 +74,6 @@ def add_extra_dependencies(ec, dep_type, extra_deps): add_extra_dependencies(ec, 'dependencies', extra_deps) - def pre_configure_hook(self, *args, **kwargs): if self.name == 'GROMACS': # HPC2N always uses -DGMX_USE_NVML=ON on GPU builds From fec8dd0190c61409ec31fd270fcd1025671307a8 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:21:42 +0100 Subject: [PATCH 068/344] Add missing blank line. --- contrib/hooks/add_delete_configopt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/hooks/add_delete_configopt.py b/contrib/hooks/add_delete_configopt.py index b187ef25c3..9a79a9bcc9 100644 --- a/contrib/hooks/add_delete_configopt.py +++ b/contrib/hooks/add_delete_configopt.py @@ -1,5 +1,6 @@ # Small example of how to add/delete a configure option. + # We need to be able to distinguish between versions of OpenMPI from distutils.version import LooseVersion From 86e923d6050a24c974df4aa11253924e8f09af35 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:22:01 +0100 Subject: [PATCH 069/344] Try to fix all the "too long line"s --- contrib/hooks/hpc2n_hooks.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index f6b96d45ab..d3853f7e6c 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -27,7 +27,8 @@ def add_extra_dependencies(ec, dep_type, extra_deps): if isinstance(extra_deps, tuple): ec[dep_type].append(extra_deps) else: - raise EasyBuildError("parse_hook: Type of extra_deps argument (%s), for 'osdependencies' must be tuple, found %s" % (extra_deps, type(extra_deps))) + raise EasyBuildError("parse_hook: Type of extra_deps argument (%s), for 'osdependencies' must be " + "tuple, found %s" % (extra_deps, type(extra_deps))) else: raise EasyBuildError("parse_hook: Incorrect dependency type in add_extra_dependencies: %s" % dep_type) @@ -98,9 +99,12 @@ def pre_configure_hook(self, *args, **kwargs): if get_software_root('libevent'): extra_opts += "--with-libevent=$EBROOTLIBEVENT " else: - raise EasyBuildError("Error in pre_configure_hook for OpenMPI: External use of PMIx requires external libevent, which was not found. Check parse_hook for dependency settings.") + raise EasyBuildError("Error in pre_configure_hook for OpenMPI: External use of PMIx requires " + "external libevent, which was not found. " + "Check parse_hook for dependency settings.") else: - raise EasyBuildError("Error in pre_configure_hook for OpenMPI: PMIx not defined in dependencies. Check parse_hook for dependency settings.") + raise EasyBuildError("Error in pre_configure_hook for OpenMPI: PMIx not defined in dependencies. " + "Check parse_hook for dependency settings.") if get_software_root('UCX'): extra_opts += "--with-ucx=$EBROOTUCX " @@ -173,7 +177,9 @@ def pre_module_hook(self, *args, **kwargs): if self.name == 'OpenBLAS': self.log.info("[pre-module hook] Set OMP_NUM_THREADS=1 in OpenBLAS module") - self.cfg.update('modluafooter', 'if ((mode() == "load" and os.getenv("OMP_NUM_THREADS") == nil) or (mode() == "unload" and os.getenv("__OpenBLAS_set_OMP_NUM_THREADS") == "1")) then setenv("OMP_NUM_THREADS","1"); setenv("__OpenBLAS_set_OMP_NUM_THREADS", "1") end') + self.cfg.update('modluafooter', 'if ((mode() == "load" and os.getenv("OMP_NUM_THREADS") == nil) ' + 'or (mode() == "unload" and os.getenv("__OpenBLAS_set_OMP_NUM_THREADS") == "1")) then ' + 'setenv("OMP_NUM_THREADS","1"); setenv("__OpenBLAS_set_OMP_NUM_THREADS", "1") end') if self.name == 'OpenMPI': if LooseVersion(self.version) < LooseVersion('2.1'): From d3d2a09874e0213f80ab4ab21d964953346bdb73 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:24:37 +0100 Subject: [PATCH 070/344] Add missing args to parse_hook. --- contrib/hooks/hpc2n_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index d3853f7e6c..461bc9ab76 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -13,7 +13,7 @@ # Add/remove dependencies and/or patches # Access to the raw values before templating and such. -def parse_hook(ec): +def parse_hook(ec, *args, **kwargs): # Internal helper function def add_extra_dependencies(ec, dep_type, extra_deps): From 37248f0ca4e55ad1cf0859067144db2be0fb3636 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:27:36 +0100 Subject: [PATCH 071/344] Fix one more too long line. --- contrib/hooks/hpc2n_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index 461bc9ab76..9a85e5aed4 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -167,7 +167,9 @@ def pre_module_hook(self, *args, **kwargs): pmix_root = get_software_root('PMIx') if pmix_root: mpi_type = 'pmix_v3' - self.cfg['modextravars'].update({'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext)}) + self.cfg['modextravars'].update({ + 'I_MPI_PMI_LIBRARY': os.path.join(pmix_root, "lib", "libpmi." + shlib_ext) + }) self.cfg['modextravars'].update({'SLURM_MPI_TYPE': mpi_type}) # Unfortunately UCX doesn't yet work for unknown reasons. Make sure it is off. self.cfg['modextravars'].update({'SLURM_PMIX_DIRECT_CONN_UCX': 'false'}) From 16ccd92f5edf9044f5c2f7ffe7d8890ccdba2c22 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 09:35:16 +0100 Subject: [PATCH 072/344] Put the blank line in the correct place for Hound. --- contrib/hooks/add_delete_configopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/hooks/add_delete_configopt.py b/contrib/hooks/add_delete_configopt.py index 9a79a9bcc9..ba64a9d92e 100644 --- a/contrib/hooks/add_delete_configopt.py +++ b/contrib/hooks/add_delete_configopt.py @@ -1,9 +1,9 @@ # Small example of how to add/delete a configure option. - # We need to be able to distinguish between versions of OpenMPI from distutils.version import LooseVersion + def pre_configure_hook(self, *args, **kwargs): # Check that we're dealing with the correct easyconfig file if self.name == 'OpenMPI': From 20c3219c3ae19680c4217cf6648f6db56951ee1e Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 20 Feb 2019 11:01:07 +0100 Subject: [PATCH 073/344] Add author info to add_delete_configopt.py --- contrib/hooks/add_delete_configopt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/hooks/add_delete_configopt.py b/contrib/hooks/add_delete_configopt.py index ba64a9d92e..b349bcbdee 100644 --- a/contrib/hooks/add_delete_configopt.py +++ b/contrib/hooks/add_delete_configopt.py @@ -1,4 +1,6 @@ # Small example of how to add/delete a configure option. +# +# Author: Åke Sandgren, HPC2N # We need to be able to distinguish between versions of OpenMPI from distutils.version import LooseVersion From 5f823ad388f9fb012cb179230cd1565be5d281af Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 21 Feb 2019 08:19:01 +0100 Subject: [PATCH 074/344] Shorten the README in contrib/hooks. --- contrib/hooks/README.rst | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/contrib/hooks/README.rst b/contrib/hooks/README.rst index 42e699443b..3821474a4c 100644 --- a/contrib/hooks/README.rst +++ b/contrib/hooks/README.rst @@ -1,20 +1,9 @@ .. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png :align: center -`EasyBuild `_ is a software build -and installation framework that allows you to manage (scientific) software -on High Performance Computing (HPC) systems in an efficient way. +https://easybuild.readthedocs.io -The **easybuild-framework** package is the core of EasyBuild. It -supports the implementation and use of so-called easyblocks which -implement the software install procedure for a particular (group of) software -package(s). +This directory contain examples of hooks used at various sites and also +a couple of small examples with explanations. -The EasyBuild documentation is available at http://easybuild.readthedocs.org/. - -The EasyBuild framework source code is hosted on GitHub, along -with an issue tracker for bug reports and feature requests, see -https://github.com/easybuilders/easybuild-framework. - -This directory, contrib/hooks, contain examples of hooks used at various -sites and also a couple of small examples with explanations. +See https://easybuild.readthedocs.io/en/latest/Hooks.html for documentation on hooks. From 203fcc9099260aeca42a386cfb4f92749845b88e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 24 Feb 2019 10:34:44 +0100 Subject: [PATCH 075/344] enhance search_file test to verify case-sensitive searching --- test/framework/filetools.py | 9 +++++++++ test/framework/robot.py | 1 + 2 files changed, 10 insertions(+) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 9788a42398..bb51befdb3 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1504,6 +1504,15 @@ def test_search_file(self): self.assertTrue(hits[3].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb')) self.assertTrue(hits[4].endswith('/hwloc-1.8-gcccuda-2018a.eb')) + # also test case-sensitive searching + var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True) + self.assertEqual(var_defs, []) + self.assertEqual(hits_bis, []) + + var_defs, hits_bis = ft.search_file([test_ecs], 'hwloc', silent=True, case_sensitive=True) + self.assertEqual(var_defs, []) + self.assertEqual(hits_bis, hits) + # check filename-only mode var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True) self.assertEqual(var_defs, []) diff --git a/test/framework/robot.py b/test/framework/robot.py index 3828233442..cbad100d19 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1472,6 +1472,7 @@ def test_search_easyconfigs(self): ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] self.assertEqual(paths, ref_paths) + def suite(): """ returns all the testcases in this module """ return TestLoaderFiltered().loadTestsFromTestCase(RobotTest, sys.argv[1:]) From 3a9f17f4703b4736d4d6444f991a6db2cd2d0f9f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 24 Feb 2019 12:42:40 +0100 Subject: [PATCH 076/344] style fixes + code cleanup --- easybuild/framework/easyconfig/tweak.py | 130 +++++++++++++----------- test/framework/tweak.py | 48 +++++---- 2 files changed, 96 insertions(+), 82 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3755a63d53..3f76454d77 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -50,6 +50,7 @@ from easybuild.framework.easyconfig.easyconfig import get_toolchain_hierarchy, ActiveMNS from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS +from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option @@ -82,7 +83,7 @@ def ec_filename_for(path): def tweak(easyconfigs, build_specs, modtool, targetdirs=None): """Tweak list of easyconfigs according to provided build specifications.""" # keep track of originally listed easyconfigs (via their path) - listed_ec_paths = [ec['spec'] for ec in easyconfigs] + listed_ec_filenames = [os.path.basename(ec['spec']) for ec in easyconfigs] tweaked_ecs_path, tweaked_ecs_deps_path = None, None if targetdirs is not None: @@ -147,7 +148,6 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # simply re-raise the exception if something else went wrong raise err - if not revert_to_regex: _log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs) orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) @@ -158,13 +158,15 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) toolchain_ec = process_easyconfig(path) toolchain_deps = resolve_dependencies(toolchain_ec, modtool, retain_all_deps=True) - toolchain_dep_specs = [dep['spec'] for dep in toolchain_deps if - dep['spec'] not in listed_ec_paths] + toolchain_dep_filenames = [os.path.basename(dep['spec']) for dep in toolchain_deps] + # only retain toolchain dependencies that are not in original list of easyconfigs to tweak + toolchain_dep_filenames = [td for td in toolchain_dep_filenames if td not in listed_ec_filenames] else: - toolchain_dep_specs = [] + toolchain_dep_filenames = [] + i = 0 while i < len(orig_ecs): - if orig_ecs[i]['spec'] in toolchain_dep_specs: + if os.path.basename(orig_ecs[i]['spec']) in toolchain_dep_filenames: # drop elements in toolchain hierarchy del orig_ecs[i] else: @@ -190,7 +192,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): new_ec_file = None verification_build_specs = copy.copy(build_specs) - if orig_ec['spec'] in listed_ec_paths: + if os.path.basename(orig_ec['spec']) in listed_ec_filenames: if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, @@ -848,7 +850,7 @@ def cache_aware_func(software_name, original_toolchain, toolchain_mapping): @map_versionsuffixes_cache def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapping): """ - Create a mapping of common versionssuffixes (like `-Python-%(pyvers)`) between toolchains + Create a mapping of common versionssuffixes (like `-Python-%(pyver)s`) between toolchains :param software_name: Name of software :param original_toolchain: original toolchain @@ -859,32 +861,30 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp versionsuffix_mappings = {} - # Find all Python versions in the original toolchain hierarchy and register what they would be mapped to + # Find all versions in the original toolchain hierarchy and register what they would be mapped to for toolchain in orig_toolchain_hierarchy: prefix_stub = '%s-' % software_name cand_paths, toolchain_suffix = get_matching_easyconfig_candidates(prefix_stub, toolchain) for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - try: - regex = '%s(.*)%s' % (prefix_stub, toolchain_suffix) - version = re.search(regex, filename).group(1) + + version, versionsuffix = fetch_parameters_from_easyconfig(read_file(path), ['version', 'versionsuffix']) + + if version is None: + raise EasyBuildError("Failed to extract 'version' value from %s", path) + else: major_version = version.split('.')[0] try: # make sure we have a have an integer value for the major version int(major_version) except ValueError: - _log.info("Cannot extract major version for %s from %s, probably a name clash", prefix_stub, - filename) - except AttributeError: - raise EasyBuildError("Somethings wrong, could not extract version from %s using %s", filename, - regex) + _log.info("Cannot extract major version for %s from %s", prefix_stub, version) + # Use these values to construct a dependency software_as_dep = { 'name': software_name, + 'toolchain': toolchain, 'version': version, - 'toolchain': toolchain + 'versionsuffix': versionsuffix or '', } # See what this dep would be mapped to version_matches = find_potential_version_mappings(software_as_dep, toolchain_mapping) @@ -916,7 +916,7 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain): if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: toolchain_suffix = EB_FORMAT_EXTENSION else: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) + toolchain_suffix = '-%s-%s' % (toolchain['name'], toolchain['version']) regex_search_query = '^%s.*' % prefix_stub + toolchain_suffix cand_paths = search_easyconfigs(regex_search_query, consider_extra_paths=False, print_result=False, case_sensitive=True) @@ -930,6 +930,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= :param ec_spec: Location of original easyconfig file :param toolchain_mapping: Mapping between source toolchain and target toolchain :param targetdir: Directory to dump the modified easyconfig file in + :param update_dep_versions: boolean indicating whether dependency versions should be updated :return: Location of the modified easyconfig file """ @@ -954,26 +955,31 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # loop over a *copy* of dependency dicts (with resolved templates); # to update the original dep dict, we need to index with idx into self._config[key][0]... for idx, dep in enumerate(parsed_ec['ec'][key]): + # reference to original dep dict, this is the one we should be updating orig_dep = parsed_ec['ec']._config[key][0][idx] + # skip dependencies that are marked as external modules if dep['external_module']: continue dep_tc_name = dep['toolchain']['name'] if dep_tc_name in toolchain_mapping: orig_dep['toolchain'] = toolchain_mapping[dep_tc_name] - # Replace the binutils version (if necessary) + dep_changed = False + + # replace the binutils version (if necessary) if 'binutils' in toolchain_mapping and (dep['name'] == 'binutils' and dep_tc_name == GCCcore.NAME): orig_dep.update(toolchain_mapping['binutils']) dep_changed = True + elif update_dep_versions: # Search for available updates for this dependency: # First get highest version candidate paths for this (include search through subtoolchains) potential_version_matches = find_potential_version_mappings(dep, toolchain_mapping, - versonsuffix_mapping=versonsuffix_mapping) - # Only highest version match is retained by default in potential_version_matches, compare that version - # to the original version and replace if appropriate (upgrades only). + versionsuffix_mapping=versonsuffix_mapping) + # Only highest version match is retained by default in potential_version_matches, + # compare that version to the original version and replace if appropriate (upgrades only). if potential_version_matches: highest_version_match = potential_version_matches[0]['version'] if LooseVersion(highest_version_match) > LooseVersion(dep['version']): @@ -1003,17 +1009,18 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= return tweaked_spec -def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping={}, highest_versions_only=True): +def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True): """ Find potential version mapping for a dependency in a new hierarchy - :param dep: dependency + :param dep: dependency specification (dict) :param toolchain_mapping: toolchain mapping used for search - :param versonsuffix_mapping: mapping of version suffixes (required by software with a special version suffix, such - as python packages) + :param versionsuffix_mapping: mapping of version suffixes + (required by software with a special version suffix, such as Python packages) + :param highest_versions_only: only return highest versions :return: list of dependencies that match """ - - potential_version_matches = [] + if versionsuffix_mapping is None: + versionsuffix_mapping = {} # Find the target toolchain and create the hierarchy to search within dep_tc_name = dep['toolchain']['name'] @@ -1022,15 +1029,18 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping else: # dummy search_toolchain = dep['toolchain'] + toolchain_hierarchy = get_toolchain_hierarchy(search_toolchain) - # Figure out what precedes the version + + # Figure out what precedes the version (i.e. name + versionprefix (if any)) versionprefix = dep.get('versionprefix', '') - prefix_to_version = ''.join([x for x in [dep['name'] + '-', versionprefix] if x]) + prefix_to_version = dep['name'] + '-' + versionprefix + # Figure out the main versionsuffix (altered depending on toolchain in the loop below) versionsuffix = dep.get('versionsuffix', '') # If versionsuffix is in our mapping then we expect it to be updated - if versionsuffix in versonsuffix_mapping: - versionsuffix = versonsuffix_mapping[versionsuffix] + if versionsuffix in versionsuffix_mapping: + versionsuffix = versionsuffix_mapping[versionsuffix] # the candidate version is a regex string, let's be conservative and search for patch upgrade first, if that doesn't # work look for a minor version upgrade and if that fails will we try a global search, i.e, a major version upgrade @@ -1040,49 +1050,45 @@ def find_potential_version_mappings(dep, toolchain_mapping, versonsuffix_mapping major_version = version_components[0] if len(version_components) > 2: # Have something like major.minor.XXX minor_version = version_components[1] - candidate_ver_list.append('%s\.%s\..*' % (major_version, minor_version)) + candidate_ver_list.append(r'%s\.%s\..*' % (major_version, minor_version)) if len(version_components) > 1: # Have at least major.minor - candidate_ver_list.append('%s\..*' % major_version) - candidate_ver_list.append('.*') # Include a major version search + candidate_ver_list.append(r'%s\..*' % major_version) + candidate_ver_list.append(r'.*') # Include a major version search + + potential_version_matches, highest_version = [], None - highest_version = None for candidate_ver in candidate_ver_list: if not potential_version_matches: for toolchain in toolchain_hierarchy: - # determine main install version based on toolchain - if toolchain['name'] != DUMMY_TOOLCHAIN_NAME: - toolchain_suffix = "-%s-%s" % (toolchain['name'], toolchain['version']) - else: - toolchain_suffix = '' - full_versionsuffix = ''.join([x for x in [toolchain_suffix, versionsuffix, EB_FORMAT_EXTENSION] - if x]) - # prepend/append version prefix/suffix - depver = ''.join([x for x in ['^', prefix_to_version, candidate_ver, full_versionsuffix] if x]) + # determine search pattern based on toolchain, version prefix/suffix & version regex + if toolchain['name'] == DUMMY_TOOLCHAIN_NAME: + toolchain_suffix = '' + else: + toolchain_suffix = '-%s-%s' % (toolchain['name'], toolchain['version']) + full_versionsuffix = toolchain_suffix + versionsuffix + EB_FORMAT_EXTENSION + depver = '^' + prefix_to_version + candidate_ver + full_versionsuffix cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, case_sensitive=True) - # Filter out easyconfigs that have been tweaked in this instance (they sit in the tempdir in a subdir - # that starts with 'tweaked_*') + + # Filter out easyconfigs that have been tweaked in this instance + # (they sit in the tempdir in a subdir that starts with 'tweaked_*') tweaked_ec_stub = os.path.join(tempfile.gettempdir(), 'tweaked_') cand_paths = [path for path in cand_paths if not path.startswith(tweaked_ec_stub)] + # Add what is left to the possibilities for path in cand_paths: - # Get the version from the path - filename = os.path.basename(path) - # Find the version sandwiched between our known values - regex = re.compile('^%s(.+?)%s' % (prefix_to_version, full_versionsuffix)) - res = regex.search(filename) - if res: - version = res.group(1) + version = fetch_parameters_from_easyconfig(read_file(path), ['version'])[0] + if version: if highest_version is None or LooseVersion(version) > LooseVersion(highest_version): highest_version = version else: - raise EasyBuildError("Failed to determine version from '%s' using regex pattern '%s'", filename, - regex.pattern) - potential_version_matches.append({'version': version, 'path': path, 'toolchain': toolchain}) + raise EasyBuildError("Failed to determine version from contents of %s", path) + + potential_version_matches.append({'path': path, 'toolchain': toolchain, 'version': version}) if highest_versions_only and highest_version is not None: potential_version_matches = [d for d in potential_version_matches if d['version'] == highest_version] - _log.debug("Found possible dependency upgrades: %s", potential_version_matches) + _log.debug("Found potential version match for %s: %s", dep, potential_version_matches) return potential_version_matches diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 3ccc3a12b9..9fc304898c 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -299,42 +299,49 @@ def test_get_matching_easyconfig_candidates(self): 'robot_path': [test_easyconfigs], }) toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} - expected_toolchain_stub = '-GCC-4.9.3-2.26' - expected_paths = [os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_stub + '.eb')] - paths, toolchain_stub = get_matching_easyconfig_candidates('gzip', toolchain) - self.assertEqual(toolchain_stub, expected_toolchain_stub) + paths, toolchain_suff = get_matching_easyconfig_candidates('gzip-', toolchain) + expected_toolchain_suff = '-GCC-4.9.3-2.26' + self.assertEqual(toolchain_suff, expected_toolchain_suff) + expected_paths = [os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.4' + expected_toolchain_suff + '.eb')] self.assertEqual(paths, expected_paths) + paths, toolchain_stub = get_matching_easyconfig_candidates('nosuchmatch', toolchain) + self.assertEqual(paths, []) + self.assertEqual(toolchain_stub, expected_toolchain_suff) + def test_map_common_versionsuffixes(self): """Test mapping between two toolchain hierarchies""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'valid_module_classes': module_classes(), 'robot_path': [test_easyconfigs], + 'silent': True, + 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() + gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} iccifort_binutils_tc = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'} + toolchain_mapping = map_toolchain_hierarchies(iccifort_binutils_tc, gcc_binutils_tc, self.modtool) possible_mappings = map_common_versionsuffixes('binutils', iccifort_binutils_tc, toolchain_mapping) - expected_mappings = {'-binutils-2.25': '-binutils-2.26'} - self.assertEqual(possible_mappings, expected_mappings) + self.assertEqual(possible_mappings, {'-binutils-2.25': '-binutils-2.26'}) # Make sure we only map upwards, here it's gzip 1.4 in gcc and 1.6 in iccifort possible_mappings = map_common_versionsuffixes('gzip', iccifort_binutils_tc, toolchain_mapping) - expected_mappings = {} - self.assertEqual(possible_mappings, expected_mappings) + self.assertEqual(possible_mappings, {}) + + # newer gzip is picked up other way around (GCC -> iccifort) toolchain_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) possible_mappings = map_common_versionsuffixes('gzip', gcc_binutils_tc, toolchain_mapping) - expected_mappings = {'-gzip-1.4': '-gzip-1.6'} - self.assertEqual(possible_mappings, expected_mappings) + self.assertEqual(possible_mappings, {'-gzip-1.4': '-gzip-1.6'}) def test_find_potential_version_mappings(self): """Test ability to find potential version mappings of a dependency for a given toolchain mapping""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ - 'valid_module_classes': module_classes(), 'robot_path': [test_easyconfigs], + 'silent': True, + 'valid_module_classes': module_classes(), }) get_toolchain_hierarchy.clear() @@ -344,8 +351,11 @@ def test_find_potential_version_mappings(self): tc_mapping = map_toolchain_hierarchies(gcc_binutils_tc, iccifort_binutils_tc, self.modtool) ec_spec = os.path.join(test_easyconfigs, 'h', 'hwloc', 'hwloc-1.6.2-GCC-4.9.3-2.26.eb') parsed_ec = process_easyconfig(ec_spec)[0] - gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'] - potential_versions = find_potential_version_mappings(gzip_dep[0], tc_mapping, []) + gzip_dep = [dep for dep in parsed_ec['ec']['dependencies'] if dep['name'] == 'gzip'][0] + self.assertEqual(gzip_dep['full_mod_name'], 'gzip/1.4-GCC-4.9.3-2.26') + + potential_versions = find_potential_version_mappings(gzip_dep, tc_mapping) + self.assertEqual(len(potential_versions), 1) # Should see version 1.6 of gzip with iccifort toolchain expected_dep_path = os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb') @@ -354,11 +364,12 @@ def test_find_potential_version_mappings(self): def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - init_config(build_options={ + build_options = { 'robot_path': [test_easyconfigs], 'silent': True, 'valid_module_classes': module_classes(), - }) + } + init_config(build_options=build_options) get_toolchain_hierarchy.clear() gcc_binutils_tc = {'name': 'GCC', 'version': '4.9.3-2.26'} @@ -378,10 +389,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): value == tweaked_dict['builddependencies'][0][key]) # Now test the case where we try to update the dependencies - init_config(build_options={ - 'valid_module_classes': module_classes(), - 'robot_path': [test_easyconfigs], - }) + init_config(build_options=build_options) get_toolchain_hierarchy.clear() tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, tc_mapping, update_dep_versions=True) tweaked_ec = process_easyconfig(tweaked_spec)[0] From 531dd7c7b770b42bb028c1a99c5454c5f7285ebb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 25 Feb 2019 09:36:08 +0100 Subject: [PATCH 077/344] potential_version_matches -> potential_version_mappings --- easybuild/framework/easyconfig/tweak.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3f76454d77..f2c4daa728 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1055,10 +1055,10 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin candidate_ver_list.append(r'%s\..*' % major_version) candidate_ver_list.append(r'.*') # Include a major version search - potential_version_matches, highest_version = [], None + potential_version_mappings, highest_version = [], None for candidate_ver in candidate_ver_list: - if not potential_version_matches: + if not potential_version_mappings: for toolchain in toolchain_hierarchy: # determine search pattern based on toolchain, version prefix/suffix & version regex @@ -1085,10 +1085,10 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin else: raise EasyBuildError("Failed to determine version from contents of %s", path) - potential_version_matches.append({'path': path, 'toolchain': toolchain, 'version': version}) + potential_version_mappings.append({'path': path, 'toolchain': toolchain, 'version': version}) if highest_versions_only and highest_version is not None: - potential_version_matches = [d for d in potential_version_matches if d['version'] == highest_version] + potential_version_mappings = [d for d in potential_version_mappings if d['version'] == highest_version] - _log.debug("Found potential version match for %s: %s", dep, potential_version_matches) - return potential_version_matches + _log.debug("Found potential version mappings for %s: %s", dep, potential_version_mappings) + return potential_version_mappings From 67f62ccbc56ad560ab5d418ac6c2eef1157091b0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 25 Feb 2019 10:45:34 +0100 Subject: [PATCH 078/344] Address some of the comments --- easybuild/framework/easyconfig/tweak.py | 20 ++++++++++---------- easybuild/tools/options.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index f2c4daa728..3b58ec73b8 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -51,6 +51,7 @@ from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths from easybuild.toolchains.gcccore import GCCcore from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option @@ -152,7 +153,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) - # We can leave any dependencies they may have as they will only be used if required (or originally listed) + # Also filter any dependencies of the hierarchy (unless they were originally listed for tweaking) _log.debug("Filtering out toolchain hierarchy and dependencies for %s", source_toolchain) if source_toolchain['name'] != DUMMY_TOOLCHAIN_NAME: path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) @@ -976,17 +977,17 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= elif update_dep_versions: # Search for available updates for this dependency: # First get highest version candidate paths for this (include search through subtoolchains) - potential_version_matches = find_potential_version_mappings(dep, toolchain_mapping, + potential_version_mappings = find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=versonsuffix_mapping) - # Only highest version match is retained by default in potential_version_matches, + # Only highest version match is retained by default in potential_version_mappings, # compare that version to the original version and replace if appropriate (upgrades only). - if potential_version_matches: - highest_version_match = potential_version_matches[0]['version'] + if potential_version_mappings: + highest_version_match = potential_version_mappings[0]['version'] if LooseVersion(highest_version_match) > LooseVersion(dep['version']): _log.info("Updating version of %s dependency from %s to %s", dep['name'], dep['version'], highest_version_match) _log.info("Depending on your configuration, this will be resolved with one of the following " - "easyconfigs: \n%s", '\n'.join(cand['path'] for cand in potential_version_matches)) + "easyconfigs: \n%s", '\n'.join(cand['path'] for cand in potential_version_mappings)) orig_dep['version'] = highest_version_match if orig_dep['versionsuffix'] in versonsuffix_mapping: dep['versionsuffix'] = versonsuffix_mapping[orig_dep['versionsuffix']] @@ -1071,10 +1072,9 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, case_sensitive=True) - # Filter out easyconfigs that have been tweaked in this instance - # (they sit in the tempdir in a subdir that starts with 'tweaked_*') - tweaked_ec_stub = os.path.join(tempfile.gettempdir(), 'tweaked_') - cand_paths = [path for path in cand_paths if not path.startswith(tweaked_ec_stub)] + # Filter out easyconfigs that have been tweaked in this instance, they are not relevant here + tweaked_ecs_paths, _ = alt_easyconfig_paths(tempfile.gettempdir(), tweaked_ecs=True) + cand_paths = [path for path in cand_paths if not path.startswith(tweaked_ecs_paths)] # Add what is left to the possibilities for path in cand_paths: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 867be47b95..878e78af79 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -318,9 +318,6 @@ def software_options(self): None, 'store', None, {'metavar': 'NAME'}), 'toolchain-version': ("Search and build with given toolchain version", None, 'store', None, {'metavar': 'VERSION'}), - 'update-deps': ("Try to update versions of the dependencies of an easyconfig based on what is available in " - "the robot path", - None, 'store_true', False), }) longopts = opts.keys() @@ -331,6 +328,9 @@ def software_options(self): opts['map-toolchains'] = ("Enable mapping of (sub)toolchains when --try-toolchain(-version) is used", None, 'store_true', True) + opts['try-update-deps'] = ("Try to update versions of the dependencies of an easyconfig based on what is " + "available in the robot path", + None, 'store_true', False) self.log.debug("software_options: descr %s opts %s" % (descr, opts)) self.add_group_parser(opts, descr) From 133b90b63e3689d567eff373ecd237d14ddb99bc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 25 Feb 2019 10:46:48 +0100 Subject: [PATCH 079/344] Appease the hound --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3b58ec73b8..3e97cf4024 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -978,7 +978,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # Search for available updates for this dependency: # First get highest version candidate paths for this (include search through subtoolchains) potential_version_mappings = find_potential_version_mappings(dep, toolchain_mapping, - versionsuffix_mapping=versonsuffix_mapping) + versionsuffix_mapping=versonsuffix_mapping) # Only highest version match is retained by default in potential_version_mappings, # compare that version to the original version and replace if appropriate (upgrades only). if potential_version_mappings: From 38e771e4933f15ae6046015c6e0d548dc9dec6ba Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 26 Feb 2019 11:36:45 +0100 Subject: [PATCH 080/344] Update tweak.py --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 3e97cf4024..a3d6b6f8b5 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -83,7 +83,7 @@ def ec_filename_for(path): def tweak(easyconfigs, build_specs, modtool, targetdirs=None): """Tweak list of easyconfigs according to provided build specifications.""" - # keep track of originally listed easyconfigs (via their path) + # keep track of originally listed easyconfigs (via their filename) listed_ec_filenames = [os.path.basename(ec['spec']) for ec in easyconfigs] tweaked_ecs_path, tweaked_ecs_deps_path = None, None From 251d2e19c77699fe8cc877603b9446882daac2c7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 26 Feb 2019 17:30:31 +0100 Subject: [PATCH 081/344] Revert to using full paths to distinguish files (since contents of files given on command line may differ from filename) --- easybuild/framework/easyconfig/tweak.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a3d6b6f8b5..3cf51833a6 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -83,8 +83,8 @@ def ec_filename_for(path): def tweak(easyconfigs, build_specs, modtool, targetdirs=None): """Tweak list of easyconfigs according to provided build specifications.""" - # keep track of originally listed easyconfigs (via their filename) - listed_ec_filenames = [os.path.basename(ec['spec']) for ec in easyconfigs] + # keep track of originally listed easyconfigs (via their path) + listed_ec_paths = [ec['spec'] for ec in easyconfigs] tweaked_ecs_path, tweaked_ecs_deps_path = None, None if targetdirs is not None: @@ -159,15 +159,15 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): path = robot_find_easyconfig(source_toolchain['name'], source_toolchain['version']) toolchain_ec = process_easyconfig(path) toolchain_deps = resolve_dependencies(toolchain_ec, modtool, retain_all_deps=True) - toolchain_dep_filenames = [os.path.basename(dep['spec']) for dep in toolchain_deps] + toolchain_dep_paths = [dep['spec'] for dep in toolchain_deps] # only retain toolchain dependencies that are not in original list of easyconfigs to tweak - toolchain_dep_filenames = [td for td in toolchain_dep_filenames if td not in listed_ec_filenames] + toolchain_dep_paths = [td for td in toolchain_dep_paths if td not in listed_ec_paths] else: - toolchain_dep_filenames = [] + toolchain_dep_paths = [] i = 0 while i < len(orig_ecs): - if os.path.basename(orig_ecs[i]['spec']) in toolchain_dep_filenames: + if orig_ecs[i]['spec'] in toolchain_dep_paths: # drop elements in toolchain hierarchy del orig_ecs[i] else: @@ -193,7 +193,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): new_ec_file = None verification_build_specs = copy.copy(build_specs) - if os.path.basename(orig_ec['spec']) in listed_ec_filenames: + if orig_ec['spec'] in listed_ec_paths: if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, From 5d7fd33b07fb9c7f1734b3aaa8e9e08aa4d0b3b4 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 30 Aug 2019 14:43:16 +0200 Subject: [PATCH 082/344] Appease hound --- easybuild/tools/filetools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4d6e66aa04..b4a875230f 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -636,10 +636,10 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen # compile regex, case-insensitive try: if case_sensitive: - query = re.compile(query, re.I) + query = re.compile(query, re.I) else: - # compile regex, case-insensitive - query = re.compile(query, re.I) + # compile regex, case-insensitive + query = re.compile(query, re.I) except re.error as err: raise EasyBuildError("Invalid search query: %s", err) From bce88b3d709201eea723926dcbb12ce2db2b6404 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 30 Aug 2019 18:23:30 +0200 Subject: [PATCH 083/344] Be more careful when checking merge conflicts! --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index add88e91f0..a5d0818554 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -947,7 +947,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= :return: Location of the modified easyconfig file """ # Fully parse the original easyconfig - parsed_ec = process_easyconfig(ec_spec, validate=False)[0] + parsed_ec = process_easyconfig(ec_spec, validate=False)[0]['ec'] versonsuffix_mapping = {} From 1824ccea113b2b44b5b532c14202c345996bbe1e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 30 Aug 2019 18:37:00 +0200 Subject: [PATCH 084/344] Be more careful when checking merge conflicts! --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index a5d0818554..4e9e20fbe8 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1017,7 +1017,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # Determine the name of the modified easyconfig and dump it to target_dir if parsed_ec['versionsuffix'] in versonsuffix_mapping: - parsed_ec['versionsuffix'] = versonsuffix_mapping[parsed_ec['ec']['versionsuffix']] + parsed_ec['versionsuffix'] = versonsuffix_mapping[parsed_ec['versionsuffix']] ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) From 1dd342030cdfeedea4c910495806c3227e34d848 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 2 Sep 2019 09:27:03 +0200 Subject: [PATCH 085/344] Case sensitive search was ignoring case --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b4a875230f..1d13f58760 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -636,7 +636,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen # compile regex, case-insensitive try: if case_sensitive: - query = re.compile(query, re.I) + query = re.compile(query) else: # compile regex, case-insensitive query = re.compile(query, re.I) From b03421fa3988ed8322a8d8c2abc765e9bcfe472f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 2 Sep 2019 11:06:55 +0200 Subject: [PATCH 086/344] Don't unnecessarily call out to MNS since some depend on easyconfigs existing (which may not be the case during the tweaking process) --- easybuild/framework/easyconfig/tweak.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 4e9e20fbe8..5b209ca950 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1012,8 +1012,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= dep_changed = True if dep_changed: - orig_dep['short_mod_name'] = ActiveMNS().det_short_module_name(dep) - orig_dep['full_mod_name'] = ActiveMNS().det_full_module_name(dep) + _log.debug("Modified dependency %s of %s", dep['name'], ec_spec) # Determine the name of the modified easyconfig and dump it to target_dir if parsed_ec['versionsuffix'] in versonsuffix_mapping: From 0260023793e98bbc7c095c19e5242ab4688635c4 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 16 Sep 2019 15:35:54 +0000 Subject: [PATCH 087/344] adding a locking feature that will prevent two parallel builds of the same installation directory, with an option --wait-on-lock which will cause EB to wait until the lock file is removed, checking every minute for the presence of the lock file. --- easybuild/framework/easyblock.py | 47 +++++++++++++++++++++++--------- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 2 ++ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 151d8d059e..1071f83066 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2943,20 +2943,41 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - try: - for (step_name, descr, step_methods, skippable) in steps: - if self._skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) - else: - if self.dry_run: - self.dry_run_msg("%s... [DRY RUN]\n", descr) - else: - print_msg("%s..." % descr, log=self.log, silent=self.silent) - self.current_step = step_name - self.run_step(step_name, step_methods) - except StopException: - pass + if not os.path.exists(build_path()): + mkdir(build_path()) + lockfile_name = os.path.join(build_path(),".%s.lock" % self.installdir.replace('/','_') ) + if os.path.exists(lockfile_name): + if build_options('wait_on_lock'): + while os.path.exists(lockfile_name): + print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name) + time.sleep(60) + else: + print_msg("Build aborted. Lock file %s exists." % lockfile_name) + return False + else: + try: + # create a new lock file + print_msg("Creating lock file %s" % lockfile_name) + f = open(lockfile_name,"w+") + f.close() + + for (step_name, descr, step_methods, skippable) in steps: + if self._skip_step(step_name, skippable): + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) + else: + if self.dry_run: + self.dry_run_msg("%s... [DRY RUN]\n", descr) + else: + print_msg("%s..." % descr, log=self.log, silent=self.silent) + self.current_step = step_name + self.run_step(step_name, step_methods) + + except StopException: + pass + finally: + print_msg("Removing lock file %s" % lockfile_name) + os.remove(lockfile_name) # return True for successfull build (or stopped build) return True diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 086a90d73f..ad4cc47a03 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -221,6 +221,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'use_f90cache', 'use_existing_modules', 'set_default_module', + 'wait_on_lock', ], True: [ 'cleanup_builddir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 22b1bc3b49..55a37e94cd 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -441,6 +441,8 @@ def override_options(self): None, 'store_true', False), 'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents", None, 'store_true', False), + 'wait-on-lock': ("Wait until lock file is removed when a lock if found", + None, 'store_true', False), 'zip-logs': ("Zip logs that are copied to install directory, using specified command", None, 'store_or_None', 'gzip'), From 431161520348f7b5d1f2efc23476135618003b15 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 16 Sep 2019 15:39:55 +0000 Subject: [PATCH 088/344] appeasing hound --- easybuild/framework/easyblock.py | 6 +++--- easybuild/tools/options.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1071f83066..22004fff66 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2946,9 +2946,9 @@ def run_all_steps(self, run_test_cases): if not os.path.exists(build_path()): mkdir(build_path()) - lockfile_name = os.path.join(build_path(),".%s.lock" % self.installdir.replace('/','_') ) + lockfile_name = os.path.join(build_path(), ".%s.lock" % self.installdir.replace('/','_')) if os.path.exists(lockfile_name): - if build_options('wait_on_lock'): + if build_option('wait_on_lock'): while os.path.exists(lockfile_name): print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name) time.sleep(60) @@ -2959,7 +2959,7 @@ def run_all_steps(self, run_test_cases): try: # create a new lock file print_msg("Creating lock file %s" % lockfile_name) - f = open(lockfile_name,"w+") + f = open(lockfile_name, "w+") f.close() for (step_name, descr, step_methods, skippable) in steps: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 55a37e94cd..c955c75142 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -442,7 +442,7 @@ def override_options(self): 'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents", None, 'store_true', False), 'wait-on-lock': ("Wait until lock file is removed when a lock if found", - None, 'store_true', False), + None, 'store_true', False), 'zip-logs': ("Zip logs that are copied to install directory, using specified command", None, 'store_or_None', 'gzip'), From 0d74bd05cc936a4092883c12754d3b9d41af1344 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 16 Sep 2019 19:20:37 +0000 Subject: [PATCH 089/344] adding silent mode for printed messages to hopefully make tests pass. --- easybuild/framework/easyblock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 22004fff66..7011de4d0a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2950,10 +2950,10 @@ def run_all_steps(self, run_test_cases): if os.path.exists(lockfile_name): if build_option('wait_on_lock'): while os.path.exists(lockfile_name): - print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name) + print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name, silent=self.silent) time.sleep(60) else: - print_msg("Build aborted. Lock file %s exists." % lockfile_name) + print_msg("Build aborted. Lock file %s exists." % lockfile_name, silent=self.silent) return False else: try: @@ -2976,7 +2976,7 @@ def run_all_steps(self, run_test_cases): except StopException: pass finally: - print_msg("Removing lock file %s" % lockfile_name) + print_msg("Removing lock file %s" % lockfile_name, silent=self.silent) os.remove(lockfile_name) # return True for successfull build (or stopped build) From e6bd0469f94ddd9f7103e6f9b34dc49452c1b323 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 16 Sep 2019 20:26:22 +0000 Subject: [PATCH 090/344] adding the option to specify the lock path using a global argument --- easybuild/framework/easyblock.py | 7 ++++--- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7011de4d0a..bbe14fa7cf 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2944,9 +2944,10 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - if not os.path.exists(build_path()): - mkdir(build_path()) - lockfile_name = os.path.join(build_path(), ".%s.lock" % self.installdir.replace('/','_')) + lockpath = build_option('lockpath') or build_path() + if not os.path.exists(lockpath): + mkdir(lockpath) + lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/','_')) if os.path.exists(lockfile_name): if build_option('wait_on_lock'): while os.path.exists(lockfile_name): diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ad4cc47a03..dea59c4318 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -162,6 +162,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'job_output_dir', 'job_polling_interval', 'job_target_resource', + 'lockpath', 'modules_footer', 'modules_header', 'mpi_cmd_template', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index c955c75142..52705a70ad 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -273,6 +273,7 @@ def basic_options(self): "and skipping check for OS dependencies", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), + 'lockpath': ("Specifies which path should be used to store lock files", None, 'store_or_None', None), 'missing-modules': ("Print list of missing modules for dependencies of specified easyconfigs", None, 'store_true', False, 'M'), 'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}), From e467ba39d2bc01bb0e7ebed51d37bf007c2bc0dd Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 16 Sep 2019 20:28:00 +0000 Subject: [PATCH 091/344] appeasing hound --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bbe14fa7cf..db2c445dac 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2947,7 +2947,7 @@ def run_all_steps(self, run_test_cases): lockpath = build_option('lockpath') or build_path() if not os.path.exists(lockpath): mkdir(lockpath) - lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/','_')) + lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/', '_')) if os.path.exists(lockfile_name): if build_option('wait_on_lock'): while os.path.exists(lockfile_name): From 9c725e7443e0b677eeb03db02c1f2926bf923d67 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 17 Sep 2019 13:13:44 +0000 Subject: [PATCH 092/344] made one more message silent. changed the default lockpath to use install_path --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index db2c445dac..0ec5f89085 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2944,7 +2944,7 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - lockpath = build_option('lockpath') or build_path() + lockpath = build_option('lockpath') or os.path.join(install_path('software'),'.locks') if not os.path.exists(lockpath): mkdir(lockpath) lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/', '_')) @@ -2959,7 +2959,7 @@ def run_all_steps(self, run_test_cases): else: try: # create a new lock file - print_msg("Creating lock file %s" % lockfile_name) + print_msg("Creating lock file %s" % lockfile_name, silent=self.silent) f = open(lockfile_name, "w+") f.close() From dfb9a7b4a3293cae8bcd14c823aa0ecc91658999 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 17 Sep 2019 13:15:20 +0000 Subject: [PATCH 093/344] appeasing hound --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 0ec5f89085..596684261b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2944,7 +2944,7 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - lockpath = build_option('lockpath') or os.path.join(install_path('software'),'.locks') + lockpath = build_option('lockpath') or os.path.join(install_path('software'), '.locks') if not os.path.exists(lockpath): mkdir(lockpath) lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/', '_')) From fa2a21b0c862b04eef72e292e563f9ab501cc9ba Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 10 Dec 2019 15:46:15 +0100 Subject: [PATCH 094/344] Use correct module for errors_found_in_log --- easybuild/framework/easyblock.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 256a9eeb18..89be65ebe8 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -61,7 +61,7 @@ from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH from easybuild.framework.easyconfig.tools import get_paths_for from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict -from easybuild.tools import config, filetools +from easybuild.tools import config, run from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning @@ -3082,7 +3082,7 @@ def build_and_install_one(ecdict, init_env): # restore original environment, and then sanitize it _log.info("Resetting environment") - filetools.errors_found_in_log = 0 + run.errors_found_in_log = 0 restore_env(init_env) sanitize_env() @@ -3233,9 +3233,9 @@ def build_and_install_one(ecdict, init_env): print_msg("%s: Installation %s %s (took %s)" % (summary, ended, succ, req_time), log=_log, silent=silent) # check for errors - if filetools.errors_found_in_log > 0: + if run.errors_found_in_log > 0: print_msg("WARNING: %d possible error(s) were detected in the " - "build logs, please verify the build." % filetools.errors_found_in_log, + "build logs, please verify the build." % run.errors_found_in_log, _log, silent=silent) if app.postmsg: From e9e3f3b0eeef65038058c6a2987edb734af27f2b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 11 Dec 2019 13:25:14 +0100 Subject: [PATCH 095/344] Fix keyword argument to print_msg --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 89be65ebe8..24f96ab946 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3236,7 +3236,7 @@ def build_and_install_one(ecdict, init_env): if run.errors_found_in_log > 0: print_msg("WARNING: %d possible error(s) were detected in the " "build logs, please verify the build." % run.errors_found_in_log, - _log, silent=silent) + log=_log, silent=silent) if app.postmsg: print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) From 1f01afe7a69c47a36bbdf6d6d997398a85024769 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Jan 2020 13:56:00 +0100 Subject: [PATCH 096/344] Write possible errors warning to log only --- easybuild/framework/easyblock.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 24f96ab946..47397c657b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3234,9 +3234,8 @@ def build_and_install_one(ecdict, init_env): # check for errors if run.errors_found_in_log > 0: - print_msg("WARNING: %d possible error(s) were detected in the " - "build logs, please verify the build." % run.errors_found_in_log, - log=_log, silent=silent) + _log.warning("%d possible error(s) were detected in the " + "build logs, please verify the build.", run.errors_found_in_log) if app.postmsg: print_msg("\nWARNING: %s\n" % app.postmsg, log=_log, silent=silent) From 1c51dd0c726040b39acaf3023b6329f1406b8404 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Jan 2020 20:16:53 +0100 Subject: [PATCH 097/344] bump version to 4.1.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index ca7d7a8e65..f2efa7ae04 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.1.1') +VERSION = LooseVersion('4.1.2.dev0') UNKNOWN = 'UNKNOWN' From 8331eb84bad8355b01ffe94f3c2a745305714c11 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 20 Jan 2020 20:06:55 +0100 Subject: [PATCH 098/344] also mention working directory + input passed via stdin (if any) in trace output of run_cmd --- easybuild/tools/run.py | 4 ++++ test/framework/run.py | 33 +++++++++++++++++++++++++++------ test/framework/toy_build.py | 9 ++++++++- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index 7e22e8c0ad..a3471abc04 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -175,6 +175,9 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True if trace: trace_txt = "running command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') + trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) + if inp: + trace_txt += "\t[input: %s]\n" % inp trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd_msg) @@ -300,6 +303,7 @@ def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, re if trace: trace_txt = "running interactive command:\n" trace_txt += "\t[started at: %s]\n" % start_time.strftime('%Y-%m-%d %H:%M:%S') + trace_txt += "\t[working dir: %s]\n" % (path or os.getcwd()) trace_txt += "\t[output logged in %s]\n" % cmd_log_fn trace_msg(trace_txt + '\t' + cmd.strip()) diff --git a/test/framework/run.py b/test/framework/run.py index a5f1000e05..1d87f609c2 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -268,6 +268,15 @@ def test_run_cmd_trace(self): init_config(build_options={'trace': True}) + pattern = [ + r"^ >> running command:", + r"\t\[started at: .*\]", + r"\t\[working dir: .*\]", + r"\t\[output logged in .*\]", + r"\techo hello", + r" >> command completed: exit 0, ran in .*", + ] + self.mock_stdout(True) self.mock_stderr(True) (out, ec) = run_cmd("echo hello") @@ -275,13 +284,24 @@ def test_run_cmd_trace(self): stderr = self.get_stderr() self.mock_stdout(False) self.mock_stderr(False) + self.assertEqual(ec, 0) self.assertEqual(stderr, '') - pattern = "^ >> running command:\n" - pattern += "\t\[started at: .*\]\n" - pattern += "\t\[output logged in .*\]\n" - pattern += "\techo hello\n" - pattern += ' >> command completed: exit 0, ran in .*' - regex = re.compile(pattern) + regex = re.compile('\n'.join(pattern)) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + + # also test with command that is fed input via stdin + self.mock_stdout(True) + self.mock_stderr(True) + (out, ec) = run_cmd('cat', inp='hello') + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(ec, 0) + self.assertEqual(stderr, '') + pattern.insert(3, r"\t\[input: hello\]") + pattern[-2] = "\tcat" + regex = re.compile('\n'.join(pattern)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) # trace output can be disabled on a per-command basis @@ -358,6 +378,7 @@ def test_run_cmd_qa_trace(self): self.assertEqual(stderr, '') pattern = "^ >> running interactive command:\n" pattern += "\t\[started at: .*\]\n" + pattern += "\t\[working dir: .*\]\n" pattern += "\t\[output logged in .*\]\n" pattern += "\techo \'n: \'; read n; seq 1 \$n\n" pattern += ' >> interactive command completed: exit 0, ran in .*' diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ef3e5d10e1..318c1e2869 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2079,7 +2079,14 @@ def test_toy_build_trace(self): "^ >> installation prefix: .*/software/toy/0\.0$", "^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", "^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", - "^ >> running command:\n\t\[started at: .*\]\n\t\[output logged in .*\]\n\tgcc toy.c -o toy\n" + + '\n'.join([ + "^ >> running command:", + "\t\[started at: .*\]", + "\t\[working dir: .*\]", + "\t\[output logged in .*\]", + "\tgcc toy.c -o toy\n" + '', + ]), " >> command completed: exit 0, ran in .*", '^' + '\n'.join([ "== sanity checking\.\.\.", From d0e1526fe326f12d64ecb6681e61c485b138c8e5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 21 Jan 2020 20:41:03 +0100 Subject: [PATCH 099/344] use raw string for regex patterns in tests (+ appease the Hound) --- test/framework/run.py | 12 ++++++------ test/framework/toy_build.py | 34 +++++++++++++++++----------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/test/framework/run.py b/test/framework/run.py index 1d87f609c2..e7d608c7b2 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -376,12 +376,12 @@ def test_run_cmd_qa_trace(self): self.mock_stdout(False) self.mock_stderr(False) self.assertEqual(stderr, '') - pattern = "^ >> running interactive command:\n" - pattern += "\t\[started at: .*\]\n" - pattern += "\t\[working dir: .*\]\n" - pattern += "\t\[output logged in .*\]\n" - pattern += "\techo \'n: \'; read n; seq 1 \$n\n" - pattern += ' >> interactive command completed: exit 0, ran in .*' + pattern = r"^ >> running interactive command:\n" + pattern += r"\t\[started at: .*\]\n" + pattern += r"\t\[working dir: .*\]\n" + pattern += r"\t\[output logged in .*\]\n" + pattern += r"\techo \'n: \'; read n; seq 1 \$n\n" + pattern += r' >> interactive command completed: exit 0, ran in .*' self.assertTrue(re.search(pattern, stdout), "Pattern '%s' found in: %s" % (pattern, stdout)) # trace output can be disabled on a per-command basis diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 318c1e2869..85f92f8689 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2076,24 +2076,24 @@ def test_toy_build_trace(self): self.assertEqual(stderr, '') patterns = [ - "^ >> installation prefix: .*/software/toy/0\.0$", - "^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", - "^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", - '\n'.join([ - "^ >> running command:", - "\t\[started at: .*\]", - "\t\[working dir: .*\]", - "\t\[output logged in .*\]", - "\tgcc toy.c -o toy\n" - '', + r"^ >> installation prefix: .*/software/toy/0\.0$", + r"^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", + r"^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", + r'\n'.join([ + r"^ >> running command:", + r"\t\[started at: .*\]", + r"\t\[working dir: .*\]", + r"\t\[output logged in .*\]", + r"\tgcc toy.c -o toy\n" + r'', ]), - " >> command completed: exit 0, ran in .*", - '^' + '\n'.join([ - "== sanity checking\.\.\.", - " >> file 'bin/yot' or 'bin/toy' found: OK", - " >> \(non-empty\) directory 'bin' found: OK", - ]) + '$', - "^== creating module\.\.\.\n >> generating module file @ .*/modules/all/toy/0\.0(?:\.lua)?$", + r" >> command completed: exit 0, ran in .*", + r'^' + r'\n'.join([ + r"== sanity checking\.\.\.", + r" >> file 'bin/yot' or 'bin/toy' found: OK", + r" >> \(non-empty\) directory 'bin' found: OK", + ]) + r'$', + r"^== creating module\.\.\.\n >> generating module file @ .*/modules/all/toy/0\.0(?:\.lua)?$", ] for pattern in patterns: regex = re.compile(pattern, re.M) From 2a2da7fb322644dcf4bd76df9e3504c6a1666afc Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 22 Jan 2020 19:29:09 +0100 Subject: [PATCH 100/344] Fix EasyConfig.update code to handle both strings and lists as input. Correctly handle allow_duplicate=False when "value" is a partial match for an item in key when key is a string. --- easybuild/framework/easyconfig/easyconfig.py | 27 +++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b3e8af1cb8..51ad8e6119 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -553,21 +553,30 @@ def update(self, key, value, allow_duplicate=True): """ Update a string configuration value with a value (i.e. append to it). """ + if isinstance(value, string_type): + lval = [value] + elif isinstance(value, list): + lval = value + else: + raise EasyBuildError("Can't update configuration value for %s, because the attempted update value, '%s', is not a string or list.", key, value) + prev_value = self[key] if isinstance(prev_value, string_type): - if allow_duplicate or value not in prev_value: - self[key] = '%s %s ' % (prev_value, value) + for item in lval: + if allow_duplicate or (not prev_value.startswith('%s ' % item) + and not prev_value.endswith(' %s' % item) + and ' %s ' % item not in prev_value): + prev_value += ' %s' % item + prev_value += ' ' elif isinstance(prev_value, list): - if allow_duplicate: - self[key] = prev_value + value - else: - for item in value: - # add only those items that aren't already in the list - if item not in prev_value: - self[key] = prev_value + [item] + for item in lval: + if allow_duplicate or item not in prev_value: + prev_value.append(item) else: raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key) + self[key] = prev_value + def set_keys(self, params): """ Set keys in this EasyConfig instance based on supplied easyconfig parameter values. From 46473d5e9c049e98bd0eae4331a02c583a889398 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 23 Jan 2020 15:13:18 +0100 Subject: [PATCH 101/344] Add probe of PREFIX and VERSION variable in external modules This commit handles metadata for external module dependencies when there is not entry in the metadata file It should look for the pair of variables definitions in the modules in the given order 1. CRAY_XXXX_PREFIX and CRAY_XXXX_VERSION 2. CRAY_XXXX_DIR and CRAY_XXXX_VERSION 2. CRAY_XXXX_ROOT and CRAY_XXXX_VERSION 5. XXXX_PREFIX and XXXX_VERSION 4. XXXX_DIR and XXXX_VERSION 5. XXXX_ROOT and XXXX_VERSION 3. XXXX_HOME and XXXX_VERSION If no pair is found, an empty dependency dictionary is returned, as is the default behaviour This avoids the need for a very large metadata file. --- easybuild/framework/easyconfig/easyconfig.py | 69 ++++++++++++++++++- .../lowercase_module_naming_scheme.py | 62 ----------------- 2 files changed, 67 insertions(+), 64 deletions(-) delete mode 100644 easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b3e8af1cb8..8a699e0300 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -36,6 +36,7 @@ :author: Alan O'Cais (Juelich Supercomputing Centre) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) :author: Maxime Boissonneault (Universite Laval, Calcul Quebec, Compute Canada) +:author: Victor Holanda (CSCS, ETH Zurich) """ import copy @@ -61,7 +62,7 @@ from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name -from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file +from easybuild.tools.filetools import convert_name, find_backup_name_candidate, find_easyconfigs, read_file, write_file from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version @@ -1152,6 +1153,68 @@ def _validate(self, attr, values): # private method if self[attr] and self[attr] not in values: raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) + def get_variable_from_modulefile(self, mod_name, var_name): + """ + Get info from the module file for the specified module. + + :param mod_name: module name + :param regex: (compiled) regular expression, with one group + """ + try: + regex = re.compile(r'^setenv\s*%s\s*(?P\S*)' % var_name, re.M) + ans =self.modules_tool.get_value_from_modulefile(mod_name, regex) + except: + return None + + return ans + + def handle_external_module_metadata_by_probing_modules(self, dep_name): + """ + helper function for handle_external_module_metadata + handles metadata for external module dependencies when there is not entry in the + metadata file + + It should look for the pair of variables definitions in the available modules + 1. CRAY_XXXX_PREFIX and CRAY_XXXX_VERSION + 2. CRAY_XXXX_DIR and CRAY_XXXX_VERSION + 2. CRAY_XXXX_ROOT and CRAY_XXXX_VERSION + 5. XXXX_PREFIX and XXXX_VERSION + 4. XXXX_DIR and XXXX_VERSION + 5. XXXX_ROOT and XXXX_VERSION + 3. XXXX_HOME and XXXX_VERSION + + If neither of the pairs is found, then an empty dictionary is returned + """ + dependency = {} + + dep_name_upper = convert_name(dep_name, upper=True) + short_ext_modname = dep_name.split('/')[0] + short_ext_modname_upper = convert_name(short_ext_modname, upper=True) + + allowed_pairs = [ + ('CRAY_%s_PREFIX' % short_ext_modname_upper, 'CRAY_%s_VERSION' % short_ext_modname_upper), + ('CRAY_%s_DIR' % short_ext_modname_upper, 'CRAY_%s_VERSION' % short_ext_modname_upper), + ('CRAY_%s_ROOT' % short_ext_modname_upper, 'CRAY_%s_VERSION' % short_ext_modname_upper), + ('%s_PREFIX' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), + ('%s_DIR' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), + ('%s_ROOT' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), + ('%s_HOME' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), + ] + + for p, v in allowed_pairs: + prefix = self.get_variable_from_modulefile(dep_name, p) + version = self.get_variable_from_modulefile(dep_name, v) + + if prefix and version: + dependency = { + 'name': [short_ext_modname], + 'version': [version], + 'prefix': p + } + break + + return dependency + def handle_external_module_metadata(self, dep_name): """ helper function for _parse_dependency @@ -1163,7 +1226,9 @@ def handle_external_module_metadata(self, dep_name): self.log.info("Updated dependency info with available metadata for external module %s: %s", dep_name, dependency['external_module_metadata']) else: - self.log.info("No metadata available for external module %s", dep_name) + self.log.info("No metadata available for external module %s. Attempting to read from available modules", + dep_name) + dependency['external_module_metadata'] = self.handle_external_module_metadata_by_probing_modules(dep_name) return dependency diff --git a/easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py b/easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py deleted file mode 100644 index 8bf090272d..0000000000 --- a/easybuild/tools/module_naming_scheme/lowercase_module_naming_scheme.py +++ /dev/null @@ -1,62 +0,0 @@ -## -# Copyright 2013-2017 Ghent University -# -# This file is part of EasyBuild, -# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), -# with support of Ghent University (http://ugent.be/hpc), -# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), -# Flemish Research Foundation (FWO) (http://www.fwo.be/en) -# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). -# -# http://github.com/hpcugent/easybuild -# -# EasyBuild is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation v2. -# -# EasyBuild is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with EasyBuild. If not, see . -## -""" -Implementation of (default) EasyBuild module naming scheme. -:author: Kenneth Hoste (Ghent University) -:author: Guilherme Peretti-Pezzi (CSCS) -""" - -import os -import re - -from easybuild.tools.module_naming_scheme import ModuleNamingScheme -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version - -class LowercaseModuleNamingScheme(ModuleNamingScheme): - """Class implementing a lowercase module naming scheme.""" - - REQUIRED_KEYS = ['name', 'version', 'versionsuffix', 'toolchain'] - - def det_full_module_name(self, ec): - """ - Determine full module name from given easyconfig, according to the EasyBuild module naming scheme. - :param ec: dict-like object with easyconfig parameter values (e.g. 'name', 'version', etc.) - :return: string with full module name /, e.g.: 'gzip/1.5-goolf-1.4.10' - """ - return os.path.join(ec['name'], det_full_ec_version(ec)).lower() - - def is_short_modname_for(self, short_modname, name): - """ - Determine whether the specified (short) module name is a module for software with the specified name. - Default implementation checks via a strict regex pattern, and assumes short module names are of the form: - /[-] - """ - modname_regex = re.compile('^%s(/\S+)?$' % re.escape(name.lower())) - res = bool(modname_regex.match(short_modname)) - - self.log.debug("Checking whether '%s' is a module name for software with name '%s' via regex %s: %s", - short_modname, name, modname_regex.pattern, res) - - return res From 9cdf2ed3e5a2df6af4852380b2b21f5838aa83c3 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 23 Jan 2020 15:40:08 +0100 Subject: [PATCH 102/344] Fix PR remarks and add support for Lua syntax --- easybuild/framework/easyconfig/easyconfig.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8a699e0300..dc60792064 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1161,10 +1161,16 @@ def get_variable_from_modulefile(self, mod_name, var_name): :param regex: (compiled) regular expression, with one group """ try: + # Tcl syntax regex = re.compile(r'^setenv\s*%s\s*(?P\S*)' % var_name, re.M) - ans =self.modules_tool.get_value_from_modulefile(mod_name, regex) - except: - return None + ans = self.modules_tool.get_value_from_modulefile(mod_name, regex) + except Exception: + try: + # Lua syntax + regex = re.compile(r'^setenv\(\"GO_ROOT\",\s*\"(?P\S*)\"\)' % var_name, re.M) + ans = self.modules_tool.get_value_from_modulefile(mod_name, regex) + except Exception: + return None return ans @@ -1187,7 +1193,6 @@ def handle_external_module_metadata_by_probing_modules(self, dep_name): """ dependency = {} - dep_name_upper = convert_name(dep_name, upper=True) short_ext_modname = dep_name.split('/')[0] short_ext_modname_upper = convert_name(short_ext_modname, upper=True) @@ -1227,7 +1232,7 @@ def handle_external_module_metadata(self, dep_name): dep_name, dependency['external_module_metadata']) else: self.log.info("No metadata available for external module %s. Attempting to read from available modules", - dep_name) + dep_name) dependency['external_module_metadata'] = self.handle_external_module_metadata_by_probing_modules(dep_name) return dependency From afe4e1103fb947d18f39665ef6bee4a67518ab67 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 23 Jan 2020 17:34:01 +0100 Subject: [PATCH 103/344] Improve the design --- easybuild/framework/easyconfig/easyconfig.py | 25 +------------ easybuild/tools/modules.py | 39 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index dc60792064..ba862994bb 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1153,27 +1153,6 @@ def _validate(self, attr, values): # private method if self[attr] and self[attr] not in values: raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) - def get_variable_from_modulefile(self, mod_name, var_name): - """ - Get info from the module file for the specified module. - - :param mod_name: module name - :param regex: (compiled) regular expression, with one group - """ - try: - # Tcl syntax - regex = re.compile(r'^setenv\s*%s\s*(?P\S*)' % var_name, re.M) - ans = self.modules_tool.get_value_from_modulefile(mod_name, regex) - except Exception: - try: - # Lua syntax - regex = re.compile(r'^setenv\(\"GO_ROOT\",\s*\"(?P\S*)\"\)' % var_name, re.M) - ans = self.modules_tool.get_value_from_modulefile(mod_name, regex) - except Exception: - return None - - return ans - def handle_external_module_metadata_by_probing_modules(self, dep_name): """ helper function for handle_external_module_metadata @@ -1207,8 +1186,8 @@ def handle_external_module_metadata_by_probing_modules(self, dep_name): ] for p, v in allowed_pairs: - prefix = self.get_variable_from_modulefile(dep_name, p) - version = self.get_variable_from_modulefile(dep_name, v) + prefix = self.modules_tool.get_variable_from_modulefile(dep_name, p) + version = self.modules_tool.get_variable_from_modulefile(dep_name, v) if prefix and version: dependency = { diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 8bfc737bc3..9746e41e7a 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -649,6 +649,15 @@ def show(self, mod_name): return ans + def get_variable_from_modulefile(self, mod_name, var_name): + """ + Get info from the module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable value to be extracted + """ + pass + def get_value_from_modulefile(self, mod_name, regex): """ Get info from the module file for the specified module. @@ -1126,6 +1135,21 @@ def update(self): """Update after new modules were added.""" pass + def get_variable_from_modulefile(self, mod_name, var_name): + """ + Get info from the module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable value to be extracted + """ + try: + # Tcl syntax + regex = re.compile(r'^setenv\s*%s\s*(?P\S*)' % var_name, re.M) + ans = self.get_value_from_modulefile(mod_name, regex) + except Exception: + return None + + return ans class EnvironmentModulesTcl(EnvironmentModulesC): """Interface to (Tcl) environment modules (modulecmd.tcl).""" @@ -1390,6 +1414,21 @@ def exist(self, mod_names, skip_avail=False, maybe_partial=True): return super(Lmod, self).exist(mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*(\.lua)?:\s*$', skip_avail=skip_avail, maybe_partial=maybe_partial) + def get_variable_from_modulefile(self, mod_name, var_name): + """ + Get info from the module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable value to be extracted + """ + try: + # Lua syntax + regex = re.compile(r'^setenv\(\"%s\",\s*\"(?P\S*)\"\)' % var_name, re.M) + ans = self.get_value_from_modulefile(mod_name, regex) + except Exception: + return None + + return ans def get_software_root_env_var_name(name): """Return name of environment variable for software root.""" From 4cf11d892f5c635467fa88661264b47879e44432 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 24 Jan 2020 13:42:06 +0100 Subject: [PATCH 104/344] Add CMAKE_PREFIX_PATH and CMAKE_LIBRARY_PATH to make_module_req Fixes #2624 --- easybuild/framework/easyblock.py | 26 ++++++++++++++++++++-- test/framework/easyblock.py | 37 ++++++++++++++++++++++++++------ test/framework/toy_build.py | 2 ++ 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 300f474217..1176e0ba5d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1289,9 +1289,13 @@ def make_module_req(self): note = "note: glob patterns are not expanded and existence checks " note += "for paths are skipped for the statements below due to dry run" lines.append(self.module_generator.comment(note)) + lib64_is_symlink = False + else: + lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) + and os.path.samefile('lib', 'lib64')) # for these environment variables, the corresponding subdirectory must include at least one file - keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH') + keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH') for key in sorted(requirements): if self.dry_run: @@ -1304,7 +1308,23 @@ def make_module_req(self): for path in reqs: # only use glob if the string is non-empty if path and not self.dry_run: - paths = sorted(glob.glob(path)) + paths = glob.glob(path) + # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates + if lib64_is_symlink: + fixed_paths = [] + for path in paths: + if (path + os.path.sep).startswith('lib64' + os.path.sep): + # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path + if key == 'CMAKE_LIBRARY_PATH': + continue + path = path.replace('lib64', 'lib', 1) + fixed_paths.append(path) + if fixed_paths != paths: + self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", + key, paths, fixed_paths) + paths = fixed_paths + # Use a set to remove duplicates + paths = sorted(set(paths)) if paths and key in keys_requiring_files: # only retain paths that contain at least one file retained_paths = [ @@ -1342,6 +1362,8 @@ def make_module_req_guess(self): 'CLASSPATH': ['*.jar'], 'XDG_DATA_DIRS': ['share'], 'GI_TYPELIB_PATH': [os.path.join(x, 'girepository-*') for x in lib_paths], + 'CMAKE_PREFIX_PATH': [''], + 'CMAKE_LIBRARY_PATH': ['lib64'], # lib and lib32 are searched through the above } def load_module(self, mod_paths=None, purge=True, extra_modules=None): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 00033c106a..27bd148464 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -51,7 +51,7 @@ from easybuild.tools.modules import reset_module_caches from easybuild.tools.utilities import time2str from easybuild.tools.version import get_git_revision, this_is_easybuild - +from easybuild.tools.py2vs3 import string_type class EasyBlockTest(EnhancedTestCase): """ Baseclass for easyblock testcases """ @@ -318,11 +318,10 @@ def test_make_module_req(self): os.makedirs(eb.installdir) open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar') open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar') - os.mkdir(os.path.join(eb.installdir, 'bin')) - os.mkdir(os.path.join(eb.installdir, 'bin', 'testdir')) - os.mkdir(os.path.join(eb.installdir, 'sbin')) - os.mkdir(os.path.join(eb.installdir, 'share')) - os.mkdir(os.path.join(eb.installdir, 'share', 'man')) + for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): + if isinstance(path, string_type): + path = (path, ) + os.mkdir(os.path.join(eb.installdir, *path)) # this is not a path that should be picked up os.mkdir(os.path.join(eb.installdir, 'CPATH')) @@ -332,6 +331,7 @@ def test_make_module_req(self): self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) self.assertTrue(re.search(r"^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M)) self.assertTrue(re.search(r"^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+CMAKE_PREFIX_PATH\s+\$root$", guess, re.M)) # bin/ is not added to $PATH if it doesn't include files self.assertFalse(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) self.assertFalse(re.search(r"^prepend-path\s+PATH\s+\$root/sbin$", guess, re.M)) @@ -341,6 +341,7 @@ def test_make_module_req(self): self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "bla.jar"\)\)$', guess, re.M)) self.assertTrue(re.search(r'^prepend_path\("CLASSPATH", pathJoin\(root, "foo.jar"\)\)$', guess, re.M)) self.assertTrue(re.search(r'^prepend_path\("MANPATH", pathJoin\(root, "share/man"\)\)$', guess, re.M)) + self.assertTrue('prepend_path("CMAKE_PREFIX_PATH", root)' in guess) # bin/ is not added to $PATH if it doesn't include files self.assertFalse(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', guess, re.M)) self.assertFalse(re.search(r'^prepend_path\("PATH", pathJoin\(root, "sbin"\)\)$', guess, re.M)) @@ -361,6 +362,30 @@ def test_make_module_req(self): else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + # Check that lib64 is only added to CMAKE_LIBRARY_PATH if there are files in there + # but only if it is not a symlink to lib + # -- No Files + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) + elif get_module_syntax() == 'Lua': + self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + # -- With files + open(os.path.join(eb.installdir, 'lib64', 'libfoo.so'), 'w').write('test') + guess = eb.make_module_req() + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) + elif get_module_syntax() == 'Lua': + self.assertTrue('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + # -- With files in lib and lib64 symlinks to lib + open(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'w').write('test') + shutil.rmtree(os.path.join(eb.installdir, 'lib64')) + os.symlink('lib', os.path.join(eb.installdir, 'lib64')) + guess = eb.make_module_req() + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search(r"^prepend-path\s+CMAKE_LIBRARY_PATH\s+\$root/lib64$", guess, re.M)) + elif get_module_syntax() == 'Lua': + self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + # check for behavior when a string value is used as dict value by make_module_req_guesses eb.make_module_req_guess = lambda: {'PATH': 'bin'} txt = eb.make_module_req() diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ef3e5d10e1..9abd4b7050 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1231,6 +1231,7 @@ def test_toy_module_fulltxt(self): r'', r'conflict\("toy"\)', r'', + r'prepend_path\("CMAKE_PREFIX_PATH", root\)', r'prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib"\)\)', r'prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib"\)\)', r'prepend_path\("PATH", pathJoin\(root, "bin"\)\)', @@ -1268,6 +1269,7 @@ def test_toy_module_fulltxt(self): r'', r'conflict toy', r'', + r'prepend-path CMAKE_PREFIX_PATH \$root', r'prepend-path LD_LIBRARY_PATH \$root/lib', r'prepend-path LIBRARY_PATH \$root/lib', r'prepend-path PATH \$root/bin', From bc4fe77c013fa2117ff09c205b7091f08105a87b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 24 Jan 2020 14:31:17 +0100 Subject: [PATCH 105/344] Fix iteration and list-hoisting in make_module_req In dry-run mode a list is assumed, but hoisting happens afterwards --- easybuild/framework/easyblock.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1176e0ba5d..840c9df309 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1297,13 +1297,12 @@ def make_module_req(self): # for these environment variables, the corresponding subdirectory must include at least one file keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH') - for key in sorted(requirements): - if self.dry_run: - self.dry_run_msg(" $%s: %s" % (key, ', '.join(requirements[key]))) - reqs = requirements[key] + for key, reqs in sorted(requirements.items()): if isinstance(reqs, string_type): self.log.warning("Hoisting string value %s into a list before iterating over it", reqs) reqs = [reqs] + if self.dry_run: + self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs))) for path in reqs: # only use glob if the string is non-empty From 17af497a915aebd9bdeca43dfa331b234d33e95b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 24 Jan 2020 14:36:08 +0100 Subject: [PATCH 106/344] Fix dry-run messages in make_module_req In dry-run installdir usually does not exist so the whole block was skipped instead of entering the dry-run blocks --- easybuild/framework/easyblock.py | 115 ++++++++++++++++--------------- 1 file changed, 59 insertions(+), 56 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 840c9df309..f0aee8cc03 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1282,66 +1282,69 @@ def make_module_req(self): lines = ['\n'] if os.path.isdir(self.installdir): - change_dir(self.installdir) + old_dir = change_dir(self.installdir) + else: + old_dir = None - if self.dry_run: - self.dry_run_msg("List of paths that would be searched and added to module file:\n") - note = "note: glob patterns are not expanded and existence checks " - note += "for paths are skipped for the statements below due to dry run" - lines.append(self.module_generator.comment(note)) - lib64_is_symlink = False - else: - lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) - and os.path.samefile('lib', 'lib64')) + if self.dry_run: + self.dry_run_msg("List of paths that would be searched and added to module file:\n") + note = "note: glob patterns are not expanded and existence checks " + note += "for paths are skipped for the statements below due to dry run" + lines.append(self.module_generator.comment(note)) + else: + lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) + and os.path.samefile('lib', 'lib64')) - # for these environment variables, the corresponding subdirectory must include at least one file - keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH') + # for these environment variables, the corresponding subdirectory must include at least one file + keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH') - for key, reqs in sorted(requirements.items()): - if isinstance(reqs, string_type): - self.log.warning("Hoisting string value %s into a list before iterating over it", reqs) - reqs = [reqs] - if self.dry_run: - self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs))) - - for path in reqs: - # only use glob if the string is non-empty - if path and not self.dry_run: - paths = glob.glob(path) - # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates - if lib64_is_symlink: - fixed_paths = [] - for path in paths: - if (path + os.path.sep).startswith('lib64' + os.path.sep): - # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path - if key == 'CMAKE_LIBRARY_PATH': - continue - path = path.replace('lib64', 'lib', 1) - fixed_paths.append(path) - if fixed_paths != paths: - self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", - key, paths, fixed_paths) - paths = fixed_paths - # Use a set to remove duplicates - paths = sorted(set(paths)) - if paths and key in keys_requiring_files: - # only retain paths that contain at least one file - retained_paths = [ - path for path in paths - if os.path.isdir(os.path.join(self.installdir, path)) - and dir_contains_files(os.path.join(self.installdir, path)) - ] - self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", - key, paths, retained_paths) - paths = retained_paths - else: - # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME) - paths = [path] - - lines.append(self.module_generator.prepend_paths(key, paths)) + for key, reqs in sorted(requirements.items()): + if isinstance(reqs, string_type): + self.log.warning("Hoisting string value %s into a list before iterating over it", reqs) + reqs = [reqs] if self.dry_run: - self.dry_run_msg('') - change_dir(self.orig_workdir) + self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs))) + + for path in reqs: + # only use glob if the string is non-empty + if path and not self.dry_run: + paths = glob.glob(path) + # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates + if lib64_is_symlink: + fixed_paths = [] + for path in paths: + if (path + os.path.sep).startswith('lib64' + os.path.sep): + # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path + if key == 'CMAKE_LIBRARY_PATH': + continue + path = path.replace('lib64', 'lib', 1) + fixed_paths.append(path) + if fixed_paths != paths: + self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", + key, paths, fixed_paths) + paths = fixed_paths + # Use a set to remove duplicates + paths = sorted(set(paths)) + if paths and key in keys_requiring_files: + # only retain paths that contain at least one file + retained_paths = [ + path for path in paths + if os.path.isdir(os.path.join(self.installdir, path)) + and dir_contains_files(os.path.join(self.installdir, path)) + ] + self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", + key, paths, retained_paths) + paths = retained_paths + else: + # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME) + paths = [path] + + lines.append(self.module_generator.prepend_paths(key, paths)) + if self.dry_run: + self.dry_run_msg('') + + if old_dir is not None: + change_dir(old_dir) return ''.join(lines) From 0aea9a4516a06faf0cd75ba286ecdc238d3f0e95 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 24 Jan 2020 15:14:01 +0100 Subject: [PATCH 107/344] Avoid duplicate paths in make_module_req Expand all globs and iterate over combined list of paths per key This allows elimination of duplicate paths and greatly simplified code. --- easybuild/framework/easyblock.py | 73 ++++++++++++++++---------------- test/framework/easyblock.py | 11 +++++ 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f0aee8cc03..05f82e2148 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1291,12 +1291,9 @@ def make_module_req(self): note = "note: glob patterns are not expanded and existence checks " note += "for paths are skipped for the statements below due to dry run" lines.append(self.module_generator.comment(note)) - else: - lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) - and os.path.samefile('lib', 'lib64')) # for these environment variables, the corresponding subdirectory must include at least one file - keys_requiring_files = ('CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH') + keys_requiring_files = {'CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH'} for key, reqs in sorted(requirements.items()): if isinstance(reqs, string_type): @@ -1304,45 +1301,47 @@ def make_module_req(self): reqs = [reqs] if self.dry_run: self.dry_run_msg(" $%s: %s" % (key, ', '.join(reqs))) - - for path in reqs: - # only use glob if the string is non-empty - if path and not self.dry_run: - paths = glob.glob(path) - # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates - if lib64_is_symlink: - fixed_paths = [] - for path in paths: - if (path + os.path.sep).startswith('lib64' + os.path.sep): - # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path - if key == 'CMAKE_LIBRARY_PATH': - continue - path = path.replace('lib64', 'lib', 1) - fixed_paths.append(path) - if fixed_paths != paths: - self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", - key, paths, fixed_paths) - paths = fixed_paths - # Use a set to remove duplicates - paths = sorted(set(paths)) - if paths and key in keys_requiring_files: - # only retain paths that contain at least one file - retained_paths = [ - path for path in paths - if os.path.isdir(os.path.join(self.installdir, path)) - and dir_contains_files(os.path.join(self.installdir, path)) - ] + # Don't expand globs or do any filtering below for dry run + paths = sorted(reqs) + else: + # Expand globs but only if the string is non-empty + # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME) + paths = sorted(sum((glob.glob(path) if path else [path] for path in reqs), [])) # sum flattens to list + + # If lib64 is just a symlink to lib we fixup the paths to avoid duplicates + lib64_is_symlink = (all(os.path.isdir(path) for path in ['lib', 'lib64']) + and os.path.samefile('lib', 'lib64')) + if lib64_is_symlink: + fixed_paths = [] + for path in paths: + if (path + os.path.sep).startswith('lib64' + os.path.sep): + # We only need CMAKE_LIBRARY_PATH if there is a separate lib64 path, so skip symlink + if key == 'CMAKE_LIBRARY_PATH': + continue + path = path.replace('lib64', 'lib', 1) + fixed_paths.append(path) + if fixed_paths != paths: + self.log.info("Fixed symlink lib64 in paths for %s: %s -> %s", key, paths, fixed_paths) + paths = fixed_paths + # Use a set to remove duplicates, e.g. by having lib64 and lib which get fixed to lib and lib above + paths = sorted(set(paths)) + if key in keys_requiring_files: + # only retain paths that contain at least one file + retained_paths = [ + path for path in paths + if os.path.isdir(os.path.join(self.installdir, path)) + and dir_contains_files(os.path.join(self.installdir, path)) + ] + if retained_paths != paths: self.log.info("Only retaining paths for %s that contain at least one file: %s -> %s", - key, paths, retained_paths) + key, paths, retained_paths) paths = retained_paths - else: - # empty string is a valid value here (i.e. to prepend the installation prefix, cfr $CUDA_HOME) - paths = [path] + if paths: lines.append(self.module_generator.prepend_paths(key, paths)) if self.dry_run: self.dry_run_msg('') - + if old_dir is not None: change_dir(old_dir) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 27bd148464..2a52200d3c 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -386,6 +386,17 @@ def test_make_module_req(self): elif get_module_syntax() == 'Lua': self.assertFalse('prepend_path("CMAKE_LIBRARY_PATH", pathJoin(root, "lib64"))' in guess) + # With files in /lib and /lib64 symlinked to /lib there should be exactly 1 entry for (LD_)LIBRARY_PATH + # pointing to /lib + for var in ('LIBRARY_PATH', 'LD_LIBRARY_PATH'): + if get_module_syntax() == 'Tcl': + self.assertFalse(re.search(r"^prepend-path\s+%s\s+\$root/lib64$" % var, guess, re.M)) + self.assertEqual(len(re.findall(r"^prepend-path\s+%s\s+\$root/lib$" % var, guess, re.M)), 1) + elif get_module_syntax() == 'Lua': + self.assertFalse(re.search(r'^prepend_path\("%s", pathJoin\(root, "lib64"\)\)$' % var, guess, re.M)) + self.assertEqual(len(re.findall(r'^prepend_path\("%s", pathJoin\(root, "lib"\)\)$' % var, + guess, re.M)), 1) + # check for behavior when a string value is used as dict value by make_module_req_guesses eb.make_module_req_guess = lambda: {'PATH': 'bin'} txt = eb.make_module_req() From 9ec25abe9d74ddde40a45a3465d57aa4e07a59b9 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 24 Jan 2020 15:55:25 +0100 Subject: [PATCH 108/344] framework/easyconfig/easyconfig.py EasyConfig.update: adjust spacing around added item to match old behaviour. --- easybuild/framework/easyconfig/easyconfig.py | 3 +-- test/framework/easyconfig.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 51ad8e6119..00e25fb0e0 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -566,8 +566,7 @@ def update(self, key, value, allow_duplicate=True): if allow_duplicate or (not prev_value.startswith('%s ' % item) and not prev_value.endswith(' %s' % item) and ' %s ' % item not in prev_value): - prev_value += ' %s' % item - prev_value += ' ' + prev_value += ' %s ' % item elif isinstance(prev_value, list): for item in lval: if allow_duplicate or item not in prev_value: diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f956900f59..ad97d3f2b9 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1515,7 +1515,7 @@ def test_update(self): ec.update('description', "- just a test") self.assertEqual(ec['description'].strip(), "Toy C program, 100% toy. - just a test") - # spaces in between multiple updates for stirng values + # spaces in between multiple updates for string values ec.update('configopts', 'CC="$CC"') ec.update('configopts', 'CXX="$CXX"') self.assertTrue(ec['configopts'].strip().endswith('CC="$CC" CXX="$CXX"')) From 92c045208452a74f4408593c980a54139356b239 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 24 Jan 2020 16:04:13 +0100 Subject: [PATCH 109/344] Appease hound. --- easybuild/framework/easyconfig/easyconfig.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 00e25fb0e0..c0afd7d9aa 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -558,14 +558,16 @@ def update(self, key, value, allow_duplicate=True): elif isinstance(value, list): lval = value else: - raise EasyBuildError("Can't update configuration value for %s, because the attempted update value, '%s', is not a string or list.", key, value) + msg = "Can't update configuration value for %s, because the " + msg += "attempted update value, '%s', is not a string or list." + raise EasyBuildError(msg, key, value) prev_value = self[key] if isinstance(prev_value, string_type): for item in lval: - if allow_duplicate or (not prev_value.startswith('%s ' % item) - and not prev_value.endswith(' %s' % item) - and ' %s ' % item not in prev_value): + if allow_duplicate or (not prev_value.startswith('%s ' % item) and + not prev_value.endswith(' %s' % item) and + ' %s ' % item not in prev_value): prev_value += ' %s ' % item elif isinstance(prev_value, list): for item in lval: From 8dfbd25d18d5010e23d2f28b0936e2e204acb711 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 24 Jan 2020 16:23:00 +0100 Subject: [PATCH 110/344] Fix Python 2.6 compatibility and add CMAKE_PREFIX_PATH to keys_requiring_files --- easybuild/framework/easyblock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 05f82e2148..fa75b17940 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1293,7 +1293,8 @@ def make_module_req(self): lines.append(self.module_generator.comment(note)) # for these environment variables, the corresponding subdirectory must include at least one file - keys_requiring_files = {'CPATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'PATH', 'CMAKE_LIBRARY_PATH'} + keys_requiring_files = set(('PATH', 'LD_LIBRARY_PATH', 'LIBRARY_PATH', 'CPATH', + 'CMAKE_PREFIX_PATH', 'CMAKE_LIBRARY_PATH')) for key, reqs in sorted(requirements.items()): if isinstance(reqs, string_type): From 27742c7d07af53c6c96e22960d7fe16ebed78a28 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Fri, 24 Jan 2020 18:08:07 +0100 Subject: [PATCH 111/344] Reduce tripple test of item against prev_value with single regexp. --- easybuild/framework/easyconfig/easyconfig.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c0afd7d9aa..f61882e6cf 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -565,9 +565,7 @@ def update(self, key, value, allow_duplicate=True): prev_value = self[key] if isinstance(prev_value, string_type): for item in lval: - if allow_duplicate or (not prev_value.startswith('%s ' % item) and - not prev_value.endswith(' %s' % item) and - ' %s ' % item not in prev_value): + if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % item, prev_value)): prev_value += ' %s ' % item elif isinstance(prev_value, list): for item in lval: From 5d87a9ad4ae555c49d0636a33a133347da44e529 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Thu, 30 Jan 2020 18:37:57 +0100 Subject: [PATCH 112/344] improve cray support --- easybuild/framework/easyconfig/easyconfig.py | 21 +++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ba862994bb..75eda133a1 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1173,6 +1173,13 @@ def handle_external_module_metadata_by_probing_modules(self, dep_name): dependency = {} short_ext_modname = dep_name.split('/')[0] + + if short_ext_modname.startswith('craype-'): + short_ext_modname = short_ext_modname.split('craype-')[1] + elif short_ext_modname.startswith('cray-'): + short_ext_modname = short_ext_modname.split('cray-')[1] + + short_ext_modname.replace('-', '_') short_ext_modname_upper = convert_name(short_ext_modname, upper=True) allowed_pairs = [ @@ -1185,15 +1192,15 @@ def handle_external_module_metadata_by_probing_modules(self, dep_name): ('%s_HOME' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), ] - for p, v in allowed_pairs: - prefix = self.modules_tool.get_variable_from_modulefile(dep_name, p) - version = self.modules_tool.get_variable_from_modulefile(dep_name, v) + for prefix, version in allowed_pairs: + module_prefix = self.modules_tool.get_variable_from_modulefile(dep_name, prefix) + module_version = self.modules_tool.get_variable_from_modulefile(dep_name, version) - if prefix and version: + if module_prefix and module_version: dependency = { - 'name': [short_ext_modname], - 'version': [version], - 'prefix': p + 'name': [short_ext_modname_upper], + 'version': [module_version], + 'prefix': module_prefix } break From 1d112fb3aac5c62b58e80dc5bd15b2ebb1188513 Mon Sep 17 00:00:00 2001 From: darkless Date: Fri, 31 Jan 2020 10:21:27 +0100 Subject: [PATCH 113/344] Fix shebang even if first line doesn't start with '#!'. --- .gitignore | 1 + easybuild/framework/easyblock.py | 3 +++ test/framework/toy_build.py | 8 ++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 31e6aff1ea..c8b95e4482 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea .pydevproject .project LICENSE_HEADER diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 300f474217..6fe972b354 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2209,6 +2209,9 @@ def fix_shebang(self): if should_patch: contents = shebang_regex.sub(shebang, contents) write_file(path, contents) + else: + contents = shebang + "\n" + contents + write_file(path, contents) def post_install_step(self): """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ef3e5d10e1..752bea8ce2 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2378,6 +2378,8 @@ def test_fix_shebang(self): " 'echo \"#! /usr/bin/env python3\\n# test\" > %(installdir)s/bin/t4.py',", # 'env python3.6' " 'echo \"#!/usr/bin/env python3.6\\n# test\" > %(installdir)s/bin/t5.py',", + # no shebang python + " 'echo \"# test\" > %(installdir)s/bin/t6.py',", # tests for perl shebang # hardcoded path to bin/perl @@ -2390,6 +2392,8 @@ def test_fix_shebang(self): " 'echo \"#!/usr/bin/perl -w\\n# test\" > %(installdir)s/bin/t4.pl',", # space after #! + 'env perl5' " 'echo \"#!/usr/bin/env perl5\\n# test\" > %(installdir)s/bin/t5.pl',", + # no shebang perl + " 'echo \"# test\" > %(installdir)s/bin/t6.pl',", "]", "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy']", @@ -2402,7 +2406,7 @@ def test_fix_shebang(self): # no re.M, this should match at start of file! py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py']: + for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py']: pybin_path = os.path.join(toy_bindir, pybin) pybin_txt = read_file(pybin_path) self.assertTrue(py_shebang_regex.match(pybin_txt), @@ -2410,7 +2414,7 @@ def test_fix_shebang(self): # no re.M, this should match at start of file! perl_shebang_regex = re.compile(r'^#!/usr/bin/env perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl']: + for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl']: perlbin_path = os.path.join(toy_bindir, perlbin) perlbin_txt = read_file(perlbin_path) self.assertTrue(perl_shebang_regex.match(perlbin_txt), From 6ae00b6b6a4d15e6f673129c2cd13677d2d320bc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 31 Jan 2020 15:34:49 +0100 Subject: [PATCH 114/344] Appease hound --- test/framework/tweak.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 2cca32db1e..af2b55357c 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -415,12 +415,12 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['builddependencies'][0] and - value == tweaked_dict['builddependencies'][0][key]) + self.assertTrue(key in tweaked_dict['builddependencies'][0] + and value == tweaked_dict['builddependencies'][0][key]) # Also check that the gzip dependency was upgraded for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['dependencies'][0] and - value == tweaked_dict['dependencies'][0][key]) + self.assertTrue(key in tweaked_dict['dependencies'][0] + and value == tweaked_dict['dependencies'][0][key]) def suite(): From 4fc151a38920212bdafda45bb38d2839afbbbb9e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 31 Jan 2020 15:44:02 +0100 Subject: [PATCH 115/344] Appease hound --- test/framework/tweak.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index af2b55357c..6720e9d936 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -415,12 +415,12 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # Also check that binutils has been mapped for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['builddependencies'][0] - and value == tweaked_dict['builddependencies'][0][key]) + self.assertTrue( + key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key] + ) # Also check that the gzip dependency was upgraded for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): - self.assertTrue(key in tweaked_dict['dependencies'][0] - and value == tweaked_dict['dependencies'][0][key]) + self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) def suite(): From 415a2ad1137fe2ca7f9bd6416e11eb1942b28cee Mon Sep 17 00:00:00 2001 From: darkless Date: Mon, 3 Feb 2020 09:37:28 +0100 Subject: [PATCH 116/344] Make fix_shebang more precise. --- easybuild/framework/easyblock.py | 4 +++- test/framework/toy_build.py | 28 ++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6fe972b354..1841007af5 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2209,7 +2209,9 @@ def fix_shebang(self): if should_patch: contents = shebang_regex.sub(shebang, contents) write_file(path, contents) - else: + elif not contents.startswith('#!'): + self.log.info("The file '%s' doesn't have any shebang present, inserting it as first line.", + path) contents = shebang + "\n" + contents write_file(path, contents) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 752bea8ce2..50fc22e954 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2378,8 +2378,12 @@ def test_fix_shebang(self): " 'echo \"#! /usr/bin/env python3\\n# test\" > %(installdir)s/bin/t4.py',", # 'env python3.6' " 'echo \"#!/usr/bin/env python3.6\\n# test\" > %(installdir)s/bin/t5.py',", + # shebang with space, should strip the space + " 'echo \"#! /usr/bin/env python\\n# test\" > %(installdir)s/bin/t6.py',", # no shebang python - " 'echo \"# test\" > %(installdir)s/bin/t6.py',", + " 'echo \"# test\" > %(installdir)s/bin/t7.py',", + # shebang bash + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b1.sh',", # tests for perl shebang # hardcoded path to bin/perl @@ -2392,12 +2396,16 @@ def test_fix_shebang(self): " 'echo \"#!/usr/bin/perl -w\\n# test\" > %(installdir)s/bin/t4.pl',", # space after #! + 'env perl5' " 'echo \"#!/usr/bin/env perl5\\n# test\" > %(installdir)s/bin/t5.pl',", + # shebang with space, should strip the space + " 'echo \"#! /usr/bin/env perl\\n# test\" > %(installdir)s/bin/t6.pl',", # no shebang perl - " 'echo \"# test\" > %(installdir)s/bin/t6.pl',", + " 'echo \"# test\" > %(installdir)s/bin/t7.pl',", + # shebang bash + " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", "]", - "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy']", - "fix_perl_shebang_for = 'bin/*.pl'", + "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy', 'bin/b1.sh']", + "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh']", ]) write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) @@ -2406,7 +2414,7 @@ def test_fix_shebang(self): # no re.M, this should match at start of file! py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$') - for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py']: + for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: pybin_path = os.path.join(toy_bindir, pybin) pybin_txt = read_file(pybin_path) self.assertTrue(py_shebang_regex.match(pybin_txt), @@ -2414,12 +2422,20 @@ def test_fix_shebang(self): # no re.M, this should match at start of file! perl_shebang_regex = re.compile(r'^#!/usr/bin/env perl\n# test$') - for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl']: + for perlbin in ['t1.pl', 't2.pl', 't3.pl', 't4.pl', 't5.pl', 't6.pl', 't7.pl']: perlbin_path = os.path.join(toy_bindir, perlbin) perlbin_txt = read_file(perlbin_path) self.assertTrue(perl_shebang_regex.match(perlbin_txt), "Pattern '%s' found in %s: %s" % (perl_shebang_regex.pattern, perlbin_path, perlbin_txt)) + # There are 2 bash files which shouldn't be influenced by fix_shebang + bash_shebang_regex = re.compile(r'^#!/usr/bin/env bash\n# test$') + for bashbin in ['b1.sh', 'b2.sh']: + bashbin_path = os.path.join(toy_bindir, bashbin) + bashbin_txt = read_file(bashbin_path) + self.assertTrue(bash_shebang_regex.match(bashbin_txt), + "Pattern '%s' found in %s: %s" % (bash_shebang_regex.pattern, bashbin_path, bashbin_txt)) + def test_toy_system_toolchain_alias(self): """Test use of 'system' toolchain alias.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From aa1221601097a498d7a27855228ff227ca64ead4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 3 Feb 2020 14:38:15 +0100 Subject: [PATCH 117/344] Make bootstrap_eb work with Python 3 --- .github/workflows/unit_tests.yml | 24 ++++---- .travis.yml | 12 ++-- easybuild/scripts/bootstrap_eb.py | 91 +++++++++++++++++++++---------- 3 files changed, 78 insertions(+), 49 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index fffb5264e5..e79e88c7f0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -153,19 +153,15 @@ jobs: EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') EB_BOOTSTRAP_SHA256SUM=$(sha256sum easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53" + EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5" test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) - # test bootstrap script (only compatible with Python 2 for now) - if [[ ${{matrix.python}} =~ '2.' ]]; then - export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap - python easybuild/scripts/bootstrap_eb.py $PREFIX - # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - unset PYTHONPATH - # simple sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now) - module use $PREFIX/modules/all - module load EasyBuild - eb --version - else - echo "Testing of bootstrap script skipped when testing with Python ${{matrix.python}}" - fi + # test bootstrap script + export PREFIX=/tmp/$USER/$GITHUB_SHA/eb_bootstrap + python easybuild/scripts/bootstrap_eb.py $PREFIX + # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module + unset PYTHONPATH + # simple sanity check on bootstrapped EasyBuild module + module use $PREFIX/modules/all + module load EasyBuild + eb --version diff --git a/.travis.yml b/.travis.yml index 7a6a0f80da..b2a8464dee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,12 +100,12 @@ script: - EB_BOOTSTRAP_VERSION=$(grep '^EB_BOOTSTRAP_VERSION' $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | sed 's/[^0-9.]//g') - EB_BOOTSTRAP_SHA256SUM=$(sha256sum $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py | cut -f1 -d' ') - EB_BOOTSTRAP_FOUND="$EB_BOOTSTRAP_VERSION $EB_BOOTSTRAP_SHA256SUM" - - EB_BOOTSTRAP_EXPECTED="20190922.01 7927513e7448d886decfb1bb5daf840e85dc7367f57cc75e51b68f21fe109d53" + - EB_BOOTSTRAP_EXPECTED="20200203.01 616bf3ce812c0844bf9ea3e690f9d88b394ed48f834ddb8424a73cf45fc64ea5" - test "$EB_BOOTSTRAP_FOUND" = "$EB_BOOTSTRAP_EXPECTED" || (echo "Version check on bootstrap script failed $EB_BOOTSTRAP_FOUND" && exit 1) - # test bootstrap script (skip when testing with Python 3 for now, since latest EasyBuild release is not compatible with Python 3 yet) - - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap; fi + # test bootstrap script + - python $TRAVIS_BUILD_DIR/easybuild/scripts/bootstrap_eb.py /tmp/$TRAVIS_JOB_ID/eb_bootstrap # unset $PYTHONPATH to avoid mixing two EasyBuild 'installations' when testing bootstrapped EasyBuild module - unset PYTHONPATH - # simply sanity check on bootstrapped EasyBuild module (skip when testing with Python 3, for now) - - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all; fi - - if [ ! "x$TRAVIS_PYTHON_VERSION" =~ x3.[0-9] ]; then module load EasyBuild; eb --version; fi + # simply sanity check on bootstrapped EasyBuild module + - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all + - module load EasyBuild; eb --version diff --git a/easybuild/scripts/bootstrap_eb.py b/easybuild/scripts/bootstrap_eb.py index cdb0afee25..2e8c487a2a 100644 --- a/easybuild/scripts/bootstrap_eb.py +++ b/easybuild/scripts/bootstrap_eb.py @@ -40,6 +40,7 @@ (via http://dubroy.com/blog/so-you-want-to-install-a-python-package/) """ +import codecs import copy import glob import os @@ -49,12 +50,19 @@ import sys import tempfile import traceback -import urllib2 from distutils.version import LooseVersion from hashlib import md5 +from platform import python_version +IS_PY3 = sys.version_info[0] == 3 -EB_BOOTSTRAP_VERSION = '20190922.01' +if not IS_PY3: + import urllib2 as std_urllib +else: + import urllib.request as std_urllib + + +EB_BOOTSTRAP_VERSION = '20200203.01' # argparse preferrred, optparse deprecated >=2.7 HAVE_ARGPARSE = False @@ -68,7 +76,9 @@ VSC_BASE = 'vsc-base' VSC_INSTALL = 'vsc-install' -EASYBUILD_PACKAGES = [VSC_INSTALL, VSC_BASE, 'easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs'] +# Python 3 is not supported by the vsc-* packages +EASYBUILD_PACKAGES = (([] if IS_PY3 else [VSC_INSTALL, VSC_BASE]) + + ['easybuild-framework', 'easybuild-easyblocks', 'easybuild-easyconfigs']) STAGE1_SUBDIR = 'eb_stage1' @@ -127,8 +137,10 @@ def error(msg, exit=True): def mock_stdout_stderr(): """Mock stdout/stderr channels""" - # cStringIO is only available in Python 2 - from cStringIO import StringIO + try: + from cStringIO import StringIO + except ImportError: + from io import StringIO orig_stdout, orig_stderr = sys.stdout, sys.stderr sys.stdout.flush() sys.stdout = StringIO() @@ -324,7 +336,7 @@ def check_setuptools(): # check setuptools version try: - os.system(cmd_tmpl % "import setuptools; print setuptools.__version__") + os.system(cmd_tmpl % "import setuptools; print(setuptools.__version__)") setuptools_ver = LooseVersion(open(outfile).read().strip()) debug("Found setuptools version %s" % setuptools_ver) @@ -336,7 +348,7 @@ def check_setuptools(): debug("Failed to check setuptools version: %s" % err) res = False - os.system(cmd_tmpl % "from setuptools.command import easy_install; print easy_install.__file__") + os.system(cmd_tmpl % "from setuptools.command import easy_install; print(easy_install.__file__)") out = open(outfile).read().strip() debug("Location of setuptools' easy_install module: %s" % out) if 'setuptools/command/easy_install' not in out: @@ -344,7 +356,7 @@ def check_setuptools(): res = False if res is None: - os.system(cmd_tmpl % "import setuptools; print setuptools.__file__") + os.system(cmd_tmpl % "import setuptools; print(setuptools.__file__)") setuptools_loc = open(outfile).read().strip() res = os.path.dirname(os.path.dirname(setuptools_loc)) debug("Location of setuptools installation: %s" % res) @@ -523,27 +535,32 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion): # install meta-package easybuild from PyPI if forcedversion: cmd.append('easybuild==%s' % forcedversion) + elif IS_PY3: + cmd.append('easybuild>=4.0') # Python 3 support added in EasyBuild 4 else: cmd.append('easybuild') - # install vsc-base again at the end, to avoid that the one available on the system is used instead - post_vsc_base = cmd[:] - post_vsc_base[-1] = VSC_BASE + '<2.9.0' + if not IS_PY3: + # install vsc-base again at the end, to avoid that the one available on the system is used instead + post_vsc_base = cmd[:] + post_vsc_base[-1] = VSC_BASE + '<2.9.0' if not print_debug: cmd.insert(0, '--quiet') - # install vsc-install version prior to 0.11.4, where mock was introduced as a dependency - # workaround for problem reported in https://github.com/easybuilders/easybuild-framework/issues/2712 - # also stick to vsc-base < 2.9.0 to avoid requiring 'future' Python package as dependency - for pkg in [VSC_INSTALL + '<0.11.4', VSC_BASE + '<2.9.0']: - precmd = cmd[:-1] + [pkg] - info("running pre-install command 'easy_install %s'" % (' '.join(precmd))) - run_easy_install(precmd) + # There is no support for Python3 in the older vsc-* packages and EasyBuild 4 includes working versions of vsc-* + if not IS_PY3: + # install vsc-install version prior to 0.11.4, where mock was introduced as a dependency + # workaround for problem reported in https://github.com/easybuilders/easybuild-framework/issues/2712 + # also stick to vsc-base < 2.9.0 to avoid requiring 'future' Python package as dependency + for pkg in [VSC_INSTALL + '<0.11.4', VSC_BASE + '<2.9.0']: + precmd = cmd[:-1] + [pkg] + info("running pre-install command 'easy_install %s'" % (' '.join(precmd))) + run_easy_install(precmd) info("installing EasyBuild with 'easy_install %s'\n" % (' '.join(cmd))) syntax_error_note = '\n'.join([ - "Note: a 'SyntaxError' may be reported for the easybuild/tools/py2vs3/py3.py module.", + "Note: a 'SyntaxError' may be reported for the easybuild/tools/py2vs3/py%s.py module." % ('3', '2')[IS_PY3], "You can safely ignore this message, it will not affect the functionality of the EasyBuild installation.", '', ]) @@ -632,8 +649,13 @@ def stage1(tmpdir, sourcepath, distribute_egg_dir, forcedversion): # make sure we're getting the expected EasyBuild packages import easybuild.framework import easybuild.easyblocks - import vsc.utils.fancylogger - for pkg in [easybuild.framework, easybuild.easyblocks, vsc.utils.fancylogger]: + pkgs_to_check = [easybuild.framework, easybuild.easyblocks] + # vsc is part of EasyBuild 4 + if LooseVersion(eb_version) < LooseVersion('4'): + import vsc.utils.fancylogger + pkgs_to_check.append(vsc.utils.fancylogger) + + for pkg in pkgs_to_check: if tmpdir not in pkg.__file__: error("Found another %s than expected: %s" % (pkg.__name__, pkg.__file__)) else: @@ -698,8 +720,8 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): # determine download URL via PyPI's 'simple' API pkg_simple = None try: - pkg_simple = urllib2.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read() - except (urllib2.URLError, urllib2.HTTPError) as err: + pkg_simple = std_urllib.urlopen('https://pypi.python.org/simple/%s' % pkg, timeout=10).read() + except (std_urllib.URLError, std_urllib.HTTPError) as err: # failing to figure out the package download URl may be OK when source tarballs are provided if sourcepath: info("Ignoring failed attempt to determine '%s' download URL since source tarballs are provided" % pkg) @@ -707,6 +729,8 @@ def stage2(tmpdir, templates, install_path, distribute_egg_dir, sourcepath): raise err if pkg_simple: + if IS_PY3: + pkg_simple = pkg_simple.decode('utf-8') pkg_url_part_regex = re.compile('/(packages/[^#]+)/%s#' % pkg_filename) res = pkg_url_part_regex.search(pkg_simple) if res: @@ -827,6 +851,8 @@ def main(): """Main script: bootstrap EasyBuild in stages.""" self_txt = open(__file__).read() + if IS_PY3: + self_txt = self_txt.encode('utf-8') info("EasyBuild bootstrap script (version %s, MD5: %s)" % (EB_BOOTSTRAP_VERSION, md5(self_txt).hexdigest())) info("Found Python %s\n" % '; '.join(sys.version.split('\n'))) @@ -866,6 +892,9 @@ def main(): forcedversion = EASYBUILD_BOOTSTRAP_FORCE_VERSION if forcedversion: info("Forcing specified version %s..." % forcedversion) + if IS_PY3 and LooseVersion(forcedversion) < LooseVersion('4'): + error('Python 3 support is only available with EasyBuild 4.x but you are trying to install EasyBuild %s' + % forcedversion) # create temporary dir for temporary installations tmpdir = tempfile.mkdtemp() @@ -982,10 +1011,12 @@ def main(): """ # check Python version -if sys.version_info[0] != 2 or sys.version_info[1] < 6: - pyver = sys.version.split(' ')[0] - sys.stderr.write("ERROR: Incompatible Python version: %s (should be Python 2 >= 2.6)\n" % pyver) - sys.stderr.write("Please try again using 'python2 %s '\n" % os.path.basename(__file__)) +loose_pyver = LooseVersion(python_version()) +min_pyver2 = LooseVersion('2.6') +min_pyver3 = LooseVersion('3.5') +if loose_pyver < min_pyver2 or (loose_pyver >= LooseVersion('3') and loose_pyver < min_pyver3): + sys.stderr.write("ERROR: Incompatible Python version: %s (should be Python 2 >= %s or Python 3 >= %s)\n" + % (python_version(), min_pyver2, min_pyver3)) sys.exit(1) # distribute_setup.py script (https://pypi.python.org/pypi/distribute) @@ -1117,8 +1148,10 @@ def main(): T4E5Gl7wpTxDXdQtzS1Hv52qHSilmOtEVO3IVjCdl5cgC5VC9T6CY1N4U4B0E1tltaqRtuYc/PyB i9tGe6+O/V0LCkGXvNkrKK2++u9qLFyTkO2sp7xSt/Bfil9os3SeOlY5fvv9mLcFj5zSNUqsRZfU 7lwukTHLpfpLDH2GT+yCCf8D2cp1xw== - -""".decode("base64").decode("zlib") +""" +if IS_PY3: + DISTRIBUTE_SETUP_PY = DISTRIBUTE_SETUP_PY.encode('ascii') +DISTRIBUTE_SETUP_PY = codecs.decode(codecs.decode(DISTRIBUTE_SETUP_PY, "base64"), "zlib") # run main function as body of script main() From a0c7e38e07cf3b2a9533321ed769de671aa18429 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 5 Feb 2020 16:27:37 +0800 Subject: [PATCH 118/344] update test_new_branch_github to also test easyblocks and framework --- easybuild/framework/easyconfig/tools.py | 6 +- easybuild/tools/github.py | 95 +++++++++++++++---------- test/framework/options.py | 51 ++++++++++++- test/framework/sandbox/a_test.py | 3 + 4 files changed, 114 insertions(+), 41 deletions(-) create mode 100644 test/framework/sandbox/a_test.py diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 7d717d6258..b9c79a8136 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -604,17 +604,21 @@ def dump_env_script(easyconfigs): def categorize_files_by_type(paths): """ - Splits list of filepaths into a 3 separate lists: easyconfigs, files to delete and patch files + Splits list of filepaths into a 4 separate lists: easyconfigs, files to delete, patch files and + files with extension .py """ res = { 'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], + 'py_files': [], } for path in paths: if path.startswith(':'): res['files_to_delete'].append(path[1:]) + elif path.endswith('.py'): + res['py_files'].append(path) # file must exist in order to check whether it's a patch file elif os.path.isfile(path) and is_patch_file(path): res['patch_files'].append(path) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 1d1999c998..4937ef95e7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -681,8 +681,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # we need files to create the PR with non_existing_paths = [] ec_paths = [] - if paths['easyconfigs']: - for path in paths['easyconfigs']: + if paths['easyconfigs'] or paths['py_files']: + for path in paths['easyconfigs'] + paths['py_files']: if not os.path.exists(path): non_existing_paths.append(path) else: @@ -696,6 +696,15 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ pr_target_repo = build_option('pr_target_repo') + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + if paths['py_files']: + raise EasyBuildError("You are submitting files with .py extension, " + "did you forget to specify --pr-target-repo?") + else: + if paths['easyconfigs'] or paths['patch_files']: + raise EasyBuildError("You are submitting easyconfigs and/or patches, " + "shouldn\'t this PR target the easyconfigs repo?") + # initialize repository git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') git_repo = init_repo(git_working_dir, pr_target_repo) @@ -731,15 +740,17 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # figure out commit message to use if commit_msg: cnt = len(file_info['paths_in_repo']) - _log.debug("Using specified commit message for all %d new/modified easyconfigs at once: %s", cnt, commit_msg) - elif all(file_info['new']) and not paths['files_to_delete']: + _log.debug("Using specified commit message for all %d new/modified files at once: %s", cnt, commit_msg) + elif pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths['files_to_delete']: # automagically derive meaningful commit message if all easyconfig files are new commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) if paths['patch_files']: commit_msg += " and patches: %s" % ', '.join(os.path.basename(p) for p in paths['patch_files']) + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and all(file_info['new']): + commit_msg = "adding easyblocks: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) else: raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " - "modifying/deleting easyconfigs") + "modifying/deleting files or targeting the framework repo.") # figure out to which software name patches relate, and copy them to the right place if paths['patch_files']: @@ -996,7 +1007,10 @@ def copy_easyblocks(paths, target_dir): mod = imp.load_source(fn, path) clsmembers = inspect.getmembers(mod, inspect.isclass) - classnames = [cl[1].__name__ for cl in clsmembers if cl[1].__module__ == mod.__name__] + if clsmembers: + classnames = [cl[1].__name__ for cl in clsmembers if cl[1].__module__ == mod.__name__] + else: + raise EasyBuildError("Invalid easyblock file") if len(classnames) > 1: raise EasyBuildError("Invalid easyblock file") @@ -1014,7 +1028,7 @@ def copy_easyblocks(paths, target_dir): full_target_path = os.path.join(target_dir, target_path) file_info['paths_in_repo'].append(full_target_path) file_info['new'].append(not os.path.exists(full_target_path)) - copy_file(path, full_target_path) + copy_file(path, full_target_path, force_in_dry_run=True) else: raise EasyBuildError("Subdir easyblocks not found") @@ -1358,11 +1372,10 @@ def new_branch_github(paths, ecs, commit_msg=None): """ Create new branch on GitHub using specified files - :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches) + :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches, files with .py extension) :param ecs: list of parsed easyconfigs, incl. for dependencies (if robot is enabled) :param commit_msg: commit message to use """ - branch_name = build_option('pr_branch_name') if commit_msg is None: commit_msg = build_option('pr_commit_msg') @@ -1473,14 +1486,14 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): file_info = det_file_info(ec_paths, target_dir) - # label easyconfigs for new software and/or new easyconfigs for existing software labels = [] - if any(file_info['new_folder']): - labels.append('new') - if any(file_info['new_file_in_existing_folder']): - labels.append('update') - if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + # label easyconfigs for new software and/or new easyconfigs for existing software + if any(file_info['new_folder']): + labels.append('new') + if any(file_info['new_file_in_existing_folder']): + labels.append('update') + # only use most common toolchain(s) in toolchain label of PR title toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']] toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)]) @@ -1490,33 +1503,39 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): classes = [ec['moduleclass'] for ec in file_info['ecs']] classes_counted = sorted([(classes.count(c), c) for c in nub(classes)]) class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]]) + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: + if any(file_info['new']): + labels.append('new') if title is None: + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + if file_info['ecs'] and all(file_info['new']) and not deleted_paths: + # mention software name/version in PR title (only first 3) + names_and_versions = nub(["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']]) + if len(names_and_versions) <= 3: + main_title = ', '.join(names_and_versions) + else: + main_title = ', '.join(names_and_versions[:3] + ['...']) + + title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) + + # if Python is listed as a dependency, then mention Python version(s) in PR title + pyver = [] + for ec in file_info['ecs']: + # iterate over all dependencies (incl. build dependencies & multi-deps) + for dep in ec.dependencies(): + if dep['name'] == 'Python': + # check whether Python is listed as a multi-dep if it's marked as a build dependency + if dep['build_only'] and 'Python' not in ec['multi_deps']: + continue + else: + pyver.append(dep['version']) + if pyver: + title += " w/ Python %s" % ' + '.join(sorted(nub(pyver))) - if file_info['ecs'] and all(file_info['new']) and not deleted_paths: - # mention software name/version in PR title (only first 3) - names_and_versions = nub(["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']]) - if len(names_and_versions) <= 3: - main_title = ', '.join(names_and_versions) else: - main_title = ', '.join(names_and_versions[:3] + ['...']) - - title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title) - - # if Python is listed as a dependency, then mention Python version(s) in PR title - pyver = [] - for ec in file_info['ecs']: - # iterate over all dependencies (incl. build dependencies & multi-deps) - for dep in ec.dependencies(): - if dep['name'] == 'Python': - # check whether Python is listed as a multi-dep if it's marked as a build dependency - if dep['build_only'] and 'Python' not in ec['multi_deps']: - continue - else: - pyver.append(dep['version']) - if pyver: - title += " w/ Python %s" % ' + '.join(sorted(nub(pyver))) - + raise EasyBuildError("Don't know how to make a PR title for this PR. " + "Please include a title (use --pr-title)") else: raise EasyBuildError("Don't know how to make a PR title for this PR. " "Please include a title (use --pr-title)") diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..1b42289140 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2940,6 +2940,8 @@ def test_new_branch_github(self): return topdir = os.path.dirname(os.path.abspath(__file__)) + + # test easyconfigs test_ecs = os.path.join(topdir, 'easyconfigs', 'test_ecs') toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') @@ -2954,11 +2956,56 @@ def test_new_branch_github(self): remote = 'git@github.com:%s/easybuild-easyconfigs.git' % GITHUB_TEST_ACCOUNT regexs = [ r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyconfigs.git\.\.\.", - r"^== copying easyconfigs to .*/easybuild-easyconfigs\.\.\.", + r"^== copying files to .*/easybuild-easyconfigs\.\.\.", + r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + ] + self._assert_regexs(regexs, txt) + + # test easyblocks + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + + args = [ + '--new-branch-github', + '--pr-target-repo=easybuild-easyblocks', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + toy_eb, + '--pr-title="add easyblock for toy"', + '-D', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + + remote = 'git@github.com:%s/easybuild-easyblocks.git' % GITHUB_TEST_ACCOUNT + regexs = [ + r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-easyblocks.git\.\.\.", + r"^== copying files to .*/easybuild-easyblocks\.\.\.", r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, ] self._assert_regexs(regexs, txt) + # test framework + test_ebs = os.path.join(topdir, 'sandbox') + toy_py = os.path.join(test_ebs, 'a_test.py') + + args = [ + '--new-branch-github', + '--pr-target-repo=easybuild-framework', + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + toy_py, + '--pr-commit-msg="a test"', + '-D', + ] + txt, _ = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) + + remote = 'git@github.com:%s/easybuild-framework.git' % GITHUB_TEST_ACCOUNT + regexs = [ + r"^== fetching branch 'develop' from https://github.com/easybuilders/easybuild-framework.git\.\.\.", + r"^== copying files to .*/easybuild-framework\.\.\.", + r"^== pushing branch '.*' to remote '.*' \(%s\) \[DRY RUN\]" % remote, + ] + self._assert_regexs(regexs, txt) + + def test_new_pr_from_branch(self): """Test --new-pr-from-branch.""" if self.github_token is None: @@ -3019,7 +3066,7 @@ def test_update_branch_github(self): full_repo = 'boegel/easybuild-easyconfigs' regexs = [ r"^== fetching branch 'develop' from https://github.com/%s.git\.\.\." % full_repo, - r"^== copying easyconfigs to .*/git-working-dir.*/easybuild-easyconfigs...", + r"^== copying files to .*/git-working-dir.*/easybuild-easyconfigs...", r"^== pushing branch 'develop' to remote '.*' \(git@github.com:%s.git\) \[DRY RUN\]" % full_repo, r"^Overview of changes:\n.*/easyconfigs/t/toy/toy-0.0.eb \| 32", r"== pushed updated branch 'develop' to boegel/easybuild-easyconfigs \[DRY RUN\]", diff --git a/test/framework/sandbox/a_test.py b/test/framework/sandbox/a_test.py new file mode 100644 index 0000000000..6d8a26f090 --- /dev/null +++ b/test/framework/sandbox/a_test.py @@ -0,0 +1,3 @@ +""" +Used for test_new_branch_github +""" From 489e801ae799b8d6f1dce97dff3c8e4868eacf07 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 5 Feb 2020 17:05:45 +0800 Subject: [PATCH 119/344] update test_categorize_files_by_type --- test/framework/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index f956900f59..d5fae16ece 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2713,7 +2713,7 @@ def test_hidden_toolchain(self): def test_categorize_files_by_type(self): """Test categorize_files_by_type""" - self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': []}, categorize_files_by_type([])) + self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, categorize_files_by_type([])) test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' From 0f62ccef2f96467d5e67ec3ec2e4d4044ca58451 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 09:30:38 +0100 Subject: [PATCH 120/344] Increase timeout for connectivity check to 30s --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 77dd8da0f5..85467cdcd2 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1611,10 +1611,10 @@ def check_github(): # check whether we're online; if not, half of the checks are going to fail... try: print_msg("Making sure we're online...", log=_log, prefix=False, newline=False) - urlopen(GITHUB_URL, timeout=5) + urlopen(GITHUB_URL, timeout=30) print_msg("OK\n", log=_log, prefix=False) except URLError as err: - print_msg("FAIL") + print_msg("FAIL", log=_log, prefix=False) raise EasyBuildError("checking status of GitHub integration must be done online") # GitHub user From 6df639ac3d2f730a0750bdb0820d9281af33ef73 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 09:42:46 +0100 Subject: [PATCH 121/344] Add reason for connectivity check failure --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 85467cdcd2..d862f0ba04 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1614,7 +1614,7 @@ def check_github(): urlopen(GITHUB_URL, timeout=30) print_msg("OK\n", log=_log, prefix=False) except URLError as err: - print_msg("FAIL", log=_log, prefix=False) + print_msg("FAIL (%s)", err, log=_log, prefix=False) raise EasyBuildError("checking status of GitHub integration must be done online") # GitHub user From 12bbbe535147314df1e0263053c89a82966d6087 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 7 Feb 2020 09:58:50 +0100 Subject: [PATCH 122/344] Restore flak8 default ignores --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e4bfd7cb81..d8f4dd91ae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,4 +19,5 @@ builtins = # ignore "Black would make changes" produced by flake8-black # see also https://github.com/houndci/hound/issues/1769 -ignore = BLK100 +# and restore current flake8 default list (Feb2020) +ignore = BLK100,E121,E123,E126,E226,E24,E704,W503,W504 From 813244c8596ac4aff14eecfce13b2fbb2ea47dc3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 12:47:25 +0100 Subject: [PATCH 123/344] Use GITHUB_API_URL to check for connectivity --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d862f0ba04..d4020a6e55 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1611,7 +1611,7 @@ def check_github(): # check whether we're online; if not, half of the checks are going to fail... try: print_msg("Making sure we're online...", log=_log, prefix=False, newline=False) - urlopen(GITHUB_URL, timeout=30) + urlopen(GITHUB_API_URL, timeout=30) print_msg("OK\n", log=_log, prefix=False) except URLError as err: print_msg("FAIL (%s)", err, log=_log, prefix=False) From 41ec934441831b8237ce500a8d60fe429d38ef25 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 10:24:17 +0100 Subject: [PATCH 124/344] Try repeatedly and with different URLs to cater for HTTP issues --- easybuild/tools/github.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d4020a6e55..2e43d9557b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1591,6 +1591,25 @@ def update_pr(pr_id, paths, ecs, commit_msg=None): print_msg(msg, log=_log) +def check_online_status(): + """ + Check whether we currently are online + Return True if online, else an URLError instance of the last failure + """ + # Try repeatedly and with different URLs to cater for flaky servers + # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly + # Timeout and repeats set to total 1 minute + urls = [GITHUB_URL, GITHUB_API_URL] + for i in range(6): + try: + urlopen(urls[i % len(urls)], timeout=10) + result = True + break + except URLError as err: + result = err + return result + + def check_github(): """ Check status of GitHub integration, and report back. @@ -1609,12 +1628,12 @@ def check_github(): print_msg("\nChecking status of GitHub integration...\n", log=_log, prefix=False) # check whether we're online; if not, half of the checks are going to fail... - try: - print_msg("Making sure we're online...", log=_log, prefix=False, newline=False) - urlopen(GITHUB_API_URL, timeout=30) + print_msg("Making sure we're online...", log=_log, prefix=False, newline=False) + online_state = check_online_status() + if online_state is True: print_msg("OK\n", log=_log, prefix=False) - except URLError as err: - print_msg("FAIL (%s)", err, log=_log, prefix=False) + else: + print_msg("FAIL (%s)", online_state, log=_log, prefix=False) raise EasyBuildError("checking status of GitHub integration must be done online") # GitHub user From 87fd425566bbe3827a7ff5ade948dbaadc40a005 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 18:16:07 +0100 Subject: [PATCH 125/344] Use better variable names --- easybuild/tools/github.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 2e43d9557b..3e2b4191fe 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1600,9 +1600,10 @@ def check_online_status(): # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly # Timeout and repeats set to total 1 minute urls = [GITHUB_URL, GITHUB_API_URL] - for i in range(6): + num_repeats = 6 + for attempt in range(num_repeats): try: - urlopen(urls[i % len(urls)], timeout=10) + urlopen(urls[attempt % len(urls)], timeout=10) result = True break except URLError as err: From 40f40fb7b0ae936b9b7fd27fa0615dc697299061 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Jan 2020 20:16:53 +0100 Subject: [PATCH 126/344] bump version to 4.1.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index ca7d7a8e65..f2efa7ae04 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.1.1') +VERSION = LooseVersion('4.1.2.dev0') UNKNOWN = 'UNKNOWN' From b8789f2fc42d82a3d8e5ab1481c7b322c2ae19fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Feb 2020 20:08:51 +0100 Subject: [PATCH 127/344] read patch files as bytestring to avoid UnicodeDecodeError for patches that include funky characters (fixes #3190) --- easybuild/tools/filetools.py | 4 +++- test/framework/filetools.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index e414ed68a7..ef5b3ce5d1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -944,7 +944,9 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False, github=False, f patched_regex = re.compile(patched_regex, re.M) if path is not None: - txt = read_file(path) + # take into account that file may contain non-UTF-8 characters; + # so, read a byyte string, and decode to ascii string (ignoring any non-ascii characters); + txt = read_file(path, mode='rb').decode('ascii', 'ignore') elif txt is None: raise EasyBuildError("Either a file path or a string representing a patch should be supplied") diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 640176ee33..332b3baecf 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -663,6 +663,18 @@ def test_det_patched_files(self): self.assertEqual(ft.det_patched_files(pf), ['b/toy-0.0/toy.source']) self.assertEqual(ft.det_patched_files(pf, omit_ab_prefix=True), ['toy-0.0/toy.source']) + # create a patch file with a non-UTF8 character in it, should not result in problems + # (see https://github.com/easybuilders/easybuild-framework/issues/3190) + test_patch = os.path.join(self.test_prefix, 'test.patch') + patch_txt = b'\n'.join([ + b"--- foo", + b"+++ foo", + b"- test line", + b"+ test line with non-UTF8 char: '\xa0'", + ]) + ft.write_file(test_patch, patch_txt) + self.assertEqual(ft.det_patched_files(test_patch), ['foo']) + def test_guess_patch_level(self): "Test guess_patch_level.""" # create dummy toy.source file so guess_patch_level can work From f09abd76b05add111e0c3dda436a6a2de8f51ea0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 09:30:38 +0100 Subject: [PATCH 128/344] Increase timeout for connectivity check to 30s --- easybuild/tools/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 77dd8da0f5..85467cdcd2 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1611,10 +1611,10 @@ def check_github(): # check whether we're online; if not, half of the checks are going to fail... try: print_msg("Making sure we're online...", log=_log, prefix=False, newline=False) - urlopen(GITHUB_URL, timeout=5) + urlopen(GITHUB_URL, timeout=30) print_msg("OK\n", log=_log, prefix=False) except URLError as err: - print_msg("FAIL") + print_msg("FAIL", log=_log, prefix=False) raise EasyBuildError("checking status of GitHub integration must be done online") # GitHub user From 881de10f716ca70f00e4258ce6c8230810216b51 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 09:42:46 +0100 Subject: [PATCH 129/344] Add reason for connectivity check failure --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 85467cdcd2..d862f0ba04 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1614,7 +1614,7 @@ def check_github(): urlopen(GITHUB_URL, timeout=30) print_msg("OK\n", log=_log, prefix=False) except URLError as err: - print_msg("FAIL", log=_log, prefix=False) + print_msg("FAIL (%s)", err, log=_log, prefix=False) raise EasyBuildError("checking status of GitHub integration must be done online") # GitHub user From d8a8be7f2c9d54ce4eea37518d05bd4aeab93f23 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 7 Feb 2020 12:47:25 +0100 Subject: [PATCH 130/344] Use GITHUB_API_URL to check for connectivity --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d862f0ba04..d4020a6e55 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1611,7 +1611,7 @@ def check_github(): # check whether we're online; if not, half of the checks are going to fail... try: print_msg("Making sure we're online...", log=_log, prefix=False, newline=False) - urlopen(GITHUB_URL, timeout=30) + urlopen(GITHUB_API_URL, timeout=30) print_msg("OK\n", log=_log, prefix=False) except URLError as err: print_msg("FAIL (%s)", err, log=_log, prefix=False) From 8b54fde362718cfbfbb11dac76ed109bbf3d2eae Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 8 Feb 2020 13:46:11 +0100 Subject: [PATCH 131/344] only ignore non-UTF-8 characters in det_patched_files --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index ef5b3ce5d1..457b5e6a42 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -945,8 +945,8 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False, github=False, f if path is not None: # take into account that file may contain non-UTF-8 characters; - # so, read a byyte string, and decode to ascii string (ignoring any non-ascii characters); - txt = read_file(path, mode='rb').decode('ascii', 'ignore') + # so, read a byte string, and decode to UTF-8 string (ignoring any non-UTF-8 characters); + txt = read_file(path, mode='rb').decode(encoding='utf-8', errors='ignore') elif txt is None: raise EasyBuildError("Either a file path or a string representing a patch should be supplied") From 3f369d6f19ad6f7e94a0f4cb818b2cd493f58cb3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 8 Feb 2020 21:06:34 +0100 Subject: [PATCH 132/344] use replace rather than ignore when decoding byte string representing patch contents, following @zao's suggestion --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 457b5e6a42..4ed13ab013 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -946,7 +946,7 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False, github=False, f if path is not None: # take into account that file may contain non-UTF-8 characters; # so, read a byte string, and decode to UTF-8 string (ignoring any non-UTF-8 characters); - txt = read_file(path, mode='rb').decode(encoding='utf-8', errors='ignore') + txt = read_file(path, mode='rb').decode(encoding='utf-8', errors='replace') elif txt is None: raise EasyBuildError("Either a file path or a string representing a patch should be supplied") From cd0c3d8c7d0e42fcb6387e1d165247f5233eb721 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 10 Feb 2020 09:02:04 +0100 Subject: [PATCH 133/344] Add test for new ways to use update function on string targets --- test/framework/easyconfig.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index ad97d3f2b9..104bae30ac 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1519,17 +1519,22 @@ def test_update(self): ec.update('configopts', 'CC="$CC"') ec.update('configopts', 'CXX="$CXX"') self.assertTrue(ec['configopts'].strip().endswith('CC="$CC" CXX="$CXX"')) + # spaces in between multiple updates for string values from list + ec.update('configopts', ['MORE_VALUE', 'EVEN_MORE']) + self.assertTrue(ec['configopts'].strip().endswith('MORE_VALUE EVEN_MORE')) # for list values: extend ec.update('patches', ['foo.patch', 'bar.patch']) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' self.assertEqual(ec['patches'], [toy_patch_fn, ('toy-extra.txt', 'toy-0.0'), 'foo.patch', 'bar.patch']) - # for unallowed duplicates + # for unallowed duplicates on string values ec.update('configopts', 'SOME_VALUE') configopts_tmp = ec['configopts'] ec.update('configopts', 'SOME_VALUE', allow_duplicate=False) self.assertEqual(ec['configopts'], configopts_tmp) + ec.update('configopts', ['CC="$CC"', 'SOME_VALUE'], allow_duplicate=False) + self.assertEqual(ec['configopts'], configopts_tmp) # for unallowed duplicates when a list is used ec.update('patches', ['foo2.patch', 'bar2.patch']) From a5ea3670892c23f295d35498f8273698a29fb4ba Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Feb 2020 09:38:23 +0100 Subject: [PATCH 134/344] Extend ignore rather than overwrite it --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d8f4dd91ae..430d761b59 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,5 +19,4 @@ builtins = # ignore "Black would make changes" produced by flake8-black # see also https://github.com/houndci/hound/issues/1769 -# and restore current flake8 default list (Feb2020) -ignore = BLK100,E121,E123,E126,E226,E24,E704,W503,W504 +extend-ignore = BLK100 From ea06c1924d342ee93b04b1ff500b56d23e4d64bd Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 10 Feb 2020 10:06:15 +0100 Subject: [PATCH 135/344] easyconfig.update: Need to regex escape the item before inclusion into the regex. --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f61882e6cf..c60476dbfc 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -565,7 +565,7 @@ def update(self, key, value, allow_duplicate=True): prev_value = self[key] if isinstance(prev_value, string_type): for item in lval: - if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % item, prev_value)): + if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % re.escape(item), prev_value)): prev_value += ' %s ' % item elif isinstance(prev_value, list): for item in lval: From 2e11b4829a6c804750a954a13846730b552d612b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 10 Feb 2020 11:12:54 +0100 Subject: [PATCH 136/344] Return list of all errors --- easybuild/tools/github.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3e2b4191fe..44b1c6d450 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1594,21 +1594,24 @@ def update_pr(pr_id, paths, ecs, commit_msg=None): def check_online_status(): """ Check whether we currently are online - Return True if online, else an URLError instance of the last failure + Return True if online, else a list of error messages """ # Try repeatedly and with different URLs to cater for flaky servers # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly # Timeout and repeats set to total 1 minute urls = [GITHUB_URL, GITHUB_API_URL] num_repeats = 6 + errors = set() # Use set to record only unique errors for attempt in range(num_repeats): + # Cycle through URLs + url = urls[attempt % len(urls)] try: - urlopen(urls[attempt % len(urls)], timeout=10) - result = True + urlopen(url, timeout=10) + errors = None break except URLError as err: - result = err - return result + errors.add('%s: %s' % (url, err)) + return sorted(errors) if errors else True def check_github(): @@ -1634,7 +1637,7 @@ def check_github(): if online_state is True: print_msg("OK\n", log=_log, prefix=False) else: - print_msg("FAIL (%s)", online_state, log=_log, prefix=False) + print_msg("FAIL (%s)", ', '.join(online_state), log=_log, prefix=False) raise EasyBuildError("checking status of GitHub integration must be done online") # GitHub user From 2976d5a9da8e52f94efc7f32470d42cb3c212e28 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 10 Feb 2020 13:24:16 +0100 Subject: [PATCH 137/344] easyconfig.update: fix assignments to not modify in place. Rename temp variable to better match usage. --- easybuild/framework/easyconfig/easyconfig.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index c60476dbfc..d6033eb7c3 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -562,19 +562,20 @@ def update(self, key, value, allow_duplicate=True): msg += "attempted update value, '%s', is not a string or list." raise EasyBuildError(msg, key, value) - prev_value = self[key] - if isinstance(prev_value, string_type): + param_value = self[key] + if isinstance(param_value, string_type): for item in lval: - if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % re.escape(item), prev_value)): - prev_value += ' %s ' % item - elif isinstance(prev_value, list): + # re.search: only add value to string if it's not there yet (surrounded by whitespace) + if allow_duplicate or (not re.search(r'(^|\s+)%s(\s+|$)' % re.escape(item), param_value)): + param_value = param_value + ' %s ' % item + elif isinstance(param_value, list): for item in lval: - if allow_duplicate or item not in prev_value: - prev_value.append(item) + if allow_duplicate or item not in param_value: + param_value = param_value + [item] else: raise EasyBuildError("Can't update configuration value for %s, because it's not a string or list.", key) - self[key] = prev_value + self[key] = param_value def set_keys(self, params): """ From 359a2bb3c40db9b14df8844452ff3e2a99c3da25 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Feb 2020 14:58:40 +0100 Subject: [PATCH 138/344] Add an option to git_config to retain the .git directory --- easybuild/tools/filetools.py | 6 +++++- test/framework/filetools.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index e414ed68a7..c9fa95628d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1864,6 +1864,7 @@ def get_source_tarball_from_git(filename, targetdir, git_config): repo_name = git_config.pop('repo_name', None) commit = git_config.pop('commit', None) recursive = git_config.pop('recursive', False) + keep_git_dir = git_config.pop('keep_git_dir', False) # input validation of git_config dict if git_config: @@ -1912,7 +1913,10 @@ def get_source_tarball_from_git(filename, targetdir, git_config): run.run_cmd(' '.join(checkout_cmd), log_all=True, log_ok=False, simple=False, regexp=False, path=repo_name) # create an archive and delete the git repo directory - tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] + if keep_git_dir: + tar_cmd = ['tar', 'cfvz', targetpath, repo_name] + else: + tar_cmd = ['tar', 'cfvz', targetpath, '--exclude', '.git', repo_name] run.run_cmd(' '.join(tar_cmd), log_all=True, log_ok=False, simple=False, regexp=False) # cleanup (repo_name dir does not exist in dry run mode) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 640176ee33..70e6408f6c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1970,6 +1970,15 @@ def run_check(): ]) run_check() + git_config['keep_git_dir'] = True + expected = '\n'.join([ + ' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', + " \(in .*/tmp.*\)", + ' running command "tar cfvz .*/target/test.tar.gz testrepository"', + " \(in .*/tmp.*\)", + ]) + run_check() + del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ From 0295225395444c7aba551f55e07c330ced58698c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Feb 2020 15:04:28 +0100 Subject: [PATCH 139/344] don't use keyword arguments in .decode in det_patched_files, since they're not supported in Python 2.6 --- easybuild/tools/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4ed13ab013..88b65ab3af 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -946,7 +946,7 @@ def det_patched_files(path=None, txt=None, omit_ab_prefix=False, github=False, f if path is not None: # take into account that file may contain non-UTF-8 characters; # so, read a byte string, and decode to UTF-8 string (ignoring any non-UTF-8 characters); - txt = read_file(path, mode='rb').decode(encoding='utf-8', errors='replace') + txt = read_file(path, mode='rb').decode('utf-8', 'replace') elif txt is None: raise EasyBuildError("Either a file path or a string representing a patch should be supplied") From 42e72aa4a56dd2ad142d84b894ee67fee94a8c97 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Feb 2020 15:10:28 +0100 Subject: [PATCH 140/344] Pass flake8 --- test/framework/filetools.py | 55 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 70e6408f6c..12a2fde32b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -268,7 +268,7 @@ def test_checksums(self): # checksum of length 32 is assumed to be MD5, length 64 to be SHA256, other lengths not allowed # checksum of length other than 32/64 yields an error - error_pattern = "Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)" + error_pattern = r"Length of checksum '.*' \(\d+\) does not match with either MD5 \(32\) or SHA256 \(64\)" for checksum in ['tooshort', 'inbetween32and64charactersisnotgoodeither', known_checksums['sha256'] + 'foo']: self.assertErrorRegex(EasyBuildError, error_pattern, ft.verify_checksum, fp, checksum) @@ -584,7 +584,7 @@ def test_read_write_file(self): txt2 = '\n'.join(['test', '123']) ft.write_file(fp, txt2, append=True) - self.assertEqual(ft.read_file(fp), txt+txt2) + self.assertEqual(ft.read_file(fp), txt + txt2) # test backing up of existing file ft.write_file(fp, 'foo', backup=True) @@ -1800,7 +1800,7 @@ def test_move_file(self): self.mock_stderr(False) # informative message printed, but file was not actually moved - regex = re.compile("^moved file .*/test\.txt to .*/new_test\.txt$") + regex = re.compile(r"^moved file .*/test\.txt to .*/new_test\.txt$") self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertEqual(stderr, '') @@ -1863,7 +1863,7 @@ def test_diff_files(self): ]) res = ft.diff_files(foo, bar) self.assertTrue(res.endswith(expected), "%s ends with %s" % (res, expected)) - regex = re.compile('^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M) + regex = re.compile(r'^--- .*/foo\s*\n\+\+\+ .*/bar\s*$', re.M) self.assertTrue(regex.search(res), "Pattern '%s' found in: %s" % (regex.pattern, res)) def test_get_source_tarball_from_git(self): @@ -1955,50 +1955,50 @@ def run_check(): } expected = '\n'.join([ ' running command "git clone --branch master git@github.com:hpcugent/testrepository.git"', - " \(in .*/tmp.*\)", + r" \(in .*/tmp.*\)", ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - " \(in .*/tmp.*\)", + r" \(in .*/tmp.*\)", ]) run_check() git_config['recursive'] = True expected = '\n'.join([ - ' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', - " \(in .*/tmp.*\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - " \(in .*/tmp.*\)", + r' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", ]) run_check() git_config['keep_git_dir'] = True expected = '\n'.join([ - ' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', - " \(in .*/tmp.*\)", - ' running command "tar cfvz .*/target/test.tar.gz testrepository"', - " \(in .*/tmp.*\)", + r' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', + r" \(in .*/tmp.*\)", + r' running command "tar cfvz .*/target/test.tar.gz testrepository"', + r" \(in .*/tmp.*\)", ]) run_check() del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - ' running command "git clone --recursive git@github.com:hpcugent/testrepository.git"', - " \(in .*/tmp.*\)", - ' running command "git checkout 8456f86 && git submodule update"', - " \(in testrepository\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - " \(in .*/tmp.*\)", + r' running command "git clone --recursive git@github.com:hpcugent/testrepository.git"', + r" \(in .*/tmp.*\)", + r' running command "git checkout 8456f86 && git submodule update"', + r" \(in testrepository\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", ]) run_check() del git_config['recursive'] expected = '\n'.join([ - ' running command "git clone git@github.com:hpcugent/testrepository.git"', - " \(in .*/tmp.*\)", - ' running command "git checkout 8456f86"', - " \(in testrepository\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', - " \(in .*/tmp.*\)", + r' running command "git clone git@github.com:hpcugent/testrepository.git"', + r" \(in .*/tmp.*\)", + r' running command "git checkout 8456f86"', + r" \(in testrepository\)", + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r" \(in .*/tmp.*\)", ]) run_check() @@ -2013,7 +2013,7 @@ def test_is_sha256_checksum(self): True, 12345, '', - (a_sha256_checksum, ), + (a_sha256_checksum,), [], ]: self.assertFalse(ft.is_sha256_checksum(not_a_sha256_checksum)) @@ -2075,7 +2075,6 @@ def test_fake_vsc(self): self.assertTrue(pkgutil.__file__.endswith('/test_fake_vsc/pkgutil.py')) - def suite(): """ returns all the testcases in this module """ return TestLoaderFiltered().loadTestsFromTestCase(FileToolsTest, sys.argv[1:]) From 259d981b1341aad545a42f8f3242e3ca70cec3da Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Feb 2020 15:12:48 +0100 Subject: [PATCH 141/344] Make things consistent --- test/framework/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 12a2fde32b..142d9d1699 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1954,9 +1954,9 @@ def run_check(): 'tag': 'master', } expected = '\n'.join([ - ' running command "git clone --branch master git@github.com:hpcugent/testrepository.git"', + r' running command "git clone --branch master git@github.com:hpcugent/testrepository.git"', r" \(in .*/tmp.*\)", - ' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', + r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", ]) run_check() From 818904ae0b5c5d592ede135562c8a7c90fc5d502 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 10 Feb 2020 15:31:25 +0100 Subject: [PATCH 142/344] Be careful to remove the key after we're finished checking it --- test/framework/filetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 142d9d1699..62ecd397c4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1978,6 +1978,7 @@ def run_check(): r" \(in .*/tmp.*\)", ]) run_check() + del git_config['keep_git_dir'] del git_config['tag'] git_config['commit'] = '8456f86' From 9039e6a1bef841b79bdd9f5d4d6dd43787197a0d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 10 Feb 2020 15:34:43 +0100 Subject: [PATCH 143/344] Don't crash when GitPython is not installed in Python3 When there is a folder (or tar file) named "git" in PYTHONPATH (includes PWD) Python3 imports that as a namespace package So to check if we actually have GitPython we need to import something from that package to trigger an import error. --- easybuild/tools/version.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index f2efa7ae04..aed5217e90 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -55,18 +55,18 @@ def get_git_revision(): relies on GitPython (see http://gitorious.org/git-python) """ try: - import git + from git import Git, GitCommandError except ImportError: return UNKNOWN try: path = os.path.dirname(__file__) - gitrepo = git.Git(path) + gitrepo = Git(path) res = gitrepo.rev_list('HEAD').splitlines()[0] # 'encode' may be required to make sure a regular string is returned rather than a unicode string # (only needed in Python 2; in Python 3, regular strings are already unicode) if not isinstance(res, str): res = res.encode('ascii') - except git.GitCommandError: + except GitCommandError: res = UNKNOWN return res From e51e4e4fc8f6390c8891a2e76290d2d35f405876 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Feb 2020 20:06:32 +0100 Subject: [PATCH 144/344] implement support creating/dumping/loading index of files in path + leverage this in search_file function --- easybuild/tools/filetools.py | 90 +++++++++++++++++++++++++++++------- test/framework/filetools.py | 45 ++++++++++++++++++ 2 files changed, 119 insertions(+), 16 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index e414ed68a7..d8b3f0e773 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -109,6 +109,7 @@ r'~': "_tilde_", } +PATH_INDEX_FILENAME = '.eb-path-index' CHECKSUM_TYPE_MD5 = 'md5' CHECKSUM_TYPE_SHA256 = 'sha256' @@ -589,6 +590,62 @@ def download_file(filename, url, path, forced=False): return None +def create_index(path, ignore_dirs=None): + """ + Create index for files in specified path. + """ + if ignore_dirs is None: + ignore_dirs = [] + + index = set() + + for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): + for filename in filenames: + # use relative paths in index + index.add(os.path.join(dirpath[len(path)+1:], filename)) + + # do not consider (certain) hidden directories + # note: we still need to consider e.g., .local ! + # replace list elements using [:], so os.walk doesn't process deleted directories + # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders + dirnames[:] = [d for d in dirnames if d not in ignore_dirs] + + return index + + +def dump_index(path): + """ + Create index for files in specified path, and dump it to file (alphabetically sorted). + """ + + index_fp = os.path.join(path, PATH_INDEX_FILENAME) + index_contents = create_index(path) + + write_file(index_fp, '\n'.join(sorted(index_contents))) + + +def load_index(path, ignore_dirs=None): + """ + Load index for specified path, and return contents (or None if no index exists). + """ + if ignore_dirs is None: + ignore_dirs = [] + + index_fp = os.path.join(path, PATH_INDEX_FILENAME) + + index, res = None, set() + + if os.path.exists(index_fp): + index = read_file(index_fp).splitlines() + + for path in index: + path_dirs = path.split(os.path.sep)[:-1] + if not any(d in path_dirs for d in ignore_dirs): + res.add(path) + + return res + + def find_easyconfigs(path, ignore_dirs=None): """ Find .eb easyconfig files in path @@ -654,22 +711,23 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen if not terse: print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) - for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): - for filename in filenames: - if query.search(filename): - if not path_hits: - var = "CFGS%d" % var_index - var_index += 1 - if filename_only: - path_hits.append(filename) - else: - path_hits.append(os.path.join(dirpath, filename)) - - # do not consider (certain) hidden directories - # note: we still need to consider e.g., .local ! - # replace list elements using [:], so os.walk doesn't process deleted directories - # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders - dirnames[:] = [d for d in dirnames if d not in ignore_dirs] + path_index = load_index(path, ignore_dirs=ignore_dirs) + if path_index: + _log.info("Cache found for %s, so using it...", path) + else: + _log.info("No index found for %s, creating one...", path) + path_index = create_index(path, ignore_dirs=ignore_dirs) + + for filepath in path_index: + filename = os.path.basename(filepath) + if query.search(filename): + if not path_hits: + var = "CFGS%d" % var_index + var_index += 1 + if filename_only: + path_hits.append(filename) + else: + path_hits.append(os.path.join(path, filepath)) path_hits = sorted(path_hits) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 640176ee33..51dfb22f0e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1652,6 +1652,51 @@ def test_remove(self): ft.adjust_permissions(self.test_prefix, stat.S_IWUSR, add=True) + def test_index_functions(self): + """Test *_index functions.""" + + test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + # first test create_index function + index = ft.create_index(test_ecs) + self.assertEqual(len(index), 79) + + expected = [ + os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), + os.path.join('t', 'toy', 'toy-0.0.eb'), + os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'), + ] + for fn in expected: + self.assertTrue(fn in index) + + for fp in index: + self.assertTrue(fp.endswith('.eb')) + + # set up some files to create actual index file for + ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) + + # test dump_index function + ft.dump_index(self.test_prefix) + + index_fp = os.path.join(self.test_prefix, '.eb-path-index') + self.assertTrue(os.path.exists(index_fp)) + + index_txt = ft.read_file(index_fp) + expected = [ + os.path.join('g', 'gzip', 'gzip-1.4.eb'), + os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'), + os.path.join('g', 'gompic', 'gompic-2018a.eb'), + ] + for fn in expected: + regex = re.compile('^%s$' % fn, re.M) + self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) + + # test load_index function + index = ft.load_index(self.test_prefix) + self.assertEqual(len(index), 24) + for fn in expected: + self.assertTrue(fn in index) + def test_search_file(self): """Test search_file function.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From ce29a3877dde2abceb766613630e5e9782478b99 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 10 Feb 2020 20:08:07 +0100 Subject: [PATCH 145/344] use path index in robot_find_easyconfig, if available (and cache it) --- easybuild/framework/easyconfig/easyconfig.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b3e8af1cb8..97e32a02a4 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -61,7 +61,8 @@ from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name -from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file +from easybuild.tools.filetools import create_index, find_backup_name_candidate, find_easyconfigs, load_index +from easybuild.tools.filetools import read_file, write_file from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version @@ -102,6 +103,7 @@ _easyconfig_files_cache = {} _easyconfigs_cache = {} +_path_indexes = {} def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): @@ -1890,10 +1892,19 @@ def robot_find_easyconfig(name, version): res = None for path in paths: + if path in _path_indexes: + path_index = _path_indexes[path] + _log.info("Found loaded index for %s", path) + else: + path_index = load_index(path) + if path_index: + _path_indexes[path] = path_index + _log.info("Loaded index for %s", path) + easyconfigs_paths = create_paths(path, name, version) for easyconfig_path in easyconfigs_paths: _log.debug("Checking easyconfig path %s" % easyconfig_path) - if os.path.isfile(easyconfig_path): + if easyconfig_path in path_index or os.path.isfile(easyconfig_path): _log.debug("Found easyconfig file for name %s, version %s at %s" % (name, version, easyconfig_path)) _easyconfig_files_cache[key] = os.path.abspath(easyconfig_path) res = _easyconfig_files_cache[key] From a4f3d673315c2bd937e1d948a32bd21f69c37ecd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2020 08:23:07 +0100 Subject: [PATCH 146/344] make create_index check whether specified path is an existing directory, load_index return None if there is no index & dump_index return path to index file --- easybuild/tools/filetools.py | 15 +++++++++++---- test/framework/filetools.py | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d8b3f0e773..af73f2bafe 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -599,6 +599,11 @@ def create_index(path, ignore_dirs=None): index = set() + if not os.path.exists(path): + raise EasyBuildError("Specified path does not exist: %s", path) + elif not os.path.isdir(path): + raise EasyBuildError("Specified path is not a directory: %s", path) + for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): for filename in filenames: # use relative paths in index @@ -623,6 +628,8 @@ def dump_index(path): write_file(index_fp, '\n'.join(sorted(index_contents))) + return index_fp + def load_index(path, ignore_dirs=None): """ @@ -643,7 +650,7 @@ def load_index(path, ignore_dirs=None): if not any(d in path_dirs for d in ignore_dirs): res.add(path) - return res + return res or None def find_easyconfigs(path, ignore_dirs=None): @@ -712,11 +719,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) path_index = load_index(path, ignore_dirs=ignore_dirs) - if path_index: - _log.info("Cache found for %s, so using it...", path) - else: + if path_index is None: _log.info("No index found for %s, creating one...", path) path_index = create_index(path, ignore_dirs=ignore_dirs) + else: + _log.info("Index found for %s, so using it...", path) for filepath in path_index: filename = os.path.basename(filepath) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 51dfb22f0e..7e1ad84138 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1657,7 +1657,17 @@ def test_index_functions(self): test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') - # first test create_index function + # create_index checks whether specified path is an existing directory + doesnotexist = os.path.join(self.test_prefix, 'doesnotexist') + self.assertErrorRegex(EasyBuildError, "Specified path does not exist", ft.create_index, doesnotexist) + + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + self.assertErrorRegex(EasyBuildError, "Specified path is not a directory", ft.create_index, toy_ec) + + # load_index just returns None if there is no index in specified directory + self.assertEqual(ft.load_index(self.test_prefix), None) + + # create index for test easyconfigs index = ft.create_index(test_ecs) self.assertEqual(len(index), 79) @@ -1676,10 +1686,9 @@ def test_index_functions(self): ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) # test dump_index function - ft.dump_index(self.test_prefix) - - index_fp = os.path.join(self.test_prefix, '.eb-path-index') + index_fp = ft.dump_index(self.test_prefix) self.assertTrue(os.path.exists(index_fp)) + self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp))) index_txt = ft.read_file(index_fp) expected = [ From 5d3b2637351e756e7b2d2969a18e65be261d0002 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2020 08:24:36 +0100 Subject: [PATCH 147/344] create index for path if no index is available in robot_find_easyconfig, and cache it --- easybuild/framework/easyconfig/easyconfig.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 97e32a02a4..2aaee7f28d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1897,10 +1897,14 @@ def robot_find_easyconfig(name, version): _log.info("Found loaded index for %s", path) else: path_index = load_index(path) - if path_index: - _path_indexes[path] = path_index + if path_index is None: + _log.info("No index found for %s, so creating it...", path) + path_index = create_index(path) + else: _log.info("Loaded index for %s", path) + _path_indexes[path] = path_index + easyconfigs_paths = create_paths(path, name, version) for easyconfig_path in easyconfigs_paths: _log.debug("Checking easyconfig path %s" % easyconfig_path) From 078f09930e81aaf8f5d3d659494992532baffb6a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2020 11:38:42 +0100 Subject: [PATCH 148/344] add support for --create-index --- easybuild/main.py | 10 +++++++++- easybuild/tools/options.py | 1 + test/framework/options.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 69c47a7293..7eeb8d9d84 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -56,7 +56,8 @@ from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, read_file, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index +from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr @@ -255,9 +256,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif options.list_software: print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) + elif options.create_index: + print_msg("Creating index for %s..." % options.create_index, prefix=False) + index_fp = dump_index(options.create_index) + index = load_index(options.create_index) + print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) + # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, + options.create_index, options.install_github_token, options.list_installed_software, options.list_software, diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d149ee3d79..efe6e644ce 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -681,6 +681,7 @@ def easyconfig_options(self): descr = ("Options for Easyconfigs", "Options that affect all specified easyconfig files.") opts = OrderedDict({ + 'create-index': ("Create index for files in specified directory", None, 'store', None), 'fix-deprecated-easyconfigs': ("Fix use of deprecated functionality in specified easyconfig files.", None, 'store_true', False), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..e3540827fa 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4676,6 +4676,25 @@ def test_cuda_compute_capabilities(self): regex = re.compile(r"^cuda-compute-capabilities\s*\(C\)\s*=\s*3\.5, 6\.2, 7\.0$", re.M) self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) + def test_create_index(self): + """Test --create-index option.""" + test_ecs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + remove_dir(self.test_prefix) + copy_dir(test_ecs, self.test_prefix) + + args = ['--create-index', self.test_prefix] + stdout, stderr = self._run_mock_eb(args, raise_error=True) + + self.assertEqual(stderr, '') + + patterns = [ + r"^Creating index for %s\.\.\.$", + r"^Index created at %s/\.eb-path-index \([0-9]+ files\)$", + ] + for pattern in patterns: + regex = re.compile(pattern % self.test_prefix, re.M) + self.assertTrue(regex.search(stdout), "Pattern %s matches in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ From 7c831bf349a0c63d2b2fdadde2c43cbf0c8e83ce Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 12 Feb 2020 12:32:39 +0100 Subject: [PATCH 149/344] Fix os_name_map for RHEL8. - RHEL8 no longer has a distinction between server and client, so the contents of /etc/redhat-release and similar files no longer contain such labels. --- easybuild/tools/systemtools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 1b285e191d..1400742d18 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -563,6 +563,7 @@ def get_os_name(): os_name_map = { 'red hat enterprise linux server': 'RHEL', + 'red hat enterprise linux': 'RHEL', # RHEL8 has no server/client 'scientific linux sl': 'SL', 'scientific linux': 'SL', 'suse linux enterprise server': 'SLES', From 80504bc0683ed460070a84ee150e5fe7b8b27c22 Mon Sep 17 00:00:00 2001 From: Terje Kvernes Date: Wed, 12 Feb 2020 12:36:32 +0100 Subject: [PATCH 150/344] The hound requires multiple spaces before inline comments. --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 1400742d18..0e114cb0dc 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -563,7 +563,7 @@ def get_os_name(): os_name_map = { 'red hat enterprise linux server': 'RHEL', - 'red hat enterprise linux': 'RHEL', # RHEL8 has no server/client + 'red hat enterprise linux': 'RHEL', # RHEL8 has no server/client 'scientific linux sl': 'SL', 'scientific linux': 'SL', 'suse linux enterprise server': 'SLES', From e70e735a4d1ad53b10f85e898f0ea44e4d3bd097 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 13 Feb 2020 14:14:59 +0100 Subject: [PATCH 151/344] Update contrib/hooks/README.rst --- contrib/hooks/README.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/contrib/hooks/README.rst b/contrib/hooks/README.rst index 3821474a4c..626d403ca7 100644 --- a/contrib/hooks/README.rst +++ b/contrib/hooks/README.rst @@ -1,9 +1,15 @@ +Example implementations of EasyBuild hooks +================================= + .. image:: https://easybuilders.github.io/easybuild/images/easybuild_logo_small.png :align: center -https://easybuild.readthedocs.io +EasyBuild website: https://easybuilders.github.io/easybuild/ +docs: https://easybuild.readthedocs.io -This directory contain examples of hooks used at various sites and also -a couple of small examples with explanations. +This directory contain examples of implementations of EasyBuild hooks +used at various sites, along with a couple of small examples with +explanations. -See https://easybuild.readthedocs.io/en/latest/Hooks.html for documentation on hooks. +See https://easybuild.readthedocs.io/en/latest/Hooks.html for +documentation on hooks in EasyBuild. From 72052ff84f0e3b5db259d881463f90de8ed08fee Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 13 Feb 2020 14:31:16 +0100 Subject: [PATCH 152/344] contrib/hooks/hpc2n_hooks.py: Fix overindentation --- contrib/hooks/hpc2n_hooks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/hooks/hpc2n_hooks.py b/contrib/hooks/hpc2n_hooks.py index 9a85e5aed4..ad93fe88e1 100644 --- a/contrib/hooks/hpc2n_hooks.py +++ b/contrib/hooks/hpc2n_hooks.py @@ -24,11 +24,11 @@ def add_extra_dependencies(ec, dep_type, extra_deps): for dep in extra_deps: ec[dep_type].append(dep) elif dep_type == 'osdependencies': - if isinstance(extra_deps, tuple): - ec[dep_type].append(extra_deps) - else: - raise EasyBuildError("parse_hook: Type of extra_deps argument (%s), for 'osdependencies' must be " - "tuple, found %s" % (extra_deps, type(extra_deps))) + if isinstance(extra_deps, tuple): + ec[dep_type].append(extra_deps) + else: + raise EasyBuildError("parse_hook: Type of extra_deps argument (%s), for 'osdependencies' must be " + "tuple, found %s" % (extra_deps, type(extra_deps))) else: raise EasyBuildError("parse_hook: Incorrect dependency type in add_extra_dependencies: %s" % dep_type) From 70d08369d2c64ddfdf52fa65a4897c1893335fc8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 14 Feb 2020 09:07:17 +0100 Subject: [PATCH 153/344] significantly speed up -D/--dry-run by avoiding useless 'module show' calls --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 7d717d6258..b255c36a03 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -90,7 +90,7 @@ def skip_available(easyconfigs, modtool): """Skip building easyconfigs for existing modules.""" module_names = [ec['full_mod_name'] for ec in easyconfigs] - modules_exist = modtool.exist(module_names) + modules_exist = modtool.exist(module_names, maybe_partial=False) retained_easyconfigs = [] for ec, mod_name, mod_exists in zip(easyconfigs, module_names, modules_exist): if mod_exists: From 3496294ac138c3e9da3fccf42abdfb7a1777957f Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Sat, 15 Feb 2020 10:15:02 -0500 Subject: [PATCH 154/344] testing coverage report gathering in travis build. Need to enable coveralls from marketplace to get report in coveralls --- .coveragerc | 26 ++++++++++++++++++++++++++ .travis.yml | 5 ++++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..cef5b1c29c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,26 @@ +# .coveragerc to control coverage.py +[run] +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True + +[html] +directory = coverage_html_report diff --git a/.travis.yml b/.travis.yml index b2a8464dee..d00ec1d2c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,7 @@ before_install: - pip --version - pip install --upgrade pip - pip --version + - pip install python-coveralls - pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" @@ -89,7 +90,7 @@ script: - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt # run test suite - - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log + - coverage run -m test.framework.suite # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) @@ -109,3 +110,5 @@ script: # simply sanity check on bootstrapped EasyBuild module - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all - module load EasyBuild; eb --version +after_success: + - coveralls From 412c5db95bc19bba1fb7af51974f35d1738b12b1 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Sat, 15 Feb 2020 10:47:44 -0500 Subject: [PATCH 155/344] Running coverage report -m in Travis to view coverage report after regtest --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d00ec1d2c6..5f805ff45f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,7 @@ script: - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt # run test suite - coverage run -m test.framework.suite + - coverage report -m # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) From d9187d2c5df2dc154aab939cba07acb9f563f9ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 16 Feb 2020 10:54:43 +0100 Subject: [PATCH 156/344] ignore errors when collecting coverage report --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5f805ff45f..bb9324b6cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,8 @@ script: - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt # run test suite - coverage run -m test.framework.suite - - coverage report -m + # ignore errors, since tests inject easyblocks for which there are no source files anymore after the tests have run + - coverage report -m --ignore-errors # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) From 022e3a7ca103fcb45f679a17eae9cd4cc15b022c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 10 Feb 2020 13:02:33 +0100 Subject: [PATCH 157/344] Optionally call log.warning in print_warning --- easybuild/tools/build_log.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 616c839531..ba45075069 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -358,10 +358,13 @@ def print_warning(msg, *args, **kwargs): if args: msg = msg % args + log = kwargs.pop('log', None) silent = kwargs.pop('silent', False) if kwargs: raise EasyBuildError("Unknown named arguments passed to print_warning: %s", kwargs) + if log: + log.warning(msg) if not silent: sys.stderr.write("\nWARNING: %s\n\n" % msg) From 66485426ef16d4c4182e828162c620ebe547826c Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 17 Feb 2020 14:36:30 +0100 Subject: [PATCH 158/344] Fix PR remarks This does not fix the style checks --- easybuild/framework/easyconfig/easyconfig.py | 72 ++++++++++++-------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 75eda133a1..1c21edb30e 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1153,7 +1153,7 @@ def _validate(self, attr, values): # private method if self[attr] and self[attr] not in values: raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) - def handle_external_module_metadata_by_probing_modules(self, dep_name): + def _handle_ext_module_metadata_by_probing_env(self, dep_name, dependency=dict()): """ helper function for handle_external_module_metadata handles metadata for external module dependencies when there is not entry in the @@ -1170,38 +1170,38 @@ def handle_external_module_metadata_by_probing_modules(self, dep_name): If neither of the pairs is found, then an empty dictionary is returned """ - dependency = {} - short_ext_modname = dep_name.split('/')[0] - - if short_ext_modname.startswith('craype-'): - short_ext_modname = short_ext_modname.split('craype-')[1] - elif short_ext_modname.startswith('cray-'): - short_ext_modname = short_ext_modname.split('cray-')[1] - - short_ext_modname.replace('-', '_') + if not 'name' in dependency: + dependency['name'] = [short_ext_modname] short_ext_modname_upper = convert_name(short_ext_modname, upper=True) allowed_pairs = [ - ('CRAY_%s_PREFIX' % short_ext_modname_upper, 'CRAY_%s_VERSION' % short_ext_modname_upper), - ('CRAY_%s_DIR' % short_ext_modname_upper, 'CRAY_%s_VERSION' % short_ext_modname_upper), - ('CRAY_%s_ROOT' % short_ext_modname_upper, 'CRAY_%s_VERSION' % short_ext_modname_upper), - ('%s_PREFIX' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), - ('%s_DIR' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), - ('%s_ROOT' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), - ('%s_HOME' % short_ext_modname_upper, '%s_VERSION' % short_ext_modname_upper), + ('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'), + ('CRAY_%s_DIR', 'CRAY_%s_VERSION'), + ('CRAY_%s_ROOT', 'CRAY_%s_VERSION'), + ('%s_PREFIX', '%s_VERSION'), + ('%s_DIR', '%s_VERSION'), + ('%s_ROOT', '%s_VERSION'), + ('%s_HOME', '%s_VERSION'), ] for prefix, version in allowed_pairs: - module_prefix = self.modules_tool.get_variable_from_modulefile(dep_name, prefix) - module_version = self.modules_tool.get_variable_from_modulefile(dep_name, version) - - if module_prefix and module_version: - dependency = { - 'name': [short_ext_modname_upper], - 'version': [module_version], - 'prefix': module_prefix - } + prefix = prefix % short_ext_modname_upper + version = version % short_ext_modname_upper + + dep_prefix = self.modules_tool.get_variable_from_modulefile(dep_name, prefix) + dep_version = self.modules_tool.get_variable_from_modulefile(dep_name, version) + + # only update missing values with both keys are found + if dep_prefix and dep_version: + # version should hold the value, not the key + if not 'version' in dependency: + dependency['version'] = [dep_version] + self.log.info('setting external module %s version to be %s' % (dep_name, dep_version)) + # prefix should hold the key, not the value + if not 'prefix' in dependency: + dependency['prefix'] = prefix + self.log.info('setting external module %s prefix to be %s' % (dep_name, dep_prefix)) break return dependency @@ -1212,14 +1212,28 @@ def handle_external_module_metadata(self, dep_name): handles metadata for external module dependencies """ dependency = {} + dep_name_no_version = dep_name.split('/')[0] + if dep_name in self.external_modules_metadata: dependency['external_module_metadata'] = self.external_modules_metadata[dep_name] - self.log.info("Updated dependency info with available metadata for external module %s: %s", - dep_name, dependency['external_module_metadata']) + if not all(d in dependency['external_module_metadata'] for d in ('name','version','prefix')): + dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_env(dep_name, + dependency=dependency['external_module_metadata']) + self.log.info("Updated dependency info with available metadata and external module %s: %s", + dep_name, dependency['external_module_metadata']) + elif dep_name_no_version in self.external_modules_metadata: + dependency['external_module_metadata'] = self.external_modules_metadata[dep_name_no_version] + if not all(d in dependency['external_module_metadata'] for d in ('name','version','prefix')): + dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_env(dep_name_no_version, + dependency=dependency['external_module_metadata']) + self.log.info("Updated dependency info with available metadata and external module %s: %s", + dep_name_no_version, dependency['external_module_metadata']) else: self.log.info("No metadata available for external module %s. Attempting to read from available modules", dep_name) - dependency['external_module_metadata'] = self.handle_external_module_metadata_by_probing_modules(dep_name) + dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_env(dep_name) + self.log.info("Updated dependency info with external module %s: %s", + dep_name, dependency['external_module_metadata']) return dependency From 7230321f7f09c66c4afd43fe85c6985d69dc513c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 20 Feb 2020 12:16:54 +0800 Subject: [PATCH 159/344] add support for --include-easyblocks-from-pr --- easybuild/main.py | 9 +++++++-- easybuild/tools/github.py | 31 ++++++++++++++++++++++++------- easybuild/tools/include.py | 5 ++++- easybuild/tools/options.py | 6 ++++-- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 69c47a7293..cf9779d7cc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -57,12 +57,13 @@ from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, read_file, write_file -from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig +from easybuild.tools.github import check_github, close_pr, new_branch_github, fetch_easyblocks_from_pr +from easybuild.tools.github import find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import set_up_configuration, use_color +from easybuild.tools.options import include_easyblocks, set_up_configuration, use_color from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs @@ -199,6 +200,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args + if options.include_easyblocks_from_pr: + included_easyblocks = fetch_easyblocks_from_pr(options.include_easyblocks_from_pr) + include_easyblocks(options.tmpdir, included_easyblocks) + global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 44b1c6d450..2846673e91 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -85,6 +85,7 @@ GITHUB_API_URL = 'https://api.github.com' GITHUB_DIR_TYPE = u'dir' GITHUB_EB_MAIN = 'easybuilders' +GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks' GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs' GITHUB_DEVELOP_BRANCH = 'develop' GITHUB_FILE_TYPE = u'file' @@ -369,8 +370,18 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ return extracted_path +def fetch_easyblocks_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) + + def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) + + +def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): + """Fetch patched easyconfig files for a particular PR.""" if github_user is None: github_user = build_option('github_user') @@ -384,9 +395,15 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): mkdir(path, parents=True) github_account = build_option('pr_target_account') - github_repo = GITHUB_EASYCONFIGS_REPO - _log.debug("Fetching easyconfigs from %s/%s PR #%s into %s", github_account, github_repo, pr, path) + if github_repo is None: + github_repo = GITHUB_EASYCONFIGS_REPO + elif github_repo not in [GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO]: + raise EasyBuildError("Don't know how to fetch files from repo %s", github_repo) + + easyfiles = 'easyconfigs' if github_repo == GITHUB_EASYCONFIGS_REPO else 'easyblocks' + + _log.debug("Fetching %s from %s/%s PR #%s into %s", easyfiles, github_account, github_repo, pr, path) pr_data, _ = fetch_pr_data(pr, github_account, github_repo, github_user) pr_merged = pr_data['merged'] @@ -429,7 +446,7 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): if final_path is None: if pr_closed: - print_warning("Using easyconfigs from closed PR #%s" % pr) + print_warning("Using %s from closed PR #%s" % (easyfiles, pr)) # obtain most recent version of patched files for patched_file in patched_files: @@ -444,21 +461,21 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): # symlink directories into expected place if they're not there yet if final_path != path: - dirpath = os.path.join(final_path, 'easybuild', 'easyconfigs') + dirpath = os.path.join(final_path, 'easybuild', easyfiles) for eb_dir in os.listdir(dirpath): symlink(os.path.join(dirpath, eb_dir), os.path.join(path, os.path.basename(eb_dir))) # sanity check: make sure all patched files are downloaded - ec_files = [] + files = [] for patched_file in [f for f in patched_files if not f.startswith('test/')]: fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:]) full_path = os.path.join(path, fn) if os.path.exists(full_path): - ec_files.append(full_path) + files.append(full_path) else: raise EasyBuildError("Couldn't find path to patched file %s", full_path) - return ec_files + return files def create_gist(txt, fn, descr=None, github_user=None, github_token=None): diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 90b9715280..e71ed7a161 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -154,7 +154,10 @@ def include_easyblocks(tmpdir, paths): easyblocks_dir = os.path.join(easyblocks_path, 'easybuild', 'easyblocks') - allpaths = [p for p in expand_glob_paths(paths) if os.path.basename(p) != '__init__.py'] + allpaths = [p for p in expand_glob_paths(paths) + if os.path.basename(p).endswith('.py') and + os.path.basename(p) != '__init__.py'] + for easyblock_module in allpaths: filename = os.path.basename(easyblock_module) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d149ee3d79..9ed4406db0 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED, GITHUB_PR_STATE_OPEN from easybuild.tools.github import GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS -from easybuild.tools.github import fetch_github_token +from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains from easybuild.tools.job.backend import avail_job_backends @@ -592,6 +592,8 @@ def github_options(self): 'git-working-dirs-path': ("Path to Git working directories for EasyBuild repositories", str, 'store', None), 'github-user': ("GitHub username", str, 'store', None), 'github-org': ("GitHub organization", str, 'store', None), + 'include-easyblocks-from-pr': ("Include easyblocks from specified PR", int, 'store', None, + {'metavar': 'PR#'}), 'install-github-token': ("Install GitHub token (requires --github-user)", None, 'store_true', False), 'close-pr': ("Close pull request", int, 'store', None, {'metavar': 'PR#'}), 'close-pr-msg': ("Custom close message for pull request closed with --close-pr; ", str, 'store', None), @@ -922,7 +924,7 @@ def _postprocess_checks(self): """Check whether (combination of) configuration options make sense.""" # fail early if required dependencies for functionality requiring using GitHub API are not available: - if self.options.from_pr or self.options.upload_test_report: + if self.options.from_pr or self.options.include_easyblocks_from_pr or self.options.upload_test_report: if not HAVE_GITHUB_API: raise EasyBuildError("Required support for using GitHub API is not available (see warnings)") From eeb1adfd9ce6a52cd0ec746e9f5e0ab8e9053a92 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 20 Feb 2020 14:47:32 +0800 Subject: [PATCH 160/344] remove unnecessary import --- easybuild/tools/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9ed4406db0..e8ed232161 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED, GITHUB_PR_STATE_OPEN from easybuild.tools.github import GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS -from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token +from easybuild.tools.github import fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains from easybuild.tools.job.backend import avail_job_backends From 12f1e570e7d5046a8b538dc0c4a522f1cd3ac6e8 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 20 Feb 2020 15:12:26 +0800 Subject: [PATCH 161/344] add test for fetch_easyblocks_from_pr --- test/framework/github.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 4b4c68c31c..251d69464f 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -245,6 +245,33 @@ def test_close_pr(self): for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) + def test_fetch_easyblocks_from_pr(self): + """Test fetch_easyblocks_from_pr function.""" + if self.skip_github_tests: + print("Skipping test_fetch_easyblocks_from_pr, no GitHub token available?") + return + + init_config(build_options={ + 'pr_target_account': gh.GITHUB_EB_MAIN, + }) + + # PR with new easyblock plus non-easyblock file + all_ebs_pr1964 = ['lammps.py'] + + # PR with changed easyblock + all_ebs_pr1967 = ['siesta.py'] + + # PR with more than one easyblock + all_ebs_pr1949 = ['configuremake.py', 'rpackage.py'] + + for pr, all_ebs in [(1964, all_ebs_pr1964), (1967, all_ebs_pr_1967), (1949, all_ebs_pr_1949)]: + try: + tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr) + eb_files = gh.fetch_easyblocks_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) + self.assertEqual(sorted(all_ebs), sorted([os.path.basename(f) for f in eb_files])) + except URLError as err: + print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) + def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" if self.skip_github_tests: From 1ba8e56100a421f6fcd7b3aaf11cb12853fc596e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 20 Feb 2020 15:14:46 +0800 Subject: [PATCH 162/344] fix typos in fetch_easyblocks_from_pr --- test/framework/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index 251d69464f..22714da9fa 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -264,13 +264,13 @@ def test_fetch_easyblocks_from_pr(self): # PR with more than one easyblock all_ebs_pr1949 = ['configuremake.py', 'rpackage.py'] - for pr, all_ebs in [(1964, all_ebs_pr1964), (1967, all_ebs_pr_1967), (1949, all_ebs_pr_1949)]: + for pr, all_ebs in [(1964, all_ebs_pr1964), (1967, all_ebs_pr1967), (1949, all_ebs_pr1949)]: try: tmpdir = os.path.join(self.test_prefix, 'pr%s' % pr) eb_files = gh.fetch_easyblocks_from_pr(pr, path=tmpdir, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(sorted(all_ebs), sorted([os.path.basename(f) for f in eb_files])) except URLError as err: - print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) + print("Ignoring URLError '%s' in test_fetch_easyblocks_from_pr" % err) def test_fetch_easyconfigs_from_pr(self): """Test fetch_easyconfigs_from_pr function.""" From 51d36baea53ea9294c1a35a54d53105fcaba7481 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 20 Feb 2020 15:39:30 +0800 Subject: [PATCH 163/344] fix test_fetch_easyblocks_from_pr --- test/framework/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index 22714da9fa..b64b98cea2 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -256,7 +256,7 @@ def test_fetch_easyblocks_from_pr(self): }) # PR with new easyblock plus non-easyblock file - all_ebs_pr1964 = ['lammps.py'] + all_ebs_pr1964 = ['.gitignore', 'lammps.py'] # PR with changed easyblock all_ebs_pr1967 = ['siesta.py'] From 4f7dcf2059652a23bafe1bfa239fec119f27c8f4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 30 Jan 2020 15:34:25 +0100 Subject: [PATCH 164/344] Fix removing temporary branch on --check-github The remote can be somethings else than "origin" so make sure we take the one used to create the branch --- easybuild/tools/github.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 44b1c6d450..f3bd87d95b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1624,6 +1624,8 @@ def check_github(): * check whether creating gists works * check whether location to local working directories for Git repositories is available (not strictly needed) """ + debug = build_option('debug') + # start by assuming that everything works, individual checks will disable action that won't work status = {} for action in ['--from-pr', '--new-pr', '--review-pr', '--upload-test-report', '--update-pr']: @@ -1716,9 +1718,9 @@ def check_github(): git_repo, res, push_err = None, None, None branch_name = 'test_branch_%s' % ''.join(random.choice(ascii_letters) for _ in range(5)) try: - git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=True) + git_repo = init_repo(git_working_dir, GITHUB_EASYCONFIGS_REPO, silent=not debug) remote_name = setup_repo(git_repo, github_account, GITHUB_EASYCONFIGS_REPO, 'master', - silent=True, git_only=True) + silent=not debug, git_only=True) git_repo.create_head(branch_name) res = getattr(git_repo.remotes, remote_name).push(branch_name) except Exception as err: @@ -1749,12 +1751,11 @@ def check_github(): print_msg(check_res, log=_log, prefix=False) # cleanup: delete test branch that was pushed to GitHub - if git_repo: + if git_repo and push_err is None: try: - if git_repo and hasattr(git_repo, 'remotes') and hasattr(git_repo.remotes, 'origin'): - git_repo.remotes.origin.push(branch_name, delete=True) + getattr(git_repo.remotes, remote_name).push(branch_name, delete=True) except GitCommandError as err: - sys.stderr.write("WARNNIG: failed to delete test branch from GitHub: %s\n" % err) + sys.stderr.write("WARNING: failed to delete test branch from GitHub: %s\n" % err) # test creating a gist print_msg("* creating gists...", log=_log, prefix=False, newline=False) From a11230492b3dabd44e2124ef0c21b7eb8c6f2050 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 20 Feb 2020 09:21:39 -0500 Subject: [PATCH 165/344] replace python-coveralls with coveralls in pip package Testing Travis Build due to similar error reported with where it fails to import Reporter https://github.com/z4r/python-coveralls/issues/73 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bb9324b6cb..3681b4592e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ before_install: - pip --version - pip install --upgrade pip - pip --version - - pip install python-coveralls + - pip install coveralls - pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" From 40a820e6b28f04dcd770ae7f333cbd9ed9b8bb2e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 20 Feb 2020 15:26:57 +0100 Subject: [PATCH 166/344] don't add shebang to binary files (fixes #3207, bug introduced in #3183) --- easybuild/framework/easyblock.py | 12 ++++++++---- easybuild/tools/filetools.py | 7 +++++++ test/framework/filetools.py | 10 ++++++++++ test/framework/toy_build.py | 17 +++++++++++++++-- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index cd734e5db2..3b5cf0466d 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -75,7 +75,7 @@ from easybuild.tools.filetools import change_dir, convert_name, compute_checksum, copy_file, derive_alt_pypi_url from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url -from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir +from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file, dir_contains_files from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP @@ -2223,20 +2223,24 @@ def fix_shebang(self): lang, shebang, glob_pattern, paths) for path in paths: # check whether file should be patched by checking whether it has a shebang we want to tweak; - # this also helps to skip binary files we may be hitting + # this also helps to skip binary files we may be hitting (but only with Python 3) try: contents = read_file(path, mode='r') should_patch = shebang_regex.match(contents) except (TypeError, UnicodeDecodeError): should_patch = False + contents = None + # if an existing shebang is found, patch it if should_patch: contents = shebang_regex.sub(shebang, contents) write_file(path, contents) - elif not contents.startswith('#!'): + + # if no shebang is present at all, add one (but only for non-binary files!) + elif contents is not None and not is_binary(contents) and not contents.startswith('#!'): self.log.info("The file '%s' doesn't have any shebang present, inserting it as first line.", path) - contents = shebang + "\n" + contents + contents = shebang + '\n' + contents write_file(path, contents) def post_install_step(self): diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index cd8554b42b..3cb7979631 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -241,6 +241,13 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over raise EasyBuildError("Failed to write to %s: %s", path, err) +def is_binary(contents): + """ + Check whether given bytestring represents the contents of a binary file or not. + """ + return isinstance(contents, bytes) and b'\00' in bytes(contents) + + def resolve_path(path): """ Return fully resolved path for given path. diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 895a8bde0e..a96ee8a6a7 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -656,6 +656,16 @@ def test_read_write_file(self): # test use of 'mode' in read_file self.assertEqual(ft.read_file(foo, mode='rb'), b'bar') + def test_is_binary(self): + """Test is_binary function.""" + + for test in ['foo', '', b'foo', b'', "This is just a test", b"This is just a test", b"\xa0"]: + self.assertFalse(ft.is_binary(test)) + + self.assertTrue(ft.is_binary(b'\00')) + self.assertTrue(ft.is_binary(b"File is binary when it includes \00 somewhere")) + self.assertTrue(ft.is_binary(ft.read_file('/bin/ls', mode='rb'))) + def test_det_patched_files(self): """Test det_patched_files function.""" toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 8512fe3a98..d4a285e7fe 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2377,6 +2377,10 @@ def test_fix_shebang(self): test_ec_txt = '\n'.join([ toy_ec_txt, "postinstallcmds = [" + # copy of bin/toy to use in fix_python_shebang_for and fix_perl_shebang_for + " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.python',", + " 'cp -a %(installdir)s/bin/toy %(installdir)s/bin/toy.perl',", + # hardcoded path to bin/python " 'echo \"#!/usr/bin/python\\n# test\" > %(installdir)s/bin/t1.py',", # hardcoded path to bin/python3.6 @@ -2413,14 +2417,23 @@ def test_fix_shebang(self): " 'echo \"#!/usr/bin/env bash\\n# test\" > %(installdir)s/bin/b2.sh',", "]", - "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy', 'bin/b1.sh']", - "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh']", + "fix_python_shebang_for = ['bin/t1.py', 'bin/*.py', 'nosuchdir/*.py', 'bin/toy.python', 'bin/b1.sh']", + "fix_perl_shebang_for = ['bin/*.pl', 'bin/b2.sh', 'bin/toy.perl']", ]) write_file(test_ec, test_ec_txt) self.test_toy_build(ec_file=test_ec, raise_error=True) toy_bindir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + # bin/toy and bin/toy2 should *not* be patched, since they're binary files + toy_txt = read_file(os.path.join(toy_bindir, 'toy'), mode='rb') + for fn in ['toy.perl', 'toy.python']: + fn_txt = read_file(os.path.join(toy_bindir, fn), mode='rb') + # no shebang added + self.assertFalse(fn_txt.startswith(b"#!/")) + # exact same file as original binary (untouched) + self.assertEqual(toy_txt, fn_txt) + # no re.M, this should match at start of file! py_shebang_regex = re.compile(r'^#!/usr/bin/env python\n# test$') for pybin in ['t1.py', 't2.py', 't3.py', 't4.py', 't5.py', 't6.py', 't7.py']: From f69be917ff43de73d2aa67194d561eec460b15be Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 20 Feb 2020 10:30:13 -0500 Subject: [PATCH 167/344] exclude coveralls for python 2.6 build. Only run coverage report and push to coveralls for all builds except for Python 2.6 --- .travis.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3681b4592e..42537c6b8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,8 @@ before_install: - pip --version - pip install --upgrade pip - pip --version - - pip install coveralls + # coveralls doesn't support Python 2.6 anymore, so don't try to install it when testing with Python 2.6 + - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then pip install coveralls; fi - pip install -r requirements.txt # git config is required to make actual git commits (cfr. tests for GitRepository) - git config --global user.name "Travis CI" @@ -89,10 +90,10 @@ script: # create file owned by root but writable by anyone (used by test_copy_file) - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite - - coverage run -m test.framework.suite - # ignore errors, since tests inject easyblocks for which there are no source files anymore after the tests have run - - coverage report -m --ignore-errors + # run coverage on all travis builds except for Python 2.6 + - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coverage run -m test.framework.suite; coverage report -m --ignore-errors; fi + # invoke the regression test for Python 2.6 the original way without coverage + - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log; fi # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|lib/python2.6/site-packages|requires Lmod as modules tool" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) @@ -113,4 +114,5 @@ script: - module use /tmp/$TRAVIS_JOB_ID/eb_bootstrap/modules/all - module load EasyBuild; eb --version after_success: - - coveralls + - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coveralls; fi + From 72138b0e32a29938c851a087edef36181c17c926 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 20 Feb 2020 11:02:45 -0500 Subject: [PATCH 168/344] capture regtest output from coverage to test_framework_suite.log --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 42537c6b8e..579863a89f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ script: - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt # run coverage on all travis builds except for Python 2.6 - - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coverage run -m test.framework.suite; coverage report -m --ignore-errors; fi + - if [ "x$TRAVIS_PYTHON_VERSION" != 'x2.6' ]; then coverage run -m test.framework.suite 2>&1 | tee test_framework_suite.log; coverage report -m --ignore-errors; fi # invoke the regression test for Python 2.6 the original way without coverage - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log; fi # try and make sure output of running tests is clean (no printed messages/warnings) From 16fcc926b3ab94e819bb0045cd22265801b8785b Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 21 Feb 2020 13:02:30 +0800 Subject: [PATCH 169/344] appease the hound --- easybuild/tools/github.py | 4 +--- test/framework/easyconfig.py | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 4937ef95e7..9a90171988 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -54,7 +54,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX from easybuild.tools.filetools import apply_patch, copy_dir, copy_file, det_patched_files, decode_class_name -from easybuild.tools.filetools import download_file, extract_file, is_patch_file, mkdir, read_file, symlink +from easybuild.tools.filetools import download_file, extract_file, mkdir, read_file, symlink from easybuild.tools.filetools import which, write_file from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version @@ -1043,12 +1043,10 @@ def copy_framework_files(paths, target_dir): 'new': [], } - dirs = [x[0] for x in os.walk(target_dir)] paths = [os.path.abspath(path) for path in paths] target_path = None for path in paths: - fn = os.path.basename(path) dirnames = os.path.dirname(path).split(os.path.sep) if 'easybuild-framework' in dirnames: diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index d5fae16ece..e9e39768ca 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2713,7 +2713,8 @@ def test_hidden_toolchain(self): def test_categorize_files_by_type(self): """Test categorize_files_by_type""" - self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, categorize_files_by_type([])) + self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, + categorize_files_by_type([])) test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',) toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' From 7cd4e9dfe5345201dd9260b4e83add5875a6d9bf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2020 09:10:01 +0100 Subject: [PATCH 170/344] add configuration option to specify maximum age of index file --- easybuild/tools/config.py | 4 ++++ easybuild/tools/options.py | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ab98bcad6d..51ff946f21 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -78,6 +78,7 @@ CONT_TYPES = [CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY] DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY +DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5 @@ -270,6 +271,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_CONT_TYPE: [ 'container_type', ], + DEFAULT_INDEX_MAX_AGE: [ + 'index_max_age', + ], DEFAULT_MAX_FAIL_RATIO_PERMS: [ 'max_fail_ratio_adjust_permissions', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index efe6e644ce..22d7299f7f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -59,9 +59,9 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE -from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_JOB_BACKEND -from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS, DEFAULT_MNS -from easybuild.tools.config import DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES +from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS +from easybuild.tools.config import DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS, ERROR from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR @@ -684,6 +684,8 @@ def easyconfig_options(self): 'create-index': ("Create index for files in specified directory", None, 'store', None), 'fix-deprecated-easyconfigs': ("Fix use of deprecated functionality in specified easyconfig files.", None, 'store_true', False), + 'index-max-age': ("Maximum age for index before it is considered stale (in seconds)", + int, 'store', DEFAULT_INDEX_MAX_AGE), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", 'choice', 'store_or_None', CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES), 'local-var-naming-check': ("Mode to use when checking whether local variables follow the recommended " From ecd10ddc38c439eb475aac061cda09f88843e0e5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2020 09:10:24 +0100 Subject: [PATCH 171/344] add support to dump_index for specifying maximum age of index + make load_index check whether index is still valid based on age --- easybuild/main.py | 2 +- easybuild/tools/filetools.py | 64 +++++++++++++++++++++++++++++------- test/framework/filetools.py | 47 ++++++++++++++++++++++++-- 3 files changed, 98 insertions(+), 15 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 7eeb8d9d84..123b39064e 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -258,7 +258,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif options.create_index: print_msg("Creating index for %s..." % options.create_index, prefix=False) - index_fp = dump_index(options.create_index) + index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age) index = load_index(options.create_index) print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index af73f2bafe..32d6dfbe53 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -56,7 +56,7 @@ from easybuild.base import fancylogger from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog -from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg +from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.py2vs3 import std_urllib, string_type from easybuild.tools.utilities import nub @@ -618,15 +618,29 @@ def create_index(path, ignore_dirs=None): return index -def dump_index(path): +def dump_index(path, max_age_sec=None): """ Create index for files in specified path, and dump it to file (alphabetically sorted). """ + if max_age_sec is None: + max_age_sec = build_option('index_max_age') index_fp = os.path.join(path, PATH_INDEX_FILENAME) index_contents = create_index(path) - write_file(index_fp, '\n'.join(sorted(index_contents))) + curr_ts = datetime.datetime.now() + if max_age_sec == 0: + end_ts = datetime.datetime.max + else: + end_ts = curr_ts + datetime.timedelta(0, max_age_sec) + + lines = [ + "# created at: %s" % str(curr_ts), + "# valid until: %s" % str(end_ts), + ] + lines.extend(sorted(index_contents)) + + write_file(index_fp, '\n'.join(lines), always_overwrite=False) return index_fp @@ -639,18 +653,46 @@ def load_index(path, ignore_dirs=None): ignore_dirs = [] index_fp = os.path.join(path, PATH_INDEX_FILENAME) - - index, res = None, set() + index = set() if os.path.exists(index_fp): - index = read_file(index_fp).splitlines() + lines = read_file(index_fp).splitlines() + + valid_ts_regex = re.compile("^# valid until: (.*)", re.M) + valid_ts = None - for path in index: - path_dirs = path.split(os.path.sep)[:-1] - if not any(d in path_dirs for d in ignore_dirs): - res.add(path) + for line in lines: - return res or None + # extract "valid until" timestamp, so we can check whether index is still valid + if valid_ts is None: + res = valid_ts_regex.match(line) + else: + res = None + + if res: + valid_ts = res.group(1) + try: + valid_ts = datetime.datetime.strptime(valid_ts, '%Y-%m-%d %H:%M:%S.%f') + except ValueError as err: + raise EasyBuildError("Failed to parse timestamp '%s' for index at %s: %s", valid_ts, path, err) + + elif line.startswith('#'): + _log.info("Ignoring unknown header line '%s' in index for %s", line, path) + + else: + # filter out files that are in an ignored directory + path_dirs = line.split(os.path.sep)[:-1] + if not any(d in path_dirs for d in ignore_dirs): + index.add(line) + + # check whether index is still valid + if valid_ts: + curr_ts = datetime.datetime.now() + if curr_ts > valid_ts: + print_warning("Index for %s is no longer valid (too old), so ignoring it...", path) + index = None + + return index or None def find_easyconfigs(path, ignore_dirs=None): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 7e1ad84138..5cd56d5e51 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -38,6 +38,7 @@ import stat import sys import tempfile +import time from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -1690,13 +1691,18 @@ def test_index_functions(self): self.assertTrue(os.path.exists(index_fp)) self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp))) - index_txt = ft.read_file(index_fp) + datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+" + expected_header = [ + "# created at: " + datestamp_pattern, + "# valid until: " + datestamp_pattern, + ] expected = [ os.path.join('g', 'gzip', 'gzip-1.4.eb'), os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'), os.path.join('g', 'gompic', 'gompic-2018a.eb'), ] - for fn in expected: + index_txt = ft.read_file(index_fp) + for fn in expected_header + expected: regex = re.compile('^%s$' % fn, re.M) self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) @@ -1704,7 +1710,42 @@ def test_index_functions(self): index = ft.load_index(self.test_prefix) self.assertEqual(len(index), 24) for fn in expected: - self.assertTrue(fn in index) + self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) + + # dump_index will not overwrite existing index without force + error_pattern = "File exists, not overwriting it without --force" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, self.test_prefix) + + ft.remove_file(index_fp) + + # test creating index file that's infinitely valid + index_fp = ft.dump_index(self.test_prefix, max_age_sec=0) + index_txt = ft.read_file(index_fp) + expected_header[1] = "# valid until: 9999-12-31 23:59:59\.9+" + for fn in expected_header + expected: + regex = re.compile('^%s$' % fn, re.M) + self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) + index = ft.load_index(self.test_prefix) + self.assertEqual(len(index), 24) + for fn in expected: + self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) + + ft.remove_file(index_fp) + + # test creating index file that's only valid for a (very) short amount of time + index_fp = ft.dump_index(self.test_prefix, max_age_sec=1) + time.sleep(3) + self.mock_stderr(True) + self.mock_stdout(True) + index = ft.load_index(self.test_prefix) + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertTrue(index is None) + self.assertFalse(stdout) + regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix) + self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) def test_search_file(self): """Test search_file function.""" From b103a9e688360f1f2b83cab6e1048eca11d619fa Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2020 10:09:20 +0100 Subject: [PATCH 172/344] take into account non-existing paths in robot_search_easyconfig while creating/loading index --- easybuild/framework/easyconfig/easyconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2aaee7f28d..6acb84ece8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1895,7 +1895,7 @@ def robot_find_easyconfig(name, version): if path in _path_indexes: path_index = _path_indexes[path] _log.info("Found loaded index for %s", path) - else: + elif os.path.exists(path): path_index = load_index(path) if path_index is None: _log.info("No index found for %s, so creating it...", path) @@ -1904,6 +1904,8 @@ def robot_find_easyconfig(name, version): _log.info("Loaded index for %s", path) _path_indexes[path] = path_index + else: + path_index = [] easyconfigs_paths = create_paths(path, name, version) for easyconfig_path in easyconfigs_paths: From 88e676bcc414588b9420ac8ffaa4ad8e7ca2cd3f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2020 10:36:22 +0100 Subject: [PATCH 173/344] extend test for --create-index --- test/framework/options.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index e3540827fa..52103c0000 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4695,6 +4695,32 @@ def test_create_index(self): regex = re.compile(pattern % self.test_prefix, re.M) self.assertTrue(regex.search(stdout), "Pattern %s matches in: %s" % (regex.pattern, stdout)) + # check contents of index + index_fp = os.path.join(self.test_prefix, '.eb-path-index') + index_txt = read_file(index_fp) + + datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+" + patterns = [ + r"^# created at: " + datestamp_pattern + '$', + r"^# valid until: " + datestamp_pattern + '$', + r"^g/GCC/GCC-7.3.0-2.30.eb", + r"^t/toy/toy-0\.0\.eb", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) + + # existing index is not overwritten without --force + error_pattern = "File exists, not overwriting it without --force: .*/.eb-path-index" + self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) + + # also test creating index that's infinitely valid + args.extend(['--index-max-ag=0', '--force']) + self._run_mock_eb(args, raise_error=True) + index_txt = read_file(index_fp) + regex = re.compile(r"^# valid until: 9999-12-31 23:59:59", re.M) + self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) + def suite(): """ returns all the testcases in this module """ From 0efe3e3737d3ad919d6e58f187d03ac887370069 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2020 10:40:38 +0100 Subject: [PATCH 174/344] print message when valid index is being used --- easybuild/tools/filetools.py | 2 ++ test/framework/filetools.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 32d6dfbe53..9d58e39330 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -691,6 +691,8 @@ def load_index(path, ignore_dirs=None): if curr_ts > valid_ts: print_warning("Index for %s is no longer valid (too old), so ignoring it...", path) index = None + else: + print_msg("found valid index for %s, so using it...", path) return index or None diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 5cd56d5e51..b3237a492a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1707,7 +1707,18 @@ def test_index_functions(self): self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) # test load_index function + self.mock_stderr(True) + self.mock_stdout(True) index = ft.load_index(self.test_prefix) + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) + self.assertEqual(len(index), 24) for fn in expected: self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) @@ -1725,7 +1736,19 @@ def test_index_functions(self): for fn in expected_header + expected: regex = re.compile('^%s$' % fn, re.M) self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) + + self.mock_stderr(True) + self.mock_stdout(True) index = ft.load_index(self.test_prefix) + stderr = self.get_stderr() + stdout = self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) + self.assertEqual(len(index), 24) for fn in expected: self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) From 91a279b242c784969312e106ff47a8204e08690a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 21 Feb 2020 10:44:33 +0100 Subject: [PATCH 175/344] take into account non-existing paths in search_file --- easybuild/tools/filetools.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 9d58e39330..b0cd86c569 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -764,8 +764,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen path_index = load_index(path, ignore_dirs=ignore_dirs) if path_index is None: - _log.info("No index found for %s, creating one...", path) - path_index = create_index(path, ignore_dirs=ignore_dirs) + if os.path.exists(path): + _log.info("No index found for %s, creating one...", path) + path_index = create_index(path, ignore_dirs=ignore_dirs) + else: + path_index = [] else: _log.info("Index found for %s, so using it...", path) From 8e67a1cc58154061bf51cc65a034878f4073e452 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 21 Feb 2020 18:08:42 +0800 Subject: [PATCH 176/344] flesh out method to determine easyblock class --- easybuild/tools/github.py | 59 ++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 9a90171988..c37fb316f0 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -48,7 +48,7 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, det_file_info -from easybuild.framework.easyconfig.easyconfig import process_easyconfig +from easybuild.framework.easyconfig.easyconfig import is_generic_easyblock, process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option @@ -58,7 +58,7 @@ from easybuild.tools.filetools import which, write_file from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version -from easybuild.tools.utilities import nub, only_if_module_is_available +from easybuild.tools.utilities import nub, only_if_module_is_available, remove_unwanted_chars _log = fancylogger.getLogger('github', fname=False) @@ -698,8 +698,14 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_target_repo == GITHUB_EASYCONFIGS_REPO: if paths['py_files']: - raise EasyBuildError("You are submitting files with .py extension, " - "did you forget to specify --pr-target-repo?") + if any([get_easyblock_class(path) for path in paths['py_files']]): + # this is not enough, we would need to change build_option('pr_target_repo') + pr_target_repo = GITHUB_EASYBLOCKS_REPO + raise EasyBuildError("You are submitting easyblock files, " + "did you forget to specify --pr-target-repo=easybuild-easyblocks?") + else: + raise EasyBuildError("You are submitting python files that are not easyblocks, " + "did you forget to specify --pr-target-repo=easybuild-framework?") else: if paths['easyconfigs'] or paths['patch_files']: raise EasyBuildError("You are submitting easyconfigs and/or patches, " @@ -993,6 +999,26 @@ def find_software_name_for_patch(patch_name, ec_dirs): return soft_name +def get_easyblock_class(path): + """Get easyblock class from file""" + fn = os.path.basename(path).split('.')[0] + mod = imp.load_source(fn, path) + clsmembers = inspect.getmembers(mod, inspect.isclass) + is_easyblock = False + for cn in clsmembers: + if cn[0] == 'EasyBlock' or cn[0].startswith(EASYBLOCK_CLASS_PREFIX): + is_easyblock = True + break + if is_easyblock: + classnames = [cl[1].__name__ for cl in clsmembers if cl[1].__module__ == mod.__name__] + if len(classnames) > 1: + return None + else: + return classnames[0] + else: + return None + + def copy_easyblocks(paths, target_dir): """ Find right location for easyblock file and copy it there""" file_info = { @@ -1003,27 +1029,16 @@ def copy_easyblocks(paths, target_dir): subdir = os.path.join('easybuild', 'easyblocks') if os.path.exists(os.path.join(target_dir, subdir)): for path in paths: - fn = os.path.basename(path).split('.')[0] - - mod = imp.load_source(fn, path) - clsmembers = inspect.getmembers(mod, inspect.isclass) - if clsmembers: - classnames = [cl[1].__name__ for cl in clsmembers if cl[1].__module__ == mod.__name__] - else: - raise EasyBuildError("Invalid easyblock file") - - if len(classnames) > 1: + cn = get_easyblock_class(path) + if not cn: raise EasyBuildError("Invalid easyblock file") - cn = classnames[0] - eb_name = decode_class_name(cn).lower() # TODO not fully right yet. - to _ (and others??) - if cn.startswith(EASYBLOCK_CLASS_PREFIX): - # regular eb file - letter = fn.lower()[0] - target_path = os.path.join(subdir, letter, "%s.%s" % (eb_name, PYTHON_EXTENSION)) - else: - # generic + eb_name = remove_unwanted_chars(decode_class_name(cn).replace('-', '_')).lower() + if is_generic_easyblock(cn): target_path = os.path.join(subdir, GENERIC_EB, "%s.%s" % (eb_name.lower(), PYTHON_EXTENSION)) + else: + letter = os.path.basename(path).lower()[0] + target_path = os.path.join(subdir, letter, "%s.%s" % (eb_name, PYTHON_EXTENSION)) full_target_path = os.path.join(target_dir, target_path) file_info['paths_in_repo'].append(full_target_path) From c0c61e68407851a1e0f62f7282cc7f407d029e8a Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 24 Feb 2020 16:02:47 +0800 Subject: [PATCH 177/344] improve detection of easyblock class --- easybuild/tools/github.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c37fb316f0..3a4264a92c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -52,7 +52,6 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX from easybuild.tools.filetools import apply_patch, copy_dir, copy_file, det_patched_files, decode_class_name from easybuild.tools.filetools import download_file, extract_file, mkdir, read_file, symlink from easybuild.tools.filetools import which, write_file @@ -698,7 +697,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_target_repo == GITHUB_EASYCONFIGS_REPO: if paths['py_files']: - if any([get_easyblock_class(path) for path in paths['py_files']]): + if any([get_easyblock_class_name(path) for path in paths['py_files']]): # this is not enough, we would need to change build_option('pr_target_repo') pr_target_repo = GITHUB_EASYBLOCKS_REPO raise EasyBuildError("You are submitting easyblock files, " @@ -999,24 +998,17 @@ def find_software_name_for_patch(patch_name, ec_dirs): return soft_name -def get_easyblock_class(path): - """Get easyblock class from file""" +def get_easyblock_class_name(path): + """Make sure file is an easyblock and get easyblock class name""" fn = os.path.basename(path).split('.')[0] mod = imp.load_source(fn, path) clsmembers = inspect.getmembers(mod, inspect.isclass) - is_easyblock = False - for cn in clsmembers: - if cn[0] == 'EasyBlock' or cn[0].startswith(EASYBLOCK_CLASS_PREFIX): - is_easyblock = True - break - if is_easyblock: - classnames = [cl[1].__name__ for cl in clsmembers if cl[1].__module__ == mod.__name__] - if len(classnames) > 1: - return None - else: - return classnames[0] - else: - return None + for cn, co in clsmembers: + if co.__module__ == mod.__name__: + ancestors = inspect.getmro(co) + if ancestors[-2].__name__ == 'EasyBlock': + return cn + return None def copy_easyblocks(paths, target_dir): @@ -1029,7 +1021,7 @@ def copy_easyblocks(paths, target_dir): subdir = os.path.join('easybuild', 'easyblocks') if os.path.exists(os.path.join(target_dir, subdir)): for path in paths: - cn = get_easyblock_class(path) + cn = get_easyblock_class_name(path) if not cn: raise EasyBuildError("Invalid easyblock file") From 68eabad1c51ea7d839a03fdd48ae23131681b114 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 24 Feb 2020 18:09:28 +0800 Subject: [PATCH 178/344] detect pr-target-repo when using --new-pr for easyblocks --- easybuild/tools/github.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 3a4264a92c..307b741b56 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -698,10 +698,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_target_repo == GITHUB_EASYCONFIGS_REPO: if paths['py_files']: if any([get_easyblock_class_name(path) for path in paths['py_files']]): - # this is not enough, we would need to change build_option('pr_target_repo') pr_target_repo = GITHUB_EASYBLOCKS_REPO - raise EasyBuildError("You are submitting easyblock files, " - "did you forget to specify --pr-target-repo=easybuild-easyblocks?") else: raise EasyBuildError("You are submitting python files that are not easyblocks, " "did you forget to specify --pr-target-repo=easybuild-framework?") @@ -839,7 +836,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ push_branch_to_github(git_repo, target_account, pr_target_repo, pr_branch) - return file_info, deleted_paths, git_repo, pr_branch, diff_stat + return file_info, deleted_paths, git_repo, pr_branch, diff_stat, pr_target_repo def create_remote(git_repo, account, repo, https=False): @@ -1014,6 +1011,7 @@ def get_easyblock_class_name(path): def copy_easyblocks(paths, target_dir): """ Find right location for easyblock file and copy it there""" file_info = { + 'eb_names': [], 'paths_in_repo': [], 'new': [], } @@ -1033,6 +1031,7 @@ def copy_easyblocks(paths, target_dir): target_path = os.path.join(subdir, letter, "%s.%s" % (eb_name, PYTHON_EXTENSION)) full_target_path = os.path.join(target_dir, target_path) + file_info['eb_names'].append(eb_name) file_info['paths_in_repo'].append(full_target_path) file_info['new'].append(not os.path.exists(full_target_path)) copy_file(path, full_target_path, force_in_dry_run=True) @@ -1392,14 +1391,15 @@ def new_branch_github(paths, ecs, commit_msg=None): @only_if_module_is_available('git', pkgname='GitPython') -def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): +def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, pr_metadata=None): """ Create new pull request from specified branch on GitHub. """ pr_target_account = build_option('pr_target_account') pr_target_branch = build_option('pr_target_branch') - pr_target_repo = build_option('pr_target_repo') + if pr_target_repo is None: + pr_target_repo = build_option('pr_target_repo') # fetch GitHub token (required to perform actions on GitHub) github_user = build_option('github_user') @@ -1537,13 +1537,14 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_metadata=None): pyver.append(dep['version']) if pyver: title += " w/ Python %s" % ' + '.join(sorted(nub(pyver))) + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO: + if file_info['eb_names'] and all(file_info['new']) and not deleted_paths: + plural = 's' if len(file_info['eb_names']) > 1 else '' + title = "new easyblock%s for %s" % (plural, (', '.join(file_info['eb_names']))) - else: - raise EasyBuildError("Don't know how to make a PR title for this PR. " - "Please include a title (use --pr-title)") - else: - raise EasyBuildError("Don't know how to make a PR title for this PR. " - "Please include a title (use --pr-title)") + if title is None: + raise EasyBuildError("Don't know how to make a PR title for this PR. " + "Please include a title (use --pr-title)") full_descr = "(created using `eb --new-pr`)\n" if descr is not None: @@ -1617,9 +1618,10 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): # create new branch in GitHub res = new_branch_github(paths, ecs, commit_msg=commit_msg) - file_info, deleted_paths, _, branch_name, diff_stat = res + file_info, deleted_paths, _, branch_name, diff_stat, pr_target_repo = res - new_pr_from_branch(branch_name, title=title, descr=descr, pr_metadata=(file_info, deleted_paths, diff_stat)) + new_pr_from_branch(branch_name, title=title, descr=descr, pr_target_repo=pr_target_repo, + pr_metadata=(file_info, deleted_paths, diff_stat)) def det_account_branch_for_pr(pr_id, github_user=None): From 484fb7767b7506df9172b969c7a127f8f7d85867 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 25 Feb 2020 13:38:18 +0800 Subject: [PATCH 179/344] remove duplicate definition of pr_target_repo in new_pr_from_branch --- easybuild/tools/github.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 307b741b56..08db5bca09 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1554,7 +1554,6 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, pr_target_branch = build_option('pr_target_branch') dry_run = build_option('dry_run') or build_option('extended_dry_run') - pr_target_repo = build_option('pr_target_repo') msg = '\n'.join([ '', "Opening pull request%s" % ('', " [DRY RUN]")[dry_run], From 94b0cae3d3407b8a838656bdb698a302aed26f61 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 25 Feb 2020 13:53:51 +0800 Subject: [PATCH 180/344] also detect pr-target-repo when using --update-pr for easyblocks --- easybuild/tools/github.py | 55 +++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 08db5bca09..178f22a320 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -693,19 +693,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if not any(paths.values()): raise EasyBuildError("No paths specified") - pr_target_repo = build_option('pr_target_repo') - - if pr_target_repo == GITHUB_EASYCONFIGS_REPO: - if paths['py_files']: - if any([get_easyblock_class_name(path) for path in paths['py_files']]): - pr_target_repo = GITHUB_EASYBLOCKS_REPO - else: - raise EasyBuildError("You are submitting python files that are not easyblocks, " - "did you forget to specify --pr-target-repo=easybuild-framework?") - else: - if paths['easyconfigs'] or paths['patch_files']: - raise EasyBuildError("You are submitting easyconfigs and/or patches, " - "shouldn\'t this PR target the easyconfigs repo?") + pr_target_repo = det_pr_target_repo(paths) # initialize repository git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') @@ -1623,7 +1611,7 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): pr_metadata=(file_info, deleted_paths, diff_stat)) -def det_account_branch_for_pr(pr_id, github_user=None): +def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None): """Determine account & branch corresponding to pull request with specified id.""" if github_user is None: @@ -1633,7 +1621,8 @@ def det_account_branch_for_pr(pr_id, github_user=None): raise EasyBuildError("GitHub username (--github-user) must be specified!") pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') + if pr_target_repo is None: + pr_target_repo = build_option('pr_target_repo') pr_data, _ = fetch_pr_data(pr_id, pr_target_account, pr_target_repo, github_user) @@ -1646,6 +1635,29 @@ def det_account_branch_for_pr(pr_id, github_user=None): return account, branch +def det_pr_target_repo(paths): + """Determine pr_target_repo from cagetorized list of files + + :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches) + """ + + pr_target_repo = build_option('pr_target_repo') + + if pr_target_repo == GITHUB_EASYCONFIGS_REPO: + if paths['py_files']: + if any([get_easyblock_class_name(path) for path in paths['py_files']]): + pr_target_repo = GITHUB_EASYBLOCKS_REPO + else: + raise EasyBuildError("You are submitting python files that are not easyblocks, " + "did you forget to specify --pr-target-repo=easybuild-framework?") + else: + if paths['easyconfigs'] or paths['patch_files']: + raise EasyBuildError("You are submitting easyconfigs and/or patches, " + "shouldn\'t this PR target the easyconfigs repo?") + + return pr_target_repo + + @only_if_module_is_available('git', pkgname='GitPython') def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None): """ @@ -1665,12 +1677,13 @@ def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None) if github_account is None: github_account = build_option('github_user') or build_option('github_org') - _, _, _, _, diff_stat = _easyconfigs_pr_common(paths, ecs, start_branch=branch_name, pr_branch=branch_name, - start_account=github_account, commit_msg=commit_msg) + _, _, _, _, diff_stat, pr_target_repo = _easyconfigs_pr_common(paths, ecs, start_branch=branch_name, + pr_branch=branch_name, start_account=github_account, + commit_msg=commit_msg) print_msg("Overview of changes:\n%s\n" % diff_stat, log=_log, prefix=False) - full_repo = '%s/%s' % (github_account, build_option('pr_target_repo')) + full_repo = '%s/%s' % (github_account, pr_target_repo) msg = "pushed updated branch '%s' to %s" % (branch_name, full_repo) if build_option('dry_run') or build_option('extended_dry_run'): msg += " [DRY RUN]" @@ -1688,11 +1701,13 @@ def update_pr(pr_id, paths, ecs, commit_msg=None): :param commit_msg: commit message to use """ - github_account, branch_name = det_account_branch_for_pr(pr_id) + pr_target_repo = det_pr_target_repo(paths) + + github_account, branch_name = det_account_branch_for_pr(pr_id, pr_target_repo=pr_target_repo) update_branch(branch_name, paths, ecs, github_account=github_account, commit_msg=commit_msg) - full_repo = '%s/%s' % (build_option('pr_target_account'), build_option('pr_target_repo')) + full_repo = '%s/%s' % (build_option('pr_target_account'), pr_target_repo) msg = "updated https://github.com/%s/pull/%s" % (full_repo, pr_id) if build_option('dry_run') or build_option('extended_dry_run'): msg += " [DRY RUN]" From 6186ba97058250d03b67a0b3c41c41a3440e9210 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 24 Feb 2020 12:00:54 +0100 Subject: [PATCH 181/344] Fix gitdb dependency on Python 2.6 --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index b7aa408d58..3a93ac826f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,10 @@ keyring==5.7.1; python_version < '2.7' keyring<=9.1; python_version >= '2.7' keyrings.alt; python_version >= '2.7' +# GitDB 4.0.1 no longer supports Python 2.6 +gitdb==0.6.4; python_version < '2.7' +gitdb; python_version >= '2.7' + # GitPython 2.1.9 no longer supports Python 2.6 GitPython==2.1.8; python_version < '2.7' GitPython; python_version >= '2.7' From d3942240b5f2bdf56ec8f8b30691dc5854edd210 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 25 Feb 2020 16:35:57 +0100 Subject: [PATCH 182/344] Allow all kinds of try-* with dependency updates --- easybuild/framework/easyconfig/tweak.py | 75 +++++++++++-------- .../h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb | 1 + test/framework/tweak.py | 32 ++++++++ 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 7262dc4ce8..161b64986e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -100,40 +100,34 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False - update_dependencies = build_specs.get('update_deps', None) - if 'toolchain_name' in build_specs or 'toolchain_version' in build_specs or update_dependencies: - keys = build_specs.keys() - - # Make sure there are no more build_specs, as combining --try-toolchain* with other options is currently not - # supported + if any(key in build_specs for key in ['toolchain', 'toolchain_name', 'toolchain_version', 'update_deps']): if not build_option('map_toolchains'): - msg = "Mapping of (sub)toolchains disabled, so falling back to regex mode, " - msg += "disabling recursion and not changing (sub)toolchains for dependencies" - _log.info(msg) - revert_to_regex = True - elif any(key not in ['toolchain_name', 'toolchain_version', 'toolchain', 'update_deps'] for key in keys): - warning_msg = "Combining --try-toolchain* or --try-update-deps with other build options " - warning_msg += "is not fully supported: using regex" - print_warning(warning_msg, silent=build_option('silent')) - revert_to_regex = True + if 'update_deps' in build_specs: + raise EasyBuildError("Cannot use --try-update-deps without setting --map-toolchains") + else: + msg = "Mapping of (sub)toolchains (with --map-toolchains) disabled, so falling back to regex mode, " + msg += "disabling recursion and not changing (sub)toolchains for dependencies" + _log.info(msg) + revert_to_regex = True if not revert_to_regex: - # we're doing something with the toolchain, - # so build specifications should be applied to whole dependency graph; + # we're doing something that involves the toolchain hierarchy; # obtain full dependency graph for specified easyconfigs; # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) + _log.debug("Updating toolchain and/or dependencies requested...applying build specifications recursively " + "(where appropriate):\n%s", build_specs) modifying_toolchains_or_deps = True + pruned_build_specs = copy.copy(build_specs) - if 'toolchain_name' in keys: - target_toolchain['name'] = build_specs['toolchain_name'] - else: - target_toolchain['name'] = source_toolchain['name'] - - if 'toolchain_version' in keys: - target_toolchain['version'] = build_specs['toolchain_version'] + update_dependencies = pruned_build_specs.pop('update_deps', None) + if 'toolchain' in pruned_build_specs: + target_toolchain = pruned_build_specs.pop('toolchain') + pruned_build_specs.pop('toolchain_name', '') + pruned_build_specs.pop('toolchain_version', '') else: - target_toolchain['version'] = source_toolchain['version'] + target_toolchain['name'] = pruned_build_specs.pop('toolchain_name', source_toolchain['name']) + target_toolchain['version'] = pruned_build_specs.pop('toolchain_version', source_toolchain['version']) try: src_to_dst_tc_mapping = map_toolchain_hierarchies(source_toolchain, target_toolchain, modtool) @@ -149,7 +143,6 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # simply re-raise the exception if something else went wrong raise err - _log.debug("Applying build specifications recursively (no software name/version found): %s", build_specs) orig_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) # Filter out the toolchain hierarchy (which would only appear if we are applying build_specs recursively) @@ -197,7 +190,8 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - tweaked_ecs_path, + targetdir=tweaked_ecs_path, + update_build_specs=pruned_build_specs, update_dep_versions=update_dependencies) # Need to update the toolchain in the build_specs to match the toolchain mapping keys = verification_build_specs.keys() @@ -217,11 +211,12 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): # Place all tweaked dependency easyconfigs in the directory appended to the robot path if modifying_toolchains_or_deps: if tc_name in src_to_dst_tc_mapping: - new_ec_file = map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, - targetdir=tweaked_ecs_deps_path, - update_dep_versions=update_dependencies) + # Note pruned_build_specs are not passed down for dependencies + map_easyconfig_to_target_tc_hierarchy(orig_ec['spec'], src_to_dst_tc_mapping, + targetdir=tweaked_ecs_deps_path, + update_dep_versions=update_dependencies) else: - new_ec_file = tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) + tweak_one(orig_ec['spec'], None, build_specs, targetdir=tweaked_ecs_deps_path) return tweaked_easyconfigs @@ -935,7 +930,8 @@ def get_matching_easyconfig_candidates(prefix_stub, toolchain): return cand_paths, toolchain_suffix -def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_dep_versions=False): +def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=None, update_build_specs=None, + update_dep_versions=False): """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out @@ -955,6 +951,19 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # We may need to update the versionsuffix if it is like, for example, `-Python-2.7.8` versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['toolchain'], toolchain_mapping) + if update_build_specs is not None: + # automagically clear out list of checksums if software version is being tweaked + if 'version' in update_build_specs: + if 'exts_list' in parsed_ec and parsed_ec['exts_list']: + raise EasyBuildError("Cannot currently handle updating version and dependencies of easyconfig with " + "exts_list entry") + elif 'checksums' not in update_build_specs: + update_build_specs['checksums'] = [] + _log.warning("Tweaking version: checksums cleared, verification disabled.") + # Update the keys according to the build specs + for key in update_build_specs: + parsed_ec[key] = update_build_specs[key] + # Replace the toolchain if the mapping exists tc_name = parsed_ec['toolchain']['name'] if tc_name in toolchain_mapping: @@ -1021,7 +1030,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= tweaked_spec = os.path.join(targetdir or tempfile.gettempdir(), ec_filename) parsed_ec.dump(tweaked_spec, always_overwrite=False, backup=True) - _log.debug("Dumped easyconfig tweaked via --try-toolchain* to %s", tweaked_spec) + _log.debug("Dumped easyconfig tweaked via --try-* to %s", tweaked_spec) return tweaked_spec diff --git a/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb index 6765b00412..86e583ee1e 100644 --- a/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb +++ b/test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.9.3-2.26.eb @@ -15,6 +15,7 @@ toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'} source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/'] sources = [SOURCE_TAR_GZ] +checksums = ['aa9d9ca75c7d7164f6bf3a52ecd77340eec02c18'] builddependencies = [('binutils', '2.26')] diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 6720e9d936..2a0e577c25 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -422,6 +422,38 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) + # Make sure there are checksums for our next test + self.assertTrue(tweaked_dict['checksums']) + + # Test the case where we also update the software version at the same time + init_config(build_options=build_options) + get_toolchain_hierarchy.clear() + new_version = '1.x.3' + tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, + tc_mapping, + update_build_specs={'version': new_version}, + update_dep_versions=True) + tweaked_ec = process_easyconfig(tweaked_spec)[0] + tweaked_dict = tweaked_ec['ec'].asdict() + # First check the mapped toolchain + key, value = 'toolchain', iccifort_binutils_tc + self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + # Also check that binutils has been mapped + for key, value in {'name': 'binutils', 'version': '2.25', 'versionsuffix': ''}.items(): + self.assertTrue( + key in tweaked_dict['builddependencies'][0] and value == tweaked_dict['builddependencies'][0][key] + ) + # Also check that the gzip dependency was upgraded + for key, value in {'name': 'gzip', 'version': '1.6', 'versionsuffix': ''}.items(): + self.assertTrue(key in tweaked_dict['dependencies'][0] and value == tweaked_dict['dependencies'][0][key]) + + # Finally check that the version was upgraded + key, value = 'version', new_version + self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) + # and that the checksum was removed + print(tweaked_dict) + self.assertFalse(tweaked_dict['checksums']) + def suite(): """ return all the tests in this file """ From d28c76b1ae9d1c987fa3985d853bed34152a44c2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 25 Feb 2020 18:19:53 +0100 Subject: [PATCH 183/344] Fix broken test --- test/framework/options.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..3b97bcf240 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1804,14 +1804,35 @@ def test_recursive_try(self): mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) - # clear fictious dependency + # recursive try also when --(try-)software(-X) is involved + for extra_args in [[], + ['--module-naming-scheme=HierarchicalMNS']]: + outtxt = self.eb_main(args + extra_args + ['--try-software-version=1.2.3'], verbose=True, raise_error=True) + # toolchain GCC/4.7.2 (subtoolchain of gompi/2018a) should be listed (and present) + + tc_regex = re.compile("^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M) + self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) + + # both toy and gzip dependency should be listed with new toolchains + # in this case we map original toolchain `dummy` to the compiler-only GCC subtoolchain of gompi/2018a + # since this subtoolchain already has sufficient capabilities (we do not map higher than necessary) + for ec_name in ['gzip-1.4', 'toy-1.2.3']: + ec = '%s-GCC-6.4.0-2.28.eb' % ec_name + if extra_args: + mod = ec_name.replace('-', '/') + else: + mod = '%s-GCC-6.4.0-2.28' % ec_name.replace('-', '/') + mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + + # clear fictitous dependency f = open(tweaked_toy_ec, 'a') f.write("dependencies = []\n") f.close() - # no recursive try if --(try-)software(-X) is involved + # no recursive try if --disable-map-toolchains is involved for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]: - outtxt = self.eb_main(args + extra_args, raise_error=True) + outtxt = self.eb_main(args + ['--disable-map-toolchains'] +extra_args, raise_error=True) for mod in ['toy/1.2.3-gompi-2018a', 'gompi/2018a', 'GCC/6.4.0-2.28']: mod_regex = re.compile("\(module: %s\)$" % mod, re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) @@ -1819,6 +1840,7 @@ def test_recursive_try(self): mod_regex = re.compile("\(module: %s\)$" % mod, re.M) self.assertFalse(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + def test_cleanup_builddir(self): """Test cleaning up of build dir and --disable-cleanup-builddir.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') From 4da4ba4dbb64e6f4cf3ea29c063ac24819f32fd5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 26 Feb 2020 08:52:25 +0100 Subject: [PATCH 184/344] Appease hound --- test/framework/options.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3b97bcf240..e63e0fe5ba 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1789,7 +1789,7 @@ def test_recursive_try(self): outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) # toolchain GCC/4.7.2 (subtoolchain of gompi/2018a) should be listed (and present) - tc_regex = re.compile("^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M) + tc_regex = re.compile(r"^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M) self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) # both toy and gzip dependency should be listed with new toolchains @@ -1801,7 +1801,7 @@ def test_recursive_try(self): mod = ec_name.replace('-', '/') else: mod = '%s-GCC-6.4.0-2.28' % ec_name.replace('-', '/') - mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) + mod_regex = re.compile(r"^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) # recursive try also when --(try-)software(-X) is involved @@ -1810,7 +1810,7 @@ def test_recursive_try(self): outtxt = self.eb_main(args + extra_args + ['--try-software-version=1.2.3'], verbose=True, raise_error=True) # toolchain GCC/4.7.2 (subtoolchain of gompi/2018a) should be listed (and present) - tc_regex = re.compile("^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M) + tc_regex = re.compile(r"^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M) self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) # both toy and gzip dependency should be listed with new toolchains @@ -1822,22 +1822,22 @@ def test_recursive_try(self): mod = ec_name.replace('-', '/') else: mod = '%s-GCC-6.4.0-2.28' % ec_name.replace('-', '/') - mod_regex = re.compile("^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) + mod_regex = re.compile(r"^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) - # clear fictitous dependency + # clear fictitious dependency f = open(tweaked_toy_ec, 'a') f.write("dependencies = []\n") f.close() # no recursive try if --disable-map-toolchains is involved for extra_args in [['--try-software-version=1.2.3'], ['--software-version=1.2.3']]: - outtxt = self.eb_main(args + ['--disable-map-toolchains'] +extra_args, raise_error=True) + outtxt = self.eb_main(args + ['--disable-map-toolchains'] + extra_args, raise_error=True) for mod in ['toy/1.2.3-gompi-2018a', 'gompi/2018a', 'GCC/6.4.0-2.28']: - mod_regex = re.compile("\(module: %s\)$" % mod, re.M) + mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) for mod in ['gompi/1.2.3', 'GCC/1.2.3']: - mod_regex = re.compile("\(module: %s\)$" % mod, re.M) + mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M) self.assertFalse(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) From c580244edc2c57aaa708abf8a9c6b54c95c4c21f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 26 Feb 2020 09:13:19 +0100 Subject: [PATCH 185/344] Update try tests to reflect new capabilities --- test/framework/options.py | 14 +++++--------- test/framework/tweak.py | 1 - 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index e63e0fe5ba..483ccc1ea8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1696,17 +1696,14 @@ def test_try(self): test_cases = [ ([], 'toy/0.0'), - # combining --try-toolchain with other build options is too complicated, in this case the code defaults back - # to doing a simple regex substitution on the toolchain - (['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a'], 'foo/1.2.3-gompi-2018a'), + # try-* only uses the subtoolchain with matching necessary features + (['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a'], 'foo/1.2.3-GCC-6.4.0-2.28'), (['--try-toolchain-name=gompi', '--try-toolchain-version=2018a'], 'toy/0.0-GCC-6.4.0.2.28'), # --try-toolchain is overridden by --toolchain (['--try-toolchain=gompi,2018a', '--toolchain=system,system'], 'toy/0.0'), (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), (['--try-toolchain-name=gompi', '--try-toolchain-version=2018a'], 'toy/0.0-GCC-6.4.0.2.28'), - # combining --try-toolchain with other build options is too complicated, in this case the code defaults back - # to doing a simple regex substitution on the toolchain - (['--try-software-version=1.2.3', '--try-toolchain=gompi,2018a'], 'toy/1.2.3-gompi-2018a'), + (['--try-software-version=1.2.3', '--try-toolchain=gompi,2018a'], 'toy/1.2.3-GCC-6.4.0.2.28'), (['--try-amend=versionsuffix=-test'], 'toy/0.0-test'), # --try-amend is overridden by --amend (['--amend=versionsuffix=', '--try-amend=versionsuffix=-test'], 'toy/0.0'), @@ -1720,9 +1717,8 @@ def test_try(self): # define extra list-typed parameter (['--try-amend=versionsuffix=-test5', '--try-amend=exts_list=1,2,3'], 'toy/0.0-test5'), # only --try causes other build specs to be included too - # --try-toolchain* has a different branch to all other try options, combining defaults back to regex - (['--try-software=foo,1.2.3', '--toolchain=gompi,2018a'], 'foo/1.2.3-gompi-2018a'), - (['--software=foo,1.2.3', '--try-toolchain=gompi,2018a'], 'foo/1.2.3-gompi-2018a'), + (['--try-software=foo,1.2.3', '--toolchain=gompi,2018a'], 'foo/1.2.3-GCC-6.4.0-2.28'), + (['--software=foo,1.2.3', '--try-toolchain=gompi,2018a'], 'foo/1.2.3-GCC-6.4.0-2.28'), (['--software=foo,1.2.3', '--try-amend=versionsuffix=-test'], 'foo/1.2.3-test'), ] diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 2a0e577c25..a1708f281f 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -451,7 +451,6 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): key, value = 'version', new_version self.assertTrue(key in tweaked_dict and value == tweaked_dict[key]) # and that the checksum was removed - print(tweaked_dict) self.assertFalse(tweaked_dict['checksums']) From 96dbfaa0c347cba36c790c42bdbb514e00028e2a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 26 Feb 2020 09:19:21 +0100 Subject: [PATCH 186/344] Fix broken test --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d4a285e7fe..b8067bfa64 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -745,7 +745,7 @@ def test_toy_hierarchical(self): '--try-toolchain=foss,2018a', # This test was created for the regex substitution of toolchains, to trigger this (rather than subtoolchain # resolution) we must add an additional build option - '--try-amend=parallel=1', + '--disable-map-toolchains', ] self.eb_main(args + extra_args, logfile=self.dummylogfn, do_build=True, verbose=True, raise_error=True) From 862ede92861d834d9b34dfd96a5a844a8e6fbbc8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 26 Feb 2020 09:48:08 +0100 Subject: [PATCH 187/344] Allow use of SYSTEM as --try-toolchain option --- easybuild/tools/options.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d149ee3d79..6a23c2d82e 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -55,6 +55,7 @@ from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict from easybuild.framework.easyconfig.format.yeb import YEB_FORMAT_EXTENSION from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, get_paths_for +from easybuild.toolchains.compiler.systemcompiler import TC_CONSTANT_SYSTEM from easybuild.tools import build_log, run # build_log should always stay there, to ensure EasyBuildLog from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror @@ -92,6 +93,7 @@ from easybuild.tools.run import run_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler +from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories from easybuild.tools.systemtools import check_python_version, get_cpu_architecture, get_cpu_family, get_cpu_features from easybuild.tools.systemtools import get_system_info @@ -738,8 +740,11 @@ def validate(self): for opt in ['software', 'try-software', 'toolchain', 'try-toolchain']: val = getattr(self.options, opt.replace('-', '_')) if val and len(val) != 2: - msg = "--%s requires NAME,VERSION (given %s)" % (opt, ','.join(val)) - error_msgs.append(msg) + if opt in ['toolchain', 'try-toolchain'] and val == [TC_CONSTANT_SYSTEM]: + setattr(self.options, opt.replace('-', '_'), [SYSTEM_TOOLCHAIN_NAME, SYSTEM_TOOLCHAIN_NAME]) + else: + msg = "--%s requires NAME,VERSION (given %s)" % (opt, ','.join(val)) + error_msgs.append(msg) if self.options.umask: umask_regex = re.compile('^[0-7]{3}$') From c440be567f54e9839e853e7cbbf78a30c9839207 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 26 Feb 2020 12:18:44 +0100 Subject: [PATCH 188/344] Add tests --- test/framework/options.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..ff1786b042 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1702,6 +1702,9 @@ def test_try(self): (['--try-toolchain-name=gompi', '--try-toolchain-version=2018a'], 'toy/0.0-GCC-6.4.0.2.28'), # --try-toolchain is overridden by --toolchain (['--try-toolchain=gompi,2018a', '--toolchain=system,system'], 'toy/0.0'), + # check we interpret SYSTEM correctly as a toolchain + (['--try-toolchain=SYSTEM'], 'toy/0.0'), + (['--toolchain=SYSTEM'], 'toy/0.0'), (['--try-software-name=foo', '--try-software-version=1.2.3'], 'foo/1.2.3'), (['--try-toolchain-name=gompi', '--try-toolchain-version=2018a'], 'toy/0.0-GCC-6.4.0.2.28'), # combining --try-toolchain with other build options is too complicated, in this case the code defaults back From 390bf3b3f02c4a994180607ab0a2ef6ba8711c00 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 26 Feb 2020 16:27:12 +0100 Subject: [PATCH 189/344] Make new option experimental --- easybuild/framework/easyconfig/tweak.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 161b64986e..76cb93ed58 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -101,6 +101,9 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): src_to_dst_tc_mapping = {} revert_to_regex = False + if 'update_deps' in build_specs: + _log.experimental("Found build spec 'update_deps': Attempting to update dependency versions.") + if any(key in build_specs for key in ['toolchain', 'toolchain_name', 'toolchain_version', 'update_deps']): if not build_option('map_toolchains'): if 'update_deps' in build_specs: From 152544e488e5a38421ef3f24af501dcdf62b0df0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 27 Feb 2020 13:31:11 +0100 Subject: [PATCH 190/344] Add ability to update software when it is also included in exts_list --- easybuild/framework/easyconfig/tweak.py | 20 +++++++++++--- .../t/toy/toy-0.0-gompi-2018a-test.eb | 1 + test/framework/tweak.py | 26 +++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 76cb93ed58..483d7e9898 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -958,9 +958,23 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= # automagically clear out list of checksums if software version is being tweaked if 'version' in update_build_specs: if 'exts_list' in parsed_ec and parsed_ec['exts_list']: - raise EasyBuildError("Cannot currently handle updating version and dependencies of easyconfig with " - "exts_list entry") - elif 'checksums' not in update_build_specs: + _log.warning("Found 'exts_list' in %s, will only update extension version of %s (if applicable)", + ec_spec, parsed_ec['name']) + for idx, extension in enumerate(parsed_ec['exts_list']): + if isinstance(extension, tuple) and extension[0] == parsed_ec['name']: + ext_as_list = list(extension) + # in the extension tuple the version is the second element + if len(ext_as_list) > 1 and ext_as_list[1] == parsed_ec['version']: + ext_as_list[1] = update_build_specs['version'] + # also need to clear the checksum (if it exists) + if len(ext_as_list) > 2: + ext_as_list[2].pop('checksums', None) + # now replace the tuple in the dict of parameters + # to update the original dep dict, we need to get a reference with templating disabled... + parsed_ec.get_ref('exts_list')[idx] = tuple(ext_as_list) + _log.info("Updated extension found in %s with new version", ec_spec) + + if 'checksums' not in update_build_specs: update_build_specs['checksums'] = [] _log.warning("Tweaking version: checksums cleared, verification disabled.") # Update the keys according to the build specs diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb index 06d1bc2233..919c5723e3 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb @@ -47,6 +47,7 @@ exts_list = [ (name, version, { 'sanity_check_paths': {'files': ['lib/libtoy.a'], 'dirs': []}, 'exts_filter': ("ls -l lib/libtoy.a", ''), + 'checksums': ['fake'] }), ] diff --git a/test/framework/tweak.py b/test/framework/tweak.py index a1708f281f..1ce5df9a67 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -453,6 +453,32 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): # and that the checksum was removed self.assertFalse(tweaked_dict['checksums']) + # Check that if we update a software version, it also updates the version if the software appears in an + # extension list (like for a PythonBundle) + ec_spec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-2018a-test.eb') + # Create the trivial toolchain mapping + toolchain = {'name': 'gompi', 'version': '2018a'} + tc_mapping = map_toolchain_hierarchies(toolchain, toolchain, self.modtool) + # Update the software version + init_config(build_options=build_options) + get_toolchain_hierarchy.clear() + new_version = '1.x.3' + tweaked_spec = map_easyconfig_to_target_tc_hierarchy(ec_spec, + tc_mapping, + update_build_specs={'version': new_version}, + update_dep_versions=False) + tweaked_ec = process_easyconfig(tweaked_spec)[0] + tweaked_dict = tweaked_ec['ec'].asdict() + extensions = tweaked_dict['exts_list'] + # check one extension with the same name exists and that the version has been updated + hit_extension = 0 + for idx, extension in enumerate(extensions): + if isinstance(extension, tuple) and extension[0] == 'toy': + self.assertEqual(extension[1], new_version) + # Make sure checksum has been purged + self.assertFalse('checksums' in extension[2]) + hit_extension += 1 + self.assertEqual(hit_extension, 1, "Should only have updated one extension") def suite(): """ return all the tests in this file """ From d816bb85fb1536ebd481e9cdd02900f47b4458d7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 27 Feb 2020 13:34:00 +0100 Subject: [PATCH 191/344] Appease hound --- test/framework/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 1ce5df9a67..f252ecfcec 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -472,7 +472,7 @@ def test_map_easyconfig_to_target_tc_hierarchy(self): extensions = tweaked_dict['exts_list'] # check one extension with the same name exists and that the version has been updated hit_extension = 0 - for idx, extension in enumerate(extensions): + for extension in extensions: if isinstance(extension, tuple) and extension[0] == 'toy': self.assertEqual(extension[1], new_version) # Make sure checksum has been purged From b3c536de48caa972a9b443660906f2bc2a11f823 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 27 Feb 2020 13:53:48 +0100 Subject: [PATCH 192/344] Use proper looking fake checksum value --- test/framework/easyblock.py | 4 ++-- .../easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 2a52200d3c..04555bbec0 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1747,11 +1747,11 @@ def run_checks(): # full check also catches checksum issues with extensions res = eb.check_checksums() - self.assertEqual(len(res), 5) + self.assertEqual(len(res), 4) run_checks() idx = 2 - for ext in ['bar', 'barbar', 'toy']: + for ext in ['bar', 'barbar']: expected = "Checksums missing for one or more sources/patches of extension %s in " % ext self.assertTrue(res[idx].startswith(expected)) idx += 1 diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb index 919c5723e3..6442171781 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb @@ -47,7 +47,7 @@ exts_list = [ (name, version, { 'sanity_check_paths': {'files': ['lib/libtoy.a'], 'dirs': []}, 'exts_filter': ("ls -l lib/libtoy.a", ''), - 'checksums': ['fake'] + 'checksums': ['f3776716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'] }), ] From 3585c224262e7cb288137e68bf345ab176871b33 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 27 Feb 2020 14:07:10 +0100 Subject: [PATCH 193/344] Use correct checksum value --- .../easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb index 6442171781..497535850c 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb @@ -47,7 +47,7 @@ exts_list = [ (name, version, { 'sanity_check_paths': {'files': ['lib/libtoy.a'], 'dirs': []}, 'exts_filter': ("ls -l lib/libtoy.a", ''), - 'checksums': ['f3776716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'] + 'checksums': ['44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc'] }), ] From e8d72353e1f18247fab5fdaa685a2a0b3a4a126f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Feb 2020 14:04:01 +0100 Subject: [PATCH 194/344] enhance test for print_warning to test passing of logger via 'log' argument --- test/framework/build_log.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/framework/build_log.py b/test/framework/build_log.py index 7af9e623f1..0c9ec6894b 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -237,11 +237,11 @@ def test_log_levels(self): def test_print_warning(self): """Test print_warning""" - def run_check(args, silent=False, expected_stderr=''): + def run_check(args, silent=False, expected_stderr='', **kwargs): """Helper function to check stdout/stderr produced via print_warning.""" self.mock_stderr(True) self.mock_stdout(True) - print_warning(*args, silent=silent) + print_warning(*args, silent=silent, **kwargs) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stdout(False) @@ -258,6 +258,14 @@ def run_check(args, silent=False, expected_stderr=''): self.assertErrorRegex(EasyBuildError, "Unknown named arguments", print_warning, 'foo', unknown_arg='bar') + # test passing of logger to print_warning + tmp_logfile = os.path.join(self.test_prefix, 'test.log') + logger, _ = init_logging(tmp_logfile, silent=True) + expected = "\nWARNING: Test log message with a logger involved.\n\n" + run_check(["Test log message with a logger involved."], expected_stderr=expected, log=logger) + log_txt = read_file(tmp_logfile) + self.assertTrue("WARNING Test log message with a logger involved." in log_txt) + def test_print_error(self): """Test print_error""" def run_check(args, silent=False, expected_stderr=''): From 8776b165b89903085c2e94c29f474054268e37a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Feb 2020 14:20:22 +0100 Subject: [PATCH 195/344] use checkout@v2 in GitHub Actions to fix broken re-triggered tests --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index e79e88c7f0..283dc6c957 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -39,7 +39,7 @@ jobs: python: 3.8 fail-fast: false steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: set up Python uses: actions/setup-python@v1 From 78b64976d095db46c75c2c7b2953c5b866e37f3d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Feb 2020 16:33:04 +0100 Subject: [PATCH 196/344] appease the Hound --- easybuild/tools/filetools.py | 2 +- test/framework/filetools.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4da98b6ddb..4645e37041 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -614,7 +614,7 @@ def create_index(path, ignore_dirs=None): for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): for filename in filenames: # use relative paths in index - index.add(os.path.join(dirpath[len(path)+1:], filename)) + index.add(os.path.join(dirpath[len(path) + 1:], filename)) # do not consider (certain) hidden directories # note: we still need to consider e.g., .local ! diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c049d96b52..f4a10d4771 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1738,7 +1738,7 @@ def test_index_functions(self): self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) self.assertEqual(len(index), 24) @@ -1754,7 +1754,7 @@ def test_index_functions(self): # test creating index file that's infinitely valid index_fp = ft.dump_index(self.test_prefix, max_age_sec=0) index_txt = ft.read_file(index_fp) - expected_header[1] = "# valid until: 9999-12-31 23:59:59\.9+" + expected_header[1] = r"# valid until: 9999-12-31 23:59:59\.9+" for fn in expected_header + expected: regex = re.compile('^%s$' % fn, re.M) self.assertTrue(regex.search(index_txt), "Pattern '%s' found in: %s" % (regex.pattern, index_txt)) @@ -1768,7 +1768,7 @@ def test_index_functions(self): self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile("^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) self.assertEqual(len(index), 24) From 6c1507567225c09ae0e3c43430d52db43b345805 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 29 Feb 2020 16:58:39 +0100 Subject: [PATCH 197/344] add support for ignoring search index via --ignore-index --- easybuild/framework/easyconfig/easyconfig.py | 6 ++- easybuild/tools/config.py | 1 + easybuild/tools/filetools.py | 2 +- easybuild/tools/options.py | 1 + test/framework/options.py | 43 ++++++++++++++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index babdda2676..f5bebaee5b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1901,7 +1901,11 @@ def robot_find_easyconfig(name, version): res = None for path in paths: - if path in _path_indexes: + + if build_option('ignore_index'): + _log.info("Ignoring index for %s...", path) + path_index = [] + elif path in _path_indexes: path_index = _path_indexes[path] _log.info("Found loaded index for %s", path) elif os.path.exists(path): diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 51ff946f21..1c16245e21 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -226,6 +226,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'group_writable_installdir', 'hidden', 'ignore_checksums', + 'ignore_index', 'install_latest_eb_release', 'lib64_fallback_sanity_check', 'logtostdout', diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 4645e37041..d651ab2b6b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -770,7 +770,7 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen print_msg("Searching (case-insensitive) for '%s' in %s " % (query.pattern, path), log=_log, silent=silent) path_index = load_index(path, ignore_dirs=ignore_dirs) - if path_index is None: + if path_index is None or build_option('ignore_index'): if os.path.exists(path): _log.info("No index found for %s, creating one...", path) path_index = create_index(path, ignore_dirs=ignore_dirs) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 22d7299f7f..1540bb18c5 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -684,6 +684,7 @@ def easyconfig_options(self): 'create-index': ("Create index for files in specified directory", None, 'store', None), 'fix-deprecated-easyconfigs': ("Fix use of deprecated functionality in specified easyconfig files.", None, 'store_true', False), + 'ignore-index': ("Ignore index when searching for files", None, 'store_true', False), 'index-max-age': ("Maximum age for index before it is considered stale (in seconds)", int, 'store', DEFAULT_INDEX_MAX_AGE), 'inject-checksums': ("Inject checksums of specified type for sources/patches into easyconfig file(s)", diff --git a/test/framework/options.py b/test/framework/options.py index 52103c0000..3b9fc440fc 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -776,6 +776,49 @@ def test_search(self): args = [opt, pattern, '--robot', test_easyconfigs_dir] self.assertErrorRegex(EasyBuildError, "Invalid search query", self.eb_main, args, raise_error=True) + def test_ignore_index(self): + """ + Test use of --ignore-index. + """ + + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') + toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') + copy_file(toy_ec, self.test_prefix) + + # install index that list more files than are actually available, + # so we can check whether it's used + index_txt = '\n'.join([ + 'toy-0.0.eb', + 'toy-1.2.3.eb', + 'toy-4.5.6.eb', + ]) + write_file(os.path.join(self.test_prefix, '.eb-path-index'), index_txt) + + args = [ + '--search=toy', + '--robot-paths=%s' % self.test_prefix, + ] + self.mock_stdout(True) + self.eb_main(args, testing=False, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + + for toy_ec_fn in ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb']: + regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + args.append('--ignore-index') + self.mock_stdout(True) + self.eb_main(args, testing=False, raise_error=True) + stdout = self.get_stdout() + self.mock_stdout(False) + + regex = re.compile(re.escape(os.path.join(self.test_prefix, 'toy-0.0.eb')), re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + for toy_ec_fn in ['toy-1.2.3.eb', 'toy-4.5.6.eb']: + regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) + self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + def test_search_archived(self): "Test searching for archived easyconfigs" args = ['--search-filename=^intel'] From bab80b3ecb4dbfd7902398c1b5a3d1aea1daf7d4 Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 2 Mar 2020 13:33:46 +0100 Subject: [PATCH 198/344] Add Cray specific mapping --- easybuild/framework/easyconfig/easyconfig.py | 31 +++++++++++++------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 1c21edb30e..643ed801ff 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1153,7 +1153,7 @@ def _validate(self, attr, values): # private method if self[attr] and self[attr] not in values: raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) - def _handle_ext_module_metadata_by_probing_env(self, dep_name, dependency=dict()): + def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=None): """ helper function for handle_external_module_metadata handles metadata for external module dependencies when there is not entry in the @@ -1170,13 +1170,22 @@ def _handle_ext_module_metadata_by_probing_env(self, dep_name, dependency=dict() If neither of the pairs is found, then an empty dictionary is returned """ + if dependency is None: + dependency = dict() + short_ext_modname = dep_name.split('/')[0] if not 'name' in dependency: dependency['name'] = [short_ext_modname] + + if short_ext_modname.startswith('cray-'): + short_ext_modname = short_ext_modname_upperdname.split('cray-')[1] + + short_ext_modname.replace('-', '_') short_ext_modname_upper = convert_name(short_ext_modname, upper=True) allowed_pairs = [ ('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'), + ('CRAY_%s_PREFIX_DIR', 'CRAY_%s_VERSION'), ('CRAY_%s_DIR', 'CRAY_%s_VERSION'), ('CRAY_%s_ROOT', 'CRAY_%s_VERSION'), ('%s_PREFIX', '%s_VERSION'), @@ -1185,6 +1194,8 @@ def _handle_ext_module_metadata_by_probing_env(self, dep_name, dependency=dict() ('%s_HOME', '%s_VERSION'), ] + dry_run = build_option('extended_dry_run') + for prefix, version in allowed_pairs: prefix = prefix % short_ext_modname_upper version = version % short_ext_modname_upper @@ -1195,11 +1206,11 @@ def _handle_ext_module_metadata_by_probing_env(self, dep_name, dependency=dict() # only update missing values with both keys are found if dep_prefix and dep_version: # version should hold the value, not the key - if not 'version' in dependency: + if 'version' not in dependency: dependency['version'] = [dep_version] self.log.info('setting external module %s version to be %s' % (dep_name, dep_version)) # prefix should hold the key, not the value - if not 'prefix' in dependency: + if 'prefix' not in dependency: dependency['prefix'] = prefix self.log.info('setting external module %s prefix to be %s' % (dep_name, dep_prefix)) break @@ -1216,22 +1227,22 @@ def handle_external_module_metadata(self, dep_name): if dep_name in self.external_modules_metadata: dependency['external_module_metadata'] = self.external_modules_metadata[dep_name] - if not all(d in dependency['external_module_metadata'] for d in ('name','version','prefix')): - dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_env(dep_name, + if not all(d in dependency['external_module_metadata'] for d in ('name', 'version', 'prefix')): + dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_modules(dep_name, dependency=dependency['external_module_metadata']) self.log.info("Updated dependency info with available metadata and external module %s: %s", dep_name, dependency['external_module_metadata']) elif dep_name_no_version in self.external_modules_metadata: dependency['external_module_metadata'] = self.external_modules_metadata[dep_name_no_version] - if not all(d in dependency['external_module_metadata'] for d in ('name','version','prefix')): - dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_env(dep_name_no_version, - dependency=dependency['external_module_metadata']) + if not all(d in dependency['external_module_metadata'] for d in ('name', 'version', 'prefix')): + dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_modules( + dep_name_no_version, dependency=dependency['external_module_metadata']) self.log.info("Updated dependency info with available metadata and external module %s: %s", dep_name_no_version, dependency['external_module_metadata']) else: self.log.info("No metadata available for external module %s. Attempting to read from available modules", - dep_name) - dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_env(dep_name) + dep_name) + dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_modules(dep_name) self.log.info("Updated dependency info with external module %s: %s", dep_name, dependency['external_module_metadata']) From b3152c75940e2d5262c252c32751130a28e0b02e Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 2 Mar 2020 14:26:24 +0100 Subject: [PATCH 199/344] Address PR remarks --- easybuild/framework/easyconfig/easyconfig.py | 46 ++++++++++++-------- easybuild/tools/modules.py | 2 +- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 643ed801ff..f0b48a984d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1178,7 +1178,7 @@ def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=No dependency['name'] = [short_ext_modname] if short_ext_modname.startswith('cray-'): - short_ext_modname = short_ext_modname_upperdname.split('cray-')[1] + short_ext_modname = short_ext_modname.split('cray-')[1] short_ext_modname.replace('-', '_') short_ext_modname_upper = convert_name(short_ext_modname, upper=True) @@ -1194,8 +1194,6 @@ def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=No ('%s_HOME', '%s_VERSION'), ] - dry_run = build_option('extended_dry_run') - for prefix, version in allowed_pairs: prefix = prefix % short_ext_modname_upper version = version % short_ext_modname_upper @@ -1224,28 +1222,40 @@ def handle_external_module_metadata(self, dep_name): """ dependency = {} dep_name_no_version = dep_name.split('/')[0] + metadata_fields = ['name', 'version', 'prefix'] + external_metadata = {} if dep_name in self.external_modules_metadata: - dependency['external_module_metadata'] = self.external_modules_metadata[dep_name] - if not all(d in dependency['external_module_metadata'] for d in ('name', 'version', 'prefix')): - dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_modules(dep_name, - dependency=dependency['external_module_metadata']) - self.log.info("Updated dependency info with available metadata and external module %s: %s", - dep_name, dependency['external_module_metadata']) + external_metadata = self.external_modules_metadata[dep_name] + if not all(d in external_metadata for d in metadata_fields): + external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name, + dependency=external_metadata) + if external_metadata: + self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", + dep_name, external_metadata) + else: + self.log.info("No metadata available for external module %s.", dep_name) elif dep_name_no_version in self.external_modules_metadata: - dependency['external_module_metadata'] = self.external_modules_metadata[dep_name_no_version] - if not all(d in dependency['external_module_metadata'] for d in ('name', 'version', 'prefix')): - dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_modules( - dep_name_no_version, dependency=dependency['external_module_metadata']) - self.log.info("Updated dependency info with available metadata and external module %s: %s", - dep_name_no_version, dependency['external_module_metadata']) + external_metadata = self.external_modules_metadata[dep_name_no_version] + if not all(d in external_metadata for d in metadata_fields): + external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name_no_version, + dependency=external_metadata) + if external_metadata: + self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", + dep_name, external_metadata) + else: + self.log.info("No metadata available for external module %s.", dep_name) else: self.log.info("No metadata available for external module %s. Attempting to read from available modules", dep_name) - dependency['external_module_metadata'] = self._handle_ext_module_metadata_by_probing_modules(dep_name) - self.log.info("Updated dependency info with external module %s: %s", - dep_name, dependency['external_module_metadata']) + external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name) + if external_metadata: + self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", + dep_name, external_metadata) + else: + self.log.info("No metadata available for external module %s.", dep_name) + dependency['external_module_metadata'] = external_metadata return dependency def handle_multi_deps(self): diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 9746e41e7a..1afd5a2ead 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1144,7 +1144,7 @@ def get_variable_from_modulefile(self, mod_name, var_name): """ try: # Tcl syntax - regex = re.compile(r'^setenv\s*%s\s*(?P\S*)' % var_name, re.M) + regex = re.compile(r'^setenv\s+%s\s+(?P\S*)' % var_name, re.M) ans = self.get_value_from_modulefile(mod_name, regex) except Exception: return None From e6f3eb7d6179af6c910ee99b2b31a9bf086f943a Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 2 Mar 2020 14:28:44 +0100 Subject: [PATCH 200/344] Address style changes --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index f0b48a984d..67212985dd 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1241,8 +1241,8 @@ def handle_external_module_metadata(self, dep_name): external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name_no_version, dependency=external_metadata) if external_metadata: - self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", - dep_name, external_metadata) + self.log.info("Updated dependency info with metadata from available modules for external module " + "%s: %s", dep_name, external_metadata) else: self.log.info("No metadata available for external module %s.", dep_name) else: From 6d3a63eb9c743fdbd907f73bb0a7fbf5715b94aa Mon Sep 17 00:00:00 2001 From: Victor Holanda Date: Mon, 2 Mar 2020 17:04:49 +0100 Subject: [PATCH 201/344] Quick fix to test external_metadata definition hypothesis --- easybuild/framework/easyconfig/easyconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 67212985dd..8116dec65b 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1233,6 +1233,7 @@ def handle_external_module_metadata(self, dep_name): if external_metadata: self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", dep_name, external_metadata) + dependency['external_module_metadata'] = external_metadata else: self.log.info("No metadata available for external module %s.", dep_name) elif dep_name_no_version in self.external_modules_metadata: @@ -1243,6 +1244,7 @@ def handle_external_module_metadata(self, dep_name): if external_metadata: self.log.info("Updated dependency info with metadata from available modules for external module " "%s: %s", dep_name, external_metadata) + dependency['external_module_metadata'] = external_metadata else: self.log.info("No metadata available for external module %s.", dep_name) else: @@ -1250,12 +1252,12 @@ def handle_external_module_metadata(self, dep_name): dep_name) external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name) if external_metadata: + dependency['external_module_metadata'] = external_metadata self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", dep_name, external_metadata) else: self.log.info("No metadata available for external module %s.", dep_name) - dependency['external_module_metadata'] = external_metadata return dependency def handle_multi_deps(self): From 0907cfbfd4e2b0aad0fc1affc90bb9fca98af268 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 3 Mar 2020 18:17:51 +0800 Subject: [PATCH 202/344] improve error messages, use eb_name consistently --- easybuild/tools/github.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 178f22a320..5094d379c1 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1009,14 +1009,13 @@ def copy_easyblocks(paths, target_dir): for path in paths: cn = get_easyblock_class_name(path) if not cn: - raise EasyBuildError("Invalid easyblock file") + raise EasyBuildError("Could not determine easyblock class from file %s" % path) eb_name = remove_unwanted_chars(decode_class_name(cn).replace('-', '_')).lower() if is_generic_easyblock(cn): - target_path = os.path.join(subdir, GENERIC_EB, "%s.%s" % (eb_name.lower(), PYTHON_EXTENSION)) + target_path = os.path.join(subdir, GENERIC_EB, "%s.%s" % (eb_name, PYTHON_EXTENSION)) else: - letter = os.path.basename(path).lower()[0] - target_path = os.path.join(subdir, letter, "%s.%s" % (eb_name, PYTHON_EXTENSION)) + target_path = os.path.join(subdir, eb_name[0], "%s.%s" % (eb_name, PYTHON_EXTENSION)) full_target_path = os.path.join(target_dir, target_path) file_info['eb_names'].append(eb_name) @@ -1025,7 +1024,7 @@ def copy_easyblocks(paths, target_dir): copy_file(path, full_target_path, force_in_dry_run=True) else: - raise EasyBuildError("Subdir easyblocks not found") + raise EasyBuildError("Could not find %s" % os.path.join(target_dir, subdir)) return file_info From 231c537530eebc3b56f8d98385c51e8996c0db4c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 16:43:27 +0100 Subject: [PATCH 203/344] don't filter out None values in to_checksums, leave them in place --- easybuild/framework/easyconfig/types.py | 2 ++ test/framework/type_checking.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index fd8f2e09a7..1ea5c31544 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -462,6 +462,8 @@ def to_checksums(checksums): for key, value in checksum.items(): validated_dict[key] = to_checksums(value) res.append(validated_dict) + else: + res.append(checksum) return res diff --git a/test/framework/type_checking.py b/test/framework/type_checking.py index 3dd60dcbd0..b1247832c5 100644 --- a/test/framework/type_checking.py +++ b/test/framework/type_checking.py @@ -658,6 +658,8 @@ def test_to_checksums(self): ['be662daa971a640e40be5c804d9d7d10', ('adler32', '0x998410035'), ('crc32', '0x1553842328'), ('md5', 'be662daa971a640e40be5c804d9d7d10'), ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), ('size', 273)], + # None values should not be filtered out, but left in place + [None, 'fa618be8435447a017fd1bf2c7ae922d0428056cfc7449f7a8641edf76b48265', None], ] for checksums in test_inputs: self.assertEqual(to_checksums(checksums), checksums) From d1c9795a40146e59f40f6f177408af8fa83f4296 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 17:06:22 +0100 Subject: [PATCH 204/344] fix broken test for --review-pr by using different PR to test with --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..c25fde2486 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2710,17 +2710,17 @@ def test_review_pr(self): self.mock_stdout(True) self.mock_stderr(True) - # PR for CMake 3.12.1 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/6660 + # PR for gzip 1.10 easyconfig, see https://github.com/easybuilders/easybuild-easyconfigs/pull/9921 args = [ '--color=never', '--github-user=%s' % GITHUB_TEST_ACCOUNT, - '--review-pr=6660', + '--review-pr=9921', ] self.eb_main(args, raise_error=True) txt = self.get_stdout() self.mock_stdout(False) self.mock_stderr(False) - regex = re.compile(r"^Comparing CMake-3.12.1-\S* with CMake-3.12.1-") + regex = re.compile(r"^Comparing gzip-1.10-\S* with gzip-1.10-") self.assertTrue(regex.search(txt), "Pattern '%s' not found in: %s" % (regex.pattern, txt)) def test_set_tmpdir(self): From a2786d66c6073e56f9782420630570e38061cecf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 17:35:21 +0100 Subject: [PATCH 205/344] update comment in to_checksums to mention None + dict --- easybuild/framework/easyconfig/types.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index 1ea5c31544..17b199cdc2 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -446,9 +446,11 @@ def to_checksums(checksums): res = [] for checksum in checksums: # each list entry can be: - # * a string (MD5 checksum) + # * None (indicates no checksum) + # * a string (MD5 or SHA256 checksum) # * a tuple with 2 elements: checksum type + checksum value # * a list of checksums (i.e. multiple checksums for a single file) + # * a dict (filename to checksum mapping) if isinstance(checksum, string_type): res.append(checksum) elif isinstance(checksum, (list, tuple)): From 830754978a345bf77b66ca5daa67cdf1fb955d64 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Mar 2020 09:21:10 +0100 Subject: [PATCH 206/344] add get_cpu_arch_name function to systemtools, using archspec to determine CPU arch name --- easybuild/tools/systemtools.py | 23 +++++++++++++++++++++++ test/framework/systemtools.py | 28 ++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 0e114cb0dc..5f8853500c 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -59,6 +59,14 @@ _log.debug("Failed to import 'distro' Python module: %s", err) HAVE_DISTRO = False +try: + from archspec.cpu import host as archspec_cpu_host + HAVE_ARCHSPEC = True +except ImportError as err: + _log.debug("Failed to import 'archspec' Python module: %s", err) + HAVE_ARCHSPEC = False + + # Architecture constants AARCH32 = 'AArch32' @@ -344,6 +352,20 @@ def get_cpu_family(): return family +def get_cpu_arch_name(): + """ + Determine CPU architecture name via archspec (if available). + """ + cpu_arch_name = None + if HAVE_ARCHSPEC: + cpu_arch_name = archspec_cpu_host() + + if cpu_arch_name is None: + cpu_arch_name = UNKNOWN + + return cpu_arch_name + + def get_cpu_model(): """ Determine CPU model, e.g., Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz @@ -746,6 +768,7 @@ def get_system_info(): return { 'core_count': get_avail_core_count(), 'total_memory': get_total_memory(), + 'cpu_arch_name': get_cpu_arch_name(), 'cpu_model': get_cpu_model(), 'cpu_speed': get_cpu_speed(), 'cpu_vendor': get_cpu_vendor(), diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index fa1f1331cb..c33d271a66 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -44,8 +44,8 @@ from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP from easybuild.tools.systemtools import check_python_version, pick_dep_version -from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor +from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_arch_name, get_cpu_architecture +from easybuild.tools.systemtools import get_cpu_family, get_cpu_features, get_cpu_model, get_cpu_speed, get_cpu_vendor from easybuild.tools.systemtools import get_gcc_version, get_glibc_version, get_os_type, get_os_name, get_os_version from easybuild.tools.systemtools import get_platform_name, get_shared_lib_ext, get_system_info, get_total_memory @@ -338,6 +338,11 @@ def setUp(self): self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version self.orig_sys_version_info = st.sys.version_info + self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC + if hasattr(st, 'archspec_cpu_host'): + self.orig_archspec_cpu_host = st.archspec_cpu_host + else: + self.orig_archspec_cpu_host = None def tearDown(self): """Cleanup after systemtools test.""" @@ -349,6 +354,9 @@ def tearDown(self): st.platform.uname = self.orig_platform_uname st.get_tool_version = self.orig_get_tool_version st.sys.version_info = self.orig_sys_version_info + st.HAVE_ARCHSPEC = self.orig_HAVE_ARCHSPEC + if self.orig_archspec_cpu_host is not None: + st.archspec_cpu_host = self.orig_archspec_cpu_host super(SystemToolsTest, self).tearDown() def test_avail_core_count_native(self): @@ -529,6 +537,22 @@ def test_cpu_architecture(self): MACHINE_NAME = name self.assertEqual(get_cpu_architecture(), machine_names[name]) + def test_cpu_arch_name_native(self): + """Test getting CPU arch name.""" + arch_name = get_cpu_arch_name() + self.assertTrue(isinstance(arch_name, string_type)) + + def test_cpu_arch_name(self): + """Test getting CPU arch name.""" + st.HAVE_ARCHSPEC = True + st.archspec_cpu_host = lambda: 'haswell' + arch_name = get_cpu_arch_name() + self.assertEqual(arch_name, 'haswell') + + st.archspec_cpu_host = lambda: None + arch_name = get_cpu_arch_name() + self.assertEqual(arch_name, 'UNKNOWN') + def test_cpu_vendor_native(self): """Test getting CPU vendor.""" cpu_vendor = get_cpu_vendor() From 707a4b71075fa337599623635a69cbb0454dc50f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 16:34:28 +0100 Subject: [PATCH 207/344] mention CPU arch name in comment for uploaded test report, if it's known --- easybuild/tools/testing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 456e7c0db2..d114992ec7 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -50,7 +50,7 @@ from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel from easybuild.tools.robot import resolve_dependencies -from easybuild.tools.systemtools import get_system_info +from easybuild.tools.systemtools import UNKNOWN, get_system_info from easybuild.tools.version import FRAMEWORK_VERSION, EASYBLOCKS_VERSION @@ -264,6 +264,11 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, 'os_version': system_info['os_version'], 'pyver': system_info['python_version'].split(' ')[0], } + + # also mention CPU architecture name, but only if it's known + if system_info['cpu_arch_name'] != UNKNOWN: + short_system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] + comment_lines = [ "Test report by @%s" % user, ('**FAILED**', '**SUCCESS**')[success], From 959f99758efbd20241d0d120b402dcfb96392bd7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 3 Mar 2020 20:21:29 +0100 Subject: [PATCH 208/344] install archspec as optional dependency when testing with Python >= 2.7 --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 3a93ac826f..0a8591c50f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,3 +50,5 @@ GC3Pie python-graph-dot python-hglib requests + +archspec; python_version >= '2.7' From 156c34cabd771a12965fbd2c42e0dec69b5cc0ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 4 Mar 2020 15:10:33 +0100 Subject: [PATCH 209/344] reorganize EasyBlock.extensions_step to ensure correct filtering of extensions (fixes #3167) --- easybuild/framework/easyblock.py | 63 +++++++++++++++++--------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 44c2965c1a..6b33bc43c2 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1473,17 +1473,18 @@ def skip_extensions(self): raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") res = [] - for ext in self.exts: - cmd, stdin = resolve_exts_filter_template(exts_filter, ext) + for ext_inst in self.ext_instances: + cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False) self.log.info("exts_filter result %s %s", cmdstdouterr, ec) if ec: - self.log.info("Not skipping %s" % ext['name']) - self.log.debug("exit code: %s, stdout/err: %s" % (ec, cmdstdouterr)) - res.append(ext) + self.log.info("Not skipping %s", ext_inst.name) + self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) + res.append(ext_inst) else: - self.log.info("Skipping %s" % ext['name']) - self.exts = res + self.log.info("Skipping %s", ext_inst.name) + + self.ext_instances = res # # MISCELLANEOUS UTILITY FUNCTIONS @@ -2077,9 +2078,6 @@ def extensions_step(self, fetch=False): self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping - if self.skip: - self.skip_extensions() - # actually install extensions self.log.debug("Installing extensions") exts_defaultclass = self.cfg['exts_defaultclass'] @@ -2100,14 +2098,8 @@ def extensions_step(self, fetch=False): # get class instances for all extensions self.ext_instances = [] - exts_cnt = len(self.exts) - for idx, ext in enumerate(self.exts): - self.log.debug("Starting extension %s" % ext['name']) - tup = (ext['name'], ext.get('version', ''), idx+1, exts_cnt) - print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) - - # always go back to original work dir to avoid running stuff from a dir that no longer exists - change_dir(self.orig_workdir) + for ext in self.exts: + self.log.debug("Creating class instance for extension %s...", ext['name']) cls, inst = None, None class_name = encode_class_name(ext['name']) @@ -2119,11 +2111,11 @@ def extensions_step(self, fetch=False): # with a similar name (e.g., Perl Extension 'GO' vs 'Go' for which 'EB_Go' is available) cls = get_easyblock_class(None, name=ext['name'], error_on_failed_import=False, error_on_missing_easyblock=False) - self.log.debug("Obtained class %s for extension %s" % (cls, ext['name'])) + self.log.debug("Obtained class %s for extension %s", cls, ext['name']) if cls is not None: inst = cls(self, ext) except (ImportError, NameError) as err: - self.log.debug("Failed to use extension-specific class for extension %s: %s" % (ext['name'], err)) + self.log.debug("Failed to use extension-specific class for extension %s: %s", ext['name'], err) # alternative attempt: use class specified in class map (if any) if inst is None and ext['name'] in exts_classmap: @@ -2141,7 +2133,7 @@ def extensions_step(self, fetch=False): if inst is None: try: cls = get_class_for(default_class_modpath, default_class) - self.log.debug("Obtained class %s for installing extension %s" % (cls, ext['name'])) + self.log.debug("Obtained class %s for installing extension %s", cls, ext['name']) inst = cls(self, ext) self.log.debug("Installing extension %s with default class %s (from %s)", ext['name'], default_class, default_class_modpath) @@ -2149,7 +2141,23 @@ def extensions_step(self, fetch=False): raise EasyBuildError("Also failed to use default class %s from %s for extension %s: %s, giving up", default_class, default_class_modpath, ext['name'], err) else: - self.log.debug("Installing extension %s with class %s (from %s)" % (ext['name'], class_name, mod_path)) + self.log.debug("Installing extension %s with class %s (from %s)", ext['name'], class_name, mod_path) + + self.ext_instances.append(inst) + + if self.skip: + self.skip_extensions() + + exts_cnt = len(self.exts) + for idx, (ext, ext_instance) in enumerate(zip(self.exts, self.ext_instances)): + + self.log.debug("Starting extension %s" % ext['name']) + + # always go back to original work dir to avoid running stuff from a dir that no longer exists + change_dir(self.orig_workdir) + + tup = (ext['name'], ext.get('version', ''), idx+1, exts_cnt) + print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) if self.dry_run: tup = (ext['name'], ext.get('version', ''), cls.__name__) @@ -2165,18 +2173,15 @@ def extensions_step(self, fetch=False): else: # don't reload modules for toolchain, there is no need since they will be loaded already; # the (fake) module for the parent software gets loaded before installing extensions - inst.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + ext_instance.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, rpath_filter_dirs=self.rpath_filter_dirs) # real work - inst.prerun() - txt = inst.run() + ext_instance.prerun() + txt = ext_instance.run() if txt: self.module_extra_extensions += txt - inst.postrun() - - # append so we can make us of it later (in sanity_check_step) - self.ext_instances.append(inst) + ext_instance.postrun() # cleanup (unload fake module, remove fake module dir) if fake_mod_data: From 047331f2b14b18ad01a51b3dfffb1c191532a37c Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 5 Mar 2020 19:44:43 +0100 Subject: [PATCH 210/344] Fix _set_mpi_variables for impi version 2019 and later. --- easybuild/toolchains/mpi/intelmpi.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index e404bf72f8..539bf25c76 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -29,7 +29,11 @@ :author: Kenneth Hoste (Ghent University) """ +import os + import easybuild.tools.toolchain as toolchain + +from distutils.version import LooseVersion from easybuild.toolchains.mpi.mpich2 import Mpich2 from easybuild.tools.toolchain.constants import COMPILER_FLAGS, COMPILER_VARIABLES from easybuild.tools.toolchain.variables import CommandFlagList @@ -67,6 +71,21 @@ def _set_mpi_compiler_variables(self): super(IntelMPI, self)._set_mpi_compiler_variables() + def _set_mpi_variables(self): + """Set the other MPI variables""" + + if (LooseVersion(self.version) >= LooseVersion('2019')): + lib_dir = [os.path.join('intel64', 'lib', 'release')] + incl_dir = [os.path.join('intel64', 'include')] + + for root in self.get_software_root(self.MPI_MODULE_NAME): + self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, filename="lib%s.a" % self.MPI_LIBRARY_NAME) + self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, filename="lib%s.so" % self.MPI_LIBRARY_NAME) + self.variables.append_exists('MPI_LIB_DIR', root, lib_dir) + self.variables.append_exists('MPI_INC_DIR', root, incl_dir) + + super(IntelMPI, self)._set_mpi_variables() + MPI_LINK_INFO_OPTION = '-show' def set_variables(self): From 22a6cc4c47305a1f18d08dbdc8fed0ddb3b4f21d Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 5 Mar 2020 19:51:45 +0100 Subject: [PATCH 211/344] Fix too long lines --- easybuild/toolchains/mpi/intelmpi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 539bf25c76..3a9867c677 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -79,8 +79,10 @@ def _set_mpi_variables(self): incl_dir = [os.path.join('intel64', 'include')] for root in self.get_software_root(self.MPI_MODULE_NAME): - self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, filename="lib%s.a" % self.MPI_LIBRARY_NAME) - self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, filename="lib%s.so" % self.MPI_LIBRARY_NAME) + self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, + filename="lib%s.a" % self.MPI_LIBRARY_NAME) + self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, + filename="lib%s.so" % self.MPI_LIBRARY_NAME) self.variables.append_exists('MPI_LIB_DIR', root, lib_dir) self.variables.append_exists('MPI_INC_DIR', root, incl_dir) From 44b68af3816f09ab8f9b38ba06cff3eade3e233f Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 5 Mar 2020 19:52:32 +0100 Subject: [PATCH 212/344] Appease Hound --- easybuild/toolchains/mpi/intelmpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 3a9867c677..9e5494aa23 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -80,9 +80,9 @@ def _set_mpi_variables(self): for root in self.get_software_root(self.MPI_MODULE_NAME): self.variables.append_exists('MPI_LIB_STATIC', root, lib_dir, - filename="lib%s.a" % self.MPI_LIBRARY_NAME) + filename="lib%s.a" % self.MPI_LIBRARY_NAME) self.variables.append_exists('MPI_LIB_SHARED', root, lib_dir, - filename="lib%s.so" % self.MPI_LIBRARY_NAME) + filename="lib%s.so" % self.MPI_LIBRARY_NAME) self.variables.append_exists('MPI_LIB_DIR', root, lib_dir) self.variables.append_exists('MPI_INC_DIR', root, incl_dir) From e456a677e32dfe615db7f2bc5cd3a9c19abd7408 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 14:31:51 +0100 Subject: [PATCH 213/344] only get cpu arch name in get_cpu_arch_name, not the whole archspec.MicroArchitecture instance --- easybuild/tools/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 5f8853500c..c155d47853 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -358,7 +358,7 @@ def get_cpu_arch_name(): """ cpu_arch_name = None if HAVE_ARCHSPEC: - cpu_arch_name = archspec_cpu_host() + cpu_arch_name = archspec_cpu_host().name if cpu_arch_name is None: cpu_arch_name = UNKNOWN From 00516f8f9613c82408817642a922b22505052014 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 14:37:15 +0100 Subject: [PATCH 214/344] use correct target account/repo when creating test report & posting comment in PR (fixes #3233) --- easybuild/tools/testing.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 456e7c0db2..d61e1611da 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -140,7 +140,10 @@ def session_state(): def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_log=False): """Create test report for easyconfigs PR, in Markdown format.""" - user = build_option('github_user') + + github_user = build_option('github_user') + pr_target_account = build_option('pr_target_account') + pr_target_repo = build_option('pr_target_repo') end_time = gmtime() @@ -148,7 +151,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l test_report = [] if pr_nr is not None: test_report.extend([ - "Test report for https://github.com/easybuilders/easybuild-easyconfigs/pull/%s" % pr_nr, + "Test report for https://github.com/%s/%s/pull/%s" % (pr_target_account, pr_target_repo, pr_nr), "", ]) test_report.extend([ @@ -182,7 +185,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l if pr_nr is not None: descr += " (PR #%s)" % pr_nr fn = '%s_partial.log' % os.path.basename(ec['spec'])[:-3] - gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=user) + gist_url = create_gist(partial_log_txt, fn, descr=descr, github_user=github_user) test_log = "(partial log available at %s)" % gist_url build_overview.append(" * **%s** _%s_ %s" % (test_result, os.path.basename(ec['spec']), test_log)) @@ -239,15 +242,16 @@ def upload_test_report_as_gist(test_report, descr=None, fn=None): if fn is None: fn = 'easybuild_test_report_%s.md' % strftime("%Y%M%d-UTC-%H-%M-%S", gmtime()) - user = build_option('github_user') + github_user = build_option('github_user') + gist_url = create_gist(test_report, descr=descr, fn=fn, github_user=github_user) - gist_url = create_gist(test_report, descr=descr, fn=fn, github_user=user) return gist_url def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, success): """Post test report in a gist, and submit comment in easyconfigs PR.""" - user = build_option('github_user') + + github_user = build_option('github_user') # create gist with test report descr = "EasyBuild test report for easyconfigs PR #%s" % pr_nr @@ -265,14 +269,18 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, 'pyver': system_info['python_version'].split(' ')[0], } comment_lines = [ - "Test report by @%s" % user, + "Test report by @%s" % github_user, ('**FAILED**', '**SUCCESS**')[success], msg, short_system_info, "See %s for a full test report." % gist_url, ] comment = '\n'.join(comment_lines) - post_comment_in_issue(pr_nr, comment, github_user=user) + + pr_target_account = build_option('pr_target_account') + pr_target_repo = build_option('pr_target_repo') + + post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) msg = "Test report uploaded to %s and mentioned in a comment in easyconfigs PR#%s" % (gist_url, pr_nr) return msg From 4442bdc9b9baa3d7602ec17b25b25abf18ad017d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 16:16:07 +0100 Subject: [PATCH 215/344] fix broken test_skip_extensions_step --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 2a52200d3c..adeb2aabc6 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -850,8 +850,8 @@ def test_skip_extensions_step(self): eb.installdir = config.install_path() eb.skip = True eb.extensions_step(fetch=True) - # 'ext1' should be in eb.exts - eb_exts = [y for x in eb.exts for y in x.values()] + # 'ext1' should be in eb.ext_instances + eb_exts = [x.name for x in eb.ext_instances] self.assertTrue('ext1' in eb_exts) # 'ext2' should not self.assertFalse('ext2' in eb_exts) From c9a3ba133d22dec1dba6687d9588f3f73e3c4d3e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 16:33:16 +0100 Subject: [PATCH 216/344] enhance test_skip_extensions_step to verify that #3167 is fixed --- test/framework/easyblock.py | 11 +++++++---- .../easybuild/easyblocks/generic/dummyextension.py | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index adeb2aabc6..0db4425664 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -833,11 +833,11 @@ def test_skip_extensions_step(self): toolchain = SYSTEM exts_list = [ "ext1", - ("ext2", "42", {"source_tmpl": "dummy.tgz"}), + ("EXT-2", "42", {"source_tmpl": "dummy.tgz"}), ("ext3", "1.1", {"source_tmpl": "dummy.tgz", "modulename": "real_ext"}), ] exts_filter = ("\ - if [ %(ext_name)s == 'ext2' ] && [ %(ext_version)s == '42' ] && [[ %(src)s == *dummy.tgz ]];\ + if [ %(ext_name)s == 'ext_2' ] && [ %(ext_version)s == '42' ] && [[ %(src)s == *dummy.tgz ]];\ then exit 0;\ elif [ %(ext_name)s == 'real_ext' ]; then exit 0;\ else exit 1; fi", "") @@ -853,8 +853,11 @@ def test_skip_extensions_step(self): # 'ext1' should be in eb.ext_instances eb_exts = [x.name for x in eb.ext_instances] self.assertTrue('ext1' in eb_exts) - # 'ext2' should not - self.assertFalse('ext2' in eb_exts) + # 'EXT-2' should not + self.assertFalse('EXT-2' in eb_exts) + self.assertFalse('EXT_2' in eb_exts) + self.assertFalse('ext-2' in eb_exts) + self.assertFalse('ext_2' in eb_exts) # 'ext3' should not self.assertFalse('ext3' in eb_exts) diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py index da16d01483..af97c3f254 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/dummyextension.py @@ -32,3 +32,11 @@ class DummyExtension(ExtensionEasyBlock): """Support for building/installing dummy extensions.""" + + def __init__(self, *args, **kwargs): + + super(DummyExtension, self).__init__(*args, **kwargs) + + # use lowercase name as default value for expected module name, and replace '-' with '_' + if 'modulename' not in self.options: + self.options['modulename'] = self.name.lower().replace('-', '_') From 0fc2a4b00845358c7e97deef40ef7064bd45657f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 17:11:07 +0100 Subject: [PATCH 217/344] take into account that result of archspec.cpu.host() may be None --- easybuild/tools/systemtools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index c155d47853..13e28cc6f3 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -358,7 +358,9 @@ def get_cpu_arch_name(): """ cpu_arch_name = None if HAVE_ARCHSPEC: - cpu_arch_name = archspec_cpu_host().name + res = archspec_cpu_host() + if res: + cpu_arch_name = res.name if cpu_arch_name is None: cpu_arch_name = UNKNOWN From a6cb6b505091f5d345f1b543c506de06f067b6b0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 17:11:22 +0100 Subject: [PATCH 218/344] fix test_cpu_arch_name --- test/framework/systemtools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index c33d271a66..bda6813014 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -544,8 +544,13 @@ def test_cpu_arch_name_native(self): def test_cpu_arch_name(self): """Test getting CPU arch name.""" + + class MicroArch(object): + def __init__(self, name): + self.name = name + st.HAVE_ARCHSPEC = True - st.archspec_cpu_host = lambda: 'haswell' + st.archspec_cpu_host = lambda: MicroArch('haswell') arch_name = get_cpu_arch_name() self.assertEqual(arch_name, 'haswell') From af2fb0e2a35d8721c6ea68a76d1fca44b4d91f47 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 20:31:12 +0100 Subject: [PATCH 219/344] make --merge-pr take into account --pr-target-branch --- easybuild/tools/github.py | 7 ++++--- test/framework/options.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index f3bd87d95b..dd6eccc15c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -991,9 +991,10 @@ def not_eligible(msg): target = '%s/%s' % (pr_data['base']['repo']['owner']['login'], pr_data['base']['repo']['name']) print_msg("Checking eligibility of %s PR #%s for merging..." % (target, pr_data['number']), prefix=False) - # check target branch, must be 'develop' - msg_tmpl = "* targets develop branch: %s" - if pr_data['base']['ref'] == 'develop': + # check target branch, must be branch name specified in --pr-target-branch (usually 'develop') + pr_target_branch = build_option('pr_target_branch') + msg_tmpl = "* targets %s branch: %%s" % pr_target_branch + if pr_data['base']['ref'] == pr_target_branch: print_msg(msg_tmpl % 'OK', prefix=False) else: res = not_eligible(msg_tmpl % "FAILED; found '%s'" % pr_data['base']['ref']) diff --git a/test/framework/options.py b/test/framework/options.py index d09bfdc17a..1bb5a18bd4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3429,6 +3429,7 @@ def test_merge_pr(self): '4781', # PR for easyconfig for EasyBuild-3.3.0.eb '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--pr-target-branch=some_branch', ] # merged PR for EasyBuild-3.3.0.eb, is missing approved review @@ -3436,12 +3437,12 @@ def test_merge_pr(self): expected_stdout = '\n'.join([ "Checking eligibility of easybuilders/easybuild-easyconfigs PR #4781 for merging...", - "* targets develop branch: OK", "* test suite passes: OK", "* last test report is successful: OK", "* milestone is set: OK (3.3.1)", ]) expected_stderr = '\n'.join([ + "* targets some_branch branch: FAILED; found 'develop' => not eligible for merging!", "* approved review: MISSING => not eligible for merging!", '', "WARNING: Review indicates this PR should not be merged (use -f/--force to do so anyway)", @@ -3449,7 +3450,8 @@ def test_merge_pr(self): self.assertEqual(stderr.strip(), expected_stderr) self.assertTrue(stdout.strip().endswith(expected_stdout), "'%s' ends with '%s'" % (stdout, expected_stdout)) - # full eligible merged PR + # full eligible merged PR, default target branch + del args[-1] args[1] = '4832' stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) From 4d044eccd453cec6679d0d41b2f46ee30d211b87 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 7 Mar 2020 20:56:03 +0100 Subject: [PATCH 220/344] fix default value for pr_target_branch build option --- easybuild/tools/config.py | 5 ++++- easybuild/tools/options.py | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ab98bcad6d..1a410d4e06 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -78,6 +78,7 @@ CONT_TYPES = [CONT_TYPE_DOCKER, CONT_TYPE_SINGULARITY] DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY +DEFAULT_BRANCH = 'develop' DEFAULT_JOB_BACKEND = 'GC3Pie' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") DEFAULT_MAX_FAIL_RATIO_PERMS = 0.5 @@ -195,7 +196,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_commit_msg', 'pr_descr', 'pr_target_account', - 'pr_target_branch', 'pr_target_repo', 'pr_title', 'rpath_filter', @@ -270,6 +270,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_CONT_TYPE: [ 'container_type', ], + DEFAULT_BRANCH: [ + 'pr_target_branch', + ], DEFAULT_MAX_FAIL_RATIO_PERMS: [ 'max_fail_ratio_adjust_permissions', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6a23c2d82e..513bf715e6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,9 +60,9 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE -from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_FORCE_DOWNLOAD, DEFAULT_JOB_BACKEND -from easybuild.tools.config import DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS, DEFAULT_MNS -from easybuild.tools.config import DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES +from easybuild.tools.config import DEFAULT_ALLOW_LOADED_MODULES, DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD +from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS +from easybuild.tools.config import DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS, ERROR from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR @@ -611,7 +611,7 @@ def github_options(self): 'pr-commit-msg': ("Commit message for new/updated pull request created with --new-pr", str, 'store', None), 'pr-descr': ("Description for new pull request created with --new-pr", str, 'store', None), 'pr-target-account': ("Target account for new PRs", str, 'store', GITHUB_EB_MAIN), - 'pr-target-branch': ("Target branch for new PRs", str, 'store', 'develop'), + 'pr-target-branch': ("Target branch for new PRs", str, 'store', DEFAULT_BRANCH), 'pr-target-repo': ("Target repository for new/updating PRs", str, 'store', GITHUB_EASYCONFIGS_REPO), 'pr-title': ("Title for new pull request created with --new-pr", str, 'store', None), 'preview-pr': ("Preview a new pull request", None, 'store_true', False), From 6bb21315a0e2f7e1918428c3a47f722817e40fb6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Mar 2020 10:35:11 +0100 Subject: [PATCH 221/344] fix determining relative paths in create_index --- easybuild/tools/filetools.py | 6 +++--- test/framework/filetools.py | 30 ++++++++++++++++-------------- test/framework/options.py | 10 ++++------ 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index d651ab2b6b..a45382a2a7 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -611,15 +611,15 @@ def create_index(path, ignore_dirs=None): elif not os.path.isdir(path): raise EasyBuildError("Specified path is not a directory: %s", path) - for (dirpath, dirnames, filenames) in os.walk(path, topdown=True): + for (dirpath, dirnames, filenames) in os.walk(path, topdown=True, followlinks=True): for filename in filenames: # use relative paths in index - index.add(os.path.join(dirpath[len(path) + 1:], filename)) + index.add(os.path.join(os.path.relpath(dirpath, path), filename)) # do not consider (certain) hidden directories # note: we still need to consider e.g., .local ! # replace list elements using [:], so os.walk doesn't process deleted directories - # see http://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders + # see https://stackoverflow.com/questions/13454164/os-walk-without-hidden-folders dirnames[:] = [d for d in dirnames if d not in ignore_dirs] return index diff --git a/test/framework/filetools.py b/test/framework/filetools.py index f4a10d4771..cd76667397 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1690,20 +1690,22 @@ def test_index_functions(self): # load_index just returns None if there is no index in specified directory self.assertEqual(ft.load_index(self.test_prefix), None) - # create index for test easyconfigs - index = ft.create_index(test_ecs) - self.assertEqual(len(index), 79) - - expected = [ - os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), - os.path.join('t', 'toy', 'toy-0.0.eb'), - os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'), - ] - for fn in expected: - self.assertTrue(fn in index) - - for fp in index: - self.assertTrue(fp.endswith('.eb')) + # create index for test easyconfigs; + # test with specified path with and without trailing '/'s + for path in [test_ecs, test_ecs + '/', test_ecs + '//']: + index = ft.create_index(path) + self.assertEqual(len(index), 79) + + expected = [ + os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), + os.path.join('t', 'toy', 'toy-0.0.eb'), + os.path.join('s', 'ScaLAPACK', 'ScaLAPACK-2.0.2-gompi-2018a-OpenBLAS-0.2.20.eb'), + ] + for fn in expected: + self.assertTrue(fn in index) + + for fp in index: + self.assertTrue(fp.endswith('.eb')) # set up some files to create actual index file for ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) diff --git a/test/framework/options.py b/test/framework/options.py index 3b9fc440fc..a7f1d9bdad 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -785,13 +785,11 @@ def test_ignore_index(self): toy_ec = os.path.join(test_ecs_dir, 'test_ecs', 't', 'toy', 'toy-0.0.eb') copy_file(toy_ec, self.test_prefix) + toy_ec_list = ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb'] + # install index that list more files than are actually available, # so we can check whether it's used - index_txt = '\n'.join([ - 'toy-0.0.eb', - 'toy-1.2.3.eb', - 'toy-4.5.6.eb', - ]) + index_txt = '\n'.join(toy_ec_list) write_file(os.path.join(self.test_prefix, '.eb-path-index'), index_txt) args = [ @@ -803,7 +801,7 @@ def test_ignore_index(self): stdout = self.get_stdout() self.mock_stdout(False) - for toy_ec_fn in ['toy-0.0.eb', 'toy-1.2.3.eb', 'toy-4.5.6.eb']: + for toy_ec_fn in toy_ec_list: regex = re.compile(re.escape(os.path.join(self.test_prefix, toy_ec_fn)), re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) From 4dc4554f637d3c4f32fdff0a067a4ef4ffce3863 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Mar 2020 13:38:16 +0100 Subject: [PATCH 222/344] avoid that relative paths start with './' in create_index --- easybuild/tools/filetools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a45382a2a7..922a8ce5d6 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -614,7 +614,11 @@ def create_index(path, ignore_dirs=None): for (dirpath, dirnames, filenames) in os.walk(path, topdown=True, followlinks=True): for filename in filenames: # use relative paths in index - index.add(os.path.join(os.path.relpath(dirpath, path), filename)) + rel_dirpath = os.path.relpath(dirpath, path) + # avoid that relative paths start with './' + if rel_dirpath == '.': + rel_dirpath = '' + index.add(os.path.join(rel_dirpath, filename)) # do not consider (certain) hidden directories # note: we still need to consider e.g., .local ! From d9eda8c7aba442ab7c759fc6c4d236db0f9b8437 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Mar 2020 15:04:36 +0100 Subject: [PATCH 223/344] enhance test for categorize_files_by_type to also cover py_files --- test/framework/easyconfig.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e9e39768ca..65b5f1b309 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2716,15 +2716,23 @@ def test_categorize_files_by_type(self): self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, categorize_files_by_type([])) - test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',) + test_dir = os.path.dirname(os.path.abspath(__file__)) + test_ecs_dir = os.path.join(test_dir, 'easyconfigs') toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch' toy_patch = os.path.join(os.path.dirname(test_ecs_dir), 'sandbox', 'sources', 'toy', toy_patch_fn) + + easyblocks_dir = os.path.join(test_dir, 'sandbox', 'easybuild', 'easyblocks') + configuremake = os.path.join(easyblocks_dir, 'generic', 'configuremake.py') + toy_easyblock = os.path.join(easyblocks_dir, 't', 'toy.py') + paths = [ 'bzip2-1.0.6.eb', + toy_easyblock, os.path.join(test_ecs_dir, 'test_ecs', 'g', 'gzip', 'gzip-1.4.eb'), toy_patch, 'foo', ':toy-0.0-deps.eb', + configuremake, ] res = categorize_files_by_type(paths) expected = [ @@ -2735,6 +2743,7 @@ def test_categorize_files_by_type(self): self.assertEqual(res['easyconfigs'], expected) self.assertEqual(res['files_to_delete'], ['toy-0.0-deps.eb']) self.assertEqual(res['patch_files'], [toy_patch]) + self.assertEqual(res['py_files'], [toy_easyblock, configuremake]) def test_resolve_template(self): """Test resolve_template function.""" From bdf437f8324bf6d7906e1bfcf069ee84350fbd6b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Mar 2020 15:21:53 +0100 Subject: [PATCH 224/344] move constant for 'generic' to config.py, avoid constant for .py + order imports alphabetically --- easybuild/framework/easyconfig/easyconfig.py | 5 +++-- easybuild/tools/config.py | 3 +++ easybuild/tools/github.py | 15 ++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b3e8af1cb8..650d29d5fc 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -58,7 +58,8 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, template_constant_dict from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN +from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file @@ -1697,7 +1698,7 @@ def get_module_path(name, generic=None, decode=True): modpath = ['easybuild', 'easyblocks'] if generic: - modpath.append('generic') + modpath.append(GENERIC_EASYBLOCK_PKG) return '.'.join(modpath + [module_name]) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ab98bcad6d..d4899a7fe3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -111,6 +111,9 @@ FORCE_DOWNLOAD_CHOICES = [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES] DEFAULT_FORCE_DOWNLOAD = FORCE_DOWNLOAD_SOURCES +# package name for generic easyblocks +GENERIC_EASYBLOCK_PKG = 'generic' + # general module class GENERAL_CLASS = 'all' diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 5094d379c1..520d796627 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -30,11 +30,11 @@ :author: Toon Willems (Ghent University) """ import base64 -import imp -import inspect import copy import getpass import glob +import imp +import inspect import os import random import re @@ -51,7 +51,7 @@ from easybuild.framework.easyconfig.easyconfig import is_generic_easyblock, process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning -from easybuild.tools.config import build_option +from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option from easybuild.tools.filetools import apply_patch, copy_dir, copy_file, det_patched_files, decode_class_name from easybuild.tools.filetools import download_file, extract_file, mkdir, read_file, symlink from easybuild.tools.filetools import which, write_file @@ -84,7 +84,6 @@ _log.warning("Failed to import 'git' Python module: %s", err) -GENERIC_EB = 'generic' GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' GITHUB_DIR_TYPE = u'dir' @@ -109,7 +108,6 @@ HTTP_STATUS_CREATED = 201 HTTP_STATUS_NO_CONTENT = 204 KEYRING_GITHUB_TOKEN = 'github_token' -PYTHON_EXTENSION = 'py' URL_SEPARATOR = '/' VALID_CLOSE_PR_REASONS = { @@ -1012,10 +1010,13 @@ def copy_easyblocks(paths, target_dir): raise EasyBuildError("Could not determine easyblock class from file %s" % path) eb_name = remove_unwanted_chars(decode_class_name(cn).replace('-', '_')).lower() + if is_generic_easyblock(cn): - target_path = os.path.join(subdir, GENERIC_EB, "%s.%s" % (eb_name, PYTHON_EXTENSION)) + pkgdir = GENERIC_EASYBLOCK_PKG else: - target_path = os.path.join(subdir, eb_name[0], "%s.%s" % (eb_name, PYTHON_EXTENSION)) + pkgdir = eb_name[0] + + target_path = os.path.join(subdir, pkgdir, eb_name + '.py') full_target_path = os.path.join(target_dir, target_path) file_info['eb_names'].append(eb_name) From 4705c0219bf811fd0c8a114761e8e21ac5ec4263 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 9 Mar 2020 07:42:04 +0100 Subject: [PATCH 225/344] First attempt at adding missing install/builddir templates to extensions. --- easybuild/framework/extension.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index a27f81dd47..271e184be9 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -37,7 +37,7 @@ import os from easybuild.framework.easyconfig.easyconfig import resolve_template -from easybuild.framework.easyconfig.templates import template_constant_dict +from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir from easybuild.tools.run import run_cmd @@ -111,6 +111,11 @@ def __init__(self, mself, ext, extra_params=None): # construct dict with template values that can be used self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) + # Add install/builddir templates, copied from update_config_template_run_step + # do we need to call self.cfg.generate_template_values() after? + for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: + self.cfg.template_values[name[0]] = str(getattr(self.master, name[0], None)) + # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) From 1f995afd87cafb641711a97828d05ba2732352b5 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 9 Mar 2020 17:06:30 +0800 Subject: [PATCH 226/344] move handling of options.include_easyblocks_from_pr to options.set_up_configuration --- easybuild/main.py | 9 ++------- easybuild/tools/options.py | 7 ++++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index cf9779d7cc..69c47a7293 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -57,13 +57,12 @@ from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, read_file, write_file -from easybuild.tools.github import check_github, close_pr, new_branch_github, fetch_easyblocks_from_pr -from easybuild.tools.github import find_easybuild_easyconfig +from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool -from easybuild.tools.options import include_easyblocks, set_up_configuration, use_color +from easybuild.tools.options import set_up_configuration, use_color from easybuild.tools.robot import check_conflicts, dry_run, missing_deps, resolve_dependencies, search_easyconfigs from easybuild.tools.package.utilities import check_pkg_support from easybuild.tools.parallelbuild import submit_jobs @@ -200,10 +199,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args - if options.include_easyblocks_from_pr: - included_easyblocks = fetch_easyblocks_from_pr(options.include_easyblocks_from_pr) - include_easyblocks(options.tmpdir, included_easyblocks) - global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e8ed232161..7d622fbb1a 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED, GITHUB_PR_STATE_OPEN from easybuild.tools.github import GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS -from easybuild.tools.github import fetch_github_token +from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS from easybuild.tools.include import include_easyblocks, include_module_naming_schemes, include_toolchains from easybuild.tools.job.backend import avail_job_backends @@ -1395,6 +1395,11 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): init(options, config_options_dict) init_build_options(build_options=build_options, cmdline_options=options) + # done here instead of in _postprocess_include because github integration requires build_options to be initialized + if eb_go.options.include_easyblocks_from_pr: + included_easyblocks = fetch_easyblocks_from_pr(eb_go.options.include_easyblocks_from_pr) + include_easyblocks(eb_go.options.tmpdir, included_easyblocks) + check_python_version() # move directory containing fake vsc namespace into temporary directory used for this session From b1368d6c1ee136c3cbe37c2fd68b3d794836d63d Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 9 Mar 2020 17:07:33 +0800 Subject: [PATCH 227/344] use tempfile.mkdtemp for included easyblocks --- easybuild/tools/include.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index e71ed7a161..2e85d99e20 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -31,6 +31,7 @@ import os import re import sys +import tempfile from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError @@ -147,7 +148,7 @@ def is_software_specific_easyblock(module): def include_easyblocks(tmpdir, paths): """Include generic and software-specific easyblocks found in specified locations.""" - easyblocks_path = os.path.join(tmpdir, 'included-easyblocks') + easyblocks_path = tempfile.mkdtemp(dir=tmpdir, prefix='included-easyblocks-') set_up_eb_package(easyblocks_path, 'easybuild.easyblocks', subpkgs=['generic'], pkg_init_body=EASYBLOCKS_PKG_INIT_BODY) From a5254e22ad290d530f797ac28594ddb2df37deae Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 9 Mar 2020 17:09:51 +0800 Subject: [PATCH 228/344] make fetch_files_from_pr also work for easyblocks also when PR is already merged --- easybuild/tools/github.py | 21 +++++++++++++-------- test/framework/github.py | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 2846673e91..e8baa141cb 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -381,7 +381,7 @@ def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): - """Fetch patched easyconfig files for a particular PR.""" + """Fetch patched files for a particular PR.""" if github_user is None: github_user = build_option('github_user') @@ -398,10 +398,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): if github_repo is None: github_repo = GITHUB_EASYCONFIGS_REPO - elif github_repo not in [GITHUB_EASYBLOCKS_REPO, GITHUB_EASYCONFIGS_REPO]: + + if github_repo == GITHUB_EASYCONFIGS_REPO: + easyfiles = 'easyconfigs' + elif github_repo == GITHUB_EASYBLOCKS_REPO: + easyfiles = 'easyblocks' + else: raise EasyBuildError("Don't know how to fetch files from repo %s", github_repo) - easyfiles = 'easyconfigs' if github_repo == GITHUB_EASYCONFIGS_REPO else 'easyblocks' + subdir = os.path.join('easybuild', easyfiles) _log.debug("Fetching %s from %s/%s PR #%s into %s", easyfiles, github_account, github_repo, pr, path) pr_data, _ = fetch_pr_data(pr, github_account, github_repo, github_user) @@ -449,9 +454,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): print_warning("Using %s from closed PR #%s" % (easyfiles, pr)) # obtain most recent version of patched files - for patched_file in patched_files: + for patched_file in [f for f in patched_files if subdir in f]: # path to patch file, incl. subdir it is in - fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:]) + fn = patched_file.split(subdir)[1].strip(os.path.sep) sha = pr_data['head']['sha'] full_url = URL_SEPARATOR.join([GITHUB_RAW, github_account, github_repo, sha, patched_file]) _log.info("Downloading %s from %s", fn, full_url) @@ -461,14 +466,14 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): # symlink directories into expected place if they're not there yet if final_path != path: - dirpath = os.path.join(final_path, 'easybuild', easyfiles) + dirpath = os.path.join(final_path, subdir) for eb_dir in os.listdir(dirpath): symlink(os.path.join(dirpath, eb_dir), os.path.join(path, os.path.basename(eb_dir))) # sanity check: make sure all patched files are downloaded files = [] - for patched_file in [f for f in patched_files if not f.startswith('test/')]: - fn = os.path.sep.join(patched_file.split(os.path.sep)[-3:]) + for patched_file in [f for f in patched_files if subdir in f]: + fn = patched_file.split(easyfiles)[1].strip(os.path.sep) full_path = os.path.join(path, fn) if os.path.exists(full_path): files.append(full_path) diff --git a/test/framework/github.py b/test/framework/github.py index b64b98cea2..22714da9fa 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -256,7 +256,7 @@ def test_fetch_easyblocks_from_pr(self): }) # PR with new easyblock plus non-easyblock file - all_ebs_pr1964 = ['.gitignore', 'lammps.py'] + all_ebs_pr1964 = ['lammps.py'] # PR with changed easyblock all_ebs_pr1967 = ['siesta.py'] From c7740906184604dd8c689ea1048ba78c250462a4 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 9 Mar 2020 10:50:11 +0100 Subject: [PATCH 229/344] Drop comment about generate_template_values which is not needed. Clarify where the values for install/builddir is coming from. --- easybuild/framework/extension.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 271e184be9..b44d5759fe 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -111,8 +111,7 @@ def __init__(self, mself, ext, extra_params=None): # construct dict with template values that can be used self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) - # Add install/builddir templates, copied from update_config_template_run_step - # do we need to call self.cfg.generate_template_values() after? + # Add install/builddir templates with values from master. for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: self.cfg.template_values[name[0]] = str(getattr(self.master, name[0], None)) From 1a5d0b013c3aadf01805ca157fd26cf04221f848 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 9 Mar 2020 18:00:19 +0800 Subject: [PATCH 230/344] fix tests after use of mkdtemp for included easyblocks --- test/framework/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index bcb3dcbe09..810291e694 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2401,7 +2401,7 @@ def test_xxx_include_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', 'foo.py') + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks', 'foo.py') foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) @@ -2444,7 +2444,7 @@ def test_xxx_include_generic_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks', 'generic', 'foobar.py') foo_regex = re.compile(r"^\|-- FooBar \(easybuild.easyblocks.generic.foobar @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) @@ -2482,7 +2482,7 @@ def test_xxx_include_generic_easyblocks(self): logtxt = read_file(self.logfile) mod_pattern = 'easybuild.easyblocks.generic.generictest' - path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks', 'easybuild', 'easyblocks', + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks', 'generic', 'generictest.py') foo_regex = re.compile(r"^\|-- GenericTest \(%s @ %s\)" % (mod_pattern, path_pattern), re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) From 476b96f6f2cdc1246bdf251472cccaf67dba4dfb Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 9 Mar 2020 18:23:13 +0800 Subject: [PATCH 231/344] appease the hound --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 810291e694..71e22e7200 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2401,7 +2401,8 @@ def test_xxx_include_easyblocks(self): self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks', 'foo.py') + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', + 'easybuild', 'easyblocks', 'foo.py') foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) From 4b1d8efe6af5c5617d530996ccdc7d16943246c3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 9 Mar 2020 15:29:34 +0100 Subject: [PATCH 232/344] make tests use easybuilders/testrepository rather than hpcugent/testrepository after it was moved --- test/framework/filetools.py | 16 ++++++++-------- test/framework/github.py | 16 ++++++++-------- test/framework/repository.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a96ee8a6a7..e5fefa80d4 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1893,7 +1893,7 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', - 'url': 'https://github.com/hpcugent', + 'url': 'https://github.com/easybuilders', 'tag': 'master', } target_dir = os.path.join(self.test_prefix, 'target') @@ -1918,7 +1918,7 @@ def test_get_source_tarball_from_git(self): git_config = { 'repo_name': 'testrepository', - 'url': 'git@github.com:hpcugent', + 'url': 'git@github.com:easybuilders', 'tag': 'master', } args = ['test.tar.gz', self.test_prefix, git_config] @@ -1972,11 +1972,11 @@ def run_check(): git_config = { 'repo_name': 'testrepository', - 'url': 'git@github.com:hpcugent', + 'url': 'git@github.com:easybuilders', 'tag': 'master', } expected = '\n'.join([ - r' running command "git clone --branch master git@github.com:hpcugent/testrepository.git"', + r' running command "git clone --branch master git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -1985,7 +1985,7 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', + r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz --exclude .git testrepository"', r" \(in .*/tmp.*\)", @@ -1994,7 +1994,7 @@ def run_check(): git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch master --recursive git@github.com:hpcugent/testrepository.git"', + r' running command "git clone --branch master --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", @@ -2005,7 +2005,7 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --recursive git@github.com:hpcugent/testrepository.git"', + r' running command "git clone --recursive git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86 && git submodule update"', r" \(in testrepository\)", @@ -2016,7 +2016,7 @@ def run_check(): del git_config['recursive'] expected = '\n'.join([ - r' running command "git clone git@github.com:hpcugent/testrepository.git"', + r' running command "git clone git@github.com:easybuilders/testrepository.git"', r" \(in .*/tmp.*\)", r' running command "git checkout 8456f86"', r" \(in testrepository\)", diff --git a/test/framework/github.py b/test/framework/github.py index 4b4c68c31c..bb1e4be68c 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -54,8 +54,8 @@ # test account, for which a token may be available GITHUB_TEST_ACCOUNT = 'easybuild_test' -# the user & repo to use in this test (https://github.com/hpcugent/testrepository) -GITHUB_USER = "hpcugent" +# the user & repo to use in this test (https://github.com/easybuilders/testrepository) +GITHUB_USER = "easybuilders" GITHUB_REPO = "testrepository" # branch to test GITHUB_BRANCH = 'master' @@ -220,10 +220,10 @@ def test_close_pr(self): self.mock_stdout(False) patterns = [ - "hpcugent/testrepository PR #2 was submitted by migueldiascosta", + "easybuilders/testrepository PR #2 was submitted by migueldiascosta", "[DRY RUN] Adding comment to testrepository issue #2: '" + "@migueldiascosta, this PR is being closed for the following reason(s): just a test", - "[DRY RUN] Closed hpcugent/testrepository PR #2", + "[DRY RUN] Closed easybuilders/testrepository PR #2", ] for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) @@ -236,11 +236,11 @@ def test_close_pr(self): self.mock_stdout(False) patterns = [ - "hpcugent/testrepository PR #2 was submitted by migueldiascosta", + "easybuilders/testrepository PR #2 was submitted by migueldiascosta", "[DRY RUN] Adding comment to testrepository issue #2: '" + "@migueldiascosta, this PR is being closed for the following reason(s): %s" % retest_msg, - "[DRY RUN] Closed hpcugent/testrepository PR #2", - "[DRY RUN] Reopened hpcugent/testrepository PR #2", + "[DRY RUN] Closed easybuilders/testrepository PR #2", + "[DRY RUN] Reopened easybuilders/testrepository PR #2", ] for pattern in patterns: self.assertTrue(pattern in stdout, "Pattern '%s' found in: %s" % (pattern, stdout)) @@ -597,7 +597,7 @@ def test_restclient(self): client = RestClient('https://api.github.com', username=GITHUB_TEST_ACCOUNT, token=self.github_token) - status, body = client.repos['hpcugent']['testrepository'].contents.a_directory['a_file.txt'].get() + status, body = client.repos['easybuilders']['testrepository'].contents.a_directory['a_file.txt'].get() self.assertEqual(status, 200) # base64.b64encode requires & produces a 'bytes' value in Python 3, # but we need a string value hence the .decode() (also works in Python 2) diff --git a/test/framework/repository.py b/test/framework/repository.py index 41a985deb2..b2326c7426 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -79,7 +79,7 @@ def test_gitrepo(self): print("(skipping GitRepository test)") return - test_repo_url = 'https://github.com/hpcugent/testrepository' + test_repo_url = 'https://github.com/easybuilders/testrepository' # URL repo = GitRepository(test_repo_url) @@ -122,7 +122,7 @@ def test_svnrepo(self): return # GitHub also supports SVN - test_repo_url = 'https://github.com/hpcugent/testrepository' + test_repo_url = 'https://github.com/easybuilders/testrepository' repo = SvnRepository(test_repo_url) repo.init() From 655d49424076e8ef64c63c54a3c492fc81d6be43 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 9 Mar 2020 22:20:36 +0000 Subject: [PATCH 233/344] fix mkdir of lockpath to include parents --- easybuild/framework/easyblock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 596684261b..7880ac6fab 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2946,7 +2946,7 @@ def run_all_steps(self, run_test_cases): lockpath = build_option('lockpath') or os.path.join(install_path('software'), '.locks') if not os.path.exists(lockpath): - mkdir(lockpath) + mkdir(lockpath, parents=True) lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/', '_')) if os.path.exists(lockfile_name): if build_option('wait_on_lock'): From d61b52adc03edf75984d165fbc5f773fcdbc5c2f Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 11 Mar 2020 14:27:42 +0800 Subject: [PATCH 234/344] make --list-easyblocks aware of --include-easyblocks-from-pr and test using both --include-easyblocks options --- easybuild/tools/options.py | 29 ++++++++++++++++++++----- test/framework/options.py | 43 ++++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 7d622fbb1a..19b1c4d9ed 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1042,8 +1042,8 @@ def _postprocess_list_avail(self): if self.options.avail_easyconfig_licenses: msg += avail_easyconfig_licenses(self.options.output_format) - # dump available easyblocks - if self.options.list_easyblocks: + # dump available easyblocks (unless including easyblocks from pr, in which case it will be done later) + if self.options.list_easyblocks and not self.options.include_easyblocks_from_pr: msg += list_easyblocks(self.options.list_easyblocks, self.options.output_format) # dump known toolchains @@ -1087,7 +1087,8 @@ def _postprocess_list_avail(self): print(msg) # cleanup tmpdir and exit - cleanup_and_exit(self.tmpdir) + if not self.options.include_easyblocks_from_pr: + cleanup_and_exit(self.tmpdir) def avail_repositories(self): """Show list of known repository types.""" @@ -1397,8 +1398,26 @@ def set_up_configuration(args=None, logfile=None, testing=False, silent=False): # done here instead of in _postprocess_include because github integration requires build_options to be initialized if eb_go.options.include_easyblocks_from_pr: - included_easyblocks = fetch_easyblocks_from_pr(eb_go.options.include_easyblocks_from_pr) - include_easyblocks(eb_go.options.tmpdir, included_easyblocks) + easyblocks_from_pr = fetch_easyblocks_from_pr(eb_go.options.include_easyblocks_from_pr) + + if eb_go.options.include_easyblocks: + # make sure we're not including the same easyblock twice + included_from_pr = set([os.path.basename(eb) for eb in easyblocks_from_pr]) + included_from_file = set([os.path.basename(eb) for eb in eb_go.options.include_easyblocks]) + included_twice = included_from_pr & included_from_file + if included_twice: + raise EasyBuildError("Multiple inclusion of %s, check your --include-easyblocks options", + ','.join(included_twice)) + + include_easyblocks(eb_go.options.tmpdir, easyblocks_from_pr) + + if eb_go.options.list_easyblocks: + msg = list_easyblocks(eb_go.options.list_easyblocks, eb_go.options.output_format) + if eb_go.options.unittest_file: + log.info(msg) + else: + print(msg) + cleanup_and_exit(tmpdir) check_python_version() diff --git a/test/framework/options.py b/test/framework/options.py index 71e22e7200..31d302dc75 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2351,7 +2351,7 @@ def generate_cmd_line(ebopts): # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... def test_xxx_include_easyblocks(self): - """Test --include-easyblocks.""" + """Test --include-easyblocks*.""" orig_local_sys_path = sys.path[:] fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -2393,25 +2393,56 @@ def test_xxx_include_easyblocks(self): # clear log write_file(self.logfile, '') + # include both extra EB_foo easyblock and an easyblock from a PR args = [ '--include-easyblocks=%s/*.py' % self.test_prefix, + '--include-easyblocks-from-pr=1915', '--list-easyblocks=detailed', '--unittest-file=%s' % self.logfile, ] self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', - 'easybuild', 'easyblocks', 'foo.py') - foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks') + + foo_pattern = os.path.join(path_pattern, 'foo.py') + foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % foo_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) - # easyblock is found via get_easyblock_class + cmm_pattern = os.path.join(path_pattern, 'generic', 'cmakemake.py') + cmm_regex = re.compile(r"\|-- CMakeMake \(easybuild.easyblocks.generic.cmakemake @ %s\)" % cmm_pattern, re.M) + self.assertTrue(cmm_regex.search(logtxt), "Pattern '%s' found in: %s" % (cmm_regex.pattern, logtxt)) + + # easyblocks are found via get_easyblock_class klass = get_easyblock_class('EB_foo') self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) - # 'undo' import of foo easyblock + klass = get_easyblock_class('CMakeMake') + self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) + + # 'undo' import of foo and cmakemake easyblock del sys.modules['easybuild.easyblocks.foo'] + del sys.modules['easybuild.easyblocks.generic.cmakemake'] + + # include extra test cmakemake easyblock + cmm_txt = '\n'.join([ + 'from easybuild.framework.easyblock import EasyBlock', + 'class CMakeMake(EasyBlock):', + ' pass', + '' + ]) + write_file(os.path.join(self.test_prefix, 'cmakemake.py'), cmm_txt) + + # including the same easyblock twice should fail + args = [ + '--include-easyblocks=%s/cmakemake.py' % self.test_prefix, + '--include-easyblocks-from-pr=1915', + '--list-easyblocks=detailed', + '--unittest-file=%s' % self.logfile, + ] + self.assertErrorRegex(EasyBuildError, + "Multiple inclusion of cmakemake.py, check your --include-easyblocks options", + self.eb_main, args, raise_error=True) # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... From cb73209c1e51fb756b56f6cc73fa1dabbac29fbb Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 11 Mar 2020 08:16:27 +0100 Subject: [PATCH 235/344] Add test to verify that %(installdir) is actually set in an extension. --- test/framework/easyconfig.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 104bae30ac..e68443369c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -464,7 +464,7 @@ def test_extensions_templates(self): ' "source_tmpl": "%(name)s-%(version_major_minor)s-py%(pymajver)s%(versionsuffix)s.tar.gz",', ' "patches": ["%(name)s-%(version)s_fix-silly-typo-in-printf-statement.patch"],', # use hacky prebuildopts that is picked up by 'EB_Toy' easyblock, to check whether templates are resolved - ' "prebuildopts": "gcc -O2 %(name)s.c -o toy-%(version)s && mv toy-%(version)s toy #",', + ' "prebuildopts": "gcc -O2 %(name)s.c -o toy-%(version)s && mv toy-%(version)s toy # echo installdir is %(installdir)s #",', ' }),', ']', ]) @@ -489,9 +489,11 @@ def test_extensions_templates(self): for patch in toy_ext.patches: patches.append(patch['path']) self.assertEqual(patches, [os.path.join(self.test_prefix, toy_patch_fn)]) + # define actual installation dir + pi_installdir = os.path.join(self.test_installpath, 'software', 'pi', '3.14-test') expected = { 'patches': ['toy-0.0_fix-silly-typo-in-printf-statement.patch'], - 'prebuildopts': 'gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy #', + 'prebuildopts': 'gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy # echo installdir is %s #' % pi_installdir, 'source_tmpl': 'toy-0.0-py3-test.tar.gz', 'source_urls': ['https://pypi.python.org/packages/source/t/toy'], } @@ -500,10 +502,9 @@ def test_extensions_templates(self): # also .cfg of Extension instance was updated correctly self.assertEqual(toy_ext.cfg['source_urls'], ['https://pypi.python.org/packages/source/t/toy']) self.assertEqual(toy_ext.cfg['patches'], [toy_patch_fn]) - self.assertEqual(toy_ext.cfg['prebuildopts'], "gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy #") + self.assertEqual(toy_ext.cfg['prebuildopts'], "gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy # echo installdir is %s #" % pi_installdir) # check whether files expected to be installed for 'toy' extension are in place - pi_installdir = os.path.join(self.test_installpath, 'software', 'pi', '3.14-test') self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'bin', 'toy'))) self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'lib', 'libtoy.a'))) From c2b6934f9ec335597c794ac69311b24d9f3b54e1 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Wed, 11 Mar 2020 08:25:44 +0100 Subject: [PATCH 236/344] Trying to fix too long lines. --- test/framework/easyconfig.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e68443369c..0f45797bc7 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -464,7 +464,8 @@ def test_extensions_templates(self): ' "source_tmpl": "%(name)s-%(version_major_minor)s-py%(pymajver)s%(versionsuffix)s.tar.gz",', ' "patches": ["%(name)s-%(version)s_fix-silly-typo-in-printf-statement.patch"],', # use hacky prebuildopts that is picked up by 'EB_Toy' easyblock, to check whether templates are resolved - ' "prebuildopts": "gcc -O2 %(name)s.c -o toy-%(version)s && mv toy-%(version)s toy # echo installdir is %(installdir)s #",', + ' "prebuildopts": "gcc -O2 %(name)s.c -o toy-%(version)s &&' + + ' mv toy-%(version)s toy # echo installdir is %(installdir)s #",', ' }),', ']', ]) @@ -491,9 +492,10 @@ def test_extensions_templates(self): self.assertEqual(patches, [os.path.join(self.test_prefix, toy_patch_fn)]) # define actual installation dir pi_installdir = os.path.join(self.test_installpath, 'software', 'pi', '3.14-test') + expected_prebuildopts = 'gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy # echo installdir is %s #' % pi_installdir expected = { 'patches': ['toy-0.0_fix-silly-typo-in-printf-statement.patch'], - 'prebuildopts': 'gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy # echo installdir is %s #' % pi_installdir, + 'prebuildopts': expected_prebuildopts, 'source_tmpl': 'toy-0.0-py3-test.tar.gz', 'source_urls': ['https://pypi.python.org/packages/source/t/toy'], } @@ -502,7 +504,7 @@ def test_extensions_templates(self): # also .cfg of Extension instance was updated correctly self.assertEqual(toy_ext.cfg['source_urls'], ['https://pypi.python.org/packages/source/t/toy']) self.assertEqual(toy_ext.cfg['patches'], [toy_patch_fn]) - self.assertEqual(toy_ext.cfg['prebuildopts'], "gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy # echo installdir is %s #" % pi_installdir) + self.assertEqual(toy_ext.cfg['prebuildopts'], expected_prebuildopts) # check whether files expected to be installed for 'toy' extension are in place self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'bin', 'toy'))) From 998c5da04cc9a2b7af5c79be9dc3757e39454c57 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 11 Mar 2020 18:31:39 +0800 Subject: [PATCH 237/344] use different paths when fetching both easyconfigs or easyblocks from PRs --- easybuild/tools/github.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d197736dc6..0dd8eced55 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -385,8 +385,15 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): if github_user is None: github_user = build_option('github_user') + + if github_repo is None: + github_repo = GITHUB_EASYCONFIGS_REPO + if path is None: - path = build_option('pr_path') + if github_repo == GITHUB_EASYCONFIGS_REPO: + path = build_option('pr_path') + elif github_repo == GITHUB_EASYBLOCKS_REPO: + path = 'ebs_pr%s' % pr if path is None: path = tempfile.mkdtemp() @@ -396,9 +403,6 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): github_account = build_option('pr_target_account') - if github_repo is None: - github_repo = GITHUB_EASYCONFIGS_REPO - if github_repo == GITHUB_EASYCONFIGS_REPO: easyfiles = 'easyconfigs' elif github_repo == GITHUB_EASYBLOCKS_REPO: From 255fcec28991bf73ba2ec5553a15bf36f93f2e47 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 12 Mar 2020 11:48:36 +0800 Subject: [PATCH 238/344] add separate test for include_easyblocks_from_pr --- test/framework/options.py | 150 ++++++++++++++++++++++++++++---------- 1 file changed, 113 insertions(+), 37 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 41ba6ce92c..1d981eb577 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2354,7 +2354,7 @@ def generate_cmd_line(ebopts): # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... def test_xxx_include_easyblocks(self): - """Test --include-easyblocks*.""" + """Test --include-easyblocks.""" orig_local_sys_path = sys.path[:] fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -2396,56 +2396,25 @@ def test_xxx_include_easyblocks(self): # clear log write_file(self.logfile, '') - # include both extra EB_foo easyblock and an easyblock from a PR args = [ '--include-easyblocks=%s/*.py' % self.test_prefix, - '--include-easyblocks-from-pr=1915', '--list-easyblocks=detailed', '--unittest-file=%s' % self.logfile, ] self.eb_main(args, logfile=dummylogfn, raise_error=True) logtxt = read_file(self.logfile) - path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks') - - foo_pattern = os.path.join(path_pattern, 'foo.py') - foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % foo_pattern, re.M) + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks', + 'foo.py') + foo_regex = re.compile(r"^\|-- EB_foo \(easybuild.easyblocks.foo @ %s\)" % path_pattern, re.M) self.assertTrue(foo_regex.search(logtxt), "Pattern '%s' found in: %s" % (foo_regex.pattern, logtxt)) - cmm_pattern = os.path.join(path_pattern, 'generic', 'cmakemake.py') - cmm_regex = re.compile(r"\|-- CMakeMake \(easybuild.easyblocks.generic.cmakemake @ %s\)" % cmm_pattern, re.M) - self.assertTrue(cmm_regex.search(logtxt), "Pattern '%s' found in: %s" % (cmm_regex.pattern, logtxt)) - - # easyblocks are found via get_easyblock_class + # easyblock is found via get_easyblock_class klass = get_easyblock_class('EB_foo') self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) - klass = get_easyblock_class('CMakeMake') - self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) - - # 'undo' import of foo and cmakemake easyblock + # 'undo' import of foo easyblock del sys.modules['easybuild.easyblocks.foo'] - del sys.modules['easybuild.easyblocks.generic.cmakemake'] - - # include extra test cmakemake easyblock - cmm_txt = '\n'.join([ - 'from easybuild.framework.easyblock import EasyBlock', - 'class CMakeMake(EasyBlock):', - ' pass', - '' - ]) - write_file(os.path.join(self.test_prefix, 'cmakemake.py'), cmm_txt) - - # including the same easyblock twice should fail - args = [ - '--include-easyblocks=%s/cmakemake.py' % self.test_prefix, - '--include-easyblocks-from-pr=1915', - '--list-easyblocks=detailed', - '--unittest-file=%s' % self.logfile, - ] - self.assertErrorRegex(EasyBuildError, - "Multiple inclusion of cmakemake.py, check your --include-easyblocks options", - self.eb_main, args, raise_error=True) # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... @@ -2528,6 +2497,113 @@ def test_xxx_include_generic_easyblocks(self): # 'undo' import of foo easyblock del sys.modules['easybuild.easyblocks.generic.generictest'] + # must be run after test for --list-easyblocks, hence the '_xxx_' + # cleaning up the imported easyblocks is quite difficult... + def test_xxx_include_easyblocks_from_pr(self): + """Test --include-easyblocks-from-pr.""" + if self.github_token is None: + print("Skipping test_preview_pr, no GitHub token available?") + return + + orig_local_sys_path = sys.path[:] + fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') + os.close(fd) + + # clear log + write_file(self.logfile, '') + + # include extra test easyblock + foo_txt = '\n'.join([ + 'from easybuild.framework.easyblock import EasyBlock', + 'class EB_foo(EasyBlock):', + ' pass', + '' + ]) + write_file(os.path.join(self.test_prefix, 'foo.py'), foo_txt) + + args = [ + '--include-easyblocks=%s/*.py' % self.test_prefix, # this shouldn't interfere + '--include-easyblocks-from-pr=1915', # a PR for CMakeMake easyblock + '--list-easyblocks=detailed', + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + + # easyblock included from pr is found + path_pattern = os.path.join(self.test_prefix, '.*', 'included-easyblocks-.*', 'easybuild', 'easyblocks') + cmm_pattern = os.path.join(path_pattern, 'generic', 'cmakemake.py') + cmm_regex = re.compile(r"\|-- CMakeMake \(easybuild.easyblocks.generic.cmakemake @ %s\)" % cmm_pattern, re.M) + self.assertTrue(cmm_regex.search(logtxt), "Pattern '%s' found in: %s" % (cmm_regex.pattern, logtxt)) + + # easyblock is found via get_easyblock_class + klass = get_easyblock_class('CMakeMake') + self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) + + # 'undo' import of easyblocks + del sys.modules['easybuild.easyblocks.foo'] + del sys.modules['easybuild.easyblocks.generic.cmakemake'] + os.remove(os.path.join(self.test_prefix, 'foo.py')) + sys.path = orig_local_sys_path + import easybuild.easyblocks + reload(easybuild.easyblocks) + import easybuild.easyblocks.generic + reload(easybuild.easyblocks.generic) + + # include test cmakemake easyblock + cmm_txt = '\n'.join([ + 'from easybuild.framework.easyblock import EasyBlock', + 'class CMakeMake(EasyBlock):', + ' pass', + '' + ]) + write_file(os.path.join(self.test_prefix, 'cmakemake.py'), cmm_txt) + + # including the same easyblock twice should fail + args = [ + '--include-easyblocks=%s/cmakemake.py' % self.test_prefix, + '--include-easyblocks-from-pr=1915', + '--list-easyblocks=detailed', + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + ] + self.assertErrorRegex(EasyBuildError, + "Multiple inclusion of cmakemake.py, check your --include-easyblocks options", + self.eb_main, args, raise_error=True) + + os.remove(os.path.join(self.test_prefix, 'cmakemake.py')) + + # clear log + write_file(self.logfile, '') + + args = [ + '--from-pr=9979', # PR for CMake easyconfig + '--include-easyblocks-from-pr=1936', # PR for EB_CMake easyblock + '--unittest-file=%s' % self.logfile, + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + '--extended-dry-run', + ] + self.eb_main(args, logfile=dummylogfn, raise_error=True) + logtxt = read_file(self.logfile) + + # easyconfig from pr is found + ec_pattern = os.path.join(self.test_prefix, '.*', 'files_pr9979', 'c', 'CMake', + 'CMake-3.16.4-GCCcore-9.2.0.eb') + ec_regex = re.compile(r"Parsing easyconfig file %s" % ec_pattern, re.M) + self.assertTrue(ec_regex.search(logtxt), "Pattern '%s' found in: %s" % (ec_regex.pattern, logtxt)) + + # easyblock included from pr is found + eb_regex = re.compile(r"Successfully obtained EB_CMake class instance from easybuild.easyblocks.cmake", re.M) + self.assertTrue(eb_regex.search(logtxt), "Pattern '%s' found in: %s" % (eb_regex.pattern, logtxt)) + + # easyblock is found via get_easyblock_class + klass = get_easyblock_class('EB_CMake') + self.assertTrue(issubclass(klass, EasyBlock), "%s is an EasyBlock derivative class" % klass) + + # 'undo' import of easyblocks + del sys.modules['easybuild.easyblocks.cmake'] + def mk_eb_test_cmd(self, args): """Construct test command for 'eb' with given options.""" From d0105a63a691fd3395bcfadd45408e8e06f6d7db Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 13 Mar 2020 10:51:56 +0100 Subject: [PATCH 239/344] make --check-contrib print a warning when None is used for checksums --- easybuild/framework/easyblock.py | 11 ++++++++--- test/framework/options.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6b33bc43c2..19c52e7cb9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1869,9 +1869,14 @@ def check_checksums_for(self, ent, sub='', source_cnt=None): else: valid_checksums = (checksum,) - if not all(is_sha256_checksum(c) for c in valid_checksums): - msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums) - checksum_issues.append(msg) + non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)] + if non_sha256_checksums: + if all(c is None for c in non_sha256_checksums): + print_warning("Found %d None checksum value(s), please make sure this is intended!" % + len(non_sha256_checksums)) + else: + msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums) + checksum_issues.append(msg) return checksum_issues diff --git a/test/framework/options.py b/test/framework/options.py index 1bb5a18bd4..90d029a45e 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4116,6 +4116,29 @@ def test_check_contrib_non_style(self): for pattern in patterns: self.assertTrue(re.search(pattern, stdout, re.M), "Pattern '%s' found in: %s" % (pattern, stdout)) + # --check-contrib passes if None values are used as checksum, but produces warning + toy = os.path.join(self.test_prefix, 'toy.eb') + copy_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb'), toy) + toytxt = read_file(toy) + toytxt = toytxt + '\n'.join([ + 'checksums = [', + " None, # toy-0.0.tar.gz", + " # toy-0.0_fix-silly-typo-in-printf-statement.patch", + " '45b5e3f9f495366830e1869bb2b8f4e7c28022739ce48d9f9ebb159b439823c5',", + " '4196b56771140d8e2468fb77f0240bc48ddbf5dabafe0713d612df7fafb1e458', # toy-extra.txt", + ']\n', + ]) + write_file(toy, toytxt) + + args = ['--check-contrib', toy] + self.mock_stdout(True) + self.mock_stderr(True) + self.eb_main(args, raise_error=True) + stderr = self.get_stderr().strip() + self.mock_stdout(False) + self.mock_stderr(False) + self.assertEqual(stderr, "WARNING: Found 1 None checksum value(s), please make sure this is intended!") + def test_allow_use_as_root(self): """Test --allow-use-as-root-and-accept-consequences""" From b819a9d3d89682969b16392dcb87fe52636b4d36 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 13 Mar 2020 11:57:09 +0100 Subject: [PATCH 240/344] Created installscript for sprint, and updated existing installscript that installs develop for all repos by removing the vsc-based repos --- .../scripts/install-EasyBuild-develop.sh | 25 +-- easybuild/scripts/install-EasyBuild-sprint.sh | 143 ++++++++++++++++++ 2 files changed, 148 insertions(+), 20 deletions(-) create mode 100755 easybuild/scripts/install-EasyBuild-sprint.sh diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index b5ea2eb3d1..4181d8c42a 100755 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -28,20 +28,11 @@ github_clone_branch() echo "=== Cloning ${GITHUB_USERNAME}/${REPO} ..." git clone --branch "${BRANCH}" "git@github.com:${GITHUB_USERNAME}/${REPO}.git" - if [[ "$REPO" == "vsc"* ]] - then - echo "=== Adding and fetching HPC-UGent GitHub repository @ hpcugent/${REPO} ..." - cd "${REPO}" - git remote add "github_hpcugent" "git@github.com:hpcugent/${REPO}.git" - git fetch github_hpcugent - git branch --set-upstream-to "github_hpcugent/${BRANCH}" "${BRANCH}" - else - echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..." - cd "${REPO}" - git remote add "github_easybuilders" "git@github.com:easybuilders/${REPO}.git" - git fetch github_easybuilders - git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}" - fi + echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..." + cd "${REPO}" + git remote add "github_easybuilders" "git@github.com:easybuilders/${REPO}.git" + git fetch github_easybuilders + git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}" } # Print the content of the module @@ -72,8 +63,6 @@ conflict EasyBuild prepend-path PATH "\$root/easybuild-framework" -prepend-path PYTHONPATH "\$root/vsc-base/lib" -prepend-path PYTHONPATH "\$root/vsc-install/lib" prepend-path PYTHONPATH "\$root/easybuild-framework" prepend-path PYTHONPATH "\$root/easybuild-easyblocks" prepend-path PYTHONPATH "\$root/easybuild-easyconfigs" @@ -112,10 +101,6 @@ mkdir -p "${INSTALL_DIR}" cd "${INSTALL_DIR}" INSTALL_DIR="${PWD}" # get the full path -# Clone repository for vsc-base dependency with 'master' branch -github_clone_branch "vsc-base" "master" -github_clone_branch "vsc-install" "master" - # Clone code repositories with the 'develop' branch github_clone_branch "easybuild-framework" "develop" github_clone_branch "easybuild-easyblocks" "develop" diff --git a/easybuild/scripts/install-EasyBuild-sprint.sh b/easybuild/scripts/install-EasyBuild-sprint.sh new file mode 100755 index 0000000000..57a0a802db --- /dev/null +++ b/easybuild/scripts/install-EasyBuild-sprint.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +# Stop in case of error +set -e + +########################### +# Helpers functions +########################### + +# Print script help +print_usage() +{ + echo "Usage: $0 " + echo + echo " github_username: username on GitHub for which the EasyBuild repositories should be cloned" + echo + echo " install_dir: directory were all the EasyBuild files will be installed" + echo + echo " easyconfigs_branch: easybuild-easyconfigs branch to check out" + echo +} + +# Clone one branch +github_clone_branch() +{ + REPO="$1" + BRANCH="$2" + + cd "${INSTALL_DIR}" + + # Check if BRANCH already exists in the ${GITHUB_USRENAME}/${REPO} + if [[ ! -z $(git ls-remote --heads "git@github.com:${GITHUB_USERNAME}/${REPO}.git" "${BRANCH}") ]]; then + echo "=== Cloning ${GITHUB_USERNAME}/${REPO} branch ${BRANCH} ..." + git clone --branch "${BRANCH}" "git@github.com:${GITHUB_USERNAME}/${REPO}.git" + + echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..." + cd "${REPO}" + git remote add "github_easybuilders" "git@github.com:easybuilders/${REPO}.git" + git fetch github_easybuilders + git branch --set-upstream-to "github_easybuilders/${BRANCH}" "${BRANCH}" + else + echo "=== Cloning ${GITHUB_USERNAME}/${REPO} ..." + git clone "git@github.com:${GITHUB_USERNAME}/${REPO}.git" + + echo "=== Adding and fetching EasyBuilders GitHub repository @ easybuilders/${REPO} ..." + cd "${REPO}" + git remote add "github_easybuilders" "git@github.com:easybuilders/${REPO}.git" + git fetch github_easybuilders + git checkout -b "${BRANCH}" "github_easybuilders/${BRANCH}" + fi +} + +# Print the content of the module +print_devel_module() +{ +cat < "${EB_DEVEL_MODULE}" +echo +echo "=== Run 'module use ${MODULES_INSTALL_DIR}' and 'module load ${EB_DEVEL_MODULE_NAME}' to use your development version of EasyBuild." +echo "=== (you can append ${MODULES_INSTALL_DIR} to your MODULEPATH to make this module always available for loading)" +echo +echo "=== To update each repository, run 'git pull origin' in each subdirectory of ${INSTALL_DIR}" +echo + +exit 0 + + From 69cf227fb6f34d6ab8b338e2472cc8de1965fe64 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Fri, 13 Mar 2020 19:13:44 +0000 Subject: [PATCH 241/344] fixed test for toy_build, given locks --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index d4a285e7fe..79a9654ff6 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2057,7 +2057,7 @@ def test_toy_modaltsoftname(self): self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) # only subdirectories for software should be created - self.assertEqual(os.listdir(software_path), ['toy']) + self.assertEqual(os.listdir(software_path), ['toy', '.locks']) self.assertEqual(sorted(os.listdir(os.path.join(software_path, 'toy'))), ['0.0-one', '0.0-two']) # only subdirectories for modules with alternative names should be created From 21900fc6196ea5479aa8ac78cbe9261123828b2f Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Fri, 13 Mar 2020 19:24:05 +0000 Subject: [PATCH 242/344] fixing test for toy_build, to take locks into account --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 79a9654ff6..c58eb6eb78 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1415,7 +1415,7 @@ def test_module_only(self): self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))) modtxt = read_file(toy_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # install (only) additional module under a hierarchical MNS From 5efac48971fd5b7242157040b39c7f6c5921e97e Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Fri, 13 Mar 2020 20:11:18 +0000 Subject: [PATCH 243/344] trying to fix test agian --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index c58eb6eb78..54f980e8fe 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1430,7 +1430,7 @@ def test_module_only(self): # existing install is reused modtxt2 = read_file(toy_core_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt2)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # make sure load statements for dependencies are included From 4edcfd98423e239a846322bb58a2cc5d90b69fd9 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Fri, 13 Mar 2020 20:35:39 +0000 Subject: [PATCH 244/344] trying to fix test agian --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 54f980e8fe..b36368e6ce 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1455,7 +1455,7 @@ def test_module_only(self): # existing install is reused modtxt3 = read_file(toy_mod + '.lua') self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # make sure load statements for dependencies are included From a9038cad6aeb7dc986e7fe21627019be2b44fcbe Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Fri, 13 Mar 2020 21:12:51 +0000 Subject: [PATCH 245/344] trying to fix test agian --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index b36368e6ce..f86f416468 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2057,7 +2057,7 @@ def test_toy_modaltsoftname(self): self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) # only subdirectories for software should be created - self.assertEqual(os.listdir(software_path), ['toy', '.locks']) + self.assertEqual(os.listdir(software_path), ['.locks', 'toy']) self.assertEqual(sorted(os.listdir(os.path.join(software_path, 'toy'))), ['0.0-one', '0.0-two']) # only subdirectories for modules with alternative names should be created From f355c1a65ecf5d4e3904d1d0c2650ea37a134f06 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Fri, 13 Mar 2020 21:15:31 +0000 Subject: [PATCH 246/344] trying to fix test agian --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index f86f416468..e5a86d735d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2057,7 +2057,7 @@ def test_toy_modaltsoftname(self): self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) # only subdirectories for software should be created - self.assertEqual(os.listdir(software_path), ['.locks', 'toy']) + self.assertEqual(sorted(os.listdir(software_path)), sorted(['.locks', 'toy'])) self.assertEqual(sorted(os.listdir(os.path.join(software_path, 'toy'))), ['0.0-one', '0.0-two']) # only subdirectories for modules with alternative names should be created From f40953f239c491b0a4113cab6eb9e606fdebcec2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 4 Mar 2020 15:46:07 +0100 Subject: [PATCH 247/344] Improve raised error in rmtree2 --- easybuild/tools/filetools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3cb7979631..0bdb97bee6 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1384,6 +1384,7 @@ def rmtree2(path, n=3): """Wrapper around shutil.rmtree to make it more robust when used on NFS mounted file systems.""" ok = False + errors = [] for i in range(0, n): try: shutil.rmtree(path) @@ -1391,12 +1392,14 @@ def rmtree2(path, n=3): break except OSError as err: _log.debug("Failed to remove path %s with shutil.rmtree at attempt %d: %s" % (path, n, err)) + errors.append(err) time.sleep(2) # make sure write permissions are enabled on entire directory adjust_permissions(path, stat.S_IWUSR, add=True, recursive=True) if not ok: - raise EasyBuildError("Failed to remove path %s with shutil.rmtree, even after %d attempts.", path, n) + raise EasyBuildError("Failed to remove path %s with shutil.rmtree, even after %d attempts.\nReason(s): %s", + path, n, errors) else: _log.info("Path %s successfully removed." % path) From d0cc8d7960525bcf456dd25d5b0d57bc99a1ad1a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 16 Mar 2020 13:21:42 +0100 Subject: [PATCH 248/344] Deprecate rmtree2 in favor of remove_dir Move the code from rmtree2 into remove_dir Add deprecation notice Change calls to rmtree2 to remove_dir --- easybuild/framework/easyblock.py | 6 ++-- easybuild/tools/containers/docker.py | 4 +-- easybuild/tools/filetools.py | 47 +++++++++++++-------------- easybuild/tools/repository/gitrepo.py | 4 +-- easybuild/tools/repository/hgrepo.py | 4 +-- easybuild/tools/repository/svnrepo.py | 4 +-- 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6b33bc43c2..bd72294b30 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -76,7 +76,7 @@ from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir -from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file, dir_contains_files +from easybuild.tools.filetools import remove_file, verify_checksum, weld_paths, write_file, dir_contains_files from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP @@ -1437,7 +1437,7 @@ def clean_up_fake_module(self, fake_mod_data): try: self.modules_tool.unload([self.short_mod_name]) self.modules_tool.remove_module_path(os.path.join(fake_mod_path, self.mod_subdir)) - rmtree2(os.path.dirname(fake_mod_path)) + remove_dir(os.path.dirname(fake_mod_path)) except OSError as err: raise EasyBuildError("Failed to clean up fake module dir %s: %s", fake_mod_path, err) elif self.short_mod_name is None: @@ -2666,7 +2666,7 @@ def cleanup_step(self): self.log.info("Cleaning up builddir %s (in %s)", self.builddir, os.getcwd()) try: - rmtree2(self.builddir) + remove_dir(self.builddir) base = os.path.dirname(self.builddir) # keep removing empty directories until we either find a non-empty one diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index bb5c2eb09e..0da773a269 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -34,7 +34,7 @@ from easybuild.tools.config import DOCKER_BASE_IMAGE_CENTOS, DOCKER_BASE_IMAGE_UBUNTU from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.containers.utils import det_os_deps -from easybuild.tools.filetools import rmtree2 +from easybuild.tools.filetools import remove_dir from easybuild.tools.run import run_cmd @@ -157,4 +157,4 @@ def build_image(self, dockerfile): run_cmd(docker_cmd, path=tempdir, stream_output=True) print_msg("Docker image created at %s" % container_name, log=self.log) - rmtree2(tempdir) + remove_dir(tempdir) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 0bdb97bee6..6ce5ebb5a1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -303,11 +303,27 @@ def remove_dir(path): dry_run_msg("directory %s removed" % path, silent=build_option('silent')) return - try: - if os.path.exists(path): - rmtree2(path) - except OSError as err: - raise EasyBuildError("Failed to remove directory %s: %s", path, err) + if os.path.exists(path): + ok = False + errors = [] + # Try multiple times to cater for temporary failures on e.g. NFS mounted paths + max_attempts = 3 + for i in range(0, max_attempts): + try: + shutil.rmtree(path) + ok = True + break + except OSError as err: + _log.debug("Failed to remove path %s with shutil.rmtree at attempt %d: %s" % (path, i, err)) + errors.append(err) + time.sleep(2) + # make sure write permissions are enabled on entire directory + adjust_permissions(path, stat.S_IWUSR, add=True, recursive=True) + if ok: + _log.info("Path %s successfully removed." % path) + else: + raise EasyBuildError("Failed to remove directory %s even after %d attempts.\nReasons: %s", + path, max_attempts, errors) def remove(paths): @@ -1383,25 +1399,8 @@ def path_matches(path, paths): def rmtree2(path, n=3): """Wrapper around shutil.rmtree to make it more robust when used on NFS mounted file systems.""" - ok = False - errors = [] - for i in range(0, n): - try: - shutil.rmtree(path) - ok = True - break - except OSError as err: - _log.debug("Failed to remove path %s with shutil.rmtree at attempt %d: %s" % (path, n, err)) - errors.append(err) - time.sleep(2) - - # make sure write permissions are enabled on entire directory - adjust_permissions(path, stat.S_IWUSR, add=True, recursive=True) - if not ok: - raise EasyBuildError("Failed to remove path %s with shutil.rmtree, even after %d attempts.\nReason(s): %s", - path, n, errors) - else: - _log.info("Path %s successfully removed." % path) + _log.deprecated("Use 'remove_dir' rather than 'rmtree2'", '5.0') + remove_dir(path) def find_backup_name_candidate(src_file): diff --git a/easybuild/tools/repository/gitrepo.py b/easybuild/tools/repository/gitrepo.py index f34a95088e..d9f84d6700 100644 --- a/easybuild/tools/repository/gitrepo.py +++ b/easybuild/tools/repository/gitrepo.py @@ -44,7 +44,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import rmtree2 +from easybuild.tools.filetools import remove_dir from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.utilities import only_if_module_is_available from easybuild.tools.version import VERSION @@ -188,6 +188,6 @@ def cleanup(self): """ try: self.wc = os.path.dirname(self.wc) - rmtree2(self.wc) + remove_dir(self.wc) except IOError as err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/repository/hgrepo.py b/easybuild/tools/repository/hgrepo.py index 000dd9b5b8..cb121f5cb2 100644 --- a/easybuild/tools/repository/hgrepo.py +++ b/easybuild/tools/repository/hgrepo.py @@ -44,7 +44,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import rmtree2 +from easybuild.tools.filetools import remove_dir from easybuild.tools.repository.filerepo import FileRepository _log = fancylogger.getLogger('hgrepo', fname=False) @@ -192,6 +192,6 @@ def cleanup(self): Clean up mercurial working copy. """ try: - rmtree2(self.wc) + remove_dir(self.wc) except IOError as err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) diff --git a/easybuild/tools/repository/svnrepo.py b/easybuild/tools/repository/svnrepo.py index 6dc0f3c7b0..24dfcb8811 100644 --- a/easybuild/tools/repository/svnrepo.py +++ b/easybuild/tools/repository/svnrepo.py @@ -44,7 +44,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import rmtree2 +from easybuild.tools.filetools import remove_dir from easybuild.tools.repository.filerepo import FileRepository from easybuild.tools.utilities import only_if_module_is_available @@ -190,6 +190,6 @@ def cleanup(self): Clean up SVN working copy. """ try: - rmtree2(self.wc) + remove_dir(self.wc) except OSError as err: raise EasyBuildError("Can't remove working copy %s: %s", self.wc, err) From ba1bdbebb119440a6887aacd6ac549836c2b4082 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 16 Mar 2020 13:34:42 +0100 Subject: [PATCH 249/344] bump version to v4.2.0dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index b454c04a80..ae1710966d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.1.2') +VERSION = LooseVersion('4.2.0.dev0') UNKNOWN = 'UNKNOWN' From 6cd7bd511f5e030afc30502183f63f1fdace6c19 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 19 Mar 2020 21:04:17 +0100 Subject: [PATCH 250/344] intelmpi.py: Make the super call to _set_mpi_variables first. --- easybuild/toolchains/mpi/intelmpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/toolchains/mpi/intelmpi.py b/easybuild/toolchains/mpi/intelmpi.py index 9e5494aa23..b4811754dc 100644 --- a/easybuild/toolchains/mpi/intelmpi.py +++ b/easybuild/toolchains/mpi/intelmpi.py @@ -74,6 +74,8 @@ def _set_mpi_compiler_variables(self): def _set_mpi_variables(self): """Set the other MPI variables""" + super(IntelMPI, self)._set_mpi_variables() + if (LooseVersion(self.version) >= LooseVersion('2019')): lib_dir = [os.path.join('intel64', 'lib', 'release')] incl_dir = [os.path.join('intel64', 'include')] @@ -86,8 +88,6 @@ def _set_mpi_variables(self): self.variables.append_exists('MPI_LIB_DIR', root, lib_dir) self.variables.append_exists('MPI_INC_DIR', root, incl_dir) - super(IntelMPI, self)._set_mpi_variables() - MPI_LINK_INFO_OPTION = '-show' def set_variables(self): From 4c5a9ba04e41b76ec88df87e9377791a6fec1c0b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Mar 2020 14:09:28 +0100 Subject: [PATCH 251/344] unset $GITHUB_TOKEN in Travis after installing token, to avoid failing test_from_pr_token_log --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 579863a89f..3b032f5291 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,14 +57,18 @@ script: - if [ ! -z $MOD_INIT ] && [ ! -z $LMOD_VERSION ]; then alias ml=foobar; fi # set up environment for modules tool (if $MOD_INIT is defined) - if [ ! -z $MOD_INIT ]; then source $MOD_INIT; type module; fi - # install GitHub token + # install GitHub token; + # unset $GITHUB_TOKEN environment variable after installing token, + # to avoid that it is included in environment dump that is included in EasyBuild debug logs, + # which causes test_from_pr_token_log to fail... - if [ ! -z $GITHUB_TOKEN ]; then if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; else SET_KEYRING="import keyrings; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; fi; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; - fi + fi; + unset GITHUB_TOKEN; - if [ ! -z $TEST_EASYBUILD_MODULES_TOOL ]; then export EASYBUILD_MODULES_TOOL=$TEST_EASYBUILD_MODULES_TOOL; fi - if [ ! -z $TEST_EASYBUILD_MODULE_SYNTAX ]; then export EASYBUILD_MODULE_SYNTAX=$TEST_EASYBUILD_MODULE_SYNTAX; fi # create 'source distribution' tarball, like we do when publishing a release to PyPI From cd49c19e1dea42f79df203ff40304d6bc01fa095 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 24 Mar 2020 11:21:55 +0100 Subject: [PATCH 252/344] Fix reporting when skipping extensions The lists ext and ext_instances become out of sync when extensions are skipped. Hence use ext_instances as the true source for reporting Also fix the documentation of skip_extensions and add verbose reporting to stdout --- easybuild/framework/easyblock.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 6b33bc43c2..dc6a007dc0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1463,7 +1463,7 @@ def prepare_for_extensions(self): def skip_extensions(self): """ Called when self.skip is True - - use this to detect existing extensions and to remove them from self.exts + - use this to detect existing extensions and to remove them from self.ext_instances - based on initial R version """ # obtaining untemplated reference value is required here to support legacy string templates like name/version @@ -1482,7 +1482,7 @@ def skip_extensions(self): self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) else: - self.log.info("Skipping %s", ext_inst.name) + print_msg("Skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) self.ext_instances = res @@ -2148,19 +2148,19 @@ def extensions_step(self, fetch=False): if self.skip: self.skip_extensions() - exts_cnt = len(self.exts) - for idx, (ext, ext_instance) in enumerate(zip(self.exts, self.ext_instances)): + exts_cnt = len(self.ext_instances) + for idx, ext in enumerate(self.ext_instances): - self.log.debug("Starting extension %s" % ext['name']) + self.log.debug("Starting extension %s" % ext.name) # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) - tup = (ext['name'], ext.get('version', ''), idx+1, exts_cnt) + tup = (ext.name, ext.version, idx+1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) if self.dry_run: - tup = (ext['name'], ext.get('version', ''), cls.__name__) + tup = (ext.name, ext.version, cls.__name__) msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup self.dry_run_msg(msg) @@ -2173,15 +2173,15 @@ def extensions_step(self, fetch=False): else: # don't reload modules for toolchain, there is no need since they will be loaded already; # the (fake) module for the parent software gets loaded before installing extensions - ext_instance.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs) + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs) # real work - ext_instance.prerun() - txt = ext_instance.run() + ext.prerun() + txt = ext.run() if txt: self.module_extra_extensions += txt - ext_instance.postrun() + ext.postrun() # cleanup (unload fake module, remove fake module dir) if fake_mod_data: From cb7e2a66f7ee430176f5adf0c0ffcf01f4f8b4ea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Mar 2020 09:52:43 +0100 Subject: [PATCH 253/344] avoid printing 'None' when there's no extension version --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index dc6a007dc0..625ad199bd 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1482,7 +1482,7 @@ def skip_extensions(self): self.log.debug("exit code: %s, stdout/err: %s", ec, cmdstdouterr) res.append(ext_inst) else: - print_msg("Skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) self.ext_instances = res @@ -2156,7 +2156,7 @@ def extensions_step(self, fetch=False): # always go back to original work dir to avoid running stuff from a dir that no longer exists change_dir(self.orig_workdir) - tup = (ext.name, ext.version, idx+1, exts_cnt) + tup = (ext.name, ext.version or '', idx+1, exts_cnt) print_msg("installing extension %s %s (%d/%d)..." % tup, silent=self.silent) if self.dry_run: From e25673751d389a67546e3eb169e87b950fc3d80b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Mar 2020 10:05:39 +0100 Subject: [PATCH 254/344] enhance test_skip_extensions_step to catch incorrect counting of extensions under --skip --- test/framework/easyblock.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 0db4425664..72f25d1931 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -822,7 +822,6 @@ def test_extensions_step(self): def test_skip_extensions_step(self): """Test the skip_extensions_step""" - init_config(build_options={'silent': True}) self.contents = cleandoc(""" easyblock = "ConfigureMake" @@ -849,7 +848,21 @@ def test_skip_extensions_step(self): eb.builddir = config.build_path() eb.installdir = config.install_path() eb.skip = True + + self.mock_stdout(True) eb.extensions_step(fetch=True) + stdout = self.get_stdout() + self.mock_stdout(False) + + patterns = [ + r"^== skipping extension EXT-2", + r"^== skipping extension ext3", + r"^== installing extension ext1 \(1/1\)\.\.\.", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + # 'ext1' should be in eb.ext_instances eb_exts = [x.name for x in eb.ext_instances] self.assertTrue('ext1' in eb_exts) From a11ea09f5a8a37305b517c13a5c179803722d7e2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 25 Mar 2020 10:24:27 +0100 Subject: [PATCH 255/344] add another extension after the ones that are skipped in test_skip_extensions_step --- test/framework/easyblock.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 72f25d1931..057bcfb56e 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -834,6 +834,7 @@ def test_skip_extensions_step(self): "ext1", ("EXT-2", "42", {"source_tmpl": "dummy.tgz"}), ("ext3", "1.1", {"source_tmpl": "dummy.tgz", "modulename": "real_ext"}), + "ext4", ] exts_filter = ("\ if [ %(ext_name)s == 'ext_2' ] && [ %(ext_version)s == '42' ] && [[ %(src)s == *dummy.tgz ]];\ @@ -857,7 +858,8 @@ def test_skip_extensions_step(self): patterns = [ r"^== skipping extension EXT-2", r"^== skipping extension ext3", - r"^== installing extension ext1 \(1/1\)\.\.\.", + r"^== installing extension ext1 \(1/2\)\.\.\.", + r"^== installing extension ext4 \(2/2\)\.\.\.", ] for pattern in patterns: regex = re.compile(pattern, re.M) From 66dee541c1289dc7e0d8c37753e587386ea55c9c Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Mar 2020 13:48:01 +0100 Subject: [PATCH 256/344] [Github] Avoid API rate limit errors on online check --- easybuild/tools/github.py | 42 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index dd6eccc15c..2efa6e15e1 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -251,7 +251,7 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): _log.warning("Error occurred while performing get request: %s", err) status, data = 0, None - _log.debug("get request result for %s: status: %d, data: %s", url, status, data) + _log.debug("get request result for %s: status: %d, data: %s", url.url, status, data) return (status, data) @@ -284,7 +284,7 @@ def github_api_put_request(request_f, github_user=None, token=None, **kwargs): else: raise EasyBuildError("FAILED: %s", data.get('message', "(unknown reason)")) - _log.debug("get request result for %s: status: %d, data: %s", url, status, data) + _log.debug("get request result for %s: status: %d, data: %s", url.url, status, data) return (status, data) @@ -1597,22 +1597,28 @@ def check_online_status(): Check whether we currently are online Return True if online, else a list of error messages """ - # Try repeatedly and with different URLs to cater for flaky servers - # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly - # Timeout and repeats set to total 1 minute - urls = [GITHUB_URL, GITHUB_API_URL] - num_repeats = 6 - errors = set() # Use set to record only unique errors - for attempt in range(num_repeats): - # Cycle through URLs - url = urls[attempt % len(urls)] - try: - urlopen(url, timeout=10) - errors = None - break - except URLError as err: - errors.add('%s: %s' % (url, err)) - return sorted(errors) if errors else True + result = True + # Try API request first to avoid running into rate limits + status, data = github_api_get_request(lambda x: x.rate_limit) + if status != HTTP_STATUS_OK or not data: + # Try repeatedly and with different URLs to cater for flaky servers + # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly + # Timeout and repeats set to total 1 minute + urls = [GITHUB_URL, GITHUB_API_URL, 'https://google.com'] + num_repeats = 6 + errors = set() # Use set to record only unique errors + for attempt in range(num_repeats): + # Cycle through URLs + url = urls[attempt % len(urls)] + try: + urlopen(url, timeout=10) + errors = None + break + except URLError as err: + errors.add('%s: %s' % (url, err)) + if errors: + result = sorted(errors) + return result def check_github(): From 9a5e74bbfa7955eb857e40b5f819fb4d82e7b615 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Mar 2020 13:56:42 +0100 Subject: [PATCH 257/344] [Github] Do a plain request to rate_limit instead --- easybuild/tools/github.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 2efa6e15e1..192f4df13e 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1597,28 +1597,22 @@ def check_online_status(): Check whether we currently are online Return True if online, else a list of error messages """ - result = True - # Try API request first to avoid running into rate limits - status, data = github_api_get_request(lambda x: x.rate_limit) - if status != HTTP_STATUS_OK or not data: - # Try repeatedly and with different URLs to cater for flaky servers - # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly - # Timeout and repeats set to total 1 minute - urls = [GITHUB_URL, GITHUB_API_URL, 'https://google.com'] - num_repeats = 6 - errors = set() # Use set to record only unique errors - for attempt in range(num_repeats): - # Cycle through URLs - url = urls[attempt % len(urls)] - try: - urlopen(url, timeout=10) - errors = None - break - except URLError as err: - errors.add('%s: %s' % (url, err)) - if errors: - result = sorted(errors) - return result + # Try repeatedly and with different URLs to cater for flaky servers + # E.g. Github returned "HTTP Error 403: Forbidden" and "HTTP Error 406: Not Acceptable" randomly + # Timeout and repeats set to total 1 minute + urls = [GITHUB_API_URL + '/rate_limit', GITHUB_URL, GITHUB_API_URL] + num_repeats = 6 + errors = set() # Use set to record only unique errors + for attempt in range(num_repeats): + # Cycle through URLs + url = urls[attempt % len(urls)] + try: + urlopen(url, timeout=10) + errors = None + break + except URLError as err: + errors.add('%s: %s' % (url, err)) + return sorted(errors) if errors else True def check_github(): From 644507c3021b97b2c4e751afb4f185337eca60c8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Mar 2020 13:23:46 +0100 Subject: [PATCH 258/344] Show EC names for parallel build --- easybuild/tools/parallelbuild.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index da824d66fd..8a7348fb18 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -68,7 +68,7 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu :param output_dir: output directory :param prepare_first: prepare by runnning fetch step first for each easyconfig """ - _log.info("going to build these easyconfigs in parallel: %s", easyconfigs) + _log.info("going to build these easyconfigs in parallel: %s", [os.path.basename(ec['spec']) for ec in easyconfigs]) active_job_backend = job_backend() if active_job_backend is None: @@ -94,7 +94,7 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu prepare_easyconfig(easyconfig) # the new job will only depend on already submitted jobs - _log.info("creating job for ec: %s" % easyconfig['ec']) + _log.info("creating job for ec: %s" % os.path.basename(easyconfig['spec'])) new_job = create_job(active_job_backend, build_command, easyconfig, output_dir=output_dir) # filter out dependencies marked as external modules From 77940585c8012a588c5b50d3e5f5f63667227b05 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Mar 2020 14:59:00 +0100 Subject: [PATCH 259/344] Check if toolchain in hierarchy before writing in dump --- easybuild/framework/easyconfig/easyconfig.py | 3 +- easybuild/framework/easyconfig/format/one.py | 30 +++++++++++++------- easybuild/framework/easyconfig/format/yeb.py | 2 +- easybuild/framework/easyconfig/parser.py | 5 ++-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index d6033eb7c3..8d59a76b9d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1126,7 +1126,8 @@ def dump(self, fp, always_overwrite=True, backup=False): templ_val[self.template_values[key]] = key try: - ectxt = self.parser.dump(self, default_values, templ_const, templ_val) + ectxt = self.parser.dump(self, default_values, templ_const, templ_val, + toolchain_hierarchy=get_toolchain_hierarchy(self.toolchain)) except NotImplementedError as err: # need to restore enable_templating value in case this method is caught in a try/except block and ignored # (the ability to dump is not a hard requirement for build success) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 8534497293..1a0ae092bc 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -65,15 +65,17 @@ _log = fancylogger.getLogger('easyconfig.format.one', fname=False) -def dump_dependency(dep, toolchain): +def dump_dependency(dep, toolchain, toolchain_hierarchy=[]): """Dump parsed dependency in tuple format""" + if not toolchain_hierarchy: + toolchain_hierarchy = [toolchain] if dep['external_module']: res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name']) else: - # mininal spec: (name, version) + # minimal spec: (name, version) tup = (dep['name'], dep['version']) - if dep['toolchain'] != toolchain: + if all(dep['toolchain'] != subtoolchain for subtoolchain in toolchain_hierarchy): if dep[SYSTEM_TOOLCHAIN_NAME]: tup += (dep['versionsuffix'], True) else: @@ -260,7 +262,7 @@ def _find_param_with_comments(self, key, val, templ_const, templ_val): return res - def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val): + def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val, toolchain_hierarchy=[]): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ @@ -279,12 +281,18 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ # the way that builddependencies are constructed with multi_deps # we just need to dump the first entry without the dependencies # that are listed in multi_deps - valstr = [dump_dependency(d, ecfg['toolchain']) for d in val[0] - if d['name'] not in ecfg['multi_deps']] + valstr = [ + dump_dependency(d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy) + for d in val[0] if d['name'] not in ecfg['multi_deps'] + ] else: - valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val] + valstr = [ + [dump_dependency(d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy) + for d in dep] for dep in val + ] else: - valstr = [dump_dependency(d, ecfg['toolchain']) for d in val] + valstr = [dump_dependency(d, ecfg['toolchain'], toolchain_hierarchy=toolchain_hierarchy) + for d in val] elif key == 'toolchain': valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key] else: @@ -299,7 +307,7 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ return eclines, printed_keys - def dump(self, ecfg, default_values, templ_const, templ_val): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=[]): """ Dump easyconfig in format v1. @@ -307,12 +315,14 @@ def dump(self, ecfg, default_values, templ_const, templ_val): :param default_values: default values for easyconfig parameters :param templ_const: known template constants :param templ_val: known template values + :param toolchain_hierarchy: hierarchy of toolchains for easyconfig """ # include header comments first dump = self.comments['header'][:] # print easyconfig parameters ordered and in groups specified above - params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val) + params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val, + toolchain_hierarchy=toolchain_hierarchy) dump.extend(params) # print other easyconfig parameters at the end diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 6215500f8e..5ccb8632de 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -126,7 +126,7 @@ def _inject_constants_dict(self, txt): return full_txt - def dump(self, ecfg, default_values, templ_const, templ_val): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=[]): """Dump parsed easyconfig in .yeb format""" raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet") diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index ee21fcd558..63fc7893de 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -226,6 +226,7 @@ def get_config_dict(self, validate=True): return cfg - def dump(self, ecfg, default_values, templ_const, templ_val): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=[]): """Dump easyconfig in format it was parsed from.""" - return self._formatter.dump(ecfg, default_values, templ_const, templ_val) + return self._formatter.dump(ecfg, default_values, templ_const, templ_val, + toolchain_hierarchy=toolchain_hierarchy) From 70afbc95cb6e100991be5d1467ded90fdfce8810 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Mar 2020 15:08:56 +0100 Subject: [PATCH 260/344] Reference toolchain not it's instance --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8d59a76b9d..802272ada6 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1127,7 +1127,7 @@ def dump(self, fp, always_overwrite=True, backup=False): try: ectxt = self.parser.dump(self, default_values, templ_const, templ_val, - toolchain_hierarchy=get_toolchain_hierarchy(self.toolchain)) + toolchain_hierarchy=get_toolchain_hierarchy(self['toolchain'])) except NotImplementedError as err: # need to restore enable_templating value in case this method is caught in a try/except block and ignored # (the ability to dump is not a hard requirement for build success) From bd9a9e168e597439e28f141b7e301830d32d1400 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Mar 2020 15:26:31 +0100 Subject: [PATCH 261/344] Default kwarg to None --- easybuild/framework/easyconfig/format/one.py | 6 +++--- easybuild/framework/easyconfig/parser.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index 1a0ae092bc..88d043bc61 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -65,7 +65,7 @@ _log = fancylogger.getLogger('easyconfig.format.one', fname=False) -def dump_dependency(dep, toolchain, toolchain_hierarchy=[]): +def dump_dependency(dep, toolchain, toolchain_hierarchy=None): """Dump parsed dependency in tuple format""" if not toolchain_hierarchy: toolchain_hierarchy = [toolchain] @@ -262,7 +262,7 @@ def _find_param_with_comments(self, key, val, templ_const, templ_val): return res - def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val, toolchain_hierarchy=[]): + def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val, toolchain_hierarchy=None): """ Determine parameters in the dumped easyconfig file which have a non-default value. """ @@ -307,7 +307,7 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_ return eclines, printed_keys - def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=[]): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """ Dump easyconfig in format v1. diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 63fc7893de..bb432724c3 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -226,7 +226,7 @@ def get_config_dict(self, validate=True): return cfg - def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=[]): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """Dump easyconfig in format it was parsed from.""" return self._formatter.dump(ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=toolchain_hierarchy) From 4530531a6636f0975617c30b773cf8b863768309 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 27 Mar 2020 15:34:32 +0100 Subject: [PATCH 262/344] Be more flexible when we fail to find a hierarchy --- easybuild/framework/easyconfig/easyconfig.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 802272ada6..8f0e1d5ebb 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1128,6 +1128,13 @@ def dump(self, fp, always_overwrite=True, backup=False): try: ectxt = self.parser.dump(self, default_values, templ_const, templ_val, toolchain_hierarchy=get_toolchain_hierarchy(self['toolchain'])) + except EasyBuildError as err: + if 'No version found for subtoolchain' in str(err): + self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump', + self['toolchain']) + ectxt = self.parser.dump(self, default_values, templ_const, templ_val) + else: + raise err except NotImplementedError as err: # need to restore enable_templating value in case this method is caught in a try/except block and ignored # (the ability to dump is not a hard requirement for build success) From 8a1115f16790afae1b7e1be59e25fb34daa133c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 16:34:34 +0200 Subject: [PATCH 263/344] fix adding of CPU arch name in PR comment for test report --- easybuild/tools/testing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index d114992ec7..d44080dd3c 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -256,6 +256,11 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, # post comment to report test result system_info = init_session_state['system_info'] + + # also mention CPU architecture name, but only if it's known + if system_info['cpu_arch_name'] != UNKNOWN: + system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] + short_system_info = "%(hostname)s - %(os_type)s %(os_name)s %(os_version)s, %(cpu_model)s, Python %(pyver)s" % { 'cpu_model': system_info['cpu_model'], 'hostname': system_info['hostname'], @@ -265,10 +270,6 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, 'pyver': system_info['python_version'].split(' ')[0], } - # also mention CPU architecture name, but only if it's known - if system_info['cpu_arch_name'] != UNKNOWN: - short_system_info['cpu_model'] += " (%s)" % system_info['cpu_arch_name'] - comment_lines = [ "Test report by @%s" % user, ('**FAILED**', '**SUCCESS**')[success], From 1d571f0b7c1e5639651e3715fb15879b2d995c94 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 16:38:22 +0200 Subject: [PATCH 264/344] mention CPU arch name in --show-system-info output, if archspec is available --- easybuild/tools/options.py | 9 ++++++++- test/framework/options.py | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6a23c2d82e..987f707be6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -1153,6 +1153,7 @@ def show_system_info(self): """Show system information.""" system_info = get_system_info() cpu_features = get_cpu_features() + cpu_arch_name = system_info['cpu_arch_name'] lines = [ "System information (%s):" % system_info['hostname'], '', @@ -1166,6 +1167,11 @@ def show_system_info(self): " -> vendor: %s" % system_info['cpu_vendor'], " -> architecture: %s" % get_cpu_architecture(), " -> family: %s" % get_cpu_family(), + ] + if cpu_arch_name: + lines.append(" -> arch name: %s" % cpu_arch_name) + + lines.extend([ " -> model: %s" % system_info['cpu_model'], " -> speed: %s" % system_info['cpu_speed'], " -> cores: %s" % system_info['core_count'], @@ -1175,7 +1181,8 @@ def show_system_info(self): " -> glibc version: %s" % system_info['glibc_version'], " -> Python binary: %s" % sys.executable, " -> Python version: %s" % sys.version.split(' ')[0], - ] + ]) + return '\n'.join(lines) def show_config(self): diff --git a/test/framework/options.py b/test/framework/options.py index d09bfdc17a..a6c544d1b2 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -59,6 +59,7 @@ from easybuild.tools.py2vs3 import URLError, reload, sort_looseversions from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import HAVE_ARCHSPEC from easybuild.tools.version import VERSION from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config @@ -4562,6 +4563,10 @@ def test_show_system_info(self): "^ -> Python binary: .*/[pP]ython[0-9]?", "^ -> Python version: [0-9.]+", ] + + if HAVE_ARCHSPEC: + patterns.append(r"^ -> arch name: \w+") + for pattern in patterns: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) From 9ebf77e47c2571adcb020ac89c775ccd405b3454 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 16:15:34 +0200 Subject: [PATCH 265/344] fix use of --copy-ec with a single argument, assume copy to current working directory (fixes #3224) --- easybuild/main.py | 14 ++++++++++--- test/framework/options.py | 43 ++++++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 69c47a7293..5f84a885bd 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -291,8 +291,12 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - # last path is target when --copy-ec is used, so remove that from the list - target_path = orig_paths.pop() if options.copy_ec else None + if len(orig_paths) == 1: + # if only one easyconfig file is specified, use current directory as target directory + target_path = os.getcwd() + elif orig_paths: + # last path is target when --copy-ec is used, so remove that from the list + target_path = orig_paths.pop() if options.copy_ec else None categorized_paths = categorize_files_by_type(orig_paths) @@ -310,8 +314,12 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.copy_ec: if len(determined_paths) == 1: copy_file(determined_paths[0], target_path) - else: + print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False) + elif len(determined_paths) > 1: copy_files(determined_paths, target_path) + print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) + else: + raise EasyBuildError("One of more files to copy should be specified!") elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) diff --git a/test/framework/options.py b/test/framework/options.py index e4eed050d2..7c3e1d57ee 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -836,6 +836,16 @@ def test_show_ec(self): def test_copy_ec(self): """Test --copy-ec.""" + def mocked_main(args): + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + return stdout.strip() + topdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs_dir = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -845,7 +855,8 @@ def test_copy_ec(self): # basic test: copying one easyconfig file to a non-existing absolute path test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', 'toy-0.0.eb', test_ec] - self.eb_main(args) + stdout = mocked_main(args) + self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_ec) self.assertTrue(os.path.exists(test_ec)) self.assertEqual(toy_ec_txt, read_file(test_ec)) @@ -858,7 +869,8 @@ def test_copy_ec(self): self.assertFalse(os.path.exists(target_fn)) args = ['--copy-ec', 'toy-0.0.eb', target_fn] - self.eb_main(args) + stdout = mocked_main(args) + self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') change_dir(cwd) @@ -869,7 +881,8 @@ def test_copy_ec(self): test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') mkdir(test_target_dir) args = ['--copy-ec', 'toy-0.0.eb', test_target_dir] - self.eb_main(args) + stdout = mocked_main(args) + self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_target_dir) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_ec)) @@ -890,7 +903,8 @@ def check_copied_files(): # copying multiple easyconfig files to a non-existing target directory (which is created automatically) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', test_target_dir] - self.eb_main(args) + stdout = mocked_main(args) + self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) check_copied_files() @@ -901,7 +915,8 @@ def check_copied_files(): args[-1] = os.path.basename(test_target_dir) self.assertFalse(os.path.exists(args[-1])) - self.eb_main(args) + stdout = mocked_main(args) + self.assertEqual(stdout, '2 file(s) copied to test_target_dir') check_copied_files() @@ -912,6 +927,24 @@ def check_copied_files(): error_pattern = ".*/test.eb exists but is not a directory" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + # test use of --copy-ec with only one argument: copy to current working directory + test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') + mkdir(test_working_dir) + change_dir(test_working_dir) + self.assertEqual(len(os.listdir(os.getcwd())), 0) + args = ['--copy-ec', 'toy-0.0.eb'] + stdout = mocked_main(args) + regex = re.compile('toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) + self.assertTrue(regex.match(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + copied_toy_cwd = os.path.join(test_working_dir, 'toy-0.0.eb') + self.assertTrue(os.path.exists(copied_toy_cwd)) + self.assertEqual(read_file(copied_toy_cwd), toy_ec_txt) + + # --copy-ec without arguments results in a proper error + args = ['--copy-ec'] + error_pattern = "One of more files to copy should be specified!" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From a6f92f5af2055a99f57754d40d00cf1ea53bfb6b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 17:06:37 +0200 Subject: [PATCH 266/344] bug fix: actually proceed with installation after waiting for lock to be released (+ some cleanup) --- easybuild/framework/easyblock.py | 60 +++++++++++++++----------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b8c6f237bd..223165764e 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3039,41 +3039,39 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - lockpath = build_option('lockpath') or os.path.join(install_path('software'), '.locks') - if not os.path.exists(lockpath): - mkdir(lockpath, parents=True) - lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/', '_')) - if os.path.exists(lockfile_name): + locks_dir = build_option('lockpath') or os.path.join(install_path('software'), '.locks') + lock_fp = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) + + # if lock already exists, either abort or wait until it disappears + if os.path.exists(lock_fp): if build_option('wait_on_lock'): - while os.path.exists(lockfile_name): - print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name, silent=self.silent) + while os.path.exists(lock_fp): + print_msg("lock file in %s exists, waiting 60 seconds..." % locks_dir, silent=self.silent) time.sleep(60) else: - print_msg("Build aborted. Lock file %s exists." % lockfile_name, silent=self.silent) - return False - else: - try: - # create a new lock file - print_msg("Creating lock file %s" % lockfile_name, silent=self.silent) - f = open(lockfile_name, "w+") - f.close() - - for (step_name, descr, step_methods, skippable) in steps: - if self._skip_step(step_name, skippable): - print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) + raise EasyBuildError("Lock file %s already exists, aborting!", lock_fp) + + # create lock file to avoid that another installation running in parallel messes things up + print_msg("creating lock file %s" % lock_fp, silent=self.silent) + write_file(lock_fp, 'lock for %s' % self.installdir) + + try: + for (step_name, descr, step_methods, skippable) in steps: + if self._skip_step(step_name, skippable): + print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent) + else: + if self.dry_run: + self.dry_run_msg("%s... [DRY RUN]\n", descr) else: - if self.dry_run: - self.dry_run_msg("%s... [DRY RUN]\n", descr) - else: - print_msg("%s..." % descr, log=self.log, silent=self.silent) - self.current_step = step_name - self.run_step(step_name, step_methods) - - except StopException: - pass - finally: - print_msg("Removing lock file %s" % lockfile_name, silent=self.silent) - os.remove(lockfile_name) + print_msg("%s..." % descr, log=self.log, silent=self.silent) + self.current_step = step_name + self.run_step(step_name, step_methods) + + except StopException: + pass + finally: + print_msg("removing lock file %s" % lock_fp, silent=self.silent) + remove_file(lock_fp) # return True for successfull build (or stopped build) return True From 41ad45b48bd97fbbabb6938e33e85b79d26bb9bd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 17:08:44 +0200 Subject: [PATCH 267/344] don't print messages when creating/removing lock, just log an info message --- easybuild/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 223165764e..cf934640b7 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3052,8 +3052,8 @@ def run_all_steps(self, run_test_cases): raise EasyBuildError("Lock file %s already exists, aborting!", lock_fp) # create lock file to avoid that another installation running in parallel messes things up - print_msg("creating lock file %s" % lock_fp, silent=self.silent) write_file(lock_fp, 'lock for %s' % self.installdir) + self.log.info("Lock file created: %s", lock_fp) try: for (step_name, descr, step_methods, skippable) in steps: @@ -3070,8 +3070,8 @@ def run_all_steps(self, run_test_cases): except StopException: pass finally: - print_msg("removing lock file %s" % lock_fp, silent=self.silent) remove_file(lock_fp) + self.log.info("Lock file removed: %s", lock_fp) # return True for successfull build (or stopped build) return True From 0e7c81882c1fd7c37f1c2db659b957f5b16d8df3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 17:16:30 +0200 Subject: [PATCH 268/344] make wait-for-lock interval configurable via --wait-for-lock --- easybuild/framework/easyblock.py | 8 +++++--- easybuild/tools/options.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index cf934640b7..745701f387 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3044,10 +3044,12 @@ def run_all_steps(self, run_test_cases): # if lock already exists, either abort or wait until it disappears if os.path.exists(lock_fp): - if build_option('wait_on_lock'): + wait_on_lock = build_option('wait_on_lock') + if wait_on_lock: while os.path.exists(lock_fp): - print_msg("lock file in %s exists, waiting 60 seconds..." % locks_dir, silent=self.silent) - time.sleep(60) + print_msg("lock file %s exists, waiting %d seconds..." % (lock_fp, wait_on_lock), + silent=self.silent) + time.sleep(wait_on_lock) else: raise EasyBuildError("Lock file %s already exists, aborting!", lock_fp) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index ec591afd39..79528c90c8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -435,8 +435,8 @@ def override_options(self): None, 'store_true', False), 'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents", None, 'store_true', False), - 'wait-on-lock': ("Wait until lock file is removed when a lock if found", - None, 'store_true', False), + 'wait-on-lock': ("Wait interval (in seconds) to use when waiting for existing lock to be removed " + "(0: implies no waiting, but exiting with an error)", int, 'store', 0), 'zip-logs': ("Zip logs that are copied to install directory, using specified command", None, 'store_or_None', 'gzip'), From fbb7c7fcf37c19bd5555aad5f9a8567e650012be Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 19:06:22 +0200 Subject: [PATCH 269/344] rename --lockpath to --locks-dir --- easybuild/framework/easyblock.py | 2 +- easybuild/tools/config.py | 2 +- easybuild/tools/options.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 745701f387..f1d76bcfe6 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3039,7 +3039,7 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - locks_dir = build_option('lockpath') or os.path.join(install_path('software'), '.locks') + locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') lock_fp = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) # if lock already exists, either abort or wait until it disappears diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 403515243f..6f24adc863 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -185,7 +185,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'job_output_dir', 'job_polling_interval', 'job_target_resource', - 'lockpath', + 'locks_dir', 'modules_footer', 'modules_header', 'mpi_cmd_template', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 79528c90c8..d8620f484c 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -257,7 +257,8 @@ def basic_options(self): "and skipping check for OS dependencies", None, 'store_true', False, 'f'), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), - 'lockpath': ("Specifies which path should be used to store lock files", None, 'store_or_None', None), + 'locks-dir': ("Directory to store lock files (should be on a shared filesystem); " + "None implies .locks subdirectory of software installation directory", None, 'store_or_None', None), 'missing-modules': ("Print list of missing modules for dependencies of specified easyconfigs", None, 'store_true', False, 'M'), 'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}), From 878d611a692951ca9cda0d71716a3909cc1f0d7c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 19:07:07 +0200 Subject: [PATCH 270/344] add test for lock that prevents two identical installations happening in parallel --- test/framework/toy_build.py | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index e5a86d735d..0363e570d3 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -34,10 +34,12 @@ import os import re import shutil +import signal import stat import sys import tempfile from distutils.version import LooseVersion +from functools import wraps from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -2516,6 +2518,84 @@ def test_toy_ghost_installdir(self): self.assertFalse(os.path.exists(toy_installdir)) + def test_toy_build_lock(self): + """Test toy installation when a lock is already in place.""" + + locks_dir = os.path.join(self.test_installpath, 'software', '.locks') + toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + toy_lock_fn = toy_installdir.replace(os.path.sep, '_') + '.lock' + + write_file(os.path.join(locks_dir, toy_lock_fn), '') + + error_pattern = "Lock file .*_software_toy_0.0.lock already exists, aborting!" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) + + locks_dir = os.path.join(self.test_prefix, 'locks') + + # no lock in place, so installation proceeds as normal + extra_args = ['--locks-dir=%s' % locks_dir] + self.test_toy_build(extra_args=extra_args, verify=True, raise_error=True) + + # put lock in place in custom locks dir, try again + toy_lock_fp = os.path.join(locks_dir, toy_lock_fn) + write_file(toy_lock_fp, '') + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, + extra_args=extra_args, raise_error=True, verbose=False) + + # define a context manager that remove a lock after a while, so we can check the use of --wait-for-lock + class remove_lock_after: + def __init__(self, seconds, lock_fp): + self.seconds = seconds + self.lock_fp = lock_fp + + def remove_lock(self, *args): + remove_file(self.lock_fp) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.remove_lock) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + pass + + # wait for lock to be removed, with 1 second interval of checking + extra_args.append('--wait-on-lock=1') + + wait_regex = re.compile("^== lock file .*_software_toy_0.0.lock exists, waiting 1 seconds", re.M) + ok_regex = re.compile("^== COMPLETED: Installation ended successfully", re.M) + + self.assertTrue(os.path.exists(toy_lock_fp)) + + # use context manager to remove lock file after 3 seconds + with remove_lock_after(3, toy_lock_fp): + self.mock_stderr(True) + self.mock_stdout(True) + self.test_toy_build(extra_args=extra_args, verify=False, raise_error=True, testing=False) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + + wait_matches = wait_regex.findall(stdout) + # we can't rely on an exact number of 'waiting' messages, so let's go with a range... + self.assertTrue(len(wait_matches) in range(2, 5)) + + self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) + + # when there is no lock in place, --wait-on-lock has no impact + self.assertFalse(os.path.exists(toy_lock_fp)) + self.mock_stderr(True) + self.mock_stdout(True) + self.test_toy_build(extra_args=extra_args, verify=False, raise_error=True, testing=False) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) + self.assertFalse(wait_regex.search(stdout), "Pattern '%s' not found in: %s" % (wait_regex.pattern, stdout)) + def suite(): """ return all the tests in this file """ From 79a880aa3d16dd5ad1f7c934e307c26144d4c49d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Mar 2020 19:20:57 +0200 Subject: [PATCH 271/344] add --ignore-locks option --- easybuild/framework/easyblock.py | 42 ++++++++++++++++++-------------- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 2 ++ test/framework/toy_build.py | 11 ++++++--- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f1d76bcfe6..23409f96f9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3039,23 +3039,28 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent) trace_msg("installation prefix: %s" % self.installdir) - locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') - lock_fp = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) - - # if lock already exists, either abort or wait until it disappears - if os.path.exists(lock_fp): - wait_on_lock = build_option('wait_on_lock') - if wait_on_lock: - while os.path.exists(lock_fp): - print_msg("lock file %s exists, waiting %d seconds..." % (lock_fp, wait_on_lock), - silent=self.silent) - time.sleep(wait_on_lock) - else: - raise EasyBuildError("Lock file %s already exists, aborting!", lock_fp) + ignore_locks = build_option('ignore_locks') + + if ignore_locks: + self.log.info("Ignoring locks...") + else: + locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') + lock_fp = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) + + # if lock already exists, either abort or wait until it disappears + if os.path.exists(lock_fp): + wait_on_lock = build_option('wait_on_lock') + if wait_on_lock: + while os.path.exists(lock_fp): + print_msg("lock file %s exists, waiting %d seconds..." % (lock_fp, wait_on_lock), + silent=self.silent) + time.sleep(wait_on_lock) + else: + raise EasyBuildError("Lock file %s already exists, aborting!", lock_fp) - # create lock file to avoid that another installation running in parallel messes things up - write_file(lock_fp, 'lock for %s' % self.installdir) - self.log.info("Lock file created: %s", lock_fp) + # create lock file to avoid that another installation running in parallel messes things up + write_file(lock_fp, 'lock for %s' % self.installdir) + self.log.info("Lock file created: %s", lock_fp) try: for (step_name, descr, step_methods, skippable) in steps: @@ -3072,8 +3077,9 @@ def run_all_steps(self, run_test_cases): except StopException: pass finally: - remove_file(lock_fp) - self.log.info("Lock file removed: %s", lock_fp) + if not ignore_locks: + remove_file(lock_fp) + self.log.info("Lock file removed: %s", lock_fp) # return True for successfull build (or stopped build) return True diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 6f24adc863..30253b6f61 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -226,6 +226,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'group_writable_installdir', 'hidden', 'ignore_checksums', + 'ignore_locks', 'install_latest_eb_release', 'lib64_fallback_sanity_check', 'logtostdout', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index d8620f484c..5f54a005a8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -255,6 +255,8 @@ def basic_options(self): 'extended-dry-run-ignore-errors': ("Ignore errors that occur during dry run", None, 'store_true', True), 'force': ("Force to rebuild software even if it's already installed (i.e. if it can be found as module), " "and skipping check for OS dependencies", None, 'store_true', False, 'f'), + 'ignore-locks': ("Ignore locks that prevent two identical installations running in parallel", + None, 'store_true', False), 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'locks-dir': ("Directory to store lock files (should be on a shared filesystem); " diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0363e570d3..9b9cc3b920 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1417,7 +1417,7 @@ def test_module_only(self): self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))) modtxt = read_file(toy_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # install (only) additional module under a hierarchical MNS @@ -1432,7 +1432,7 @@ def test_module_only(self): # existing install is reused modtxt2 = read_file(toy_core_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt2)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # make sure load statements for dependencies are included @@ -1457,7 +1457,7 @@ def test_module_only(self): # existing install is reused modtxt3 = read_file(toy_mod + '.lua') self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # make sure load statements for dependencies are included @@ -2059,7 +2059,7 @@ def test_toy_modaltsoftname(self): self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) # only subdirectories for software should be created - self.assertEqual(sorted(os.listdir(software_path)), sorted(['.locks', 'toy'])) + self.assertEqual(os.listdir(software_path), ['toy']) self.assertEqual(sorted(os.listdir(os.path.join(software_path, 'toy'))), ['0.0-one', '0.0-two']) # only subdirectories for modules with alternative names should be created @@ -2542,6 +2542,9 @@ def test_toy_build_lock(self): self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=extra_args, raise_error=True, verbose=False) + # also test use of --ignore-locks + self.test_toy_build(extra_args=extra_args + ['--ignore-locks'], verify=True, raise_error=True) + # define a context manager that remove a lock after a while, so we can check the use of --wait-for-lock class remove_lock_after: def __init__(self, seconds, lock_fp): From 9724ed9bea0679a7b52000620c6118c41e6bff7f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 09:30:17 +0200 Subject: [PATCH 272/344] fix broken toy tests, take into account .locks dir in software directory --- test/framework/toy_build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 9b9cc3b920..95bce1a232 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1417,7 +1417,7 @@ def test_module_only(self): self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0-deps', 'bin'))) modtxt = read_file(toy_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # install (only) additional module under a hierarchical MNS @@ -1432,7 +1432,7 @@ def test_module_only(self): # existing install is reused modtxt2 = read_file(toy_core_mod) self.assertTrue(re.search("set root %s" % prefix, modtxt2)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # make sure load statements for dependencies are included @@ -2059,7 +2059,7 @@ def test_toy_modaltsoftname(self): self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) # only subdirectories for software should be created - self.assertEqual(os.listdir(software_path), ['toy']) + self.assertEqual(os.listdir(software_path), ['toy', '.locks']) self.assertEqual(sorted(os.listdir(os.path.join(software_path, 'toy'))), ['0.0-one', '0.0-two']) # only subdirectories for modules with alternative names should be created From 983fcafaf40e88bf5a16ce0dc9779ff69d2da120 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 09:45:35 +0200 Subject: [PATCH 273/344] handle unknown CPU arch name in --show-system-info --- easybuild/tools/options.py | 8 +++++--- test/framework/options.py | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 987f707be6..dd92f897f8 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -95,8 +95,8 @@ from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories -from easybuild.tools.systemtools import check_python_version, get_cpu_architecture, get_cpu_family, get_cpu_features -from easybuild.tools.systemtools import get_system_info +from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_cpu_features, get_system_info from easybuild.tools.version import this_is_easybuild @@ -1168,7 +1168,9 @@ def show_system_info(self): " -> architecture: %s" % get_cpu_architecture(), " -> family: %s" % get_cpu_family(), ] - if cpu_arch_name: + if cpu_arch_name == UNKNOWN: + lines.append(" -> arch name: UNKNOWN (archspec is not installed?)") + else: lines.append(" -> arch name: %s" % cpu_arch_name) lines.extend([ diff --git a/test/framework/options.py b/test/framework/options.py index a6c544d1b2..17f390d1f5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -4565,7 +4565,9 @@ def test_show_system_info(self): ] if HAVE_ARCHSPEC: - patterns.append(r"^ -> arch name: \w+") + patterns.append(r"^ -> arch name: \w+$") + else: + patterns.append(r"^ -> arch name: UNKNOWN \(archspec is not installed\?\)$") for pattern in patterns: regex = re.compile(pattern, re.M) From 2c396e2166a5ad30a3914da05c804ffa7741112f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 10:32:30 +0200 Subject: [PATCH 274/344] Add template for mpi_cmd_prefix --- easybuild/framework/easyconfig/easyconfig.py | 2 +- easybuild/framework/easyconfig/templates.py | 9 +++++- easybuild/tools/toolchain/mpi.py | 30 ++++++++++++++++---- test/framework/toolchain.py | 26 +++++++++++++++++ 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index d6033eb7c3..e3bf620301 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1469,7 +1469,7 @@ def _generate_template_values(self, ignore=None): # disable templating with creating dict with template values to avoid looping back to here via __getitem__ prev_enable_templating = self.enable_templating self.enable_templating = False - template_values = template_constant_dict(self, ignore=ignore) + template_values = template_constant_dict(self, ignore=ignore, toolchain=self.toolchain) self.enable_templating = prev_enable_templating # update the template_values dict diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index f024866fd7..6da5174907 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -145,7 +145,7 @@ # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) -def template_constant_dict(config, ignore=None, skip_lower=None): +def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config """ @@ -257,6 +257,13 @@ def template_constant_dict(config, ignore=None, skip_lower=None): except Exception: _log.warning("Failed to get .lower() for name %s value %s (type %s)", name, value, type(value)) + # step 5. add additional conditional templates + if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): + # get prefix for commands to be run with mpi runtime using default number of ranks + mpi_cmd_prefix = toolchain.mpi_cmd_prefix() + if mpi_cmd_prefix is not None: + template_values['mpi_cmd_prefix'] = mpi_cmd_prefix + return template_values diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 9a30baa33f..c2329f48f2 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -166,6 +166,19 @@ def mpi_family(self): else: raise EasyBuildError("mpi_family: MPI_FAMILY is undefined.") + def mpi_cmd_prefix(self, nr_ranks=1): + """Construct an MPI command prefix to precede an executable""" + + # Verify that the command appears at the end of mpi_cmd_for + if self.mpi_cmd_for('xcommandx', '1').rstrip().endswith('xcommandx'): + result = self.mpi_cmd_for('', nr_ranks) + else: + self.log.warning("mpi_cmd_for cannot be used to construct mpi_cmd_prefix, requires that cmd template " + "appears last in result") + result = None + + return result + def mpi_cmd_for(self, cmd, nr_ranks): """Construct an MPI command for the given command and number of ranks.""" @@ -180,10 +193,10 @@ def mpi_cmd_for(self, cmd, nr_ranks): self.log.info("Using specified template for MPI commands: %s", mpi_cmd_template) else: # different known mpirun commands - mpirun_n_cmd = "mpirun -n %(nr_ranks)d %(cmd)s" + mpirun_n_cmd = "mpirun -n %(nr_ranks)s %(cmd)s" mpi_cmds = { toolchain.OPENMPI: mpirun_n_cmd, - toolchain.QLOGICMPI: "mpirun -H localhost -np %(nr_ranks)d %(cmd)s", + toolchain.QLOGICMPI: "mpirun -H localhost -np %(nr_ranks)s %(cmd)s", toolchain.INTELMPI: mpirun_n_cmd, toolchain.MVAPICH2: mpirun_n_cmd, toolchain.MPICH: mpirun_n_cmd, @@ -201,7 +214,7 @@ def mpi_cmd_for(self, cmd, nr_ranks): impi_ver = self.get_software_version(self.MPI_MODULE_NAME)[0] if LooseVersion(impi_ver) <= LooseVersion('4.1'): - mpi_cmds[toolchain.INTELMPI] = "mpirun %(mpdbf)s %(nodesfile)s -np %(nr_ranks)d %(cmd)s" + mpi_cmds[toolchain.INTELMPI] = "mpirun %(mpdbf)s %(nodesfile)s -np %(nr_ranks)s %(cmd)s" # set temporary dir for MPD # note: this needs to be kept *short*, @@ -230,7 +243,7 @@ def mpi_cmd_for(self, cmd, nr_ranks): # create nodes file nodes = os.path.join(tmpdir, 'nodes') - write_file(nodes, "localhost\n" * nr_ranks) + write_file(nodes, "localhost\n" * int(nr_ranks)) params.update({'nodesfile': "-machinefile %s" % nodes}) @@ -243,6 +256,13 @@ def mpi_cmd_for(self, cmd, nr_ranks): try: res = mpi_cmd_template % params except KeyError as err: - raise EasyBuildError("Failed to complete MPI cmd template '%s' with %s: %s", mpi_cmd_template, params, err) + missing = [] + for key in params: + tmpl = '%(' + key + ')s' + if tmpl not in mpi_cmd_template: + missing.append(tmpl) + if missing: + raise EasyBuildError("Missing templates in mpi-cmd-template value '%s': %s", mpi_cmd_template, + ', '.join(missing)) return res diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 909bb2f070..2cfd49a93a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -948,6 +948,32 @@ def test_nosuchtoolchain(self): tc = self.get_toolchain('intel', version='1970.01') self.assertErrorRegex(EasyBuildError, "No module found for toolchain", tc.prepare) + def test_mpi_cmd_prefix(self): + """Test mpi_exec_nranks function.""" + self.modtool.prepend_module_path(self.test_prefix) + + tc = self.get_toolchain('gompi', version='2018a') + tc.prepare() + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ") + self.modtool.purge() + + self.setup_sandbox_for_intel_fftw(self.test_prefix) + tc = self.get_toolchain('intel', version='2018a') + tc.prepare() + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ") + self.modtool.purge() + + self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038') + tc = self.get_toolchain('intel', version='2012a') + tc.prepare() + + mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 ") + self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix(nr_ranks=4))) + + # test specifying custom template for MPI commands + init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s", 'silent': True}) + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), "mpiexec -np 7 -- ") + def test_mpi_cmd_for(self): """Test mpi_cmd_for function.""" self.modtool.prepend_module_path(self.test_prefix) From 5adfc376d60e09a95b3cb17aa51ef2b34006a73a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 11:00:08 +0200 Subject: [PATCH 275/344] fix failing test_module_only --- test/framework/toy_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 95bce1a232..4d777a1a41 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1443,7 +1443,7 @@ def test_module_only(self): os.remove(toy_core_mod) # test installing (only) additional module in Lua syntax (if Lmod is available) - lmod_abspath = which('lmod') + lmod_abspath = os.environ.get('LMOD_CMD') or which('lmod') if lmod_abspath is not None: args = common_args[:-1] + [ '--allow-modules-tool-mismatch', @@ -1457,7 +1457,7 @@ def test_module_only(self): # existing install is reused modtxt3 = read_file(toy_mod + '.lua') self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) - self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 2) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 3) self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) # make sure load statements for dependencies are included From 1f8f4d85caf0db86da43d624f8d8331b9aad747f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 13:49:12 +0200 Subject: [PATCH 276/344] Be less strict about being able to create the MPI prefix template (don't fail hard) --- easybuild/framework/easyconfig/templates.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 6da5174907..441d69590b 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -259,10 +259,17 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # step 5. add additional conditional templates if toolchain is not None and hasattr(toolchain, 'mpi_cmd_prefix'): - # get prefix for commands to be run with mpi runtime using default number of ranks - mpi_cmd_prefix = toolchain.mpi_cmd_prefix() - if mpi_cmd_prefix is not None: - template_values['mpi_cmd_prefix'] = mpi_cmd_prefix + try: + # get prefix for commands to be run with mpi runtime using default number of ranks + mpi_cmd_prefix = toolchain.mpi_cmd_prefix() + if mpi_cmd_prefix is not None: + template_values['mpi_cmd_prefix'] = mpi_cmd_prefix + except EasyBuildError as err: + # don't fail just because we couldn't resolve this template + if "get_software_version software version for" in str(err): + _log.warning("Failed to create mpi_cmd_prefix template, error was:\n" + str(err)) + else: + raise err return template_values From 2523c4aa503adc4f5fc5f3c681eea3de628bdbda Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 13:59:27 +0200 Subject: [PATCH 277/344] Add some additional tests --- test/framework/toolchain.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 2cfd49a93a..870f4d3b13 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -955,12 +955,16 @@ def test_mpi_cmd_prefix(self): tc = self.get_toolchain('gompi', version='2018a') tc.prepare() self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2 ") + self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1 ") self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix) tc = self.get_toolchain('intel', version='2018a') tc.prepare() self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2 ") + self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1 ") self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038') @@ -969,10 +973,18 @@ def test_mpi_cmd_prefix(self): mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 ") self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix(nr_ranks=4))) + mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 1 ") + self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix())) # test specifying custom template for MPI commands init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s", 'silent': True}) self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), "mpiexec -np 7 -- ") + self.assertEqual(tc.mpi_cmd_prefix(), "mpiexec -np 1 -- ") + + # check that we return none when command does not appear at the end of the template + init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s option", 'silent': True}) + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), None) + self.assertEqual(tc.mpi_cmd_prefix(), None) def test_mpi_cmd_for(self): """Test mpi_cmd_for function.""" From 534c7cdf5fc56450ff2dad1ea2de4454f1179df5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 14:24:12 +0200 Subject: [PATCH 278/344] Update test so it doesn't have crazy stuff in unhelpful places! --- test/framework/robot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/framework/robot.py b/test/framework/robot.py index fc94a84850..549e9382cd 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -424,15 +424,15 @@ def test_resolve_dependencies_minimal(self): # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", - " ('OpenMPI', '2.1.2'),", # available with GCC/6.4.0-2.28 + # the use of %(version_minor)s here is mainly to check if templates are being handled correctly + # (it doesn't make much sense, but it serves the purpose) + " ('OpenMPI', '%(version_minor)s.1.2'),", # available with GCC/6.4.0-2.28 " ('OpenBLAS', '0.2.20'),", # available with GCC/6.4.0-2.28 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.20'),", # available with gompi/2018a " ('SQLite', '3.8.10.2'),", "]", # toolchain as list line, for easy modification later; - # the use of %(version_minor)s here is mainly to check if templates are being handled correctly - # (it doesn't make much sense, but it serves the purpose) - "toolchain = {'name': 'foss', 'version': '%(version_minor)s018a'}", + "toolchain = {'name': 'foss', 'version': '2018a'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] From 5b50556441a6716ae04642e4b92ff3f8accb84bc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 14:56:23 +0200 Subject: [PATCH 279/344] Be more forgiving if we can't generate a hierarchy --- easybuild/framework/easyconfig/easyconfig.py | 17 +++++++++-------- easybuild/framework/easyconfig/format/yeb.py | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 8f0e1d5ebb..b380f1821d 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1125,16 +1125,17 @@ def dump(self, fp, always_overwrite=True, backup=False): if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2: templ_val[self.template_values[key]] = key + toolchain_hierarchy = None try: - ectxt = self.parser.dump(self, default_values, templ_const, templ_val, - toolchain_hierarchy=get_toolchain_hierarchy(self['toolchain'])) + toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain']) except EasyBuildError as err: - if 'No version found for subtoolchain' in str(err): - self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump', - self['toolchain']) - ectxt = self.parser.dump(self, default_values, templ_const, templ_val) - else: - raise err + # don't fail hardjust because we can't get the hierarchy + self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, ' + 'error:\n%s', self['toolchain'], str(err)) + + try: + ectxt = self.parser.dump(self, default_values, templ_const, templ_val, + toolchain_hierarchy=toolchain_hierarchy) except NotImplementedError as err: # need to restore enable_templating value in case this method is caught in a try/except block and ignored # (the ability to dump is not a hard requirement for build success) diff --git a/easybuild/framework/easyconfig/format/yeb.py b/easybuild/framework/easyconfig/format/yeb.py index 5ccb8632de..4e59b4892b 100644 --- a/easybuild/framework/easyconfig/format/yeb.py +++ b/easybuild/framework/easyconfig/format/yeb.py @@ -126,7 +126,7 @@ def _inject_constants_dict(self, txt): return full_txt - def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=[]): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """Dump parsed easyconfig in .yeb format""" raise NotImplementedError("Dumping of .yeb easyconfigs not supported yet") From 20fd0372e3a021395a58da92c85f442cc910b838 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 14:57:15 +0200 Subject: [PATCH 280/344] Be more forgiving if we can't generate a hierarchy --- easybuild/framework/easyconfig/easyconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b380f1821d..054a9c7f96 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1129,7 +1129,7 @@ def dump(self, fp, always_overwrite=True, backup=False): try: toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain']) except EasyBuildError as err: - # don't fail hardjust because we can't get the hierarchy + # don't fail hard just because we can't get the hierarchy self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, ' 'error:\n%s', self['toolchain'], str(err)) From e8bff528cf3ccdcb1d5233e10181a3e3fe118a56 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 30 Mar 2020 14:59:10 +0200 Subject: [PATCH 281/344] Update templates.py --- easybuild/framework/easyconfig/templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 441d69590b..b7f0345fdf 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -267,7 +267,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) except EasyBuildError as err: # don't fail just because we couldn't resolve this template if "get_software_version software version for" in str(err): - _log.warning("Failed to create mpi_cmd_prefix template, error was:\n" + str(err)) + _log.warning("Failed to create mpi_cmd_prefix template, error was:\n%s", str(err)) else: raise err From 457b95a33f134a2bf3c01cb2e7e27fb302403c9e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 15:32:16 +0200 Subject: [PATCH 282/344] Add test for toolchain hierarchy aware dump --- test/framework/easyconfig.py | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 0f45797bc7..0dfa0b587d 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1642,6 +1642,50 @@ def test_dump(self): if param in ec: self.assertEqual(ec[param], dumped_ec[param]) + def test_toolchain_hierarchy_aware_dump(self): + """Test that EasyConfig's dump() method is aware of the toolchain hierarchy.""" + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + build_options = { + 'check_osdeps': False, + 'robot_path': [test_ecs_dir], + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + rawtxt = '\n'.join([ + "easyblock = 'EB_foo'", + '', + "name = 'foo'", + "version = '0.0.1'", + '', + "toolchain = {'name': 'foss', 'version': '2018a'}", + '', + "homepage = 'http://foo.com/'", + 'description = "foo description"', + '', + 'sources = [SOURCE_TAR_GZ]', + 'source_urls = ["http://example.com"]', + 'checksums = ["6af6ab95ce131c2dd467d2ebc8270e9c265cc32496210b069e51d3749f335f3d"]', + '', + "dependencies = [", + " ('toy', '0.0', '', ('gompi', '2018a')),", + " ('bar', '1.0'),", + " ('foobar/1.2.3', EXTERNAL_MODULE),", + "]", + '', + "foo_extra1 = 'foobar'", + '', + 'moduleclass = "tools"', + ]) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + ec = EasyConfig(None, rawtxt=rawtxt) + ecdict = ec.asdict() + ec.dump(test_ec) + # dict representation of EasyConfig instance should not change after dump + self.assertEqual(ecdict, ec.asdict()) + ectxt = read_file(test_ec) + self.assertTrue(r"'toy', '0.0')," in ectxt) + def test_dump_order(self): """Test order of easyconfig parameters in dumped easyconfig.""" rawtxt = '\n'.join([ From b6ecfb362dd78a276aacedfe14f74e06372aa1d6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 15:38:22 +0200 Subject: [PATCH 283/344] Missed initial class definition of dump method --- easybuild/framework/easyconfig/format/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 0ac6c380f8..219d842d9d 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -632,7 +632,7 @@ def parse(self, txt, **kwargs): """Parse the txt according to this format. This is highly version specific""" raise NotImplementedError - def dump(self, ecfg, default_values, templ_const, templ_val): + def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): """Dump easyconfig according to this format. This is higly version specific""" raise NotImplementedError From d5c69e5c10bf3c82fce19189e27efc225af5daf6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 30 Mar 2020 15:40:10 +0200 Subject: [PATCH 284/344] Typo --- easybuild/framework/easyconfig/format/format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 219d842d9d..85b2f239df 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -633,7 +633,7 @@ def parse(self, txt, **kwargs): raise NotImplementedError def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy=None): - """Dump easyconfig according to this format. This is higly version specific""" + """Dump easyconfig according to this format. This is highly version specific""" raise NotImplementedError def extract_comments(self, rawtxt): From f77ced5a4fdffe973c70dc458bfd1c8f3c549aba Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 16:09:22 +0200 Subject: [PATCH 285/344] use directory as lock --- easybuild/framework/easyblock.py | 21 +++++++++++---------- test/framework/toy_build.py | 21 +++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 23409f96f9..54f1772ae0 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3045,22 +3045,23 @@ def run_all_steps(self, run_test_cases): self.log.info("Ignoring locks...") else: locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') - lock_fp = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) + lock_path = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) # if lock already exists, either abort or wait until it disappears - if os.path.exists(lock_fp): + if os.path.exists(lock_path): wait_on_lock = build_option('wait_on_lock') if wait_on_lock: - while os.path.exists(lock_fp): - print_msg("lock file %s exists, waiting %d seconds..." % (lock_fp, wait_on_lock), + while os.path.exists(lock_path): + print_msg("lock %s exists, waiting %d seconds..." % (lock_path, wait_on_lock), silent=self.silent) time.sleep(wait_on_lock) else: - raise EasyBuildError("Lock file %s already exists, aborting!", lock_fp) + raise EasyBuildError("Lock %s already exists, aborting!", lock_path) - # create lock file to avoid that another installation running in parallel messes things up - write_file(lock_fp, 'lock for %s' % self.installdir) - self.log.info("Lock file created: %s", lock_fp) + # create lock to avoid that another installation running in parallel messes things up; + # we use a directory as a lock, since that's atomically created + mkdir(lock_path, parents=True) + self.log.info("Lock created: %s", lock_path) try: for (step_name, descr, step_methods, skippable) in steps: @@ -3078,8 +3079,8 @@ def run_all_steps(self, run_test_cases): pass finally: if not ignore_locks: - remove_file(lock_fp) - self.log.info("Lock file removed: %s", lock_fp) + remove_dir(lock_path) + self.log.info("Lock removed: %s", lock_path) # return True for successfull build (or stopped build) return True diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 4d777a1a41..ea17174a89 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2525,9 +2525,10 @@ def test_toy_build_lock(self): toy_installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') toy_lock_fn = toy_installdir.replace(os.path.sep, '_') + '.lock' - write_file(os.path.join(locks_dir, toy_lock_fn), '') + toy_lock_path = os.path.join(locks_dir, toy_lock_fn) + mkdir(toy_lock_path, parents=True) - error_pattern = "Lock file .*_software_toy_0.0.lock already exists, aborting!" + error_pattern = "Lock .*_software_toy_0.0.lock already exists, aborting!" self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, raise_error=True, verbose=False) locks_dir = os.path.join(self.test_prefix, 'locks') @@ -2537,8 +2538,8 @@ def test_toy_build_lock(self): self.test_toy_build(extra_args=extra_args, verify=True, raise_error=True) # put lock in place in custom locks dir, try again - toy_lock_fp = os.path.join(locks_dir, toy_lock_fn) - write_file(toy_lock_fp, '') + toy_lock_path = os.path.join(locks_dir, toy_lock_fn) + mkdir(toy_lock_path, parents=True) self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=extra_args, raise_error=True, verbose=False) @@ -2552,7 +2553,7 @@ def __init__(self, seconds, lock_fp): self.lock_fp = lock_fp def remove_lock(self, *args): - remove_file(self.lock_fp) + remove_dir(self.lock_fp) def __enter__(self): signal.signal(signal.SIGALRM, self.remove_lock) @@ -2564,13 +2565,13 @@ def __exit__(self, type, value, traceback): # wait for lock to be removed, with 1 second interval of checking extra_args.append('--wait-on-lock=1') - wait_regex = re.compile("^== lock file .*_software_toy_0.0.lock exists, waiting 1 seconds", re.M) + wait_regex = re.compile("^== lock .*_software_toy_0.0.lock exists, waiting 1 seconds", re.M) ok_regex = re.compile("^== COMPLETED: Installation ended successfully", re.M) - self.assertTrue(os.path.exists(toy_lock_fp)) + self.assertTrue(os.path.exists(toy_lock_path)) - # use context manager to remove lock file after 3 seconds - with remove_lock_after(3, toy_lock_fp): + # use context manager to remove lock after 3 seconds + with remove_lock_after(3, toy_lock_path): self.mock_stderr(True) self.mock_stdout(True) self.test_toy_build(extra_args=extra_args, verify=False, raise_error=True, testing=False) @@ -2587,7 +2588,7 @@ def __exit__(self, type, value, traceback): self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) # when there is no lock in place, --wait-on-lock has no impact - self.assertFalse(os.path.exists(toy_lock_fp)) + self.assertFalse(os.path.exists(toy_lock_path)) self.mock_stderr(True) self.mock_stdout(True) self.test_toy_build(extra_args=extra_args, verify=False, raise_error=True, testing=False) From 81f783338e3fe16cf4154f87a850e826f2e5db6e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 16:49:14 +0200 Subject: [PATCH 286/344] ensure cleaner error when lock dir can't be created --- easybuild/framework/easyblock.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 54f1772ae0..ad89b383fb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3060,7 +3060,13 @@ def run_all_steps(self, run_test_cases): # create lock to avoid that another installation running in parallel messes things up; # we use a directory as a lock, since that's atomically created - mkdir(lock_path, parents=True) + try: + mkdir(lock_path, parents=True) + except EasyBuildError as err: + # clean up the error message a bit, get rid of the "Failed to create directory" part + quotes + stripped_err = str(err).split(':', 1)[1].strip().replace("'", '').replace('"', '') + raise EasyBuildError("Failed to create lock %s: %s", lock_path, stripped_err) + self.log.info("Lock created: %s", lock_path) try: From af2f6e551f6b9e65c0cbc45926a5aa4a3717f101 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 16:52:52 +0200 Subject: [PATCH 287/344] enhance lock test to check for clean error --- test/framework/toy_build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ea17174a89..770c8f16de 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2600,6 +2600,12 @@ def __exit__(self, type, value, traceback): self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) self.assertFalse(wait_regex.search(stdout), "Pattern '%s' not found in: %s" % (wait_regex.pattern, stdout)) + # check for clean error on creation of lock + extra_args = ['--locks-dir=/'] + error_pattern = "Failed to create lock /.*_software_toy_0.0.lock:.* Read-only file system" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, + extra_args=extra_args, raise_error=True, verbose=False) + def suite(): """ return all the tests in this file """ From 5c5d2428fdaad08d60e2348660a73db4561ec04c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 16:54:05 +0200 Subject: [PATCH 288/344] appease the Hound --- easybuild/tools/options.py | 5 +++-- test/framework/toy_build.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5f54a005a8..17ddd9cd59 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -260,7 +260,8 @@ def basic_options(self): 'job': ("Submit the build as a job", None, 'store_true', False), 'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'), 'locks-dir': ("Directory to store lock files (should be on a shared filesystem); " - "None implies .locks subdirectory of software installation directory", None, 'store_or_None', None), + "None implies .locks subdirectory of software installation directory", + None, 'store_or_None', None), 'missing-modules': ("Print list of missing modules for dependencies of specified easyconfigs", None, 'store_true', False, 'M'), 'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}), @@ -439,7 +440,7 @@ def override_options(self): 'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents", None, 'store_true', False), 'wait-on-lock': ("Wait interval (in seconds) to use when waiting for existing lock to be removed " - "(0: implies no waiting, but exiting with an error)", int, 'store', 0), + "(0: implies no waiting, but exiting with an error)", int, 'store', 0), 'zip-logs': ("Zip logs that are copied to install directory, using specified command", None, 'store_or_None', 'gzip'), diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 770c8f16de..0e828ac194 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -39,7 +39,6 @@ import sys import tempfile from distutils.version import LooseVersion -from functools import wraps from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered from test.framework.package import mock_fpm from unittest import TextTestRunner From 774f5892e5d0752d58ac3db68b46b381dd30f14b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 17:49:38 +0200 Subject: [PATCH 289/344] also accept 'Permission denied' error in test for failing lock creation --- test/framework/toy_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 0e828ac194..8031d67f8d 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2601,7 +2601,8 @@ def __exit__(self, type, value, traceback): # check for clean error on creation of lock extra_args = ['--locks-dir=/'] - error_pattern = "Failed to create lock /.*_software_toy_0.0.lock:.* Read-only file system" + error_pattern = r"Failed to create lock /.*_software_toy_0.0.lock:.* " + error_pattern += r"(Read-only file system|Permission denied)" self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=extra_args, raise_error=True, verbose=False) From 1b85909459149fd19122b69cd1a9773ac7856209 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Mar 2020 20:09:43 +0200 Subject: [PATCH 290/344] sort lists before comparing them in test_toy_modaltsoftname --- test/framework/toy_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 8031d67f8d..3145981ca7 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -2058,7 +2058,7 @@ def test_toy_modaltsoftname(self): self.assertTrue(os.path.exists(os.path.join(modules_path, 'yot', yot_name))) # only subdirectories for software should be created - self.assertEqual(os.listdir(software_path), ['toy', '.locks']) + self.assertEqual(sorted(os.listdir(software_path)), sorted(['toy', '.locks'])) self.assertEqual(sorted(os.listdir(os.path.join(software_path, 'toy'))), ['0.0-one', '0.0-two']) # only subdirectories for modules with alternative names should be created From 1b7e8fd36d3d881ff6a2bdc5a48f64ce0cc1bc76 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 31 Mar 2020 20:10:25 +0200 Subject: [PATCH 291/344] cleanup, changes and enhanced tests for implementation for mpi_cmd_for/mpi_cmd_prefix functions --- easybuild/framework/easyconfig/templates.py | 5 +-- easybuild/tools/toolchain/mpi.py | 30 ++++++++++------- test/framework/toolchain.py | 36 ++++++++++++++------- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index b7f0345fdf..b6e6fe1393 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -266,10 +266,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['mpi_cmd_prefix'] = mpi_cmd_prefix except EasyBuildError as err: # don't fail just because we couldn't resolve this template - if "get_software_version software version for" in str(err): - _log.warning("Failed to create mpi_cmd_prefix template, error was:\n%s", str(err)) - else: - raise err + _log.warning("Failed to create mpi_cmd_prefix template, error was:\n%s", err) return template_values diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index c2329f48f2..052c3e061d 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -170,11 +170,14 @@ def mpi_cmd_prefix(self, nr_ranks=1): """Construct an MPI command prefix to precede an executable""" # Verify that the command appears at the end of mpi_cmd_for - if self.mpi_cmd_for('xcommandx', '1').rstrip().endswith('xcommandx'): - result = self.mpi_cmd_for('', nr_ranks) + test_cmd = 'xxx_command_xxx' + mpi_cmd = self.mpi_cmd_for(test_cmd, nr_ranks) + if mpi_cmd.rstrip().endswith(test_cmd): + result = mpi_cmd.replace(test_cmd, '').rstrip() else: - self.log.warning("mpi_cmd_for cannot be used to construct mpi_cmd_prefix, requires that cmd template " - "appears last in result") + warning_msg = "mpi_cmd_for cannot be used by mpi_cmd_prefix, " + warning_msg += "requires that %(cmd)s template appears at the end" + self.log.warning(warning_msg) result = None return result @@ -253,16 +256,19 @@ def mpi_cmd_for(self, cmd, nr_ranks): else: raise EasyBuildError("Don't know which template MPI command to use for MPI family '%s'", mpi_family) + missing = [] + for key in sorted(params.keys()): + tmpl = '%(' + key + ')s' + if tmpl not in mpi_cmd_template: + missing.append(tmpl) + if missing: + raise EasyBuildError("Missing templates in mpi-cmd-template value '%s': %s", + mpi_cmd_template, ', '.join(missing)) + try: res = mpi_cmd_template % params except KeyError as err: - missing = [] - for key in params: - tmpl = '%(' + key + ')s' - if tmpl not in mpi_cmd_template: - missing.append(tmpl) - if missing: - raise EasyBuildError("Missing templates in mpi-cmd-template value '%s': %s", mpi_cmd_template, - ', '.join(missing)) + raise EasyBuildError("Failed to complete MPI cmd template '%s' with %s: KeyError %s", + mpi_cmd_template, params, err) return res diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 870f4d3b13..924017888a 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -954,38 +954,42 @@ def test_mpi_cmd_prefix(self): tc = self.get_toolchain('gompi', version='2018a') tc.prepare() - self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ") - self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2 ") - self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1 ") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2") + self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1") self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix) tc = self.get_toolchain('intel', version='2018a') tc.prepare() - self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2 ") - self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2 ") - self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1 ") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks=2), "mpirun -n 2") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks='2'), "mpirun -n 2") + self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -n 1") self.modtool.purge() self.setup_sandbox_for_intel_fftw(self.test_prefix, imklver='10.2.6.038') tc = self.get_toolchain('intel', version='2012a') tc.prepare() - mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4 ") + mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 4") self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix(nr_ranks=4))) - mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 1 ") + mpi_exec_nranks_re = re.compile("^mpirun --file=.*/mpdboot -machinefile .*/nodes -np 1") self.assertTrue(mpi_exec_nranks_re.match(tc.mpi_cmd_prefix())) # test specifying custom template for MPI commands init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s", 'silent': True}) - self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), "mpiexec -np 7 -- ") - self.assertEqual(tc.mpi_cmd_prefix(), "mpiexec -np 1 -- ") + self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), "mpiexec -np 7 --") + self.assertEqual(tc.mpi_cmd_prefix(), "mpiexec -np 1 --") - # check that we return none when command does not appear at the end of the template + # check that we return None when command does not appear at the end of the template init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s option", 'silent': True}) self.assertEqual(tc.mpi_cmd_prefix(nr_ranks="7"), None) self.assertEqual(tc.mpi_cmd_prefix(), None) + # template with extra spaces at the end if fine though + init_config(build_options={'mpi_cmd_template': "mpirun -np %(nr_ranks)s %(cmd)s ", 'silent': True}) + self.assertEqual(tc.mpi_cmd_prefix(), "mpirun -np 1") + def test_mpi_cmd_for(self): """Test mpi_cmd_for function.""" self.modtool.prepend_module_path(self.test_prefix) @@ -1012,6 +1016,16 @@ def test_mpi_cmd_for(self): init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s", 'silent': True}) self.assertEqual(tc.mpi_cmd_for('test123', '7'), "mpiexec -np 7 -- test123") + # check whether expected error is raised when a template with missing keys is used; + # %(ranks)s should be %(nr_ranks)s + init_config(build_options={'mpi_cmd_template': "mpiexec -np %(ranks)s -- %(cmd)s", 'silent': True}) + error_pattern = r"Missing templates in mpi-cmd-template value 'mpiexec -np %\(ranks\)s -- %\(cmd\)s': %\(nr_ranks\)s" + self.assertErrorRegex(EasyBuildError, error_pattern, tc.mpi_cmd_for, 'test', 1) + + init_config(build_options={'mpi_cmd_template': "mpirun %(foo)s -np %(nr_ranks)s %(cmd)s", 'silent': True}) + error_pattern = "Failed to complete MPI cmd template .* with .*: KeyError 'foo'" + self.assertErrorRegex(EasyBuildError, error_pattern, tc.mpi_cmd_for, 'test', 1) + def test_prepare_deps(self): """Test preparing for a toolchain when dependencies are involved.""" tc = self.get_toolchain('GCC', version='6.4.0-2.28') From 262bca2263304638da1040d70a440df459b46a6b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 31 Mar 2020 20:19:17 +0200 Subject: [PATCH 292/344] enhance test_templating to check whether %(mpi_cmd_prefix)s works as expected --- test/framework/easyconfig.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 0f45797bc7..ad95b6fc9a 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1008,6 +1008,19 @@ def test_templating(self): eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s" self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s") + # test use of %(mpi_cmd_prefix)s template + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + gompi_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-2018a.eb') + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, read_file(gompi_ec) + "\nsanity_check_commands = ['%(mpi_cmd_prefix)s toy']") + + ec = EasyConfig(test_ec) + self.assertEqual(ec['sanity_check_commands'], ['mpirun -n 1 toy']) + + init_config(build_options={'mpi_cmd_template': "mpiexec -np %(nr_ranks)s -- %(cmd)s "}) + ec = EasyConfig(test_ec) + self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy']) + def test_templating_doc(self): """test templating documentation""" doc = avail_easyconfig_templates() From e9847690c634c5795fdaaaa9f27cf3b09c2cec4a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 1 Apr 2020 08:49:52 +0200 Subject: [PATCH 293/344] Keep fully resolved dependencies in the reprod easyconfigs --- easybuild/framework/easyblock.py | 2 +- easybuild/framework/easyconfig/easyconfig.py | 15 ++++++++------- test/framework/easyconfig.py | 9 +++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8ba572464a..b111a0b076 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3295,7 +3295,7 @@ def reproduce_build(app, reprod_dir_root): reprod_dir = find_backup_name_candidate(os.path.join(reprod_dir_root, REPROD)) reprod_spec = os.path.join(reprod_dir, ec_filename) try: - app.cfg.dump(reprod_spec) + app.cfg.dump(reprod_spec, explicit_toolchains=True) _log.info("Dumped easyconfig instance to %s", reprod_spec) except NotImplementedError as err: _log.warning("Unable to dump easyconfig instance to %s: %s", reprod_spec, err) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 054a9c7f96..013c18900c 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1096,7 +1096,7 @@ def all_dependencies(self): return self._all_dependencies - def dump(self, fp, always_overwrite=True, backup=False): + def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=False): """ Dump this easyconfig to file, with the given filename. @@ -1126,12 +1126,13 @@ def dump(self, fp, always_overwrite=True, backup=False): templ_val[self.template_values[key]] = key toolchain_hierarchy = None - try: - toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain']) - except EasyBuildError as err: - # don't fail hard just because we can't get the hierarchy - self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, ' - 'error:\n%s', self['toolchain'], str(err)) + if not explicit_toolchains: + try: + toolchain_hierarchy = get_toolchain_hierarchy(self['toolchain']) + except EasyBuildError as err: + # don't fail hard just because we can't get the hierarchy + self.log.warning('Could not generate toolchain hierarchy for %s to use in easyconfig dump method, ' + 'error:\n%s', self['toolchain'], str(err)) try: ectxt = self.parser.dump(self, default_values, templ_const, templ_val, diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 0dfa0b587d..eba9109211 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1684,7 +1684,16 @@ def test_toolchain_hierarchy_aware_dump(self): # dict representation of EasyConfig instance should not change after dump self.assertEqual(ecdict, ec.asdict()) ectxt = read_file(test_ec) + dumped_ec = EasyConfig(test_ec) + self.assertEqual(ecdict, dumped_ec.asdict()) self.assertTrue(r"'toy', '0.0')," in ectxt) + # test case where we ask for explicit toolchains + ec.dump(test_ec, explicit_toolchains=True) + self.assertEqual(ecdict, ec.asdict()) + ectxt = read_file(test_ec) + dumped_ec = EasyConfig(test_ec) + self.assertEqual(ecdict, dumped_ec.asdict()) + self.assertTrue(r"'toy', '0.0', '', ('gompi', '2018a'))," in ectxt) def test_dump_order(self): """Test order of easyconfig parameters in dumped easyconfig.""" From c6b00483b2d51837aaf78b85154f5d042d0f6168 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 1 Apr 2020 08:55:42 +0200 Subject: [PATCH 294/344] Appease the hound --- test/framework/toolchain.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 924017888a..2b0fc84634 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1019,7 +1019,8 @@ def test_mpi_cmd_for(self): # check whether expected error is raised when a template with missing keys is used; # %(ranks)s should be %(nr_ranks)s init_config(build_options={'mpi_cmd_template': "mpiexec -np %(ranks)s -- %(cmd)s", 'silent': True}) - error_pattern = r"Missing templates in mpi-cmd-template value 'mpiexec -np %\(ranks\)s -- %\(cmd\)s': %\(nr_ranks\)s" + error_pattern = \ + r"Missing templates in mpi-cmd-template value 'mpiexec -np %\(ranks\)s -- %\(cmd\)s': %\(nr_ranks\)s" self.assertErrorRegex(EasyBuildError, error_pattern, tc.mpi_cmd_for, 'test', 1) init_config(build_options={'mpi_cmd_template': "mpirun %(foo)s -np %(nr_ranks)s %(cmd)s", 'silent': True}) From 7aab622649da54e88c996ebdf2d2bb2b27311bee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 1 Apr 2020 20:12:11 +0200 Subject: [PATCH 295/344] determine initial set of template values without passing Toolchain instance, roll back workaround in test_resolve_dependencies_minimal w.r.t. use of %(version_minor)s template to define toolchain version --- easybuild/framework/easyconfig/easyconfig.py | 23 ++++++++++++++++---- test/framework/robot.py | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index e3bf620301..50c3ee1ffd 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1459,17 +1459,32 @@ def generate_template_values(self): def _generate_template_values(self, ignore=None): """Actual code to generate the template values""" - if self.template_values is None: - self.template_values = {} # step 0. self.template_values can/should be updated from outside easyconfig - # (eg the run_setp code in EasyBlock) + # (eg the run_step code in EasyBlock) # step 1-3 work with easyconfig.templates constants # disable templating with creating dict with template values to avoid looping back to here via __getitem__ prev_enable_templating = self.enable_templating + + self.enable_templating = False + + if self.template_values is None: + # if no template values are set yet, initiate with a minimal set of template values; + # this is important for easyconfig that use %(version_minor)s to define 'toolchain', + # which is a pretty weird use case, but fine... + self.template_values = template_constant_dict(self, ignore=ignore) + + self.enable_templating = prev_enable_templating + + # grab toolchain instance with templating support enabled, + # which is important in case the Toolchain instance was not created yet + toolchain = self.toolchain + + # get updated set of template values, now with toolchain instance + # (which is used to define the %(mpi_cmd_prefix)s template) self.enable_templating = False - template_values = template_constant_dict(self, ignore=ignore, toolchain=self.toolchain) + template_values = template_constant_dict(self, ignore=ignore, toolchain=toolchain) self.enable_templating = prev_enable_templating # update the template_values dict diff --git a/test/framework/robot.py b/test/framework/robot.py index 549e9382cd..41df63a315 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -432,7 +432,7 @@ def test_resolve_dependencies_minimal(self): " ('SQLite', '3.8.10.2'),", "]", # toolchain as list line, for easy modification later; - "toolchain = {'name': 'foss', 'version': '2018a'}", + "toolchain = {'name': 'foss', 'version': '%(version_minor)s018a'}", ] write_file(barec, '\n'.join(barec_lines)) bar = process_easyconfig(barec)[0] From 76ef5f80991f89af17dd312f3cae23913baeda05 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 4 Apr 2020 17:35:16 +0200 Subject: [PATCH 296/344] use subdir in temp dir for easyblocks downloaded from PR --- easybuild/tools/github.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 0dd8eced55..400d226a0c 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -393,7 +393,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): if github_repo == GITHUB_EASYCONFIGS_REPO: path = build_option('pr_path') elif github_repo == GITHUB_EASYBLOCKS_REPO: - path = 'ebs_pr%s' % pr + path = os.path.join(tempfile.gettempdir(), 'ebs_pr%s' % pr) + else: + raise EasyBuildError("Unknown repo: %s" % github_repo) if path is None: path = tempfile.mkdtemp() From 9569fc31b2b595d4153ef9a63c6e11111c160e0e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 5 Apr 2020 18:20:09 +0200 Subject: [PATCH 297/344] change implementation of det_pr_target_repo to always honor --pr-target-repo, make it return None if guessing didn't work + use None as default for --pr-target-repo --- easybuild/tools/github.py | 60 ++++++++++++++++++++++++++------------ easybuild/tools/options.py | 3 +- test/framework/github.py | 53 +++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 20 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8164f25c7d..61039b60fa 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -692,6 +692,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ raise EasyBuildError("No paths specified") pr_target_repo = det_pr_target_repo(paths) + if pr_target_repo is None: + raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!") # initialize repository git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') @@ -1232,7 +1234,7 @@ def close_pr(pr, motivation_msg=None): raise EasyBuildError("GitHub user must be specified to use --close-pr") pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) @@ -1305,7 +1307,7 @@ def list_prs(params, per_page=GITHUB_MAX_PER_PAGE, github_user=None): print_msg("Listing PRs with parameters: %s" % ', '.join(k + '=' + str(parameters[k]) for k in sorted(parameters))) pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO pr_data, _ = fetch_pr_data(None, pr_target_account, pr_target_repo, github_user, **parameters) @@ -1325,7 +1327,7 @@ def merge_pr(pr): raise EasyBuildError("GitHub user must be specified to use --merge-pr") pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO pr_data, pr_url = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) @@ -1388,7 +1390,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, pr_target_account = build_option('pr_target_account') pr_target_branch = build_option('pr_target_branch') if pr_target_repo is None: - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO # fetch GitHub token (required to perform actions on GitHub) github_user = build_option('github_user') @@ -1623,7 +1625,7 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None): pr_target_account = build_option('pr_target_account') if pr_target_repo is None: - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO pr_data, _ = fetch_pr_data(pr_id, pr_target_account, pr_target_repo, github_user) @@ -1637,24 +1639,42 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None): def det_pr_target_repo(paths): - """Determine pr_target_repo from cagetorized list of files + """Determine target repository for pull request from given cagetorized list of files - :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches) + :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches, .py files) """ - pr_target_repo = build_option('pr_target_repo') - if pr_target_repo == GITHUB_EASYCONFIGS_REPO: - if paths['py_files']: - if any([get_easyblock_class_name(path) for path in paths['py_files']]): + # determine target repository for PR based on which files are provided + # (see categorize_files_by_type function) + if pr_target_repo is None: + + _log.info("Trying to derive target repository based on specified files...") + + easyconfigs, files_to_delete, patch_files, py_files = [paths[key] for key in sorted(paths.keys())] + + # Python files provided, and no easyconfig files or patches + if py_files and not (easyconfigs or patch_files): + + _log.info("Only Python files provided, no easyconfig files or patches...") + + # if all Python files are easyblocks, target repo should be easyblocks; + # otherwise, target repo is assumed to be framework + if all([get_easyblock_class_name(path) for path in py_files]): pr_target_repo = GITHUB_EASYBLOCKS_REPO + _log.info("All Python files are easyblocks, target repository is assumed to be %s", pr_target_repo) else: - raise EasyBuildError("You are submitting python files that are not easyblocks, " - "did you forget to specify --pr-target-repo=easybuild-framework?") - else: - if paths['easyconfigs'] or paths['patch_files']: - raise EasyBuildError("You are submitting easyconfigs and/or patches, " - "shouldn\'t this PR target the easyconfigs repo?") + pr_target_repo = GITHUB_FRAMEWORK_REPO + _log.info("Not all Python files are easyblocks, target repository is assumed to be %s", pr_target_repo) + + # if no Python files are provided, only easyconfigs & patches, or if files to delete are .eb files, + # then target repo is assumed to be easyconfigs + elif easyconfigs or patch_files or (files_to_delete and all(x.endswith('.eb') for x in files_to_delete)): + pr_target_repo = GITHUB_EASYCONFIGS_REPO + _log.info("Only easyconfig and patch files found, target repository is assumed to be %s", pr_target_repo) + + else: + _log.info("No Python files, easyconfigs or patches found, can't derive target repository...") return pr_target_repo @@ -1703,6 +1723,8 @@ def update_pr(pr_id, paths, ecs, commit_msg=None): """ pr_target_repo = det_pr_target_repo(paths) + if pr_target_repo is None: + raise EasyBuildError("Failed to determine target repository, please specify it via --pr-target-repo!") github_account, branch_name = det_account_branch_for_pr(pr_id, pr_target_repo=pr_target_repo) @@ -2163,7 +2185,7 @@ def sync_pr_with_develop(pr_id): raise EasyBuildError("GitHub user must be specified to use --sync-pr-with-develop") target_account = build_option('pr_target_account') - target_repo = build_option('pr_target_repo') + target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO pr_account, pr_branch = det_account_branch_for_pr(pr_id) @@ -2186,7 +2208,7 @@ def sync_branch_with_develop(branch_name): raise EasyBuildError("GitHub user must be specified to use --sync-branch-with-develop") target_account = build_option('pr_target_account') - target_repo = build_option('pr_target_repo') + target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO # initialize repository git_working_dir = tempfile.mkdtemp(prefix='git-working-dir') diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 513bf715e6..0532f2846b 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -612,7 +612,8 @@ def github_options(self): 'pr-descr': ("Description for new pull request created with --new-pr", str, 'store', None), 'pr-target-account': ("Target account for new PRs", str, 'store', GITHUB_EB_MAIN), 'pr-target-branch': ("Target branch for new PRs", str, 'store', DEFAULT_BRANCH), - 'pr-target-repo': ("Target repository for new/updating PRs", str, 'store', GITHUB_EASYCONFIGS_REPO), + 'pr-target-repo': ("Target repository for new/updating PRs (default: auto-detect based on provided files)", + str, 'store', None), 'pr-title': ("Title for new pull request created with --new-pr", str, 'store', None), 'preview-pr': ("Preview a new pull request", None, 'store_true', False), 'sync-branch-with-develop': ("Sync branch with current 'develop' branch", str, 'store', None), diff --git a/test/framework/github.py b/test/framework/github.py index 4b4c68c31c..5e9837d6d2 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -37,6 +37,7 @@ from unittest import TextTestRunner from easybuild.base.rest import RestClient +from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj @@ -666,6 +667,58 @@ def test_det_account_branch_for_pr(self): self.assertEqual(account, 'migueldiascosta') self.assertEqual(branch, 'fix_inject_checksums') + def test_det_pr_target_repo(self): + """Test det_pr_target_repo.""" + + # no files => return default target repo (None) + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), None) + + # easyconfigs/patches (incl. files to delete) => easyconfigs repo + # this is solely based on filenames, actual files are not opened + test_cases = [ + ['toy.eb'], + ['toy.patch'], + ['toy.eb', 'toy.patch'], + [':toy.eb'], # deleting toy.eb + ['one.eb', 'two.eb'], + ['one.eb', 'two.eb', 'toy.patch', ':todelete.eb'], + ] + for test_case in test_cases: + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(test_case)), 'easybuild-easyconfigs') + + # if only Python files are involved, result is easyblocks or framework repo; + # all Python files are easyblocks => easyblocks repo, otherwise => framework repo; + # files are opened and inspected here to discriminate between easyblocks & other Python files, so must exist! + github_py = os.path.abspath(__file__) + testdir = os.path.dirname(github_py) + + configuremake = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') + self.assertTrue(os.path.exists(configuremake)) + toy_eb = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + self.assertTrue(os.path.exists(toy_eb)) + + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([github_py])), 'easybuild-framework') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([configuremake])), 'easybuild-easyblocks') + py_files = [github_py, configuremake] + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'easybuild-framework') + py_files[0] = toy_eb + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'easybuild-easyblocks') + py_files.append(github_py) + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'easybuild-framework') + + # as soon as an easyconfig file or patch files is involved => result is easybuild-easyconfigs repo + for fn in ['toy.eb', 'toy.patch']: + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files + [fn])), 'easybuild-easyconfigs') + + # if --pr-target-repo is specified, we always get this value (no guessing anymore) + init_config(build_options={'pr_target_repo': 'thisisjustatest'}) + + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), 'thisisjustatest') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(['toy.eb', 'toy.patch'])), 'thisisjustatest') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type(py_files)), 'thisisjustatest') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([configuremake])), 'thisisjustatest') + self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([toy_eb])), 'thisisjustatest') + def test_push_branch_to_github(self): """Test push_branch_to_github.""" From 653cae3d4ce40bbc9662d173a3e57614d2041c1d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Apr 2020 12:07:44 +0200 Subject: [PATCH 298/344] fix get_easyblock_class_name for easyblocks that derive from ExtensionEasyblock (+ add dedicated test) --- easybuild/tools/github.py | 2 +- test/framework/github.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 61039b60fa..cbed9f2cab 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -991,7 +991,7 @@ def get_easyblock_class_name(path): for cn, co in clsmembers: if co.__module__ == mod.__name__: ancestors = inspect.getmro(co) - if ancestors[-2].__name__ == 'EasyBlock': + if any(a.__name__ == 'EasyBlock' for a in ancestors): return cn return None diff --git a/test/framework/github.py b/test/framework/github.py index 5e9837d6d2..311ef988b9 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -748,6 +748,21 @@ def test_push_branch_to_github(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) + def test_get_easyblock_class_name(self): + """Test for get_easyblock_class_name function.""" + + topdir = os.path.dirname(os.path.abspath(__file__)) + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + + configuremake = os.path.join(test_ebs, 'generic', 'configuremake.py') + self.assertEqual(gh.get_easyblock_class_name(configuremake), 'ConfigureMake') + + gcc_eb = os.path.join(test_ebs, 'g', 'gcc.py') + self.assertEqual(gh.get_easyblock_class_name(gcc_eb), 'EB_GCC') + + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + self.assertEqual(gh.get_easyblock_class_name(toy_eb), 'EB_toy') + def suite(): """ returns all the testcases in this module """ From 0e2f33d3c7d2d0ff83e3562b893eb02afa6c0c0a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Apr 2020 14:43:16 +0200 Subject: [PATCH 299/344] use test_module_naming_scheme.py rather than a dedicated new file a_test.py in test_new_branch_github --- test/framework/options.py | 16 ++++++++++------ test/framework/sandbox/a_test.py | 3 --- 2 files changed, 10 insertions(+), 9 deletions(-) delete mode 100644 test/framework/sandbox/a_test.py diff --git a/test/framework/options.py b/test/framework/options.py index 3ef2ed2cb9..ad5ab94ad9 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2970,7 +2970,6 @@ def test_new_branch_github(self): args = [ '--new-branch-github', - '--pr-target-repo=easybuild-easyblocks', '--github-user=%s' % GITHUB_TEST_ACCOUNT, toy_eb, '--pr-title="add easyblock for toy"', @@ -2986,15 +2985,20 @@ def test_new_branch_github(self): ] self._assert_regexs(regexs, txt) - # test framework - test_ebs = os.path.join(topdir, 'sandbox') - toy_py = os.path.join(test_ebs, 'a_test.py') + # test framework with tweaked copy of test_module_naming_scheme.py + test_mns_py = os.path.join(topdir, 'sandbox', 'easybuild', 'tools', 'module_naming_scheme', + 'test_module_naming_scheme.py') + target_dir = os.path.join(self.test_prefix, 'easybuild-framework', 'test', 'framework', 'sandbox', + 'easybuild', 'tools', 'module_naming_scheme') + mkdir(target_dir, parents=True) + copy_file(test_mns_py, target_dir) + test_mns_py = os.path.join(target_dir, os.path.basename(test_mns_py)) + write_file(test_mns_py, '\n\n', append=True) args = [ '--new-branch-github', - '--pr-target-repo=easybuild-framework', '--github-user=%s' % GITHUB_TEST_ACCOUNT, - toy_py, + test_mns_py, '--pr-commit-msg="a test"', '-D', ] diff --git a/test/framework/sandbox/a_test.py b/test/framework/sandbox/a_test.py deleted file mode 100644 index 6d8a26f090..0000000000 --- a/test/framework/sandbox/a_test.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -Used for test_new_branch_github -""" From b6d0441ba9244ea83c1c906dfb79d7e421cd79d5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Apr 2020 20:20:33 +0200 Subject: [PATCH 300/344] use apt-spy2 before using apt in GitHub Actions --- .github/workflows/unit_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 283dc6c957..94dd61e29b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -49,6 +49,14 @@ jobs: - name: install OS & Python packages run: | + # use apt-spy2 to select closest apt mirror, + # which helps avoid connectivity issues in Azure; + # see https://github.com/actions/virtual-environments/issues/675 + sudo gem install apt-spy2 + sudo apt-spy2 check + sudo apt-spy2 fix --commit + # after selecting a specific mirror, we need to run 'apt-get update' + sudo apt-get update # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 From bf4ebe1baffe8aa6954704ee065b9a4bea77bf53 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 6 Apr 2020 17:06:14 +0200 Subject: [PATCH 301/344] move get_easyblock_class_name, copy_easyblocks and copy_framework_files to easybuild/tools/filetools.py, doesn't really belong in github.py + add tests for copy_easyblocks and copy_framework_files --- easybuild/framework/easyconfig/easyconfig.py | 9 +- easybuild/tools/filetools.py | 97 +++++++++++++- easybuild/tools/github.py | 95 +------------- test/framework/easyconfig.py | 3 + test/framework/filetools.py | 127 +++++++++++++++++++ test/framework/github.py | 15 --- 6 files changed, 236 insertions(+), 110 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 2a2fb3103c..4b04c73aa5 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -45,6 +45,7 @@ import re from distutils.version import LooseVersion +import easybuild.tools.filetools as filetools from easybuild.base import fancylogger from easybuild.framework.easyconfig import MANDATORY from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER @@ -61,7 +62,7 @@ from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme -from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name +from easybuild.tools.filetools import copy_file, decode_class_name, encode_class_name from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, read_file, write_file from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX @@ -1682,8 +1683,8 @@ def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error def is_generic_easyblock(easyblock): """Return whether specified easyblock name is a generic easyblock or not.""" - - return easyblock and not easyblock.startswith(EASYBLOCK_CLASS_PREFIX) + _log.deprecated("is_generic_easyblock function was moved to easybuild.tools.filetools", '5.0') + return filetools.is_generic_easyblock(easyblock) def get_module_path(name, generic=None, decode=True): @@ -1698,7 +1699,7 @@ def get_module_path(name, generic=None, decode=True): return None if generic is None: - generic = is_generic_easyblock(name) + generic = filetools.is_generic_easyblock(name) # example: 'EB_VSC_minus_tools' should result in 'vsc_tools' if decode: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3cb7979631..cd64bfda8e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -43,6 +43,8 @@ import fileinput import glob import hashlib +import imp +import inspect import os import re import shutil @@ -57,9 +59,9 @@ from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg -from easybuild.tools.config import build_option +from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option from easybuild.tools.py2vs3 import std_urllib, string_type -from easybuild.tools.utilities import nub +from easybuild.tools.utilities import nub, remove_unwanted_chars try: import requests @@ -2009,3 +2011,94 @@ def install_fake_vsc(): sys.path.insert(0, fake_vsc_path) return fake_vsc_path + + +def get_easyblock_class_name(path): + """Make sure file is an easyblock and get easyblock class name""" + fn = os.path.basename(path).split('.')[0] + mod = imp.load_source(fn, path) + clsmembers = inspect.getmembers(mod, inspect.isclass) + for cn, co in clsmembers: + if co.__module__ == mod.__name__: + ancestors = inspect.getmro(co) + if any(a.__name__ == 'EasyBlock' for a in ancestors): + return cn + return None + + +def is_generic_easyblock(easyblock): + """Return whether specified easyblock name is a generic easyblock or not.""" + + return easyblock and not easyblock.startswith(EASYBLOCK_CLASS_PREFIX) + + +def copy_easyblocks(paths, target_dir): + """ Find right location for easyblock file and copy it there""" + file_info = { + 'eb_names': [], + 'paths_in_repo': [], + 'new': [], + } + + subdir = os.path.join('easybuild', 'easyblocks') + if os.path.exists(os.path.join(target_dir, subdir)): + for path in paths: + cn = get_easyblock_class_name(path) + if not cn: + raise EasyBuildError("Could not determine easyblock class from file %s" % path) + + eb_name = remove_unwanted_chars(decode_class_name(cn).replace('-', '_')).lower() + + if is_generic_easyblock(cn): + pkgdir = GENERIC_EASYBLOCK_PKG + else: + pkgdir = eb_name[0] + + target_path = os.path.join(subdir, pkgdir, eb_name + '.py') + + full_target_path = os.path.join(target_dir, target_path) + file_info['eb_names'].append(eb_name) + file_info['paths_in_repo'].append(full_target_path) + file_info['new'].append(not os.path.exists(full_target_path)) + copy_file(path, full_target_path, force_in_dry_run=True) + + else: + raise EasyBuildError("Could not find %s subdir in %s", subdir, target_dir) + + return file_info + + +def copy_framework_files(paths, target_dir): + """ Find right location for framework file and copy it there""" + file_info = { + 'paths_in_repo': [], + 'new': [], + } + + paths = [os.path.abspath(path) for path in paths] + + framework_topdir = 'easybuild-framework' + + for path in paths: + target_path = None + dirnames = os.path.dirname(path).split(os.path.sep) + + print('[copy_framework_files] %s' % dirnames) + if framework_topdir in dirnames: + ind = dirnames.index(framework_topdir) + 1 + print(ind) + subdirs = dirnames[ind:] + print(subdirs) + parent_dir = os.path.join(*subdirs) if subdirs else '' + target_path = os.path.join(target_dir, parent_dir, os.path.basename(path)) + else: + raise EasyBuildError("Specified path '%s' does not include a '%s' directory!", path, framework_topdir) + + if target_path: + file_info['paths_in_repo'].append(target_path) + file_info['new'].append(not os.path.exists(target_path)) + copy_file(path, target_path) + else: + raise EasyBuildError("Couldn't find parent folder of updated file: %s", path) + + return file_info diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index cbed9f2cab..b9c92e0d8b 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -33,8 +33,6 @@ import copy import getpass import glob -import imp -import inspect import os import random import re @@ -48,16 +46,16 @@ from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR from easybuild.framework.easyconfig.easyconfig import copy_easyconfigs, copy_patch_files, det_file_info -from easybuild.framework.easyconfig.easyconfig import is_generic_easyblock, process_easyconfig +from easybuild.framework.easyconfig.easyconfig import process_easyconfig from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning -from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option -from easybuild.tools.filetools import apply_patch, copy_dir, copy_file, det_patched_files, decode_class_name -from easybuild.tools.filetools import download_file, extract_file, mkdir, read_file, symlink -from easybuild.tools.filetools import which, write_file +from easybuild.tools.config import build_option +from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files +from easybuild.tools.filetools import det_patched_files, decode_class_name, download_file, extract_file +from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version -from easybuild.tools.utilities import nub, only_if_module_is_available, remove_unwanted_chars +from easybuild.tools.utilities import nub, only_if_module_is_available _log = fancylogger.getLogger('github', fname=False) @@ -983,87 +981,6 @@ def find_software_name_for_patch(patch_name, ec_dirs): return soft_name -def get_easyblock_class_name(path): - """Make sure file is an easyblock and get easyblock class name""" - fn = os.path.basename(path).split('.')[0] - mod = imp.load_source(fn, path) - clsmembers = inspect.getmembers(mod, inspect.isclass) - for cn, co in clsmembers: - if co.__module__ == mod.__name__: - ancestors = inspect.getmro(co) - if any(a.__name__ == 'EasyBlock' for a in ancestors): - return cn - return None - - -def copy_easyblocks(paths, target_dir): - """ Find right location for easyblock file and copy it there""" - file_info = { - 'eb_names': [], - 'paths_in_repo': [], - 'new': [], - } - - subdir = os.path.join('easybuild', 'easyblocks') - if os.path.exists(os.path.join(target_dir, subdir)): - for path in paths: - cn = get_easyblock_class_name(path) - if not cn: - raise EasyBuildError("Could not determine easyblock class from file %s" % path) - - eb_name = remove_unwanted_chars(decode_class_name(cn).replace('-', '_')).lower() - - if is_generic_easyblock(cn): - pkgdir = GENERIC_EASYBLOCK_PKG - else: - pkgdir = eb_name[0] - - target_path = os.path.join(subdir, pkgdir, eb_name + '.py') - - full_target_path = os.path.join(target_dir, target_path) - file_info['eb_names'].append(eb_name) - file_info['paths_in_repo'].append(full_target_path) - file_info['new'].append(not os.path.exists(full_target_path)) - copy_file(path, full_target_path, force_in_dry_run=True) - - else: - raise EasyBuildError("Could not find %s" % os.path.join(target_dir, subdir)) - - return file_info - - -def copy_framework_files(paths, target_dir): - """ Find right location for framework file and copy it there""" - file_info = { - 'paths_in_repo': [], - 'new': [], - } - - paths = [os.path.abspath(path) for path in paths] - - target_path = None - for path in paths: - dirnames = os.path.dirname(path).split(os.path.sep) - - if 'easybuild-framework' in dirnames: - ind = dirnames.index('easybuild-framework') + 1 - parent_dir = os.path.join(*dirnames[ind:]) - - if os.path.exists(os.path.join(target_dir, parent_dir)): - target_path = os.path.join(target_dir, parent_dir) - - if target_path is None: - raise EasyBuildError("Couldn't find parent folder of updated file: %s" % path) - - full_target_path = os.path.join(target_path, os.path.basename(path)) - - file_info['paths_in_repo'].append(full_target_path) - file_info['new'].append(not os.path.exists(full_target_path)) - copy_file(path, full_target_path) - - return file_info - - def check_pr_eligible_to_merge(pr_data): """ Check whether PR is eligible for merging. diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 0545010e68..c23101359c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2983,6 +2983,9 @@ def test_get_paths_for(self): def test_is_generic_easyblock(self): """Test for is_generic_easyblock function.""" + # is_generic_easyblock in easyconfig.py is deprecated, moved to filetools.py + self.allow_deprecated_behaviour() + for name in ['Binary', 'ConfigureMake', 'CMakeMake', 'PythonPackage', 'JAR']: self.assertTrue(is_generic_easyblock(name)) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a96ee8a6a7..416f62235e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2097,6 +2097,133 @@ def test_fake_vsc(self): from test_fake_vsc import pkgutil self.assertTrue(pkgutil.__file__.endswith('/test_fake_vsc/pkgutil.py')) + def test_is_generic_easyblock(self): + """Test for is_generic_easyblock function.""" + + for name in ['Binary', 'ConfigureMake', 'CMakeMake', 'PythonPackage', 'JAR']: + self.assertTrue(ft.is_generic_easyblock(name)) + + for name in ['EB_bzip2', 'EB_DL_underscore_POLY_underscore_Classic', 'EB_GCC', 'EB_WRF_minus_Fire']: + self.assertFalse(ft.is_generic_easyblock(name)) + + def test_get_easyblock_class_name(self): + """Test for get_easyblock_class_name function.""" + + topdir = os.path.dirname(os.path.abspath(__file__)) + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + + configuremake = os.path.join(test_ebs, 'generic', 'configuremake.py') + self.assertEqual(ft.get_easyblock_class_name(configuremake), 'ConfigureMake') + + gcc_eb = os.path.join(test_ebs, 'g', 'gcc.py') + self.assertEqual(ft.get_easyblock_class_name(gcc_eb), 'EB_GCC') + + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + self.assertEqual(ft.get_easyblock_class_name(toy_eb), 'EB_toy') + + def test_copy_easyblocks(self): + """Test for copy_easyblocks function.""" + + topdir = os.path.dirname(os.path.abspath(__file__)) + test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') + + # easybuild/easyblocks subdirectory must exist in target directory + error_pattern = "Could not find easybuild/easyblocks subdir in .*" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_easyblocks, [], self.test_prefix) + + easyblocks_dir = os.path.join(self.test_prefix, 'easybuild', 'easyblocks') + + # passing empty list works fine + ft.mkdir(easyblocks_dir, parents=True) + res = ft.copy_easyblocks([], self.test_prefix) + self.assertEqual(os.listdir(easyblocks_dir), []) + self.assertEqual(res, {'eb_names': [], 'new': [], 'paths_in_repo': []}) + + # check with different types of easyblocks + configuremake = os.path.join(test_ebs, 'generic', 'configuremake.py') + gcc_eb = os.path.join(test_ebs, 'g', 'gcc.py') + toy_eb = os.path.join(test_ebs, 't', 'toy.py') + test_ebs = [gcc_eb, configuremake, toy_eb] + + # copy them straight into tmpdir first, to check whether correct subdir is derived correctly + ft.copy_files(test_ebs, self.test_prefix) + + # touch empty toy.py easyblock, to check whether 'new' aspect is determined correctly + ft.write_file(os.path.join(easyblocks_dir, 't', 'toy.py'), '') + + # check whether easyblocks were copied as expected, and returned dict is correct + test_ebs = [os.path.join(self.test_prefix, os.path.basename(e)) for e in test_ebs] + res = ft.copy_easyblocks(test_ebs, self.test_prefix) + + self.assertEqual(sorted(res.keys()), ['eb_names', 'new', 'paths_in_repo']) + self.assertEqual(res['eb_names'], ['gcc', 'configuremake', 'toy']) + self.assertEqual(res['new'], [True, True, False]) # toy.py is not new + + self.assertEqual(sorted(os.listdir(easyblocks_dir)), ['g', 'generic', 't']) + + g_dir = os.path.join(easyblocks_dir, 'g') + self.assertEqual(sorted(os.listdir(g_dir)), ['gcc.py']) + copied_gcc_eb = os.path.join(g_dir, 'gcc.py') + self.assertEqual(ft.read_file(copied_gcc_eb), ft.read_file(gcc_eb)) + self.assertTrue(os.path.samefile(res['paths_in_repo'][0], copied_gcc_eb)) + + gen_dir = os.path.join(easyblocks_dir, 'generic') + self.assertEqual(sorted(os.listdir(gen_dir)), ['configuremake.py']) + copied_configuremake = os.path.join(gen_dir, 'configuremake.py') + self.assertEqual(ft.read_file(copied_configuremake), ft.read_file(configuremake)) + self.assertTrue(os.path.samefile(res['paths_in_repo'][1], copied_configuremake)) + + t_dir = os.path.join(easyblocks_dir, 't') + self.assertEqual(sorted(os.listdir(t_dir)), ['toy.py']) + copied_toy_eb = os.path.join(t_dir, 'toy.py') + self.assertEqual(ft.read_file(copied_toy_eb), ft.read_file(toy_eb)) + self.assertTrue(os.path.samefile(res['paths_in_repo'][2], copied_toy_eb)) + + def test_copy_framework_files(self): + """Test for copy_framework_files function.""" + + target_dir = os.path.join(self.test_prefix, 'target') + ft.mkdir(target_dir) + + res = ft.copy_framework_files([], target_dir) + + self.assertEqual(os.listdir(target_dir), []) + self.assertEqual(res, {'paths_in_repo': [], 'new': []}) + + foo_py = os.path.join(self.test_prefix, 'foo.py') + ft.write_file(foo_py, '') + + error_pattern = "Specified path '.*/foo.py' does not include a 'easybuild-framework' directory!" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_framework_files, [foo_py], self.test_prefix) + + # create empty test/framework/modules.py, to check whether 'new' is set correctly in result + ft.write_file(os.path.join(target_dir, 'test', 'framework', 'modules.py'), '') + + topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + test_files = [ + os.path.join('easybuild', 'tools', 'filetools.py'), + os.path.join('test', 'framework', 'modules.py'), + os.path.join('test', 'framework', 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz'), + os.path.join('setup.py'), + ] + test_paths = [os.path.join(topdir, f) for f in test_files] + res = ft.copy_framework_files(test_paths, target_dir) + + self.assertEqual(sorted(os.listdir(target_dir)), ['easybuild', 'setup.py', 'test']) + + self.assertEqual(sorted(res.keys()), ['new', 'paths_in_repo']) + + for idx, test_file in enumerate(test_files): + orig_path = os.path.join(topdir, test_file) + copied_path = os.path.join(target_dir, test_file) + + self.assertTrue(os.path.exists(copied_path)) + self.assertEqual(ft.read_file(orig_path), ft.read_file(copied_path)) + + self.assertTrue(os.path.samefile(copied_path, res['paths_in_repo'][idx])) + + self.assertEqual(res['new'], [True, False, True, True]) # test/framework/moduels.py is not new + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/github.py b/test/framework/github.py index 311ef988b9..5e9837d6d2 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -748,21 +748,6 @@ def test_push_branch_to_github(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) - def test_get_easyblock_class_name(self): - """Test for get_easyblock_class_name function.""" - - topdir = os.path.dirname(os.path.abspath(__file__)) - test_ebs = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks') - - configuremake = os.path.join(test_ebs, 'generic', 'configuremake.py') - self.assertEqual(gh.get_easyblock_class_name(configuremake), 'ConfigureMake') - - gcc_eb = os.path.join(test_ebs, 'g', 'gcc.py') - self.assertEqual(gh.get_easyblock_class_name(gcc_eb), 'EB_GCC') - - toy_eb = os.path.join(test_ebs, 't', 'toy.py') - self.assertEqual(gh.get_easyblock_class_name(toy_eb), 'EB_toy') - def suite(): """ returns all the testcases in this module """ From ad6e6f8cf997b89c2908c9f73f558339fd4d1aeb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 08:16:47 +0200 Subject: [PATCH 302/344] fix composing of subdirectory of easybuild-framework/ in copy_framework_files --- easybuild/tools/filetools.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index a02ecba9c5..5d179a04cb 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2085,12 +2085,12 @@ def copy_framework_files(paths, target_dir): target_path = None dirnames = os.path.dirname(path).split(os.path.sep) - print('[copy_framework_files] %s' % dirnames) if framework_topdir in dirnames: - ind = dirnames.index(framework_topdir) + 1 - print(ind) - subdirs = dirnames[ind:] - print(subdirs) + # construct subdirectory by grabbing last entry in dirnames until we hit 'easybuild-framework' dir + subdirs = [] + while(dirnames[-1] != framework_topdir): + subdirs.insert(0, dirnames.pop()) + parent_dir = os.path.join(*subdirs) if subdirs else '' target_path = os.path.join(target_dir, parent_dir, os.path.basename(path)) else: From f994a831450a88046be870fb38da09b469e8a0e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 09:14:33 +0200 Subject: [PATCH 303/344] silence deprecation warnings for is_generic_easyblock in easyconfig.py tests --- test/framework/easyconfig.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c2e26ce0f7..3b720f687c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3002,12 +3002,16 @@ def test_is_generic_easyblock(self): # is_generic_easyblock in easyconfig.py is deprecated, moved to filetools.py self.allow_deprecated_behaviour() + self.mock_stderr(True) + for name in ['Binary', 'ConfigureMake', 'CMakeMake', 'PythonPackage', 'JAR']: self.assertTrue(is_generic_easyblock(name)) for name in ['EB_bzip2', 'EB_DL_underscore_POLY_underscore_Classic', 'EB_GCC', 'EB_WRF_minus_Fire']: self.assertFalse(is_generic_easyblock(name)) + self.mock_stderr(False) + def test_get_module_path(self): """Test get_module_path function.""" self.assertEqual(get_module_path('EB_bzip2', generic=False), 'easybuild.easyblocks.bzip2') From 9e480601393341efbb6b472c9df2eccf70f6f2a8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 09:42:28 +0200 Subject: [PATCH 304/344] read files in binary mode in test_copy_framework_files to avoid broken test in Python 3 --- test/framework/filetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index deda12439c..8f9aa5312b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2218,7 +2218,7 @@ def test_copy_framework_files(self): copied_path = os.path.join(target_dir, test_file) self.assertTrue(os.path.exists(copied_path)) - self.assertEqual(ft.read_file(orig_path), ft.read_file(copied_path)) + self.assertEqual(ft.read_file(orig_path, mode='rb'), ft.read_file(copied_path, mode='rb')) self.assertTrue(os.path.samefile(copied_path, res['paths_in_repo'][idx])) From e549a963d4a5908ffd0699cb771c22e1a4c56101 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 10:20:23 +0200 Subject: [PATCH 305/344] appease the Hound --- easybuild/tools/github.py | 4 ++-- test/framework/filetools.py | 2 +- test/framework/options.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index f2d2ef4b60..d24d87e7ca 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -50,8 +50,8 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option -from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_file, copy_framework_files -from easybuild.tools.filetools import det_patched_files, decode_class_name, download_file, extract_file +from easybuild.tools.filetools import apply_patch, copy_dir, copy_easyblocks, copy_framework_files +from easybuild.tools.filetools import det_patched_files, download_file, extract_file from easybuild.tools.filetools import get_easyblock_class_name, mkdir, read_file, symlink, which, write_file from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8f9aa5312b..1940f29946 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2166,7 +2166,7 @@ def test_copy_easyblocks(self): copied_gcc_eb = os.path.join(g_dir, 'gcc.py') self.assertEqual(ft.read_file(copied_gcc_eb), ft.read_file(gcc_eb)) self.assertTrue(os.path.samefile(res['paths_in_repo'][0], copied_gcc_eb)) - + gen_dir = os.path.join(easyblocks_dir, 'generic') self.assertEqual(sorted(os.listdir(gen_dir)), ['configuremake.py']) copied_configuremake = os.path.join(gen_dir, 'configuremake.py') diff --git a/test/framework/options.py b/test/framework/options.py index 668d8103e4..84409ff8ea 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3162,7 +3162,7 @@ def test_new_branch_github(self): # test framework with tweaked copy of test_module_naming_scheme.py test_mns_py = os.path.join(topdir, 'sandbox', 'easybuild', 'tools', 'module_naming_scheme', - 'test_module_naming_scheme.py') + 'test_module_naming_scheme.py') target_dir = os.path.join(self.test_prefix, 'easybuild-framework', 'test', 'framework', 'sandbox', 'easybuild', 'tools', 'module_naming_scheme') mkdir(target_dir, parents=True) From cc38187698d7723493624bba4db4d1620da863d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 12:47:07 +0200 Subject: [PATCH 306/344] make sure files being copied are in 'easybuild-framework' directory in test_copy_framework_files --- test/framework/filetools.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 1940f29946..c40b454507 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2206,7 +2206,15 @@ def test_copy_framework_files(self): os.path.join('test', 'framework', 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz'), os.path.join('setup.py'), ] - test_paths = [os.path.join(topdir, f) for f in test_files] + + # files being copied are expected to be in a directory named 'easybuild-framework', + # so we need to make sure that's the case here as well (may not be in workspace dir on Travis from example) + framework_dir = os.path.join(self.test_prefix, 'easybuild-framework') + for test_file in test_files: + ft.copy_file(os.path.join(topdir, test_file), os.path.join(framework_dir, test_file)) + + test_paths = [os.path.join(framework_dir, f) for f in test_files] + res = ft.copy_framework_files(test_paths, target_dir) self.assertEqual(sorted(os.listdir(target_dir)), ['easybuild', 'setup.py', 'test']) From bd91e0a2b7e63a82deec9573bd883be7e09ece26 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 12:51:19 +0200 Subject: [PATCH 307/344] make sure pr_target_repo build option is what we expect it to be in test_det_pr_target_repo --- test/framework/github.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/framework/github.py b/test/framework/github.py index 8906e7f78d..4ddce65e38 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -39,7 +39,7 @@ from easybuild.base.rest import RestClient from easybuild.framework.easyconfig.tools import categorize_files_by_type from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import module_classes +from easybuild.tools.config import build_option, module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import VALID_CLOSE_PR_REASONS @@ -697,6 +697,8 @@ def test_det_account_branch_for_pr(self): def test_det_pr_target_repo(self): """Test det_pr_target_repo.""" + self.assertEqual(build_option('pr_target_repo'), None) + # no files => return default target repo (None) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([])), None) @@ -724,6 +726,7 @@ def test_det_pr_target_repo(self): toy_eb = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') self.assertTrue(os.path.exists(toy_eb)) + self.assertEqual(build_option('pr_target_repo'), None) self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([github_py])), 'easybuild-framework') self.assertEqual(gh.det_pr_target_repo(categorize_files_by_type([configuremake])), 'easybuild-easyblocks') py_files = [github_py, configuremake] From aab059c5dd18c8991fe56aa90c2dae54fffd2826 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 13:16:53 +0200 Subject: [PATCH 308/344] conditionally include setup.py in list of test files in test_copy_framework_files, since it may not be there... --- test/framework/filetools.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index c40b454507..9e95d45779 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2204,8 +2204,21 @@ def test_copy_framework_files(self): os.path.join('easybuild', 'tools', 'filetools.py'), os.path.join('test', 'framework', 'modules.py'), os.path.join('test', 'framework', 'sandbox', 'sources', 'toy', 'toy-0.0.tar.gz'), - os.path.join('setup.py'), ] + expected_entries = ['easybuild', 'test'] + # test/framework/modules.py is not new + expected_new = [True, False, True] + + # we include setup.py conditionally because it may not be there, + # for example when running the tests on an actual easybuild-framework instalation, + # as opposed to when running from a repository checkout... + # setup.py is an important test case, since it has no parent directory + # (it's straight in the easybuild-framework directory) + setup_py = 'setup.py' + if os.path.exists(os.path.join(topdir, setup_py)): + test_files.append(os.path.join(setup_py)) + expected_entries.append(setup_py) + expected_new.append(True) # files being copied are expected to be in a directory named 'easybuild-framework', # so we need to make sure that's the case here as well (may not be in workspace dir on Travis from example) @@ -2217,7 +2230,7 @@ def test_copy_framework_files(self): res = ft.copy_framework_files(test_paths, target_dir) - self.assertEqual(sorted(os.listdir(target_dir)), ['easybuild', 'setup.py', 'test']) + self.assertEqual(sorted(os.listdir(target_dir)), sorted(expected_entries)) self.assertEqual(sorted(res.keys()), ['new', 'paths_in_repo']) @@ -2230,7 +2243,7 @@ def test_copy_framework_files(self): self.assertTrue(os.path.samefile(copied_path, res['paths_in_repo'][idx])) - self.assertEqual(res['new'], [True, False, True, True]) # test/framework/moduels.py is not new + self.assertEqual(res['new'], expected_new) def suite(): From 1ac9137f117a0e4dfa54c9f4e7f7c2a0ffa727c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 7 Apr 2020 13:20:18 +0200 Subject: [PATCH 309/344] fix test_det_pr_target_repo, take into account that __file__ may point to github.pyc... --- test/framework/github.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index 4ddce65e38..bd1e7cecd4 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -718,8 +718,8 @@ def test_det_pr_target_repo(self): # if only Python files are involved, result is easyblocks or framework repo; # all Python files are easyblocks => easyblocks repo, otherwise => framework repo; # files are opened and inspected here to discriminate between easyblocks & other Python files, so must exist! - github_py = os.path.abspath(__file__) - testdir = os.path.dirname(github_py) + testdir = os.path.dirname(os.path.abspath(__file__)) + github_py = os.path.join(testdir, 'github.py') configuremake = os.path.join(testdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'configuremake.py') self.assertTrue(os.path.exists(configuremake)) From cca1c795d59bf8ec07fe4fa0b77202626aa8438a Mon Sep 17 00:00:00 2001 From: darkless Date: Tue, 7 Apr 2020 17:31:03 +0200 Subject: [PATCH 310/344] Added `dirs_exist_ok` option to filetools.copy_dir() --- easybuild/tools/filetools.py | 34 ++++++++++++++++++++++++++++++---- test/framework/filetools.py | 18 ++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 6ce5ebb5a1..c2813cc6c7 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1808,24 +1808,50 @@ def copy_files(paths, target_dir, force_in_dry_run=False): copy_file(path, target_dir) -def copy_dir(path, target_path, force_in_dry_run=False, **kwargs): +def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): """ Copy a directory from specified location to specified location :param path: the original directory path :param target_path: path to copy the directory to :param force_in_dry_run: force running the command during dry run + :param dirs_exist_ok: wrapper around shutil.copytree option, which was added in Python 3.8 - Additional specified named arguments are passed down to shutil.copytree + On Python >= 3.8 shutil.copytree is always used + On Python < 3.8 if 'dirs_exist_ok' is False - shutil.copytree is used + On Python < 3.8 if 'dirs_exist_ok' is True - distutils.dir_util.copy_tree is used + + Additional specified named arguments are passed down to shutil.copytree if used. + + Because distutils.dir_util.copy_tree supports only 'symlinks' named argument, + using any other will raise EasyBuildError. """ if not force_in_dry_run and build_option('extended_dry_run'): dry_run_msg("copied directory %s to %s" % (path, target_path)) else: try: - if os.path.exists(target_path): + if not dirs_exist_ok and os.path.exists(target_path): raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path) - shutil.copytree(path, target_path, **kwargs) + # On Python >= 3.8 + if (sys.version_info[0] == 3 and sys.version_info[1] >= 8) or sys.version_info[0] > 3: + # Use the shutil.copytree WITH 'dirs_exist_ok' + shutil.copytree(path, target_path, dirs_exist_ok=dirs_exist_ok, **kwargs) + + elif dirs_exist_ok: + preserve_symlinks = False + # Get symlinks named argument and use distutils.dir_util.copy_tree instead. + if 'symlinks' in kwargs: + preserve_symlinks = kwargs.pop('symlinks', False) + # Check if there are other named arguments + if len(kwargs) > 0: + raise EasyBuildError("You can't use 'dirs_exist_ok=True' with other named arguments: %s", kwargs) + distutils.dir_util.copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) + + else: + # Use shutil.copytree WITHOUT 'dirs_exist_ok' + shutil.copytree(path, target_path, **kwargs) + _log.info("%s copied to %s", path, target_path) except (IOError, OSError) as err: raise EasyBuildError("Failed to copy directory %s to %s: %s", path, target_path, err) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index e5fefa80d4..ebb48df18b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1470,6 +1470,24 @@ def test_copy_dir(self): ft.mkdir(testdir) self.assertErrorRegex(EasyBuildError, "Target location .* already exists", ft.copy_dir, to_copy, testdir) + # if the directory already exists and 'dirs_exist_ok' is True, copy_dir should succeed + ft.copy_dir(to_copy, testdir, dirs_exist_ok=True) + self.assertTrue(sorted(os.listdir(to_copy)) == sorted(os.listdir(testdir))) + + # if the directory already exists and 'dirs_exist_ok' is True and there is another named argument (ignore) + # we expect clean error on Python < 3.8 and pass the test on Python >= 3.8 + # NOTE: reused ignore from previous test + shutil.rmtree(testdir) + ft.mkdir(testdir) + if (sys.version_info[0] == 3 and sys.version_info[1] >= 8) or sys.version_info[0] > 3: + ft.copy_dir(to_copy, testdir, dirs_exist_ok=True, + ignore=lambda src, names: [x for x in names if '6.4.0-2.28' in x]) + self.assertEqual(sorted(os.listdir(testdir)), expected) + self.assertFalse(os.path.exists(os.path.join(testdir, 'GCC-6.4.0-2.28.eb'))) + else: + self.assertErrorRegex(EasyBuildError, "You can't use 'dirs_exist_ok=True' with other named arguments .*", + ft.copy_dir, to_copy, testdir, dirs_exist_ok=True, ignore=lambda src, names: []) + # also test behaviour of copy_file under --dry-run build_options = { 'extended_dry_run': True, From 009047ea29d33bc3fe888e05f399e1626af7b525 Mon Sep 17 00:00:00 2001 From: darkless Date: Tue, 7 Apr 2020 19:33:46 +0200 Subject: [PATCH 311/344] Added import --- easybuild/tools/filetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index c2813cc6c7..52a288d3fd 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -40,6 +40,7 @@ """ import datetime import difflib +import distutils import fileinput import glob import hashlib From 552b958dd5c25b081ed3d15d5d14d66336960ed5 Mon Sep 17 00:00:00 2001 From: darkless Date: Tue, 7 Apr 2020 19:52:14 +0200 Subject: [PATCH 312/344] Fixed import --- easybuild/tools/filetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 52a288d3fd..3812830081 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -40,7 +40,6 @@ """ import datetime import difflib -import distutils import fileinput import glob import hashlib @@ -52,6 +51,7 @@ import tempfile import time import zlib +from distutils.dir_util import copy_tree from xml.etree import ElementTree from easybuild.base import fancylogger @@ -1847,7 +1847,7 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k # Check if there are other named arguments if len(kwargs) > 0: raise EasyBuildError("You can't use 'dirs_exist_ok=True' with other named arguments: %s", kwargs) - distutils.dir_util.copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) + copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) else: # Use shutil.copytree WITHOUT 'dirs_exist_ok' From fce83661e4bfc07f3e5f31263703f22ad4117207 Mon Sep 17 00:00:00 2001 From: darkless Date: Wed, 8 Apr 2020 08:37:43 +0200 Subject: [PATCH 313/344] Fixed test --- easybuild/tools/filetools.py | 3 ++- test/framework/filetools.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 3812830081..162b597bda 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1846,7 +1846,8 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k preserve_symlinks = kwargs.pop('symlinks', False) # Check if there are other named arguments if len(kwargs) > 0: - raise EasyBuildError("You can't use 'dirs_exist_ok=True' with other named arguments: %s", kwargs) + raise EasyBuildError("You can't use 'dirs_exist_ok=True' with other named arguments: %s", + list(kwargs.keys())) copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) else: diff --git a/test/framework/filetools.py b/test/framework/filetools.py index ebb48df18b..980a30ecf1 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1485,7 +1485,7 @@ def test_copy_dir(self): self.assertEqual(sorted(os.listdir(testdir)), expected) self.assertFalse(os.path.exists(os.path.join(testdir, 'GCC-6.4.0-2.28.eb'))) else: - self.assertErrorRegex(EasyBuildError, "You can't use 'dirs_exist_ok=True' with other named arguments .*", + self.assertErrorRegex(EasyBuildError, "You can't use 'dirs_exist_ok=True' with other named arguments:.*", ft.copy_dir, to_copy, testdir, dirs_exist_ok=True, ignore=lambda src, names: []) # also test behaviour of copy_file under --dry-run From 72c6b3c011f8dafd4f5f376a1ce3d013dcafbb98 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 10:31:57 +0200 Subject: [PATCH 314/344] fix minor typo in test_create_index --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 9740759008..8f681b0cab 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5016,7 +5016,7 @@ def test_create_index(self): self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) # also test creating index that's infinitely valid - args.extend(['--index-max-ag=0', '--force']) + args.extend(['--index-max-age=0', '--force']) self._run_mock_eb(args, raise_error=True) index_txt = read_file(index_fp) regex = re.compile(r"^# valid until: 9999-12-31 23:59:59", re.M) From f64ae2c95690d8bf7c5174be664d2eee3612195a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 10:34:44 +0200 Subject: [PATCH 315/344] fix typo in import + appease the Hound --- easybuild/framework/easyconfig/easyconfig.py | 4 ++-- easybuild/tools/filetools.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 3e8d6b1487..a47c3618d0 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -62,8 +62,8 @@ from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN from easybuild.tools.config import Singleton, build_option, get_module_naming_scheme -from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, copy_file, decode_class_name, encode_class_name -from easybuild.tools.filetools import create_index, find_backup_name_candidate, find_easyconfigs, load_index +from easybuild.tools.filetools import copy_file, create_index, decode_class_name, encode_class_name +from easybuild.tools.filetools import find_backup_name_candidate, find_easyconfigs, load_index from easybuild.tools.filetools import read_file, write_file from easybuild.tools.hooks import PARSE, load_hooks, run_hook from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 41b59892a7..bc8ed98652 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -59,7 +59,7 @@ from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning -from easybuild.tools.config import ,GENERIC_EASYBLOCK_PKG build_option +from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option from easybuild.tools.py2vs3 import std_urllib, string_type from easybuild.tools.utilities import nub, remove_unwanted_chars From b31bfa84af621f627ad5c71c9203f4ee958a2cee Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 10:59:00 +0200 Subject: [PATCH 316/344] take into account --ignore-index in load_index + check for it in tests --- easybuild/tools/filetools.py | 5 ++++- test/framework/filetools.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index bc8ed98652..8f357d9c6b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -684,7 +684,10 @@ def load_index(path, ignore_dirs=None): index_fp = os.path.join(path, PATH_INDEX_FILENAME) index = set() - if os.path.exists(index_fp): + if build_option('ignore_index'): + _log.info("Ignoring index for %s...", path) + + elif os.path.exists(index_fp): lines = read_file(index_fp).splitlines() valid_ts_regex = re.compile("^# valid until: (.*)", re.M) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index d3e1519b4e..f03d126e8f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1794,6 +1794,10 @@ def test_index_functions(self): regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) + # check whether load_index takes into account --ignore-index + init_config(build_options={'ignore_index': True}) + self.assertEqual(ft.load_index(self.test_prefix), None) + def test_search_file(self): """Test search_file function.""" test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 30198fcce0473393e92259ac146171adcd82ace2 Mon Sep 17 00:00:00 2001 From: darkless Date: Wed, 8 Apr 2020 12:17:59 +0200 Subject: [PATCH 317/344] Fixed remarks and imports --- easybuild/tools/filetools.py | 13 ++++++------- test/framework/filetools.py | 9 +++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 162b597bda..92b22d5e8e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -40,6 +40,7 @@ """ import datetime import difflib +import distutils.dir_util import fileinput import glob import hashlib @@ -51,7 +52,6 @@ import tempfile import time import zlib -from distutils.dir_util import copy_tree from xml.etree import ElementTree from easybuild.base import fancylogger @@ -1835,20 +1835,19 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path) # On Python >= 3.8 - if (sys.version_info[0] == 3 and sys.version_info[1] >= 8) or sys.version_info[0] > 3: + if sys.version_info >= (3, 8): # Use the shutil.copytree WITH 'dirs_exist_ok' shutil.copytree(path, target_path, dirs_exist_ok=dirs_exist_ok, **kwargs) elif dirs_exist_ok: - preserve_symlinks = False # Get symlinks named argument and use distutils.dir_util.copy_tree instead. - if 'symlinks' in kwargs: - preserve_symlinks = kwargs.pop('symlinks', False) + preserve_symlinks = kwargs.pop('symlinks', False) + # Check if there are other named arguments - if len(kwargs) > 0: + if kwargs: raise EasyBuildError("You can't use 'dirs_exist_ok=True' with other named arguments: %s", list(kwargs.keys())) - copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) + distutils.dir_util.copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) else: # Use shutil.copytree WITHOUT 'dirs_exist_ok' diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 980a30ecf1..8f394e9b89 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1477,16 +1477,17 @@ def test_copy_dir(self): # if the directory already exists and 'dirs_exist_ok' is True and there is another named argument (ignore) # we expect clean error on Python < 3.8 and pass the test on Python >= 3.8 # NOTE: reused ignore from previous test + def ignore_func(_, names): + return [x for x in names if '6.4.0-2.28' in x] shutil.rmtree(testdir) ft.mkdir(testdir) - if (sys.version_info[0] == 3 and sys.version_info[1] >= 8) or sys.version_info[0] > 3: - ft.copy_dir(to_copy, testdir, dirs_exist_ok=True, - ignore=lambda src, names: [x for x in names if '6.4.0-2.28' in x]) + if sys.version_info >= (3, 8): + ft.copy_dir(to_copy, testdir, dirs_exist_ok=True, ignore=ignore_func) self.assertEqual(sorted(os.listdir(testdir)), expected) self.assertFalse(os.path.exists(os.path.join(testdir, 'GCC-6.4.0-2.28.eb'))) else: self.assertErrorRegex(EasyBuildError, "You can't use 'dirs_exist_ok=True' with other named arguments:.*", - ft.copy_dir, to_copy, testdir, dirs_exist_ok=True, ignore=lambda src, names: []) + ft.copy_dir, to_copy, testdir, dirs_exist_ok=True, ignore=ignore_func) # also test behaviour of copy_file under --dry-run build_options = { From 26413c03270603882755cdfa9bf205006edadf17 Mon Sep 17 00:00:00 2001 From: darkless Date: Wed, 8 Apr 2020 12:35:44 +0200 Subject: [PATCH 318/344] updated comments --- easybuild/tools/filetools.py | 15 ++++++++------- test/framework/filetools.py | 6 ++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 92b22d5e8e..17794c6751 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1834,23 +1834,24 @@ def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **k if not dirs_exist_ok and os.path.exists(target_path): raise EasyBuildError("Target location %s to copy %s to already exists", target_path, path) - # On Python >= 3.8 if sys.version_info >= (3, 8): - # Use the shutil.copytree WITH 'dirs_exist_ok' + # on Python >= 3.8, shutil.copytree works fine, thanks to availability of dirs_exist_ok named argument shutil.copytree(path, target_path, dirs_exist_ok=dirs_exist_ok, **kwargs) elif dirs_exist_ok: - # Get symlinks named argument and use distutils.dir_util.copy_tree instead. + # use distutils.dir_util.copy_tree with Python < 3.8 if dirs_exist_ok is enabled + + # first get value for symlinks named argument (if any) preserve_symlinks = kwargs.pop('symlinks', False) - # Check if there are other named arguments + # check if there are other named arguments (there shouldn't be, only 'symlinks' is supported) if kwargs: - raise EasyBuildError("You can't use 'dirs_exist_ok=True' with other named arguments: %s", - list(kwargs.keys())) + raise EasyBuildError("Unknown named arguments passed to copy_dir with dirs_exist_ok=True: %s", + ', '.join(sorted(kwargs.keys()))) distutils.dir_util.copy_tree(path, target_path, preserve_symlinks=preserve_symlinks) else: - # Use shutil.copytree WITHOUT 'dirs_exist_ok' + # if dirs_exist_ok is not enabled, just use shutil.copytree shutil.copytree(path, target_path, **kwargs) _log.info("%s copied to %s", path, target_path) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8f394e9b89..3167c8749e 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1479,6 +1479,7 @@ def test_copy_dir(self): # NOTE: reused ignore from previous test def ignore_func(_, names): return [x for x in names if '6.4.0-2.28' in x] + shutil.rmtree(testdir) ft.mkdir(testdir) if sys.version_info >= (3, 8): @@ -1486,8 +1487,9 @@ def ignore_func(_, names): self.assertEqual(sorted(os.listdir(testdir)), expected) self.assertFalse(os.path.exists(os.path.join(testdir, 'GCC-6.4.0-2.28.eb'))) else: - self.assertErrorRegex(EasyBuildError, "You can't use 'dirs_exist_ok=True' with other named arguments:.*", - ft.copy_dir, to_copy, testdir, dirs_exist_ok=True, ignore=ignore_func) + error_pattern = "Unknown named arguments passed to copy_dir with dirs_exist_ok=True: ignore" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_dir, to_copy, testdir, + dirs_exist_ok=True, ignore=ignore_func) # also test behaviour of copy_file under --dry-run build_options = { From e36cde289b9941df89e036ce45d51ab57ee73dc9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 16:00:48 +0200 Subject: [PATCH 319/344] rename ModulesTool.get_variable_from_modulefile method to ModulesTool.get_setenv_value_from_modulefile, and add dedicated test for it --- easybuild/framework/easyconfig/easyconfig.py | 4 +- easybuild/tools/modules.py | 73 +++++++++++--------- test/framework/modules.py | 28 ++++++++ 3 files changed, 71 insertions(+), 34 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index e836239169..20e05736bf 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1211,8 +1211,8 @@ def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=No prefix = prefix % short_ext_modname_upper version = version % short_ext_modname_upper - dep_prefix = self.modules_tool.get_variable_from_modulefile(dep_name, prefix) - dep_version = self.modules_tool.get_variable_from_modulefile(dep_name, version) + dep_prefix = self.modules_tool.get_setenv_value_from_modulefile(dep_name, prefix) + dep_version = self.modules_tool.get_setenv_value_from_modulefile(dep_name, version) # only update missing values with both keys are found if dep_prefix and dep_version: diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 1afd5a2ead..1578a91b71 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -649,16 +649,7 @@ def show(self, mod_name): return ans - def get_variable_from_modulefile(self, mod_name, var_name): - """ - Get info from the module file for the specified module. - - :param mod_name: module name - :param var_name: name of the variable value to be extracted - """ - pass - - def get_value_from_modulefile(self, mod_name, regex): + def get_value_from_modulefile(self, mod_name, regex, strict=True): """ Get info from the module file for the specified module. @@ -669,13 +660,17 @@ def get_value_from_modulefile(self, mod_name, regex): modinfo = self.show(mod_name) res = regex.search(modinfo) if res: - return res.group(1) - else: + value = res.group(1) + elif strict: raise EasyBuildError("Failed to determine value from 'show' (pattern: '%s') in %s", regex.pattern, modinfo) + else: + value = None else: raise EasyBuildError("Can't get value from a non-existing module %s", mod_name) + return value + def modulefile_path(self, mod_name, strip_ext=False): """ Get the path of the module file for the specified module @@ -1095,6 +1090,15 @@ def path_to_top_of_module_tree(self, top_paths, mod_name, full_mod_subdir, deps, self.log.debug("Path to top of module tree from %s: %s" % (mod_name, path)) return path + def get_setenv_value_from_modulefile(self, mod_name, var_name): + """ + Get value for specific 'setenv' statement from module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable being set for which value should be returned + """ + raise NotImplementedError + def update(self): """Update after new modules were added.""" raise NotImplementedError @@ -1135,21 +1139,23 @@ def update(self): """Update after new modules were added.""" pass - def get_variable_from_modulefile(self, mod_name, var_name): + def get_setenv_value_from_modulefile(self, mod_name, var_name): """ - Get info from the module file for the specified module. + Get value for specific 'setenv' statement from module file for the specified module. :param mod_name: module name - :param var_name: name of the variable value to be extracted + :param var_name: name of the variable being set for which value should be returned """ - try: - # Tcl syntax - regex = re.compile(r'^setenv\s+%s\s+(?P\S*)' % var_name, re.M) - ans = self.get_value_from_modulefile(mod_name, regex) - except Exception: - return None + # Tcl-based module tools produce "module show" output with setenv statements like: + # "setenv GCC_PATH /opt/gcc/8.3.0" + # - line starts with 'setenv' + # - whitespace (spaces & tabs) around variable name + # - no quotes or parentheses around value (which can contain spaces!) + regex = re.compile(r'^setenv\s+%s\s+(?P.+)' % var_name, re.M) + value = self.get_value_from_modulefile(mod_name, regex, strict=False) + + return value - return ans class EnvironmentModulesTcl(EnvironmentModulesC): """Interface to (Tcl) environment modules (modulecmd.tcl).""" @@ -1414,21 +1420,24 @@ def exist(self, mod_names, skip_avail=False, maybe_partial=True): return super(Lmod, self).exist(mod_names, mod_exists_regex_template=r'^\s*\S*/%s.*(\.lua)?:\s*$', skip_avail=skip_avail, maybe_partial=maybe_partial) - def get_variable_from_modulefile(self, mod_name, var_name): + def get_setenv_value_from_modulefile(self, mod_name, var_name): """ - Get info from the module file for the specified module. + Get value for specific 'setenv' statement from module file for the specified module. :param mod_name: module name - :param var_name: name of the variable value to be extracted + :param var_name: name of the variable being set for which value should be returned """ - try: - # Lua syntax - regex = re.compile(r'^setenv\(\"%s\",\s*\"(?P\S*)\"\)' % var_name, re.M) - ans = self.get_value_from_modulefile(mod_name, regex) - except Exception: - return None + # Lmod produces "module show" output with setenv statements like: + # setenv("EBROOTBZIP2","/tmp/software/bzip2/1.0.6") + # - line starts with setenv( + # - both variable name and value are enclosed in double quotes, separated by comma + # - value can contain spaces! + # - line ends with ) + regex = re.compile(r'^setenv\("%s"\s*,\s*"(?P.+)"\)' % var_name, re.M) + value = self.get_value_from_modulefile(mod_name, regex, strict=False) + + return value - return ans def get_software_root_env_var_name(name): """Return name of environment variable for software root.""" diff --git a/test/framework/modules.py b/test/framework/modules.py index ca3d826e7c..2500dc4dd1 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -42,9 +42,11 @@ import easybuild.tools.modules as mod from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file +from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches @@ -1207,6 +1209,32 @@ def test_modulecmd_strip_source(self): modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') + def test_get_setenv_value_from_modulefile(self): + """Test for ModulesTool.get_setenv_value_from_modulefile method.""" + + topdir = os.path.dirname(os.path.abspath(__file__)) + eb_path = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, read_file(eb_path)) + write_file(test_ec, "\nmodextravars = {'FOO': 'value with spaces'}", append=True) + + toy_eb = EasyBlock(EasyConfig(test_ec)) + toy_eb.make_module_step() + + expected_root = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + ebroot = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'EBROOTTOY') + self.assertTrue(os.path.samefile(ebroot, expected_root)) + + ebversion = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'EBVERSIONTOY') + self.assertEqual(ebversion, '0.0') + + foo = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'FOO') + self.assertEqual(foo, "value with spaces") + + res = self.modtool.get_setenv_value_from_modulefile('toy/0.0', 'NO_SUCH_VARIABLE_SET') + self.assertEqual(res, None) + def suite(): """ returns all the testcases in this module """ From 1507e621676d29ab1101a221af6f75be1b186dd7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 17:48:47 +0200 Subject: [PATCH 320/344] reimplement handle_external_module_metadata + clean up probe_external_module_metadata methods in EasyConfig class --- easybuild/framework/easyconfig/easyconfig.py | 188 +++++++++++-------- easybuild/tools/modules.py | 6 +- 2 files changed, 108 insertions(+), 86 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 20e05736bf..d7cd206613 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1166,37 +1166,52 @@ def _validate(self, attr, values): # private method if self[attr] and self[attr] not in values: raise EasyBuildError("%s provided '%s' is not valid: %s", attr, self[attr], values) - def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=None): + def probe_external_module_metadata(self, mod_name, existing_metadata=None): """ - helper function for handle_external_module_metadata - handles metadata for external module dependencies when there is not entry in the - metadata file - - It should look for the pair of variables definitions in the available modules - 1. CRAY_XXXX_PREFIX and CRAY_XXXX_VERSION - 2. CRAY_XXXX_DIR and CRAY_XXXX_VERSION - 2. CRAY_XXXX_ROOT and CRAY_XXXX_VERSION - 5. XXXX_PREFIX and XXXX_VERSION - 4. XXXX_DIR and XXXX_VERSION - 5. XXXX_ROOT and XXXX_VERSION - 3. XXXX_HOME and XXXX_VERSION - - If neither of the pairs is found, then an empty dictionary is returned + Helper function for handle_external_module_metadata. + + Tries to determine metadata for external module when there is not entry in the metadata file, + by looking at the variables defined by the module file. + + This is mainly intended for modules provided in the Cray Programming Environment, + but it could also be useful in other contexts. + + The following pairs of variables are considered (in order, first hit wins), + where 'XXX' is the software name in capitals: + 1. $CRAY_XXX_PREFIX and $CRAY_XXX_VERSION + 1. $CRAY_XXX_PREFIX_DIR and $CRAY_XXX_VERSION + 2. $CRAY_XXX_DIR and $CRAY_XXX_VERSION + 2. $CRAY_XXX_ROOT and $CRAY_XXX_VERSION + 5. $XXX_PREFIX and $XXX_VERSION + 4. $XXX_DIR and $XXX_VERSION + 5. $XXX_ROOT and $XXX_VERSION + 3. $XXX_HOME and $XXX_VERSION + + If none of the pairs is found, then an empty dictionary is returned. + + :param mod_name: name of the external module + :param metadata: already available metadata for this external module (if any) """ - if dependency is None: - dependency = dict() + res = {} - short_ext_modname = dep_name.split('/')[0] - if not 'name' in dependency: - dependency['name'] = [short_ext_modname] + if existing_metadata is None: + existing_metadata = {} - if short_ext_modname.startswith('cray-'): - short_ext_modname = short_ext_modname.split('cray-')[1] + soft_name = existing_metadata.get('name') - short_ext_modname.replace('-', '_') - short_ext_modname_upper = convert_name(short_ext_modname, upper=True) + if soft_name is None: + # if the software name is not known yet, use the first part of the module name as software name, + # but strip off the leading 'cray-' part first (examples: cray-netcdf/4.6.1.3, cray-fftw/3.3.8.2) + soft_name = mod_name.split('/')[0] - allowed_pairs = [ + cray_prefix = 'cray-' + if soft_name.startswith(cray_prefix): + soft_name = soft_name[len(cray_prefix):] + + # determine software name to use in names of environment variables (upper case, '-' becomes '_') + soft_name_in_mod_name = convert_name(soft_name.replace('-', '_'), upper=True) + + var_name_pairs = [ ('CRAY_%s_PREFIX', 'CRAY_%s_VERSION'), ('CRAY_%s_PREFIX_DIR', 'CRAY_%s_VERSION'), ('CRAY_%s_DIR', 'CRAY_%s_VERSION'), @@ -1207,71 +1222,78 @@ def _handle_ext_module_metadata_by_probing_modules(self, dep_name, dependency=No ('%s_HOME', '%s_VERSION'), ] - for prefix, version in allowed_pairs: - prefix = prefix % short_ext_modname_upper - version = version % short_ext_modname_upper - - dep_prefix = self.modules_tool.get_setenv_value_from_modulefile(dep_name, prefix) - dep_version = self.modules_tool.get_setenv_value_from_modulefile(dep_name, version) - - # only update missing values with both keys are found - if dep_prefix and dep_version: - # version should hold the value, not the key - if 'version' not in dependency: - dependency['version'] = [dep_version] - self.log.info('setting external module %s version to be %s' % (dep_name, dep_version)) - # prefix should hold the key, not the value - if 'prefix' not in dependency: - dependency['prefix'] = prefix - self.log.info('setting external module %s prefix to be %s' % (dep_name, dep_prefix)) + for prefix_var_name, version_var_name in var_name_pairs: + prefix_var_name = prefix_var_name % soft_name_in_mod_name + version_var_name = version_var_name % soft_name_in_mod_name + + prefix = self.modules_tool.get_setenv_value_from_modulefile(mod_name, prefix_var_name) + version = self.modules_tool.get_setenv_value_from_modulefile(mod_name, version_var_name) + + # we only have a hit when values for *both* variables are found + if prefix and version: + + if 'name' not in existing_metadata: + res['name'] = [soft_name] + + # 'version' metadata should hold the *value* of the corresponding variable; + # if a version is already set in the available metadata, we retain it + if 'version' not in existing_metadata: + res['version'] = [version] + self.log.info('setting external module %s version to be %s', mod_name, version) + + # 'prefix' should hold the name of the variable, not the value + # if a prefix is already set in the available metadata, we retain it + # FIXME? + if 'prefix' not in existing_metadata: + res['prefix'] = prefix_var_name + self.log.info('setting external module %s prefix to be %s', mod_name, prefix_var_name) break - return dependency + return res - def handle_external_module_metadata(self, dep_name): + def handle_external_module_metadata(self, mod_name): """ - helper function for _parse_dependency - handles metadata for external module dependencies + Helper function for _parse_dependency; collects metadata for external module dependencies. + + :param mod_name: name of external module to collect metadata for """ - dependency = {} - dep_name_no_version = dep_name.split('/')[0] - metadata_fields = ['name', 'version', 'prefix'] - external_metadata = {} - - if dep_name in self.external_modules_metadata: - external_metadata = self.external_modules_metadata[dep_name] - if not all(d in external_metadata for d in metadata_fields): - external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name, - dependency=external_metadata) - if external_metadata: - self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", - dep_name, external_metadata) - dependency['external_module_metadata'] = external_metadata + partial_mod_name = mod_name.split('/')[0] + + # check whether existing metadata for external modules already has metadata for this module; + # first using full module name (as it is provided), for example 'cray-netcdf/4.6.1.3', + # then with partial module name, for example 'cray-netcdf' + metadata = self.external_modules_metadata.get(mod_name, {}) + self.log.info("Available metadata for external module %s: %s", mod_name, metadata) + + partial_mod_name_metadata = self.external_modules_metadata.get(partial_mod_name, {}) + self.log.info("Available metadata for external module using partial module name %s: %s", + partial_mod_name, partial_mod_name_metadata) + + for key in partial_mod_name_metadata: + if key not in metadata: + metadata[key] = partial_mod_name_metadata[key] + + self.log.info("Combined available metadata for external module %s: %s", mod_name, metadata) + + # if not all metadata is available (name/version/prefix), probe external module to collect more metadata; + # first with full module name, and then with partial module name if first probe didn't return anything; + # note: result of probe_external_module_metadata only contains metadata for keys that were not set yet + if not all(key in metadata for key in ['name', 'prefix', 'version']): + self.log.info("Not all metadata found yet for external module %s, probing module...", mod_name) + probed_metadata = self.probe_external_module_metadata(mod_name, existing_metadata=metadata) + if probed_metadata: + self.log.info("Extra metadata found by probing external module %s: %s", mod_name, probed_metadata) + metadata.update(probed_metadata) else: - self.log.info("No metadata available for external module %s.", dep_name) - elif dep_name_no_version in self.external_modules_metadata: - external_metadata = self.external_modules_metadata[dep_name_no_version] - if not all(d in external_metadata for d in metadata_fields): - external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name_no_version, - dependency=external_metadata) - if external_metadata: - self.log.info("Updated dependency info with metadata from available modules for external module " - "%s: %s", dep_name, external_metadata) - dependency['external_module_metadata'] = external_metadata - else: - self.log.info("No metadata available for external module %s.", dep_name) - else: - self.log.info("No metadata available for external module %s. Attempting to read from available modules", - dep_name) - external_metadata = self._handle_ext_module_metadata_by_probing_modules(dep_name) - if external_metadata: - dependency['external_module_metadata'] = external_metadata - self.log.info("Updated dependency info with metadata from available modules for external module %s: %s", - dep_name, external_metadata) - else: - self.log.info("No metadata available for external module %s.", dep_name) + self.log.info("No extra metadata found by probing %s, trying with partial module name...", mod_name) + probed_metadata = self.probe_external_module_metadata(partial_mod_name, existing_metadata=metadata) + self.log.info("Extra metadata for external module %s found by probing partial module name %s: %s", + mod_name, partial_mod_name, probed_metadata) + metadata.update(probed_metadata) - return dependency + self.log.info("Obtained metadata after module probing: %s", metadata) + + return {'external_module_metadata': metadata} def handle_multi_deps(self): """ diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 1578a91b71..70a73724af 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -656,6 +656,8 @@ def get_value_from_modulefile(self, mod_name, regex, strict=True): :param mod_name: module name :param regex: (compiled) regular expression, with one group """ + value = None + if self.exist([mod_name], skip_avail=True)[0]: modinfo = self.show(mod_name) res = regex.search(modinfo) @@ -664,9 +666,7 @@ def get_value_from_modulefile(self, mod_name, regex, strict=True): elif strict: raise EasyBuildError("Failed to determine value from 'show' (pattern: '%s') in %s", regex.pattern, modinfo) - else: - value = None - else: + elif strict: raise EasyBuildError("Can't get value from a non-existing module %s", mod_name) return value From 0277662f3c36dd1463b86209e22c56c7e800fc50 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 19:39:07 +0200 Subject: [PATCH 321/344] enhance test_external_dependencies to check probing of external modules for missing metadata --- easybuild/framework/easyconfig/easyconfig.py | 6 +- test/framework/easyconfig.py | 112 ++++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index d7cd206613..ac79a59dda 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1198,8 +1198,10 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None): existing_metadata = {} soft_name = existing_metadata.get('name') - - if soft_name is None: + if soft_name: + # software name is a list of names in metadata, just grab first one + soft_name = soft_name[0] + else: # if the software name is not known yet, use the first part of the module name as software name, # but strip off the leading 'cray-' part first (examples: cray-netcdf/4.6.1.3, cray-fftw/3.3.8.2) soft_name = mod_name.split('/')[0] diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 3b720f687c..8693da0de7 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -64,8 +64,8 @@ from easybuild.tools.config import module_classes from easybuild.tools.configobj import ConfigObj from easybuild.tools.docs import avail_easyconfig_constants, avail_easyconfig_templates -from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file, remove_file -from easybuild.tools.filetools import symlink, write_file +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_file, mkdir, read_file +from easybuild.tools.filetools import remove_dir, remove_file, symlink, write_file from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata @@ -1445,14 +1445,114 @@ def test_external_dependencies(self): deps = ec.dependencies() self.assertEqual(len(deps), 7) - correct_deps = ['somebuilddep/0.1', 'intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'test/9.7.5', 'pi/3.14', - 'hidden/.1.2.3'] + correct_deps = ['somebuilddep/0.1', 'intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', + 'test/9.7.5', 'pi/3.14', 'hidden/.1.2.3'] self.assertEqual([d['short_mod_name'] for d in deps], correct_deps) self.assertEqual([d['full_mod_name'] for d in deps], correct_deps) self.assertEqual([d['external_module'] for d in deps], [True, False, True, True, True, True, True]) self.assertEqual([d['hidden'] for d in deps], [False, False, False, False, False, False, True]) + # no metadata available for deps + expected = [{}] * len(deps) + self.assertEqual([d['external_module_metadata'] for d in deps], expected) + + # test probing done by handle_external_module_metadata via probe_external_module_metadata, + # by adding a couple of matching module files with some useful data in them + # (use Tcl syntax, so it works with all varieties of module tools) + mod_dir = os.path.join(self.test_prefix, 'modules') + self.modtool.use(mod_dir) + + pi_mod_txt = '\n'.join([ + "#%Module", + "setenv PI_ROOT /software/pi/3.14", + "setenv PI_VERSION 3.14", + ]) + write_file(os.path.join(mod_dir, 'pi/3.14'), pi_mod_txt) + + # foobar module with different version than the one used as an external dep; + # will still be used for probing (as a fallback) + foobar_mod_txt = '\n'.join([ + "#%Module", + "setenv CRAY_FOOBAR_DIR /software/foobar/2.3.4", + "setenv CRAY_FOOBAR_VERSION 2.3.4", + ]) + write_file(os.path.join(mod_dir, 'foobar/2.3.4'), foobar_mod_txt) + + ec = EasyConfig(toy_ec) + deps = ec.dependencies() + + self.assertEqual(len(deps), 7) + + for idx in [0, 1, 2, 4, 6]: + self.assertEqual(deps[idx]['external_module_metadata'], {}) + + self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') + foobar_metadata = { + 'name': ['foobar'], + 'prefix': 'CRAY_FOOBAR_DIR', + 'version': ['2.3.4'], + } + self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) + self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') + pi_metadata = { + 'name': ['pi'], + 'prefix': 'PI_ROOT', + 'version': ['3.14'], + } + self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) + + # provide file with partial metadata for some external modules; + # metadata obtained from probing modules should be added to it... metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') + metadatatxt = '\n'.join([ + '[pi/3.14]', + 'name = PI', + 'version = 3.14.0', + '[foobar]', + 'version = 1.0', + '[foobar/1.2.3]', + 'version = 1.2.3', + '[test]', + 'name = TEST', + ]) + write_file(metadata, metadatatxt) + build_options = { + 'external_modules_metadata': parse_external_modules_metadata([metadata]), + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + ec = EasyConfig(toy_ec) + deps = ec.dependencies() + + self.assertEqual(len(deps), 7) + + for idx in [0, 1, 2, 6]: + self.assertEqual(deps[idx]['external_module_metadata'], {}) + + self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') + foobar_metadata = { + 'name': ['foobar'], # probed from 'foobar' module + 'prefix': 'CRAY_FOOBAR_DIR', # probed from 'foobar' module + 'version': ['1.2.3'], # from [foobar/1.2.3] entry in metadata file + } + self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) + + self.assertEqual(deps[4]['full_mod_name'], 'test/9.7.5') + self.assertEqual(deps[4]['external_module_metadata'], { + # from [test] entry in metadata file + 'name': ['TEST'], + }) + + self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') + pi_metadata = { + 'name': ['PI'], # from [pi/3.14] entry in metadata file + 'prefix': 'PI_ROOT', # probed from 'pi/3.14' module + 'version': ['3.14.0'], # from [pi/3.14] entry in metadata file + } + self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) + + # provide file with full metadata for external modules; + # this data wins over probed metadata from modules (for backwards compatibility) metadatatxt = '\n'.join([ '[pi/3.14]', 'name = PI', @@ -1501,6 +1601,10 @@ def test_external_dependencies(self): } self.assertEqual(ec.dependencies()[5]['external_module_metadata'], metadata) + # get rid of modules first + self.modtool.unuse(mod_dir) + remove_dir(mod_dir) + # check whether $EBROOT*/$EBVERSION* environment variables are defined correctly for external modules os.environ['PI_PREFIX'] = '/test/prefix/PI' os.environ['TEST_INC'] = '/test/prefix/test/include' From 12573ab9c30f454fca70ea73b88f3433f029ef7c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 19:53:28 +0200 Subject: [PATCH 322/344] fix broken test_resolve_dependencies test by providing dummy implementation in MockModule class used in robot tests --- test/framework/robot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/robot.py b/test/framework/robot.py index 41df63a315..7ead419e91 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -88,6 +88,10 @@ def show(self, modname): txt = 'Module %s not found' % modname return txt + def get_setenv_value_from_modulefile(self, mod_name, var_name): + """Dummy implementation of get_setenv_value_from_modulefile, always returns None.""" + return None + def mock_module(mod_paths=None): """Get mock module instance.""" From ca07edc4d47c1264eba69bc7b0f0bfe6104310ec Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 20:08:21 +0200 Subject: [PATCH 323/344] fix posting of comment in PR with --upload-test-report --- easybuild/tools/github.py | 11 +++++++++-- easybuild/tools/testing.py | 14 +++++++------- test/framework/github.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d24d87e7ca..9eb9219dd8 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -493,6 +493,9 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): def create_gist(txt, fn, descr=None, github_user=None, github_token=None): """Create a gist with the provided text.""" + + dry_run = build_option('dry_run') or build_option('extended_dry_run') + if descr is None: descr = "(none)" @@ -508,8 +511,12 @@ def create_gist(txt, fn, descr=None, github_user=None, github_token=None): } } } - g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) - status, data = g.gists.post(body=body) + + if dry_run: + status, data = HTTP_STATUS_CREATED, {'html_url': 'https://gist.github.com/DRY_RUN'} + else: + g = RestClient(GITHUB_API_URL, username=github_user, token=github_token) + status, data = g.gists.post(body=body) if status != HTTP_STATUS_CREATED: raise EasyBuildError("Failed to create gist; status %s, data: %s", status, data) diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 73c1ea92a1..ec7d83ba37 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -46,7 +46,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import find_easyconfigs, mkdir, read_file, write_file -from easybuild.tools.github import create_gist, post_comment_in_issue +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, create_gist, post_comment_in_issue from easybuild.tools.jenkins import aggregate_xml_in_dirs from easybuild.tools.parallelbuild import build_easyconfigs_in_parallel from easybuild.tools.robot import resolve_dependencies @@ -143,7 +143,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nr=None, gist_l github_user = build_option('github_user') pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO end_time = gmtime() @@ -252,10 +252,13 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, """Post test report in a gist, and submit comment in easyconfigs PR.""" github_user = build_option('github_user') + pr_target_account = build_option('pr_target_account') + pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO # create gist with test report - descr = "EasyBuild test report for easyconfigs PR #%s" % pr_nr - fn = 'easybuild_test_report_easyconfigs_pr%s_%s.md' % (pr_nr, strftime("%Y%M%d-UTC-%H-%M-%S", gmtime())) + descr = "EasyBuild test report for %s/%s PR #%s" % (pr_target_account, pr_target_repo, pr_nr) + timestamp = strftime("%Y%M%d-UTC-%H-%M-%S", gmtime()) + fn = 'easybuild_test_report_%s_%s_pr%s_%s.md' % (pr_nr, pr_target_account, pr_target_repo, timestamp) gist_url = upload_test_report_as_gist(test_report, descr=descr, fn=fn) # post comment to report test result @@ -283,9 +286,6 @@ def post_easyconfigs_pr_test_report(pr_nr, test_report, msg, init_session_state, ] comment = '\n'.join(comment_lines) - pr_target_account = build_option('pr_target_account') - pr_target_repo = build_option('pr_target_repo') - post_comment_in_issue(pr_nr, comment, account=pr_target_account, repo=pr_target_repo, github_user=github_user) msg = "Test report uploaded to %s and mentioned in a comment in easyconfigs PR#%s" % (gist_url, pr_nr) diff --git a/test/framework/github.py b/test/framework/github.py index bd1e7cecd4..b0b20bfea9 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -43,6 +43,7 @@ from easybuild.tools.configobj import ConfigObj from easybuild.tools.filetools import read_file, write_file from easybuild.tools.github import VALID_CLOSE_PR_REASONS +from easybuild.tools.testing import post_easyconfigs_pr_test_report, session_state from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters import easybuild.tools.github as gh @@ -778,6 +779,39 @@ def test_push_branch_to_github(self): regex = re.compile(pattern) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' doesn't match: %s" % (regex.pattern, stdout)) + def test_post_easyconfigs_pr_test_report(self): + """Test for post_easyconfigs_pr_test_report function.""" + if self.skip_github_tests: + print("Skipping test_post_easyconfigs_pr_test_report, no GitHub token available?") + return + + init_config(build_options={ + 'dry_run': True, + 'github_user': GITHUB_TEST_ACCOUNT, + }) + + test_report = os.path.join(self.test_prefix, 'test_report.txt') + write_file(test_report, "This is a test report!") + + init_session_state = session_state() + + self.mock_stderr(True) + self.mock_stdout(True) + post_easyconfigs_pr_test_report('1234', test_report, "OK!", init_session_state, True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + + patterns = [ + r"^\[DRY RUN\] Adding comment to easybuild-easyconfigs issue #1234: 'Test report by @easybuild_test", + r"^See https://gist.github.com/DRY_RUN for a full test report.'", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ From a9facb30f80cac9ca5473cd8b9fe82bb829321c9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 20:41:42 +0200 Subject: [PATCH 324/344] strip off leading/trailing whitespace in get_setenv_value_from_modulefile method --- easybuild/tools/modules.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 70a73724af..8a5323434d 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -1154,6 +1154,9 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name): regex = re.compile(r'^setenv\s+%s\s+(?P.+)' % var_name, re.M) value = self.get_value_from_modulefile(mod_name, regex, strict=False) + if value: + value = value.strip() + return value @@ -1436,6 +1439,9 @@ def get_setenv_value_from_modulefile(self, mod_name, var_name): regex = re.compile(r'^setenv\("%s"\s*,\s*"(?P.+)"\)' % var_name, re.M) value = self.get_value_from_modulefile(mod_name, regex, strict=False) + if value: + value = value.strip() + return value From 227a9fdcf4e8031a6f54f15a0c1772a11894f610 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Apr 2020 20:56:08 +0200 Subject: [PATCH 325/344] use prefix value rather than name of environment variable that contains prefix value in probe_external_module_metadata --- easybuild/framework/easyconfig/easyconfig.py | 5 +---- test/framework/easyconfig.py | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ac79a59dda..10d6af13e4 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1237,17 +1237,14 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None): if 'name' not in existing_metadata: res['name'] = [soft_name] - # 'version' metadata should hold the *value* of the corresponding variable; # if a version is already set in the available metadata, we retain it if 'version' not in existing_metadata: res['version'] = [version] self.log.info('setting external module %s version to be %s', mod_name, version) - # 'prefix' should hold the name of the variable, not the value # if a prefix is already set in the available metadata, we retain it - # FIXME? if 'prefix' not in existing_metadata: - res['prefix'] = prefix_var_name + res['prefix'] = prefix self.log.info('setting external module %s prefix to be %s', mod_name, prefix_var_name) break diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 8693da0de7..a5ae7c184c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1488,7 +1488,7 @@ def test_external_dependencies(self): self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') foobar_metadata = { 'name': ['foobar'], - 'prefix': 'CRAY_FOOBAR_DIR', + 'prefix': '/software/foobar/2.3.4', 'version': ['2.3.4'], } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) @@ -1496,7 +1496,7 @@ def test_external_dependencies(self): self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') pi_metadata = { 'name': ['pi'], - 'prefix': 'PI_ROOT', + 'prefix': '/software/pi/3.14', 'version': ['3.14'], } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) @@ -1532,7 +1532,7 @@ def test_external_dependencies(self): self.assertEqual(deps[3]['full_mod_name'], 'foobar/1.2.3') foobar_metadata = { 'name': ['foobar'], # probed from 'foobar' module - 'prefix': 'CRAY_FOOBAR_DIR', # probed from 'foobar' module + 'prefix': '/software/foobar/2.3.4', # probed from 'foobar' module 'version': ['1.2.3'], # from [foobar/1.2.3] entry in metadata file } self.assertEqual(deps[3]['external_module_metadata'], foobar_metadata) @@ -1546,7 +1546,7 @@ def test_external_dependencies(self): self.assertEqual(deps[5]['full_mod_name'], 'pi/3.14') pi_metadata = { 'name': ['PI'], # from [pi/3.14] entry in metadata file - 'prefix': 'PI_ROOT', # probed from 'pi/3.14' module + 'prefix': '/software/pi/3.14', # probed from 'pi/3.14' module 'version': ['3.14.0'], # from [pi/3.14] entry in metadata file } self.assertEqual(deps[5]['external_module_metadata'], pi_metadata) From 3aed7938723369a152cd3c94c88e786646da5590 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Apr 2020 09:04:11 +0200 Subject: [PATCH 326/344] remove unused imports in test/framework/modules.py --- test/framework/modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index 2500dc4dd1..93015b4a07 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -46,7 +46,6 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, copy_file, copy_dir, mkdir from easybuild.tools.filetools import read_file, remove_dir, remove_file, symlink, write_file -from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches From 1e9819a498c0b3ef377370caa2fcdfdb8011b94e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Apr 2020 18:19:28 +0200 Subject: [PATCH 327/344] take into account dependencies marked as external modules when composing template values like %(pyver)s (fixes #3263) --- easybuild/framework/easyconfig/templates.py | 14 +++++- test/framework/easyconfig.py | 51 +++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index b6e6fe1393..8df02f0830 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -222,12 +222,24 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) for dep in deps: if isinstance(dep, dict): dep_name, dep_version = dep['name'], dep['version'] + + # take into account dependencies marked as external modules, + # where name/version may have to be harvested from metadata available for that external module + if dep.get('external_module', False): + metadata = dep.get('external_module_metadata', {}) + if dep_name is None: + # name is a list in metadata, just take first value (if any) + dep_name = metadata.get('name', [None])[0] + if dep_version is None: + # version is a list in metadata, just take first value (if any) + dep_version = metadata.get('version', [None])[0] + elif isinstance(dep, (list, tuple)): dep_name, dep_version = dep[0], dep[1] else: raise EasyBuildError("Unexpected type for dependency: %s", dep) - if isinstance(dep_name, string_type) and dep_name.lower() == name.lower(): + if isinstance(dep_name, string_type) and dep_name.lower() == name.lower() and dep_version: dep_version = pick_dep_version(dep_version) template_values['%sver' % pref] = dep_version dep_version_parts = dep_version.split('.') diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index a5ae7c184c..c10e49a533 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1622,6 +1622,57 @@ def test_external_dependencies(self): self.assertEqual(os.environ.get('EBVERSIONPI'), '3.14') self.assertEqual(os.environ.get('EBVERSIONTEST'), '9.7.5') + def test_external_dependencies_templates(self): + """Test use of templates for dependencies marked as external modules.""" + + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ectxt = read_file(toy_ec) + + extra_ectxt = '\n'.join([ + "versionsuffix = '-Python-%(pyver)s-Perl-%(perlshortver)s'", + '', + "dependencies = [", + " ('cray-python/3.6.5.7', EXTERNAL_MODULE),", + " ('perl/5.30.0-1', EXTERNAL_MODULE),", + "]", + ]) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, toy_ectxt + '\n' + extra_ectxt) + + # put metadata in place so templates can be defined + metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') + metadatatxt = '\n'.join([ + '[cray-python]', + 'name = Python', + '[cray-python/3.6.5.7]', + 'version = 3.6.5', + '[perl/5.30.0-1]', + 'name = Perl', + 'version = 5.30.0', + ]) + write_file(metadata, metadatatxt) + build_options = { + 'external_modules_metadata': parse_external_modules_metadata([metadata]), + 'valid_module_classes': module_classes(), + } + init_config(build_options=build_options) + + ec = EasyConfig(test_ec) + + expected_template_values = { + 'perlmajver': '5', + 'perlshortver': '5.30', + 'perlver': '5.30.0', + 'pymajver': '3', + 'pyshortver': '3.6', + 'pyver': '3.6.5', + } + for key in expected_template_values: + self.assertEqual(ec.template_values[key], expected_template_values[key]) + + self.assertEqual(ec['versionsuffix'], '-Python-3.6.5-Perl-5.30') + def test_update(self): """Test use of update() method for EasyConfig instances.""" topdir = os.path.abspath(os.path.dirname(__file__)) From 33ec98fb23cf3248766266c95ff2595f0910ad09 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 10 Apr 2020 13:00:05 +0200 Subject: [PATCH 328/344] support template name for CUDA version --- easybuild/framework/easyconfig/templates.py | 1 + test/framework/easyconfig.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 8df02f0830..9629759dae 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -77,6 +77,7 @@ # software names for which to define ver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ # software name, prefix for *ver and *shortver + ('CUDA', 'cuda'), ('Java', 'java'), ('Perl', 'perl'), ('Python', 'py'), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c10e49a533..7391678a21 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -961,12 +961,14 @@ def test_templating(self): ' "dirs": ["libfoo.%%s" %% SHLIB_EXT, "lib/%%(arch)s"],', '}', 'dependencies = [', + ' ("CUDA", "10.1.105"),' ' ("Java", "1.7.80"),' ' ("Perl", "5.22.0"),' ' ("Python", "2.7.10"),' ' ("R", "3.2.3"),' ']', 'modloadmsg = "%s"' % '; '.join([ + 'CUDA: %%(cudaver)s, %%(cudamajver)s, %%(cudashortver)s', 'Java: %%(javaver)s, %%(javamajver)s, %%(javashortver)s', 'Python: %%(pyver)s, %%(pymajver)s, %%(pyshortver)s', 'Perl: %%(perlver)s, %%(perlmajver)s, %%(perlshortver)s', @@ -1000,7 +1002,7 @@ def test_templating(self): dirs1 = eb['sanity_check_paths']['dirs'][1] self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' matches '%s'" % (lib_arch_regex.pattern, dirs1)) self.assertEqual(eb['homepage'], "http://example.com/P/p/v3/") - expected = "Java: 1.7.80, 1, 1.7; Python: 2.7.10, 2, 2.7; Perl: 5.22.0, 5, 5.22; R: 3.2.3, 3, 3.2" + expected = "CUDA: 10.1.105, 10, 10.1; Java: 1.7.80, 1, 1.7; Python: 2.7.10, 2, 2.7; Perl: 5.22.0, 5, 5.22; R: 3.2.3, 3, 3.2" self.assertEqual(eb['modloadmsg'], expected) self.assertEqual(eb['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) From 721a344d6ca92cbc04874dcd329f2120550577cf Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 10 Apr 2020 13:17:56 +0200 Subject: [PATCH 329/344] split long line --- test/framework/easyconfig.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7391678a21..9f9012b944 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1002,7 +1002,11 @@ def test_templating(self): dirs1 = eb['sanity_check_paths']['dirs'][1] self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' matches '%s'" % (lib_arch_regex.pattern, dirs1)) self.assertEqual(eb['homepage'], "http://example.com/P/p/v3/") - expected = "CUDA: 10.1.105, 10, 10.1; Java: 1.7.80, 1, 1.7; Python: 2.7.10, 2, 2.7; Perl: 5.22.0, 5, 5.22; R: 3.2.3, 3, 3.2" + expected = "CUDA: 10.1.105, 10, 10.1; " + "Java: 1.7.80, 1, 1.7; " + "Python: 2.7.10, 2, 2.7; " + "Perl: 5.22.0, 5, 5.22; " + "R: 3.2.3, 3, 3.2" self.assertEqual(eb['modloadmsg'], expected) self.assertEqual(eb['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) From e6b5ed63b4233da15020ffeec082bd60fbb9dbc6 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Fri, 10 Apr 2020 13:26:04 +0200 Subject: [PATCH 330/344] fix string splitting --- test/framework/easyconfig.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 9f9012b944..f84a346f77 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1002,11 +1002,11 @@ def test_templating(self): dirs1 = eb['sanity_check_paths']['dirs'][1] self.assertTrue(lib_arch_regex.match(dirs1), "Pattern '%s' matches '%s'" % (lib_arch_regex.pattern, dirs1)) self.assertEqual(eb['homepage'], "http://example.com/P/p/v3/") - expected = "CUDA: 10.1.105, 10, 10.1; " - "Java: 1.7.80, 1, 1.7; " - "Python: 2.7.10, 2, 2.7; " - "Perl: 5.22.0, 5, 5.22; " - "R: 3.2.3, 3, 3.2" + expected = ("CUDA: 10.1.105, 10, 10.1; " + "Java: 1.7.80, 1, 1.7; " + "Python: 2.7.10, 2, 2.7; " + "Perl: 5.22.0, 5, 5.22; " + "R: 3.2.3, 3, 3.2") self.assertEqual(eb['modloadmsg'], expected) self.assertEqual(eb['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) From b661ffd418fbd9a84c36cc0481bc282f2adb9fc1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 10:40:23 +0200 Subject: [PATCH 331/344] also honor filename_only if print_only is disabled in search_easyconfigs --- easybuild/tools/robot.py | 10 +++++----- test/framework/robot.py | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index cefbcc9d73..6871b41ac4 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -506,12 +506,12 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con else: hits.append(hit) - if print_result: - # check whether only filenames should be printed - if filename_only: - hits = [os.path.basename(hit) for hit in hits] - archived_hits = [os.path.basename(hit) for hit in archived_hits] + # check whether only filenames should be used + if filename_only: + hits = [os.path.basename(hit) for hit in hits] + archived_hits = [os.path.basename(hit) for hit in archived_hits] + if print_result: # prepare output format if terse: lines, tmpl = [], '%s' diff --git a/test/framework/robot.py b/test/framework/robot.py index a59ed01811..5293c88618 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1494,11 +1494,15 @@ def test_search_easyconfigs(self): ] self.assertEqual(paths, ref_paths) - # Now do a case sensitive search + # now do a case sensitive search paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False, case_sensitive=True) ref_paths = [os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.8-gcccuda-2018a.eb')] self.assertEqual(paths, ref_paths) + # test use of filename_only + paths = search_easyconfigs('hwloc-1.8', consider_extra_paths=False, print_result=False, filename_only=True) + self.assertEqual(paths, ['hwloc-1.8-gcccuda-2018a.eb']) + def suite(): """ returns all the testcases in this module """ From 8ee7738a7de07e46a76eb9c233eb81d73d41b8d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 10:49:11 +0200 Subject: [PATCH 332/344] enhance test_search_easyconfigs to also cover print_result option --- test/framework/robot.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/framework/robot.py b/test/framework/robot.py index 5293c88618..33f4dd72b0 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1503,6 +1503,31 @@ def test_search_easyconfigs(self): paths = search_easyconfigs('hwloc-1.8', consider_extra_paths=False, print_result=False, filename_only=True) self.assertEqual(paths, ['hwloc-1.8-gcccuda-2018a.eb']) + # test use of print_result (enabled by default) + for filename_only in [None, False, True]: + self.mock_stderr(True) + self.mock_stdout(True) + kwargs = {} + if filename_only is not None: + kwargs['filename_only'] = filename_only + search_easyconfigs('binutils-.*-GCCcore-4.9.3', **kwargs) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + self.assertEqual(len(stdout.splitlines()), 2) + pattern = [] + for ec_fn in ['binutils-2.25-GCCcore-4.9.3.eb', 'binutils-2.26-GCCcore-4.9.3.eb']: + if filename_only: + path = ec_fn + else: + path = os.path.join('test', 'framework', 'easyconfigs', 'test_ecs', 'b', 'binutils', ec_fn) + pattern.append(r"^ \* .*%s$" % path) + + regex = re.compile('\n'.join(pattern), re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def suite(): """ returns all the testcases in this module """ From 89a2b9e51073ec41d21a3a4192c712d328827844 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 11:03:50 +0200 Subject: [PATCH 333/344] enhance test_search_easyconfigs to also covered having enabled consider_extra_paths option --- easybuild/tools/robot.py | 4 +++- test/framework/robot.py | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index 6871b41ac4..cd3a2dca55 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -490,7 +490,9 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False, con extra_search_paths = build_option('search_paths') # If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths if extra_search_paths and consider_extra_paths: - search_path.extend(extra_search_paths) + # we shouldn't use += or .extend here but compose a new list, + # to avoid adding a path to the list returned by build_option('robot_path') ! + search_path = search_path + extra_search_paths ignore_dirs = build_option('ignore_dirs') diff --git a/test/framework/robot.py b/test/framework/robot.py index 33f4dd72b0..481c460d57 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -1478,12 +1478,26 @@ def test_search_easyconfigs(self): test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'robot_path': [test_ecs], + 'search_paths': [self.test_prefix], }) + + # copy some files to search_paths location + copy_file(os.path.join(test_ecs, 'b', 'binutils', 'binutils-2.25-GCCcore-4.9.3.eb'), self.test_prefix) + copy_file(os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), self.test_prefix) + paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', consider_extra_paths=False, print_result=False) ref_paths = [os.path.join(test_ecs, 'b', 'binutils', x) for x in ['binutils-2.25-GCCcore-4.9.3.eb', 'binutils-2.26-GCCcore-4.9.3.eb']] + self.assertEqual(len(paths), 2) self.assertEqual(paths, ref_paths) + # search_paths location is considered by default + paths = search_easyconfigs('binutils-.*-GCCcore-4.9.3', print_result=False) + self.assertEqual(len(paths), 3) + self.assertEqual(paths[:2], ref_paths) + # last hit is the one from search_paths + self.assertTrue(os.path.samefile(paths[2], os.path.join(self.test_prefix, 'binutils-2.25-GCCcore-4.9.3.eb'))) + paths = search_easyconfigs('8-gcc', consider_extra_paths=False, print_result=False) ref_paths = [ os.path.join(test_ecs, 'h', 'hwloc', 'hwloc-1.11.8-GCC-4.6.4.eb'), @@ -1507,7 +1521,7 @@ def test_search_easyconfigs(self): for filename_only in [None, False, True]: self.mock_stderr(True) self.mock_stdout(True) - kwargs = {} + kwargs = {'consider_extra_paths': False} if filename_only is not None: kwargs['filename_only'] = filename_only search_easyconfigs('binutils-.*-GCCcore-4.9.3', **kwargs) From fd953e62e19d5736c5a146f0f7d95134ff0cc5f8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 11:10:46 +0200 Subject: [PATCH 334/344] minor code cleanup in test_recursive_try --- test/framework/options.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 483ccc1ea8..09ed4a2bef 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1804,8 +1804,8 @@ def test_recursive_try(self): for extra_args in [[], ['--module-naming-scheme=HierarchicalMNS']]: outtxt = self.eb_main(args + extra_args + ['--try-software-version=1.2.3'], verbose=True, raise_error=True) - # toolchain GCC/4.7.2 (subtoolchain of gompi/2018a) should be listed (and present) + # toolchain GCC/6.4.0-2.28 (subtoolchain of gompi/2018a) should be listed (and present) tc_regex = re.compile(r"^ \* \[x\] .*/GCC-6.4.0-2.28.eb \(module: .*GCC/6.4.0-2.28\)$", re.M) self.assertTrue(tc_regex.search(outtxt), "Pattern %s found in %s" % (tc_regex.pattern, outtxt)) @@ -1814,10 +1814,9 @@ def test_recursive_try(self): # since this subtoolchain already has sufficient capabilities (we do not map higher than necessary) for ec_name in ['gzip-1.4', 'toy-1.2.3']: ec = '%s-GCC-6.4.0-2.28.eb' % ec_name - if extra_args: - mod = ec_name.replace('-', '/') - else: - mod = '%s-GCC-6.4.0-2.28' % ec_name.replace('-', '/') + mod = ec_name.replace('-', '/') + if not extra_args: + mod += '-GCC-6.4.0-2.28' mod_regex = re.compile(r"^ \* \[ \] \S+/eb-\S+/%s \(module: .*%s\)$" % (ec, mod), re.M) self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) From d65d12dd3ccf2887e3bb75b60ac85e70a290b33c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 11:20:50 +0200 Subject: [PATCH 335/344] also document update_build_specs named argument of map_easyconfig_to_target_tc_hierarchy in docstring --- easybuild/framework/easyconfig/tweak.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 483d7e9898..791e74bc04 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -938,9 +938,10 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= """ Take an easyconfig spec, parse it, map it to a target toolchain and dump it out - :param ec_spec: Location of original easyconfig file - :param toolchain_mapping: Mapping between source toolchain and target toolchain - :param targetdir: Directory to dump the modified easyconfig file in + :param ec_spec: location of original easyconfig file + :param toolchain_mapping: mapping between source toolchain and target toolchain + :param targetdir: directory to dump the modified easyconfig file in + :param update_build_specs: dict with names and values of easyconfig parameters to tweak :param update_dep_versions: boolean indicating whether dependency versions should be updated :return: Location of the modified easyconfig file From 10f834d888252ca8f8c22d36bdb505fed05d3c7a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 11:32:36 +0200 Subject: [PATCH 336/344] change log.info to log.warning in map_common_versionsuffixes + complete docstring for get_matching_easyconfig_candidates --- easybuild/framework/easyconfig/tweak.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 791e74bc04..0d65fed2d3 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -887,7 +887,7 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp # make sure we have a have an integer value for the major version int(major_version) except ValueError: - _log.info("Cannot extract major version for %s from %s", prefix_stub, version) + _log.warning("Cannot extract major version for %s from %s", prefix_stub, version) # Use these values to construct a dependency software_as_dep = { @@ -918,6 +918,7 @@ def map_common_versionsuffixes(software_name, original_toolchain, toolchain_mapp def get_matching_easyconfig_candidates(prefix_stub, toolchain): """ + Find easyconfigs that match specified requirements w.r.t. toolchain and partial filename pattern. :param prefix_stub: stub used in regex (e.g., 'Python-' or 'Python-2') :param toolchain: the toolchain to use with the search From 9f460a3334d1a925217e28a8790d19c6161103ca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 11:42:03 +0200 Subject: [PATCH 337/344] tweak comments in find_potential_version_mappings + minor enhancement to test --- easybuild/framework/easyconfig/tweak.py | 16 ++++++++++------ test/framework/tweak.py | 9 ++++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 0d65fed2d3..ed7545b921 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -1057,6 +1057,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=None, highest_versions_only=True): """ Find potential version mapping for a dependency in a new hierarchy + :param dep: dependency specification (dict) :param toolchain_mapping: toolchain mapping used for search :param versionsuffix_mapping: mapping of version suffixes @@ -1087,13 +1088,13 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin if versionsuffix in versionsuffix_mapping: versionsuffix = versionsuffix_mapping[versionsuffix] - # the candidate version is a regex string, let's be conservative and search for patch upgrade first, if that doesn't - # work look for a minor version upgrade and if that fails will we try a global search, i.e, a major version upgrade - # (assumes major.minor.XXX versioning) + # the candidate version is a regex string, let's be conservative and search for patch upgrade first; + # if that doesn't work look for a minor version upgrade and if that fails will we try a global search, + # i.e, a major version upgrade (assumes major.minor.xxx versioning) candidate_ver_list = [] version_components = dep['version'].split('.') major_version = version_components[0] - if len(version_components) > 2: # Have something like major.minor.XXX + if len(version_components) > 2: # Have something like major.minor.xxx minor_version = version_components[1] candidate_ver_list.append(r'%s\.%s\..*' % (major_version, minor_version)) if len(version_components) > 1: # Have at least major.minor @@ -1103,7 +1104,10 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin potential_version_mappings, highest_version = [], None for candidate_ver in candidate_ver_list: + + # if any potential version mappings were found already at this point, we don't add more if not potential_version_mappings: + for toolchain in toolchain_hierarchy: # determine search pattern based on toolchain, version prefix/suffix & version regex @@ -1116,11 +1120,11 @@ def find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mappin cand_paths = search_easyconfigs(depver, consider_extra_paths=False, print_result=False, case_sensitive=True) - # Filter out easyconfigs that have been tweaked in this instance, they are not relevant here + # filter out easyconfigs that have been tweaked in this instance, they are not relevant here tweaked_ecs_paths, _ = alt_easyconfig_paths(tempfile.gettempdir(), tweaked_ecs=True) cand_paths = [path for path in cand_paths if not path.startswith(tweaked_ecs_paths)] - # Add what is left to the possibilities + # add what is left to the possibilities for path in cand_paths: version = fetch_parameters_from_easyconfig(read_file(path), ['version'])[0] if version: diff --git a/test/framework/tweak.py b/test/framework/tweak.py index f252ecfcec..0797e76de5 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -373,9 +373,12 @@ def test_find_potential_version_mappings(self): potential_versions = find_potential_version_mappings(gzip_dep, tc_mapping) self.assertEqual(len(potential_versions), 1) # Should see version 1.6 of gzip with iccifort toolchain - expected_dep_path = os.path.join(test_easyconfigs, 'g', 'gzip', - 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb') - self.assertEqual(potential_versions[0]['path'], expected_dep_path) + expected = { + 'path': os.path.join(test_easyconfigs, 'g', 'gzip', 'gzip-1.6-iccifort-2016.1.150-GCC-4.9.3-2.25.eb'), + 'toolchain': {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}, + 'version': '1.6', + } + self.assertEqual(potential_versions[0], expected) def test_map_easyconfig_to_target_tc_hierarchy(self): """Test mapping of easyconfig to target hierarchy""" From 88be361d39fe36a02eb3e0048ccc6b3a1181710f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 11:53:19 +0200 Subject: [PATCH 338/344] minor comments cleanup in map_easyconfig_to_target_tc_hierarchy --- easybuild/framework/easyconfig/tweak.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index ed7545b921..e39dfae559 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -957,8 +957,9 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= versonsuffix_mapping = map_common_versionsuffixes('Python', parsed_ec['toolchain'], toolchain_mapping) if update_build_specs is not None: - # automagically clear out list of checksums if software version is being tweaked if 'version' in update_build_specs: + + # take into account that version in exts_list may have to be updated as well if 'exts_list' in parsed_ec and parsed_ec['exts_list']: _log.warning("Found 'exts_list' in %s, will only update extension version of %s (if applicable)", ec_spec, parsed_ec['name']) @@ -976,10 +977,12 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= parsed_ec.get_ref('exts_list')[idx] = tuple(ext_as_list) _log.info("Updated extension found in %s with new version", ec_spec) + # automagically clear out list of checksums if software version is being tweaked if 'checksums' not in update_build_specs: update_build_specs['checksums'] = [] _log.warning("Tweaking version: checksums cleared, verification disabled.") - # Update the keys according to the build specs + + # update the keys according to the build specs for key in update_build_specs: parsed_ec[key] = update_build_specs[key] @@ -1003,8 +1006,10 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= orig_val = flatten(orig_val) for idx, dep in enumerate(val): + # reference to original dep dict, this is the one we should be updating orig_dep = orig_val[idx] + # skip dependencies that are marked as external modules if dep['external_module']: continue @@ -1020,11 +1025,11 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= dep_changed = True elif update_dep_versions: - # Search for available updates for this dependency: - # First get highest version candidate paths for this (include search through subtoolchains) + # search for available updates for this dependency: + # first get highest version candidate paths for this (include search through subtoolchains) potential_version_mappings = find_potential_version_mappings(dep, toolchain_mapping, versionsuffix_mapping=versonsuffix_mapping) - # Only highest version match is retained by default in potential_version_mappings, + # only highest version match is retained by default in potential_version_mappings, # compare that version to the original version and replace if appropriate (upgrades only). if potential_version_mappings: highest_version_match = potential_version_mappings[0]['version'] @@ -1042,7 +1047,7 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir= if dep_changed: _log.debug("Modified dependency %s of %s", dep['name'], ec_spec) - # Determine the name of the modified easyconfig and dump it to target_dir + # determine the name of the modified easyconfig and dump it to target_dir if parsed_ec['versionsuffix'] in versonsuffix_mapping: parsed_ec['versionsuffix'] = versonsuffix_mapping[parsed_ec['versionsuffix']] ec_filename = '%s-%s.eb' % (parsed_ec['name'], det_full_ec_version(parsed_ec)) From a05f962c3de99f17a8219eac11970d3f93f18427 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 12:35:27 +0200 Subject: [PATCH 339/344] add (simple) end-to-end test for --try-update-deps --- test/framework/options.py | 47 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 3b2e163340..a755b7d7c4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1178,6 +1178,53 @@ def test_try_toolchain_mapping(self): regex = re.compile(anti_pattern, re.M) self.assertFalse(regex.search(outtxt), "Pattern '%s' NOT found in: %s" % (regex.pattern, outtxt)) + def test_try_update_deps(self): + """Test for --try-update-deps.""" + + # first, construct a toy easyconfig that is well suited for testing (multiple deps) + test_ectxt = '\n'.join([ + "easyblock = 'ConfigureMake'", + '', + "name = 'test'", + "version = '1.2.3'", + '' + "homepage = 'https://test.org'", + "description = 'this is just a test'", + '', + "toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}", + '', + "builddependencies = [('gzip', '1.4')]", + "dependencies = [('hwloc', '1.6.2')]", + ]) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ectxt) + + args = [ + test_ec, + '--try-toolchain-version=6.4.0-2.28', + '--try-update-deps', + '-D', + ] + + self.assertErrorRegex(EasyBuildError, "Experimental functionality", self.eb_main, args, raise_error=True) + + args.append('--experimental') + outtxt = self.eb_main(args, raise_error=True, do_build=True) + + patterns = [ + # toolchain got updated + r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$", + # no version update for gzip (because there's no gzip easyconfig using GCC/6.4.0-2.28 (sub)toolchain) + r"^ \* \[ \] .*/tweaked_dep_easyconfigs/gzip-1.4-GCC-6.4.0-2.28.eb \(module: gzip/1.4-GCC-6.4.0-2.28\)$", + # hwloc was updated to 1.11.8, thanks to available easyconfig + r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", + # also generated easyconfig for test/1.2.3 with expected toolchain + r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + def test_dry_run_hierarchical(self): """Test dry run using a hierarchical module naming scheme.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From ee6327127b233a132cafe897c0fa9a744d929d84 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 12:36:38 +0200 Subject: [PATCH 340/344] take into account additional test easyconfigs in test_index_functions --- test/framework/filetools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 7e5502c78e..6595a7884d 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1715,7 +1715,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 79) + self.assertEqual(len(index), 81) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), @@ -1764,7 +1764,7 @@ def test_index_functions(self): regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 24) + self.assertEqual(len(index), 26) for fn in expected: self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) @@ -1794,7 +1794,7 @@ def test_index_functions(self): regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 24) + self.assertEqual(len(index), 26) for fn in expected: self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index))) From 5135389c0fc11b22977c7af7f0146728de85f8bd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 11 Apr 2020 17:23:18 +0200 Subject: [PATCH 341/344] stop using apt-spy2 to pick an alternative mirror, it's expensive and may result in picking a faulty mirror... --- .github/workflows/unit_tests.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 94dd61e29b..c905a6a342 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -49,14 +49,9 @@ jobs: - name: install OS & Python packages run: | - # use apt-spy2 to select closest apt mirror, - # which helps avoid connectivity issues in Azure; - # see https://github.com/actions/virtual-environments/issues/675 - sudo gem install apt-spy2 - sudo apt-spy2 check - sudo apt-spy2 fix --commit - # after selecting a specific mirror, we need to run 'apt-get update' - sudo apt-get update + # disable apt-get update, we don't really need it, + # and it does more harm than good (it's fairly expensive, and it results in flaky test runs) + # sudo apt-get update # for modules tool sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082 From aa9506a28899973e2281d45c202ade03060b8826 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 13 Apr 2020 13:47:26 +0800 Subject: [PATCH 342/344] prepare release notes for eb420 --- RELEASE_NOTES | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8239f714bc..c022abb4c8 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,67 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.2.0 (April 14th 2020) +------------------------ + +feature release + +- various enhancements, including: + - add support for --try-update-deps, to upgrade dependencies based on available easyconfigs (#2599) + - add a contrib/hooks dir with some examples of hooks used (#2777) + - adding locking to prevent two parallel builds of the same installation directory (#3009) + - also mention working directory + input passed via stdin (if any) in trace output of run_cmd (#3168) + - add probe of PREFIX and VERSION variable in external modules (#3174) + - also update $CMAKE_PREFIX_PATH and $CMAKE_LIBRARY_PATH in generated module file (#3176) + - add support for targeting easyblocks and framework in --new-pr (#3189) + - optionally call log.warning in print_warning (#3195) + - add an option to git_config to retain the .git directory (#3197) + - add support for --include-easyblocks-from-pr (#3206) + - add support for creating an index & using it when searching for easyconfigs (#3210) + - allow use of SYSTEM as --try-toolchain option (#3213) + - mention CPU arch name in comment for uploaded test report, if it's known by archspec (#3227) + - make --merge-pr take into account --pr-target-branch (#3236) + - make --check-contrib print a warning when None is used for checksums (#3244) + - update install-EasyBuild-develop.sh script and create script for 2020a merge sprint (#3245) + - add template for mpi_cmd_prefix (#3264) + - update copy_dir to include option to merge directories (#3270) + - support template name for CUDA version (#3274) +- various bug fixes, including: + - use correct module for errors_found_in_log (#3119) + - fix EasyConfig.update code to handle both strings and lists as input. (#3170) + - fix removing temporary branch on --check-github (#3182) + - fix shebang even if first line doesn't start with '#!' (#3183) + - make boostrap script work with Python 3 (#3186) + - read patch files as bytestring to avoid UnicodeDecodeError for patches that include funky characters (#3191) + - fix online check in --check-github: try repeatedly and with different URLs to cater for HTTP issues (#3194) + - don't crash when GitPython is not installed in Python3 (#3198) + - fix os_name_map for RHEL8 (#3201) + - significantly speed up -D/--dry-run by avoiding useless 'module show' (#3203) + - don't add shebang to binary files (#3208) + - use checkout@v2 in GitHub Actions to fix broken re-triggered tests (#3219) + - don't filter out None values in to_checksums, leave them in place (#3225) + - fix defining of $MPI_INC_* and $MPI_LIB_* environment variables for impi version 2019 and later (#3229) + - use correct target account/repo when creating test report & posting comment in PR (#3234) + - reorganize EasyBlock.extensions_step to ensure correct filtering of extensions (#3235) + - also support %(installdir)s and %(builddir)s templates for extensions (#3237) + - unset $GITHUB_TOKEN in Travis after installing token, to avoid failing test_from_pr_token_log (#3252) + - fix reporting when skipping extensions (#3254) + - avoid API rate limit errors on online check in --check-github (#3257) + - show easyconfig filenames for parallel build (#3258) + - make EasyConfig.dump aware of toolchain hierarchy, to avoid hardcoded subtoolchains in dependencies easyconfig parameters (#3261) + - fix use of --copy-ec with a single argument, assume copy to current working directory (#3262) + - use apt-spy2 before using apt in GitHub Actions (#3268) + - fix posting of comment in PR with --upload-test-report (#3272) + - take into account dependencies marked as external modules when composing template values like %(pyver)s (#3273) +- other changes: + - increase timeout & use api.github.com for connectivity check in check_github (#3192) + - restore flake8 default ignores (#3193) + - enable tracking of test suite coverage using coveralls (#3204) + - make tests use easybuilders/testrepository rather than hpcugent/testrepository after it was moved (#3238) + - improve raised error in remove_dir and deprecate rmtree2 (#3228) + - stop using apt-spy2 to pick an alternative mirror, it's expensive and may result in picking a faulty mirror (#3276) + + v4.1.2 (March 16th 2020) ------------------------ @@ -12,6 +73,7 @@ bugfix release - fix broken test for --review-pr by using different PR to test with (#3226) - censor authorization part of headers before logging ReST API request (#3248) + v4.1.1 (January 16th 2020) -------------------------- From 94c0cc964ff80097a0da504fc1857685f63e7b56 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Apr 2020 11:08:45 +0200 Subject: [PATCH 343/344] update version to 4.2.0 --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index ae1710966d..79824ff42b 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.2.0.dev0') +VERSION = LooseVersion('4.2.0') UNKNOWN = 'UNKNOWN' From 57fb8e19e589d2637e6b752d7654a3b3aed84ec9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 13 Apr 2020 13:47:43 +0200 Subject: [PATCH 344/344] tweak release notes for EasyBuild v4.2.0: move up most prominent new features + refer to docs --- RELEASE_NOTES | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c022abb4c8..1b71f89630 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -8,18 +8,23 @@ v4.2.0 (April 14th 2020) feature release -- various enhancements, including: - - add support for --try-update-deps, to upgrade dependencies based on available easyconfigs (#2599) +- add support for --try-update-deps (experimental feature), to upgrade dependencies based on available easyconfigs (#2599) +- adding locking to prevent two parallel builds of the same installation directory (#3009) + - for more information, see https://easybuild.readthedocs.io/en/latest/Locks.html +- significantly speed up -D/--dry-run by avoiding useless 'module show' (#3203) +- add support for creating an index & using it when searching for easyconfigs (#3210) + - for more information, see https://easybuild.readthedocs.io/en/latest/Easyconfigs_index.html +- additional GitHub integration features: + - add support for targeting easyblocks and framework repositories in --new-pr (#1876, #3189) + - add support for --include-easyblocks-from-pr (#3206) + - for more information, https://easybuild.readthedocs.io/en/latest/Integration_with_GitHub.html +- various other enhancements, including: - add a contrib/hooks dir with some examples of hooks used (#2777) - - adding locking to prevent two parallel builds of the same installation directory (#3009) - also mention working directory + input passed via stdin (if any) in trace output of run_cmd (#3168) - - add probe of PREFIX and VERSION variable in external modules (#3174) + - probe external modules for missing metadata that is not provided via extermal module metadata file (#3174) - also update $CMAKE_PREFIX_PATH and $CMAKE_LIBRARY_PATH in generated module file (#3176) - - add support for targeting easyblocks and framework in --new-pr (#3189) - optionally call log.warning in print_warning (#3195) - add an option to git_config to retain the .git directory (#3197) - - add support for --include-easyblocks-from-pr (#3206) - - add support for creating an index & using it when searching for easyconfigs (#3210) - allow use of SYSTEM as --try-toolchain option (#3213) - mention CPU arch name in comment for uploaded test report, if it's known by archspec (#3227) - make --merge-pr take into account --pr-target-branch (#3236) @@ -30,15 +35,14 @@ feature release - support template name for CUDA version (#3274) - various bug fixes, including: - use correct module for errors_found_in_log (#3119) - - fix EasyConfig.update code to handle both strings and lists as input. (#3170) + - fix EasyConfig.update code to handle both strings and lists as input (#3170) - fix removing temporary branch on --check-github (#3182) - fix shebang even if first line doesn't start with '#!' (#3183) - make boostrap script work with Python 3 (#3186) - read patch files as bytestring to avoid UnicodeDecodeError for patches that include funky characters (#3191) - fix online check in --check-github: try repeatedly and with different URLs to cater for HTTP issues (#3194) - - don't crash when GitPython is not installed in Python3 (#3198) + - don't crash when GitPython is not installed when using Python 3 (#3198) - fix os_name_map for RHEL8 (#3201) - - significantly speed up -D/--dry-run by avoiding useless 'module show' (#3203) - don't add shebang to binary files (#3208) - use checkout@v2 in GitHub Actions to fix broken re-triggered tests (#3219) - don't filter out None values in to_checksums, leave them in place (#3225) @@ -52,7 +56,6 @@ feature release - show easyconfig filenames for parallel build (#3258) - make EasyConfig.dump aware of toolchain hierarchy, to avoid hardcoded subtoolchains in dependencies easyconfig parameters (#3261) - fix use of --copy-ec with a single argument, assume copy to current working directory (#3262) - - use apt-spy2 before using apt in GitHub Actions (#3268) - fix posting of comment in PR with --upload-test-report (#3272) - take into account dependencies marked as external modules when composing template values like %(pyver)s (#3273) - other changes: @@ -61,7 +64,6 @@ feature release - enable tracking of test suite coverage using coveralls (#3204) - make tests use easybuilders/testrepository rather than hpcugent/testrepository after it was moved (#3238) - improve raised error in remove_dir and deprecate rmtree2 (#3228) - - stop using apt-spy2 to pick an alternative mirror, it's expensive and may result in picking a faulty mirror (#3276) v4.1.2 (March 16th 2020) @@ -72,6 +74,7 @@ bugfix release - fix gitdb dependency on Python 2.6 in test configuration (#3212) - fix broken test for --review-pr by using different PR to test with (#3226) - censor authorization part of headers before logging ReST API request (#3248) + - see also https://github.com/easybuilders/easybuild-framework/security/advisories/GHSA-2wx6-wc87-rmjm v4.1.1 (January 16th 2020)