diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index ef310a3816..540fafcea1 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -73,7 +73,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz - name: run test diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 77d2a4a395..f7040000f9 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -73,7 +73,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz - name: run test diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml index ff9a5eaa4f..220258cd9f 100644 --- a/.github/workflows/eb_command.yml +++ b/.github/workflows/eb_command.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] fail-fast: false steps: - uses: actions/checkout@v3 @@ -68,7 +68,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz - name: run tests for 'eb' command env: diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index d722761208..e7c25b22d7 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11'] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c79b515e42..c9f42891ec 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -37,8 +37,6 @@ jobs: lc_all: [""] include: # Test different Python 3 versions with Lmod 8.x - - python: 3.5 - modules_tool: ${{needs.setup.outputs.lmod8}} - python: 3.7 modules_tool: ${{needs.setup.outputs.lmod8}} - python: 3.8 @@ -140,7 +138,7 @@ jobs: python setup.py sdist ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA - pip install --prefix $PREFIX dist/easybuild-framework*tar.gz + pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz - name: run test suite env: diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 07c9756e67..5a45be8a56 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,28 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.2 (12 June 2024) +--------------------- + +update/bugfix release + +- various enhancements, including: + - improve behavior when using extension which has 'nosource' enabled (#4506) + - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513) + - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553) + - add support for easyconfig parameter 'module_only' (#4537) +- various bug fixes, including: + - fix typo in patch_step logging (#4505) + - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507) + - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512) + - fix resolved (template) values in case of failure (#4532) + - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551) + - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552) +- other changes: + - code cleanup in 'easyblock.py' (#4519) + - stop running unit tests on Python 3.5 (#4530) + + v4.9.1 (5 April 2024) --------------------- diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index d2727d61ee..80cddf8819 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -1068,7 +1068,7 @@ def make_builddir(self): self.log.info("Overriding 'cleanupoldinstall' (to False), 'cleanupoldbuild' (to True) " "and 'keeppreviousinstall' because we're building in the installation directory.") # force cleanup before installation - if build_option('module_only'): + if build_option('module_only') or self.cfg['module_only']: self.log.debug("Disabling cleanupoldbuild because we run as module-only") self.cfg['cleanupoldbuild'] = False else: @@ -1139,7 +1139,7 @@ def make_dir(self, dir_name, clean, dontcreateinstalldir=False): if self.cfg['keeppreviousinstall']: self.log.info("Keeping old directory %s (hopefully you know what you are doing)", dir_name) return - elif build_option('module_only'): + elif build_option('module_only') or self.cfg['module_only']: self.log.info("Not touching existing directory %s in module-only mode...", dir_name) elif clean: remove_dir(dir_name) @@ -1365,7 +1365,7 @@ def make_module_dep(self, unload_info=None): multi_dep_mod_names[dep['name']].append(dep['short_mod_name']) multi_dep_load_defaults = [] - for depname, depmods in sorted(multi_dep_mod_names.items()): + for _, depmods in sorted(multi_dep_mod_names.items()): stmt = self.module_generator.load_module(depmods[0], multi_dep_mods=depmods, recursive_unload=recursive_unload, depends_on=depends_on) @@ -1769,10 +1769,7 @@ def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True): return ext_sep.join(exts_list) def prepare_for_extensions(self): - """ - Also do this before (eg to set the template) - """ - pass + """Ran before installing extensions (eg to set templates)""" def skip_extensions(self): """ @@ -2114,7 +2111,7 @@ def guess_start_dir(self): start_dir = '' # do not use the specified 'start_dir' when running as --module-only as # the directory will not exist (extract_step is skipped) - if self.start_dir and not build_option('module_only'): + if self.start_dir and not build_option('module_only') and not self.cfg['module_only']: start_dir = self.start_dir if not os.path.isabs(start_dir): @@ -2198,9 +2195,9 @@ def handle_iterate_opts(self): self.log.info("Current iteration index: %s", self.iter_idx) # pop first element from all iterative easyconfig parameters as next value to use - for opt in self.iter_opts: - if len(self.iter_opts[opt]) > self.iter_idx: - self.cfg[opt] = self.iter_opts[opt][self.iter_idx] + for opt, value in self.iter_opts.items(): + if len(value) > self.iter_idx: + self.cfg[opt] = value[self.iter_idx] else: self.cfg[opt] = '' # empty list => empty option as next value self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt]))) @@ -2212,12 +2209,12 @@ def post_iter_step(self): """Restore options that were iterated over""" # disable templating, since we're messing about with values in self.cfg with self.cfg.disable_templating(): - for opt in self.iter_opts: - self.cfg[opt] = self.iter_opts[opt] + for opt, value in self.iter_opts.items(): + self.cfg[opt] = value # also need to take into account extensions, since those were iterated over as well for ext in self.ext_instances: - ext.cfg[opt] = self.iter_opts[opt] + ext.cfg[opt] = value self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt]) @@ -2349,7 +2346,7 @@ def check_readiness_step(self): self.log.info("No module %s found. Not skipping anything." % self.full_mod_name) # remove existing module file under --force (but only if --skip is not used) - elif build_option('force') or build_option('rebuild'): + elif (build_option('force') or build_option('rebuild')) and not build_option('dump_env_script'): self.remove_module_file() def fetch_step(self, skip_checksums=False): @@ -2607,7 +2604,7 @@ def patch_step(self, beginpath=None, patches=None): copy_patch = 'copy' in patch and 'sourcepath' not in patch self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s", - srcind, level, srcpathsuffix, copy) + srcind, level, srcpathsuffix, copy_patch) if beginpath is None: try: @@ -2751,10 +2748,7 @@ def _test_step(self): self.report_test_failure(err) def stage_install_step(self): - """ - Install in a stage directory before actual installation. - """ - pass + """Install in a stage directory before actual installation.""" def install_step(self): """Install built software (abstract method).""" @@ -3248,7 +3242,7 @@ def sanity_check_linked_shared_libs(self, subdirs=None): required_libs.extend(self.cfg['required_linked_shared_libs']) # early return if there are no banned/required libraries - if not (banned_libs + required_libs): + if not banned_libs + required_libs: self.log.info("No banned/required libraries specified") return [] else: @@ -3795,7 +3789,7 @@ def make_module_step(self, fake=False): try: self.make_devel_module() except EasyBuildError as error: - if build_option('module_only'): + if build_option('module_only') or self.cfg['module_only']: self.log.info("Using --module-only so can recover from error: %s", error) else: raise error @@ -3903,7 +3897,7 @@ def skip_step(self, step, skippable): """Dedice whether or not to skip the specified step.""" skip = False force = build_option('force') - module_only = build_option('module_only') + module_only = build_option('module_only') or self.cfg['module_only'] sanity_check_only = build_option('sanity_check_only') skip_extensions = build_option('skip_extensions') skip_test_step = build_option('skip_test_step') @@ -4463,7 +4457,7 @@ def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): else: easyblock_paths.add(easyblock_path) for easyblock_path in easyblock_paths: - easyblock_basedir, easyblock_filename = os.path.split(easyblock_path) + easyblock_filename = os.path.basename(easyblock_path) copy_file(easyblock_path, os.path.join(reprod_easyblock_dir, easyblock_filename)) _log.info("Dumped easyblock %s required for reproduction to %s", easyblock_filename, reprod_easyblock_dir) @@ -4594,10 +4588,7 @@ def build_easyconfigs(easyconfigs, output_dir, test_results): class StopException(Exception): - """ - StopException class definition. - """ - pass + """Exception thrown to stop running steps""" def inject_checksums_to_json(ecs, checksum_type): @@ -4645,14 +4636,14 @@ def inject_checksums_to_json(ecs, checksum_type): # actually inject new checksums or overwrite existing ones (if --force) existing_checksums = app.get_checksums_from_json(always_read=True) - for filename in checksums: + for filename, checksum in checksums.items(): if filename not in existing_checksums: - existing_checksums[filename] = checksums[filename] + existing_checksums[filename] = checksum # don't do anything if the checksum already exist and is the same - elif checksums[filename] != existing_checksums[filename]: + elif checksum != existing_checksums[filename]: if build_option('force'): print_warning("Found existing checksums for %s, overwriting them (due to --force)..." % ec_fn) - existing_checksums[filename] = checksums[filename] + existing_checksums[filename] = checksum else: raise EasyBuildError("Found existing checksum for %s, use --force to overwrite them" % filename) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 965e6db037..ce11d0457b 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -109,6 +109,7 @@ 'hidden': [False, "Install module file as 'hidden' by prefixing its version with '.'", BUILD], 'installopts': ['', 'Extra options for installation', BUILD], 'maxparallel': [None, 'Max degree of parallelism', BUILD], + 'module_only': [False, 'Only generate module file', BUILD], 'parallel': [None, ('Degree of parallelism for e.g. make (default: based on the number of ' 'cores, active cpuset and restrictions in ulimit)'), BUILD], 'patches': [[], "List of patches to apply", BUILD], diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 73182da7ac..2b6aaa751a 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2026,12 +2026,14 @@ def resolve_template(value, tmpl_dict): # '%(name)s' -> '%(name)s' # '%%(name)s' -> '%%(name)s' if '%' in value: + orig_value = value value = re.sub(re.compile(r'(%)(?!%*\(\w+\)s)'), r'\1\1', value) try: value = value % tmpl_dict except KeyError: _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + value = orig_value # Undo "%"-escaping else: # this block deals with references to objects and returns other references # for reading this is ok, but for self['x'] = {} diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index 69c824fe7e..efd7c349f7 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -126,10 +126,12 @@ def _set_start_dir(self): elif ext_start_dir is None: # This may be on purpose, e.g. for Python WHL files which do not get extracted self.log.debug("Start dir is not set.") - else: + elif self.start_dir: # non-existing start dir means wrong input from user - warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name, - ext_start_dir) + raise EasyBuildError("Provided start dir (%s) for extension %s does not exist: %s", + self.start_dir, self.name, ext_start_dir) + else: + warn_msg = 'Failed to determine start dir for extension %s: %s' % (self.name, ext_start_dir) self.log.warning(warn_msg) print_warning(warn_msg, silent=build_option('silent')) @@ -137,7 +139,10 @@ def run(self, unpack_src=False): """Common operations for extensions: unpacking sources, patching, ...""" # unpack file if desired - if unpack_src: + if self.options.get('nosource', False): + # If no source wanted use the start_dir from the main EC + self.ext_dir = self.master.start_dir + elif unpack_src: targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name)) self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options, change_into_dir=False, cmd=self.src_extract_cmd) @@ -146,10 +151,9 @@ def run(self, unpack_src=False): # because start_dir value is usually a relative path (if it is set) change_dir(self.ext_dir) - self._set_start_dir() + self._set_start_dir() + if self.start_dir: change_dir(self.start_dir) - else: - self._set_start_dir() # patch if needed EasyBlock.patch_step(self, beginpath=self.ext_dir) diff --git a/easybuild/main.py b/easybuild/main.py index df5f23a460..451152f322 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -83,6 +83,8 @@ from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_easybuild_deps from easybuild.tools.testing import create_test_report, overall_test_report, regtest, session_state +from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION, UNKNOWN_EASYBLOCKS_VERSION +from easybuild.tools.version import different_major_versions _log = None @@ -440,9 +442,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules keep_available_modules = any(( - forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums, - options.sanity_check_only, options.inject_checksums_to_json) - ) + forced, dry_run_mode, any_pr_option_set, options.copy_ec, options.dump_env_script, options.extended_dry_run, + options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only + )) # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: @@ -618,6 +620,15 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, pr (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, from_pr_list, tweaked_ecs_paths) = cfg_settings + # compare running Framework and EasyBlocks versions + if EASYBLOCKS_VERSION == UNKNOWN_EASYBLOCKS_VERSION: + # most likely reason is running framework unit tests with no easyblocks installation + # so log a warning, to avoid test related issues + _log.warning("Unable to determine EasyBlocks version, so we'll assume it is not different from Framework") + elif different_major_versions(FRAMEWORK_VERSION, EASYBLOCKS_VERSION): + raise EasyBuildError("Framework (%s) and EasyBlock (%s) major versions are different." % (FRAMEWORK_VERSION, + EASYBLOCKS_VERSION)) + # load hook implementations (if any) hooks = load_hooks(options.hooks) diff --git a/easybuild/toolchains/linalg/libsci.py b/easybuild/toolchains/linalg/libsci.py index 07ea64ed82..6d7e93c16d 100644 --- a/easybuild/toolchains/linalg/libsci.py +++ b/easybuild/toolchains/linalg/libsci.py @@ -65,13 +65,20 @@ def _get_software_root(self, name, required=True): """Get install prefix for specified software name; special treatment for Cray modules.""" if name == 'cray-libsci': # Cray-provided LibSci module - env_var = 'CRAY_LIBSCI_PREFIX_DIR' - root = os.getenv(env_var, None) + root = None + # consider both $CRAY_LIBSCI_PREFIX_DIR and $CRAY_PE_LIBSCI_PREFIX_DIR, + # cfr. https://github.com/easybuilders/easybuild-framework/issues/4536 + env_vars = ('CRAY_LIBSCI_PREFIX_DIR', 'CRAY_PE_LIBSCI_PREFIX_DIR') + for env_var in env_vars: + root = os.getenv(env_var, None) + if root is not None: + self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + break + if root is None: if required: - raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_var) - else: - self.log.debug("Obtained install prefix for %s via $%s: %s", name, env_var, root) + env_vars_str = ', '.join('$' + e for e in env_vars) + raise EasyBuildError("Failed to determine install prefix for %s via $%s", name, env_vars_str) else: root = super(LibSci, self)._get_software_root(name, required=required) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 3a5cbecb55..6bec64764c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -278,6 +278,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'debug', 'debug_lmod', 'dump_autopep8', + 'dump_env_script', 'enforce_checksums', 'experimental', 'extended_dry_run', diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ae9b78b5ed..6d4a1300fc 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -60,6 +60,7 @@ from easybuild.tools.py2vs3 import HTTPError, URLError, ascii_letters, urlopen from easybuild.tools.systemtools import UNKNOWN, get_tool_version from easybuild.tools.utilities import nub, only_if_module_is_available +from easybuild.tools.version import FRAMEWORK_VERSION, different_major_versions _log = fancylogger.getLogger('github', fname=False) @@ -587,9 +588,45 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, gi else: raise EasyBuildError("Couldn't find path to patched file %s", full_path) + if github_repo == GITHUB_EASYCONFIGS_REPO: + ver_file = os.path.join(final_path, 'setup.py') + elif github_repo == GITHUB_EASYBLOCKS_REPO: + ver_file = os.path.join(final_path, 'easybuild', 'easyblocks', '__init__.py') + else: + raise EasyBuildError("Don't know how to determine version for repo %s", github_repo) + + # take into account that the file we need to determine repo version may not be available, + # for example when a closed PR is used (since then we only download files patched by the PR) + if os.path.exists(ver_file): + ver = _get_version_for_repo(ver_file) + if different_major_versions(FRAMEWORK_VERSION, ver): + raise EasyBuildError("Framework (%s) is a different major version than used in %s/%s PR #%s (%s)", + FRAMEWORK_VERSION, github_account, github_repo, pr, ver) + return files +def _get_version_for_repo(filename): + """Extract version from filename.""" + _log.debug("Extract version from %s" % filename) + + try: + ver_line = "" + with open(filename) as f: + for line in f.readlines(): + if line.startswith("VERSION "): + ver_line = line + break + + # version can be a string or LooseVersion + res = re.search(r"""^VERSION = .*['"](.*)['"].?$""", ver_line) + + _log.debug("PR target version is %s" % res.group(1)) + return res.group(1) + except Exception: + raise EasyBuildError("Couldn't determine version of PR from %s" % filename) + + def fetch_easyblocks_from_pr(pr, path=None, github_user=None): """Fetch patched easyblocks for a particular PR.""" return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) @@ -612,6 +649,13 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, if github_repo is None: github_repo = GITHUB_EASYCONFIGS_REPO + if github_repo == GITHUB_EASYCONFIGS_REPO: + easybuild_subdir = os.path.join('easybuild', 'easyconfigs') + elif github_repo == GITHUB_EASYBLOCKS_REPO: + easybuild_subdir = os.path.join('easybuild', 'easyblocks') + else: + raise EasyBuildError("Unknown repo: %s", github_repo) + if path is None: if github_repo == GITHUB_EASYCONFIGS_REPO: extra_ec_paths = build_option('extra_ec_paths') @@ -655,6 +699,12 @@ def fetch_files_from_commit(commit, files=None, path=None, github_account=None, else: raise EasyBuildError("Unknown repo: %s" % github_repo) + # symlink subdirectories of 'easybuild/easy{blocks,configs}' into path that gets added to robot search path + mkdir(path, parents=True) + dirpath = os.path.join(repo_commit, easybuild_subdir) + for subdir in os.listdir(dirpath): + symlink(os.path.join(dirpath, subdir), os.path.join(path, subdir)) + # copy specified files to directory where they're expected to be found file_paths = [] for file in files: diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index fc21a564b9..7318be4f4f 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -37,6 +37,7 @@ * Jens Timmerman (Ghent University) * David Brown (Pacific Northwest National Laboratory) """ +import glob import os import re import shlex @@ -52,6 +53,7 @@ from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.py2vs3 import subprocess_popen_text from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.utilities import get_subclasses, nub # software root/version environment variable name prefixes @@ -1671,6 +1673,7 @@ def get_software_libdir(name, only_one=True, fs=None): Returns the library subdirectory, relative to software root. It fails if multiple library subdirs are found, unless only_one is False which yields a list of all library subdirs. + If only_one is True and fs is None, select the one subdirectory with shared or static libraries, if possible. :param name: name of the software package :param only_one: indicates whether only one lib path is expected to be found @@ -1703,6 +1706,16 @@ def get_software_libdir(name, only_one=True, fs=None): if len(res) == 1: res = res[0] else: + if fs is None and len(res) == 2: + # if both lib and lib64 were found, check if only one (exactly) has libraries; + # this is needed for software with library archives in lib64 but other files/directories in lib + lib_glob = ['*.%s' % ext for ext in ['a', get_shared_lib_ext()]] + has_libs = [any(glob.glob(os.path.join(root, subdir, f)) for f in lib_glob) for subdir in res] + if has_libs[0] and not has_libs[1]: + return res[0] + elif has_libs[1] and not has_libs[0]: + return res[1] + raise EasyBuildError("Multiple library subdirectories found for %s in %s: %s", name, root, ', '.join(res)) return res diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 0e82ff2ce9..bbe7f386ad 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -45,8 +45,9 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.9.1') +VERSION = LooseVersion('4.9.2') UNKNOWN = 'UNKNOWN' +UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' def get_git_revision(): @@ -87,7 +88,7 @@ def get_git_revision(): try: from easybuild.easyblocks import VERBOSE_VERSION as EASYBLOCKS_VERSION except Exception: - EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' # make sure it is smaller then anything + EASYBLOCKS_VERSION = UNKNOWN_EASYBLOCKS_VERSION # make sure it is smaller then anything def this_is_easybuild(): @@ -103,3 +104,14 @@ def this_is_easybuild(): msg = msg.encode('ascii') return msg + + +def different_major_versions(v1, v2): + """Compare major versions""" + # Deal with version instances being either strings or LooseVersion + if isinstance(v1, str): + v1 = LooseVersion(v1) + if isinstance(v2, str): + v2 = LooseVersion(v2) + + return v1.version[0] != v2.version[0] diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index e6b54e0bc8..19218eca6a 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1685,7 +1685,7 @@ def test_fetch_patches(self): self.assertEqual(eb.patches[1]['level'], 4) self.assertEqual(eb.patches[2]['name'], toy_patch) self.assertEqual(eb.patches[2]['sourcepath'], 'foobar') - self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'), + self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz') self.assertEqual(eb.patches[3]['copy'], 'some/path') self.assertEqual(eb.patches[4]['name'], toy_patch) self.assertEqual(eb.patches[4]['level'], 0) @@ -2200,11 +2200,16 @@ def test_extension_set_start_dir(self): cwd = os.getcwd() self.assertExists(cwd) - def check_ext_start_dir(expected_start_dir, unpack_src=True): + def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=None): """Check start dir.""" # make sure we're in an existing directory at the start change_dir(cwd) + eb = EasyBlock(ec['ec']) + if not os.path.exists(eb.builddir): + eb.make_builddir() # Required to exist for samefile + eb.cfg['start_dir'] = parent_startdir + eb.extensions_step(fetch=True, install=False) # extract sources of the extension ext = eb.ext_instances[-1] @@ -2212,6 +2217,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): if expected_start_dir is None: self.assertIsNone(ext.start_dir) + # Without a start dir we don't change the CWD + self.assertEqual(os.getcwd(), cwd) else: self.assertTrue(os.path.isabs(ext.start_dir)) if ext.start_dir != os.sep: @@ -2221,14 +2228,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): else: abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir) self.assertEqual(ext.start_dir, abs_expected_start_dir) - if not os.path.exists(eb.builddir): - eb.make_builddir() # Required to exist for samefile self.assertTrue(os.path.samefile(ext.start_dir, abs_expected_start_dir)) - if unpack_src: self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir)) - else: - # When not unpacking we don't change the CWD - self.assertEqual(os.getcwd(), cwd) remove_dir(eb.builddir) ec['ec']['exts_defaultclass'] = 'DummyExtension' @@ -2257,11 +2258,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): 'start_dir': 'nonexistingdir'}), ] with self.mocked_stdout_stderr(): - err_pattern = "Failed to change from .*barbar/barbar-0.0 to nonexistingdir.*" + err_pattern = r"Provided start dir \(nonexistingdir\) for extension barbar does not exist:.*" self.assertErrorRegex(EasyBuildError, err_pattern, check_ext_start_dir, 'whatever') - stderr = self.get_stderr() - warning_pattern = "WARNING: Provided start dir (nonexistingdir) for extension barbar does not exist" - self.assertIn(warning_pattern, stderr) # No error when using relative path in non-extracted source for some reason ec['ec']['exts_list'] = [ @@ -2291,6 +2289,15 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True): check_ext_start_dir(os.sep, unpack_src=False) self.assertFalse(self.get_stderr()) + # Go to ECs start dir if nosource is used + ec['ec']['exts_list'] = [ + ('barbar', '0.0', { + 'nosource': True}), + ] + with self.mocked_stdout_stderr(): + check_ext_start_dir(self.test_prefix, parent_startdir=self.test_prefix) + self.assertFalse(self.get_stderr()) + def test_prepare_step(self): """Test prepare step (setting up build environment).""" test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 45e2852221..4b63dc605b 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -3654,6 +3654,30 @@ def test_resolve_template(self): # '%(name)' is not a correct template spec (missing trailing 's') self.assertEqual(resolve_template('%(name)', tmpl_dict), '%(name)') + # Correct (un)escaping + values = ( + ('10%', '10%'), + ('%of', '%of'), + ('10%of', '10%of'), + ('%s', '%s'), + ('%%(name)s', '%(name)s'), + ('%%%(name)s', '%FooBar'), + ('%%%%(name)s', '%%(name)s'), + # It doesn't matter what is resolved + ('%%(invalid)s', '%(invalid)s'), + ('%%%%(invalid)s', '%%(invalid)s'), + ) + for value, expected in values: + self.assertEqual(resolve_template(value, tmpl_dict), expected) + # Templates are resolved + value += ' %(name)s' + expected += ' FooBar' + self.assertEqual(resolve_template(value, tmpl_dict), expected) + + # On unknown values the value is returned unchanged + for value in ('%(invalid)s', '%(name)s %(invalid)s', '%%%(invalid)s', '% %(invalid)s', '%s %(invalid)s'): + self.assertEqual(resolve_template(value, tmpl_dict), value) + def test_det_subtoolchain_version(self): """Test det_subtoolchain_version function""" _, all_tc_classes = search_toolchain('') diff --git a/test/framework/modules.py b/test/framework/modules.py index 82d03f6440..195bf0339c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -51,6 +51,7 @@ from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches from easybuild.tools.run import run_cmd +from easybuild.tools.systemtools import get_shared_lib_ext # number of modules included for testing purposes @@ -674,10 +675,23 @@ def test_get_software_root_version_libdir(self): os.environ.pop('EBROOT%s' % env_var_name) os.environ.pop('EBVERSION%s' % env_var_name) - # check expected result of get_software_libdir with multiple lib subdirs + # if only 'lib' has a library archive, use it root = os.path.join(tmpdir, name) mkdir(os.path.join(root, 'lib64')) os.environ['EBROOT%s' % env_var_name] = root + write_file(os.path.join(root, 'lib', 'libfoo.a'), 'foo') + self.assertEqual(get_software_libdir(name), 'lib') + + remove_file(os.path.join(root, 'lib', 'libfoo.a')) + + # also check vice versa with *shared* library in lib64 + shlib_ext = get_shared_lib_ext() + write_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext), 'foo') + self.assertEqual(get_software_libdir(name), 'lib64') + + remove_file(os.path.join(root, 'lib64', 'libfoo.' + shlib_ext)) + + # check expected result of get_software_libdir with multiple lib subdirs self.assertErrorRegex(EasyBuildError, "Multiple library subdirectories found.*", get_software_libdir, name) self.assertEqual(get_software_libdir(name, only_one=False), ['lib', 'lib64']) diff --git a/test/framework/options.py b/test/framework/options.py index b2f3da43c1..94c4372f91 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -366,6 +366,25 @@ def test_skip(self): self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + # check use of module_only parameter + remove_dir(os.path.join(self.test_installpath, 'modules', 'all', 'toy')) + remove_dir(os.path.join(self.test_installpath, 'software', 'toy', '0.0')) + args = [ + test_ec, + '--rebuild', + ] + test_ec_txt += "\nmodule_only = True\n" + write_file(test_ec, test_ec_txt) + self.eb_main(args, do_build=True, raise_error=True) + + self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + + # check that no software was installed + installdir = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + installdir_glob = glob.glob(os.path.join(installdir, '*')) + easybuild_dir = os.path.join(installdir, 'easybuild') + self.assertEqual(installdir_glob, [easybuild_dir]) + def test_skip_test_step(self): """Test skipping testing the build (--skip-test-step).""" @@ -1908,7 +1927,7 @@ def test_github_from_pr(self): # make sure that *only* these modules are listed, no others regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) - self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr6424') regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) @@ -1943,12 +1962,12 @@ def test_github_from_pr(self): # make sure that *only* these modules are listed, no others regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) - self.assertTrue(sorted(regex.findall(outtxt)), sorted(modules)) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) for pr in ('12150', '12366'): pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_pr%s' % pr) regex = re.compile(r"Extended list of robot search paths with .*%s.*:" % pr_tmpdir, re.M) - self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + self.assertTrue(regex.search(outtxt), "Found pattern '%s' in: %s" % (regex.pattern, outtxt)) except URLError as err: print("Ignoring URLError '%s' in test_from_pr" % err) @@ -2125,6 +2144,50 @@ def test_from_commit(self): print("Ignoring URLError '%s' in test_from_commit" % err) shutil.rmtree(tmpdir) + easyblock_template = '\n'.join([ + "from easybuild.framework.easyblock import EasyBlock", + "class %s(EasyBlock):", + " pass", + ]) + + # create fake custom easyblock for CMake that is required by easyconfig used in test below + easyblock_file = os.path.join(self.test_prefix, 'easyblocks', 'cmake.py') + write_file(easyblock_file, easyblock_template % 'EB_CMake') + + # also test with an easyconfig that requires additional easyconfigs to resolve dependencies, + # cfr. https://github.com/easybuilders/easybuild-framework/issues/4540; + # using commit that adds CMake-3.18.4.eb (which requires ncurses-6.2.eb), + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/13156 + test_commit = '41eee3fe2e5102f52319481ca8dde16204dab590' + args = [ + '--from-commit=%s' % test_commit, + '--dry-run', + '--tmpdir=%s' % tmpdir, + '--include-easyblocks=' + os.path.join(self.test_prefix, 'easyblocks', '*.py'), + ] + try: + outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) + modules = [ + (tmpdir, 'ncurses/6.2'), + (tmpdir, 'CMake/3.18.4'), + ] + for path_prefix, module in modules: + ec_fn = "%s.eb" % '-'.join(module.split('/')) + path = '.*%s' % os.path.dirname(path_prefix) + regex = re.compile(r"^ \* \[.\] %s.*%s \(module: %s\)$" % (path, ec_fn, module), re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + + # make sure that *only* these modules are listed, no others + regex = re.compile(r"^ \* \[.\] .*/(?P.*) \(module: (?P.*)\)$", re.M) + self.assertEqual(sorted(x[1] for x in regex.findall(outtxt)), sorted(x[1] for x in modules)) + + pr_tmpdir = os.path.join(tmpdir, r'eb-\S{6,8}', 'files_commit_%s' % test_commit) + regex = re.compile(r"Extended list of robot search paths with \['%s'\]:" % pr_tmpdir, re.M) + self.assertTrue(regex.search(outtxt), "Found pattern %s in %s" % (regex.pattern, outtxt)) + except URLError as err: + print("Ignoring URLError '%s' in test_from_commit" % err) + shutil.rmtree(tmpdir) + # must be run after test for --list-easyblocks, hence the '_xxx_' # cleaning up the imported easyblocks is quite difficult... def test_xxx_include_easyblocks_from_commit(self): @@ -4522,7 +4585,7 @@ def test_github_new_update_pr(self): res = [d for d in res if os.path.basename(d) != os.path.basename(git_working_dir)] if len(res) == 1: unstaged_file_full = os.path.join(res[0], unstaged_file) - self.assertNotExists(unstaged_file_full), "%s not found in %s" % (unstaged_file, res[0]) + self.assertNotExists(unstaged_file_full) else: self.fail("Found copy of easybuild-easyconfigs working copy") @@ -5249,6 +5312,45 @@ def test_dump_env_script(self): ]) self.assertEqual(out.strip(), expected_out) + def test_dump_env_script_existing_module(self): + toy_ec = 'toy-0.0.eb' + + os.chdir(self.test_prefix) + self._run_mock_eb([toy_ec, '--force'], do_build=True) + env_script = os.path.join(self.test_prefix, os.path.splitext(toy_ec)[0] + '.env') + test_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + if get_module_syntax() == 'Lua': + test_module += '.lua' + self.assertExists(test_module) + self.assertNotExists(env_script) + + args = [toy_ec, '--dump-env'] + os.chdir(self.test_prefix) + self._run_mock_eb(args, do_build=True, raise_error=True) + self.assertExists(env_script) + self.assertExists(test_module) + module_content = read_file(test_module) + env_file_content = read_file(env_script) + + error_msg = (r"Script\(s\) already exists, not overwriting them \(unless --force is used\): " + + os.path.basename(env_script)) + os.chdir(self.test_prefix) + self.assertErrorRegex(EasyBuildError, error_msg, self._run_mock_eb, args, do_build=True, raise_error=True) + self.assertExists(env_script) + self.assertExists(test_module) + # Unchanged module and env file + self.assertEqual(read_file(test_module), module_content) + self.assertEqual(read_file(env_script), env_file_content) + + args.append('--force') + os.chdir(self.test_prefix) + self._run_mock_eb(args, do_build=True, raise_error=True) + self.assertExists(env_script) + self.assertExists(test_module) + # Unchanged module and env file + self.assertEqual(read_file(test_module), module_content) + self.assertEqual(read_file(env_script), env_file_content) + def test_stop(self): """Test use of --stop.""" args = ['toy-0.0.eb', '--force', '--stop=configure']