Skip to content

Commit

Permalink
Merge pull request #2599 from ocaisa/allow_dep_upgrades
Browse files Browse the repository at this point in the history
Add ability to upgrade dependencies based on available easyconfigs
  • Loading branch information
boegel authored Apr 11, 2020
2 parents 33aa08a + ee63271 commit faa3e2a
Show file tree
Hide file tree
Showing 15 changed files with 810 additions and 127 deletions.
2 changes: 1 addition & 1 deletion easybuild/framework/easyconfig/format/one.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def dump_dependency(dep, toolchain):
if dep['external_module']:
res = "(%s, EXTERNAL_MODULE)" % quote_py_str(dep['full_mod_name'])
else:
# mininal spec: (name, version)
# minimal spec: (name, version)
tup = (dep['name'], dep['version'])
if dep['toolchain'] != toolchain:
if dep[SYSTEM_TOOLCHAIN_NAME]:
Expand Down
404 changes: 337 additions & 67 deletions easybuild/framework/easyconfig/tweak.py

Large diffs are not rendered by default.

9 changes: 7 additions & 2 deletions easybuild/tools/filetools.py
Original file line number Diff line number Diff line change
Expand Up @@ -758,7 +758,8 @@ def find_easyconfigs(path, ignore_dirs=None):
return files


def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False):
def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filename_only=False, terse=False,
case_sensitive=False):
"""
Search for files using in specified paths using specified search query (regular expression)
Expand All @@ -782,7 +783,11 @@ def search_file(paths, query, short=False, ignore_dirs=None, silent=False, filen

# compile regex, case-insensitive
try:
query = re.compile(query, re.I)
if case_sensitive:
query = re.compile(query)
else:
# compile regex, case-insensitive
query = re.compile(query, re.I)
except re.error as err:
raise EasyBuildError("Invalid search query: %s", err)

Expand Down
4 changes: 4 additions & 0 deletions easybuild/tools/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,9 @@ def software_options(self):

opts['map-toolchains'] = ("Enable mapping of (sub)toolchains when --try-toolchain(-version) is used",
None, 'store_true', True)
opts['try-update-deps'] = ("Try to update versions of the dependencies of an easyconfig based on what is "
"available in the robot path",
None, 'store_true', False)

self.log.debug("software_options: descr %s opts %s" % (descr, opts))
self.add_group_parser(opts, descr)
Expand Down Expand Up @@ -1481,6 +1484,7 @@ def process_software_build_specs(options):
'version': options.try_software_version,
'toolchain_name': options.try_toolchain_name,
'toolchain_version': options.try_toolchain_version,
'update_deps': options.try_update_deps
}

# process easy options
Expand Down
84 changes: 55 additions & 29 deletions easybuild/tools/robot.py
Original file line number Diff line number Diff line change
Expand Up @@ -469,20 +469,36 @@ def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_erro
return ordered_ecs


def search_easyconfigs(query, short=False, filename_only=False, terse=False):
"""Search for easyconfigs, if a query is provided."""
def search_easyconfigs(query, short=False, filename_only=False, terse=False, consider_extra_paths=True,
print_result=True, case_sensitive=False):
"""
Search for easyconfigs, if a query is provided.
:param query: regex query string
:param short: figure out common prefix of hits, use variable to factor it out
:param filename_only: only print filenames, not paths
:param terse: stick to terse (machine-readable) output, as opposed to pretty-printing
:param consider_extra_paths: consider all paths when searching
:param print_result: print the list of easyconfigs
:param case_sensitive: boolean to decide whether search is case sensitive
:return: return a list of paths for the query
"""
search_path = build_option('robot_path')
if not search_path:
search_path = [os.getcwd()]
extra_search_paths = build_option('search_paths')
if extra_search_paths:
search_path.extend(extra_search_paths)
# If we're returning a list of possible resolutions by the robot, don't include the extra_search_paths
if extra_search_paths and consider_extra_paths:
# we shouldn't use += or .extend here but compose a new list,
# to avoid adding a path to the list returned by build_option('robot_path') !
search_path = search_path + extra_search_paths

ignore_dirs = build_option('ignore_dirs')

# note: don't pass down 'filename_only' here, we need the full path to filter out archived easyconfigs
var_defs, _hits = search_file(search_path, query, short=short, ignore_dirs=ignore_dirs, terse=terse,
silent=True, filename_only=False)
silent=True, filename_only=False, case_sensitive=case_sensitive)

# filter out archived easyconfigs, these are handled separately
hits, archived_hits = [], []
Expand All @@ -492,32 +508,42 @@ def search_easyconfigs(query, short=False, filename_only=False, terse=False):
else:
hits.append(hit)

# check whether only filenames should be printed
# check whether only filenames should be used
if filename_only:
hits = [os.path.basename(hit) for hit in hits]
archived_hits = [os.path.basename(hit) for hit in archived_hits]

# prepare output format
if terse:
lines, tmpl = [], '%s'
if print_result:
# prepare output format
if terse:
lines, tmpl = [], '%s'
else:
lines = ['%s=%s' % var_def for var_def in var_defs]
tmpl = ' * %s'

# non-archived hits are shown first
lines.extend(tmpl % hit for hit in hits)

# also take into account archived hits
if archived_hits:
if build_option('consider_archived_easyconfigs'):
if not terse:
lines.extend(['', "Matching archived easyconfigs:", ''])
lines.extend(tmpl % hit for hit in archived_hits)
elif not terse:
cnt = len(archived_hits)
lines.extend([
'',
"Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them"
% cnt,
])

print('\n'.join(lines))

# if requested return the matches as a list
if build_option('consider_archived_easyconfigs'):
final_hits = hits + archived_hits
else:
lines = ['%s=%s' % var_def for var_def in var_defs]
tmpl = ' * %s'

# non-archived hits are shown first
lines.extend(tmpl % hit for hit in hits)

# also take into account archived hits
if archived_hits:
if build_option('consider_archived_easyconfigs'):
if not terse:
lines.extend(['', "Matching archived easyconfigs:", ''])
lines.extend(tmpl % hit for hit in archived_hits)
elif not terse:
cnt = len(archived_hits)
lines.extend([
'',
"Note: %d matching archived easyconfig(s) found, use --consider-archived-easyconfigs to see them" % cnt,
])

print('\n'.join(lines))
final_hits = hits

return final_hits
4 changes: 2 additions & 2 deletions test/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -1765,11 +1765,11 @@ def run_checks():

# full check also catches checksum issues with extensions
res = eb.check_checksums()
self.assertEqual(len(res), 5)
self.assertEqual(len(res), 4)
run_checks()

idx = 2
for ext in ['bar', 'barbar', 'toy']:
for ext in ['bar', 'barbar']:
expected = "Checksums missing for one or more sources/patches of extension %s in " % ext
self.assertTrue(res[idx].startswith(expected))
idx += 1
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
##
# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
#
# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC
# Authors:: Thekla Loizou <[email protected]>
# License:: MIT/GPL
# $Id$
#
# This work implements a part of the HPCBIOS project and is a component of the policy:
# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html
##
easyblock = 'ConfigureMake'

name = 'gzip'
version = '1.4'

homepage = "http://www.gzip.org/"
description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"

# test toolchain specification
toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}

# source tarball filename
sources = ['%(name)s-%(version)s.tar.gz']

# download location for source files
source_urls = [GNU_SOURCE]

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
'files': ["bin/gunzip", "bin/gzip"],
'dirs': [],
}

# run 'gzip -h' and 'gzip --version' after installation
sanity_check_commands = [True, ('gzip', '--version')]

software_license = GPLv3

moduleclass = 'tools'
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
##
# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
#
# Copyright:: Copyright (c) 2012-2013 Cyprus Institute / CaSToRC
# Authors:: Thekla Loizou <[email protected]>
# License:: MIT/GPL
# $Id$
#
# This work implements a part of the HPCBIOS project and is a component of the policy:
# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_06-19.html
##
easyblock = 'ConfigureMake'

name = 'gzip'
version = '1.6'

homepage = "http://www.gzip.org/"
description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"

# test toolchain specification
toolchain = {'name': 'iccifort', 'version': '2016.1.150-GCC-4.9.3-2.25'}

# source tarball filename
sources = ['%(name)s-%(version)s.tar.gz']

# download location for source files
source_urls = [GNU_SOURCE]

# make sure the gzip and gunzip binaries are available after installation
sanity_check_paths = {
'files': ["bin/gunzip", "bin/gzip"],
'dirs': [],
}

# run 'gzip -h' and 'gzip --version' after installation
sanity_check_commands = [True, ('gzip', '--version')]

software_license = GPLv3

moduleclass = 'tools'
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@ toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}

source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
sources = [SOURCE_TAR_GZ]
checksums = ['aa9d9ca75c7d7164f6bf3a52ecd77340eec02c18']

builddependencies = [('binutils', '2.26')]

# introduce fake dependency for testing dep upgrades
dependencies = [('gzip', '1.4')]

moduleclass = 'system'
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ exts_list = [
(name, version, {
'sanity_check_paths': {'files': ['lib/libtoy.a'], 'dirs': []},
'exts_filter': ("ls -l lib/libtoy.a", ''),
'checksums': ['44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc']
}),
]

Expand Down
15 changes: 12 additions & 3 deletions test/framework/filetools.py
Original file line number Diff line number Diff line change
Expand Up @@ -1715,7 +1715,7 @@ def test_index_functions(self):
# test with specified path with and without trailing '/'s
for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
index = ft.create_index(path)
self.assertEqual(len(index), 79)
self.assertEqual(len(index), 81)

expected = [
os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
Expand Down Expand Up @@ -1764,7 +1764,7 @@ def test_index_functions(self):
regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))

self.assertEqual(len(index), 24)
self.assertEqual(len(index), 26)
for fn in expected:
self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index)))

Expand Down Expand Up @@ -1794,7 +1794,7 @@ def test_index_functions(self):
regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix)
self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout))

self.assertEqual(len(index), 24)
self.assertEqual(len(index), 26)
for fn in expected:
self.assertTrue(fn in index, "%s should be found in %s" % (fn, sorted(index)))

Expand Down Expand Up @@ -1834,6 +1834,15 @@ def test_search_file(self):
self.assertTrue(hits[3].endswith('/hwloc-1.6.2-GCC-4.9.3-2.26.eb'))
self.assertTrue(hits[4].endswith('/hwloc-1.8-gcccuda-2018a.eb'))

# also test case-sensitive searching
var_defs, hits_bis = ft.search_file([test_ecs], 'HWLOC', silent=True, case_sensitive=True)
self.assertEqual(var_defs, [])
self.assertEqual(hits_bis, [])

var_defs, hits_bis = ft.search_file([test_ecs], 'hwloc', silent=True, case_sensitive=True)
self.assertEqual(var_defs, [])
self.assertEqual(hits_bis, hits)

# check filename-only mode
var_defs, hits = ft.search_file([test_ecs], 'HWLOC', silent=True, filename_only=True)
self.assertEqual(var_defs, [])
Expand Down
Loading

0 comments on commit faa3e2a

Please sign in to comment.