From ab933f1a6c4b6d8d72df89698b97bb0640fcdbde Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 09:37:37 +0200 Subject: [PATCH 001/115] Allow use of --copy-ec with --from-pr --- easybuild/main.py | 49 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index bd7a0b0f5e..79f009b0cd 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -176,6 +176,26 @@ def run_contrib_style_checks(ecs, check_contrib, check_style): return check_contrib or check_style +def copy_ecs_to_target(determined_paths, target_path, prefix=False): + """ + Copy list of easyconfigs to specified path + + :param determined_paths: paths to ecs to copy + :param target_path: target to copy files to + :param prefix: include message prefix characters + """ + if not target_path: + raise EasyBuildError("No target path specified to copy files to!") + if len(determined_paths) == 1: + copy_file(determined_paths[0], target_path) + print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) + elif len(determined_paths) > 1: + copy_files(determined_paths, target_path) + print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=prefix) + else: + raise EasyBuildError("One of more files to copy should be specified!") + + def clean_exit(logfile, tmpdir, testing, silent=False): """Small utility function to perform a clean exit.""" cleanup(logfile, tmpdir, testing, silent=silent) @@ -303,12 +323,16 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - if len(orig_paths) == 1: + if not orig_paths and options.copy_ec and options.from_pr: + # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory + target_path = os.getcwd() + elif len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): # if only one easyconfig file is specified, use current directory as target directory target_path = os.getcwd() elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list - target_path = orig_paths.pop() if options.copy_ec else None + # use the absolute path as some use cases occur in a temporary directory + target_path = os.path.abspath(orig_paths.pop()) if options.copy_ec else None categorized_paths = categorize_files_by_type(orig_paths) @@ -321,17 +345,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: + if (options.copy_ec and not tweaked_ecs_paths) or \ + (options.copy_ec and not options.from_pr) or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: - if len(determined_paths) == 1: - copy_file(determined_paths[0], target_path) - print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False) - elif len(determined_paths) > 1: - copy_files(determined_paths, target_path) - print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) - else: - raise EasyBuildError("One of more files to copy should be specified!") + copy_ecs_to_target(determined_paths, target_path) elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) @@ -369,6 +387,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) + # if we are only copying ec's for a specific PR we can do that now + if options.copy_ec and options.from_pr: + copy_ecs_to_target(determined_paths,target_path) + clean_exit(logfile, eb_tmpdir, testing) + # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): clean_exit(logfile, eb_tmpdir, testing) @@ -429,8 +452,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) - copy_files(tweaked_ecs_in_all_ecs, target_path) - print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) + copy_ecs_to_target(tweaked_ecs_in_all_ecs, target_path) + clean_exit(logfile, eb_tmpdir, testing) # creating/updating PRs if pr_options: From 9accd5dc826af6f8bf382bf5e2af5abf588e2b3a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 09:40:31 +0200 Subject: [PATCH 002/115] Appease the hound --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 79f009b0cd..ccfcc58fd7 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -389,7 +389,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # if we are only copying ec's for a specific PR we can do that now if options.copy_ec and options.from_pr: - copy_ecs_to_target(determined_paths,target_path) + copy_ecs_to_target(determined_paths, target_path) clean_exit(logfile, eb_tmpdir, testing) # handle --check-contrib & --check-style options From e2f270af2486711d6926585cb49d2a05ee253f25 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 10:07:23 +0200 Subject: [PATCH 003/115] Fix broken tests --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index e2c90a22be..200c818250 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1002,7 +1002,7 @@ def mocked_main(args): args = ['--copy-ec', 'toy-0.0.eb', target_fn] stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') + self.assertEqual(stdout, 'toy-0.0.eb copied to %s/test.eb' % cwd) change_dir(cwd) @@ -2072,7 +2072,7 @@ def test_try_with_copy(self): verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.assertTrue(r'1 file(s) copied to ' + tweaked_ecs_dir in outtxt) + self.assertTrue( + r'foo-1.2.3-gompi-2018a.eb copied to ' + tweaked_ecs_dir in outtxt) self.assertFalse(errtxt) self.mock_stdout(False) self.mock_stderr(False) From 64e95f23d6b6d811f3e46186f883a4f6c7aa8fef Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 10:08:41 +0200 Subject: [PATCH 004/115] Fix broken tests --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 200c818250..12c26339a8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2072,7 +2072,7 @@ def test_try_with_copy(self): verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.assertTrue( + r'foo-1.2.3-gompi-2018a.eb copied to ' + tweaked_ecs_dir in outtxt) + self.assertTrue(r'foo-1.2.3-gompi-2018a.eb copied to ' + tweaked_ecs_dir in outtxt) self.assertFalse(errtxt) self.mock_stdout(False) self.mock_stderr(False) From 4e6980fe819a940b5e9e856c8f7191706df9cc22 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 12:55:40 +0200 Subject: [PATCH 005/115] Fix broken tests (2nd attempt) --- easybuild/main.py | 10 ++++------ test/framework/options.py | 14 +++++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index ccfcc58fd7..609fed00d5 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -184,8 +184,6 @@ def copy_ecs_to_target(determined_paths, target_path, prefix=False): :param target_path: target to copy files to :param prefix: include message prefix characters """ - if not target_path: - raise EasyBuildError("No target path specified to copy files to!") if len(determined_paths) == 1: copy_file(determined_paths[0], target_path) print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) @@ -323,9 +321,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - if not orig_paths and options.copy_ec and options.from_pr: + if not orig_paths: # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory - target_path = os.getcwd() + target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None elif len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): # if only one easyconfig file is specified, use current directory as target directory target_path = os.getcwd() @@ -345,8 +343,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if (options.copy_ec and not tweaked_ecs_paths) or \ - (options.copy_ec and not options.from_pr) or options.fix_deprecated_easyconfigs or options.show_ec: + if (options.copy_ec and not (tweaked_ecs_paths or options.from_pr)) or options.fix_deprecated_easyconfigs or \ + options.show_ec: if options.copy_ec: copy_ecs_to_target(determined_paths, target_path) diff --git a/test/framework/options.py b/test/framework/options.py index 12c26339a8..488640d9dd 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1002,7 +1002,7 @@ def mocked_main(args): args = ['--copy-ec', 'toy-0.0.eb', target_fn] stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s/test.eb' % cwd) + self.assertEqual(stdout, 'toy-0.0.eb copied to %s/test.eb' % self.test_prefix) change_dir(cwd) @@ -1048,7 +1048,7 @@ def check_copied_files(): self.assertFalse(os.path.exists(args[-1])) stdout = mocked_main(args) - self.assertEqual(stdout, '2 file(s) copied to test_target_dir') + self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) check_copied_files() @@ -2067,15 +2067,15 @@ def test_try_with_copy(self): self.mock_stdout(True) self.mock_stderr(True) - tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') - self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], - verbose=True, raise_error=True) + tweaked_ecs_dir = os.path.join(self.test_buildpath) + os.chdir(tweaked_ecs_dir) + self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a'], verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.assertTrue(r'foo-1.2.3-gompi-2018a.eb copied to ' + tweaked_ecs_dir in outtxt) - self.assertFalse(errtxt) self.mock_stdout(False) self.mock_stderr(False) + self.assertTrue(r'foo-1.2.3-GCC-6.4.0-2.28.eb copied to ' + tweaked_ecs_dir in outtxt) + self.assertFalse(errtxt) self.assertTrue( os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) ) From fceea89aa1fac7ad2e774fcf1b172d57a07dc54b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 13:41:37 +0200 Subject: [PATCH 006/115] Reinstate the copy of the easyconfigs from the PR --- easybuild/main.py | 1 + test/framework/options.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 609fed00d5..8507e492df 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -387,6 +387,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # if we are only copying ec's for a specific PR we can do that now if options.copy_ec and options.from_pr: + determined_paths = [ec['spec'] for ec in easyconfigs] copy_ecs_to_target(determined_paths, target_path) clean_exit(logfile, eb_tmpdir, testing) diff --git a/test/framework/options.py b/test/framework/options.py index 488640d9dd..cf384d171a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1052,7 +1052,7 @@ def check_copied_files(): check_copied_files() - # copying multiple easyconfig to an existing target file resuts in an error + # copying multiple easyconfig to an existing target file results in an error target = os.path.join(self.test_prefix, 'test.eb') self.assertTrue(os.path.isfile(target)) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', target] From 51f0b068e49621951d9e7d03e5e682aec44869f3 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 14:29:00 +0200 Subject: [PATCH 007/115] Add test for --from-pr and --copy-ec --- easybuild/main.py | 12 ++++++------ test/framework/options.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 8507e492df..a668c5deb7 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -373,6 +373,12 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s", paths) + # if we are only copying ec's for a specific PR we can do that now + if options.copy_ec and options.from_pr: + determined_paths = [path[0] for path in paths] + copy_ecs_to_target(determined_paths, target_path) + clean_exit(logfile, eb_tmpdir, testing) + # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") @@ -385,12 +391,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) - # if we are only copying ec's for a specific PR we can do that now - if options.copy_ec and options.from_pr: - determined_paths = [ec['spec'] for ec in easyconfigs] - copy_ecs_to_target(determined_paths, target_path) - clean_exit(logfile, eb_tmpdir, testing) - # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): clean_exit(logfile, eb_tmpdir, testing) diff --git a/test/framework/options.py b/test/framework/options.py index cf384d171a..fa230982c3 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1072,6 +1072,20 @@ def check_copied_files(): self.assertTrue(os.path.exists(copied_toy_cwd)) self.assertEqual(read_file(copied_toy_cwd), toy_ec_txt) + # Test --copy-ec coupled with --from-pr + all_ecs_pr8007 = [ + 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', + 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', + 'bat-0.3.3-fix-pyspark.patch', + ] + # copying multiple easyconfig files to a non-existing target directory (which is created automatically) + args = ['--copy-ec', '--from-pr', '8007', test_target_dir] + stdout = mocked_main(args) + self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) + # Check that the two easyconfigs exist + self.assertTrue(os.path.exists(os.path.join(test_target_dir, 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb'))) + self.assertTrue(os.path.exists(os.path.join(test_target_dir, 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'))) + # --copy-ec without arguments results in a proper error args = ['--copy-ec'] error_pattern = "One of more files to copy should be specified!" From d633ece19f4fbd53338781fdceedd7a823df90d1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 16:42:19 +0200 Subject: [PATCH 008/115] Fix linting and add additional tests --- test/framework/options.py | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index fa230982c3..9ef1ab6035 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1075,16 +1075,45 @@ def check_copied_files(): # Test --copy-ec coupled with --from-pr all_ecs_pr8007 = [ 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', - 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', 'bat-0.3.3-fix-pyspark.patch', + 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', ] + + # test use of `--copy-ec` with `--from-pr` to the cwd + test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') + mkdir(test_working_dir) + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007'] + stdout = mocked_main(args) + self.assertEqual(stdout, '2 file(s) copied to %s' % test_working_dir) + # check that the two easyconfigs exist + self.assertTrue(os.path.exists(os.path.join(test_working_dir, all_ecs_pr8007[0]))) + remove_file(os.path.join(test_working_dir, all_ecs_pr8007[0])) + self.assertTrue(os.path.exists(os.path.join(test_working_dir, all_ecs_pr8007[2]))) + remove_file(os.path.join(test_working_dir, all_ecs_pr8007[2])) + # ...but the patch doesn't + self.assertFalse(os.path.exists(os.path.join(test_working_dir, all_ecs_pr8007[1]))) + remove_file(os.path.join(test_working_dir, all_ecs_pr8007[1])) + # copying multiple easyconfig files to a non-existing target directory (which is created automatically) args = ['--copy-ec', '--from-pr', '8007', test_target_dir] stdout = mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) - # Check that the two easyconfigs exist - self.assertTrue(os.path.exists(os.path.join(test_target_dir, 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb'))) - self.assertTrue(os.path.exists(os.path.join(test_target_dir, 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'))) + # check that the two easyconfigs exist + self.assertTrue(os.path.exists(os.path.join(test_target_dir, all_ecs_pr8007[0]))) + self.assertTrue(os.path.exists(os.path.join(test_target_dir, all_ecs_pr8007[2]))) + # ...but the patch doesn't + self.assertFalse(os.path.exists(os.path.join(test_target_dir, all_ecs_pr8007[1]))) + remove_dir(test_target_dir) + + # test with only one ec in the PR (final argument is taken as a filename) + test_ec = os.path.join(self.test_prefix, 'test.eb') + args = ['--copy-ec', '--from-pr', '11521', test_ec] + ec_pr11521 = "ExifTool-12.00-GCCcore-9.3.0.eb" + stdout = mocked_main(args) + self.assertEqual(stdout, '%s copied to %s' % (ec_pr11521, test_ec)) + self.assertTrue(os.path.exists(test_ec)) + remove_file(test_ec) # --copy-ec without arguments results in a proper error args = ['--copy-ec'] From f8db57dca8c7ba2899ad26265cbcfdac577a0c98 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 18:31:28 +0200 Subject: [PATCH 009/115] Address most comments in the review --- easybuild/framework/easyconfig/tools.py | 18 +++++++++++ easybuild/main.py | 43 ++++++++----------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2a3260ae80..51537b4b0d 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -728,3 +728,21 @@ def avail_easyblocks(): easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks + + +def copy_ecs_to_target(determined_paths, target_path, prefix=False): + """ + Copy list of easyconfigs to specified path + + :param determined_paths: paths to ecs to copy + :param target_path: target to copy files to + :param prefix: include message prefix characters + """ + if len(determined_paths) == 1: + copy_file(determined_paths[0], target_path) + print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) + elif len(determined_paths) > 1: + copy_files(determined_paths, target_path) + print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=prefix) + else: + raise EasyBuildError("One of more files to copy should be specified!") \ No newline at end of file diff --git a/easybuild/main.py b/easybuild/main.py index a668c5deb7..823346b093 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -50,7 +50,7 @@ from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph +from easybuild.framework.easyconfig.tools import categorize_files_by_type, copy_ecs_to_target, dep_graph from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak @@ -176,24 +176,6 @@ def run_contrib_style_checks(ecs, check_contrib, check_style): return check_contrib or check_style -def copy_ecs_to_target(determined_paths, target_path, prefix=False): - """ - Copy list of easyconfigs to specified path - - :param determined_paths: paths to ecs to copy - :param target_path: target to copy files to - :param prefix: include message prefix characters - """ - if len(determined_paths) == 1: - copy_file(determined_paths[0], target_path) - print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) - elif len(determined_paths) > 1: - copy_files(determined_paths, target_path) - print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=prefix) - else: - raise EasyBuildError("One of more files to copy should be specified!") - - def clean_exit(logfile, tmpdir, testing, silent=False): """Small utility function to perform a clean exit.""" cleanup(logfile, tmpdir, testing, silent=silent) @@ -321,16 +303,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - if not orig_paths: - # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory - target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None - elif len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): - # if only one easyconfig file is specified, use current directory as target directory + # if only one easyconfig is specified, or if none are specified and we are using --from-pr, + # use current directory as target directory + if len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): target_path = os.getcwd() elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list - # use the absolute path as some use cases occur in a temporary directory + # use the absolute path as the --from-pr case drops us into a temporary directory target_path = os.path.abspath(orig_paths.pop()) if options.copy_ec else None + else: + # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory + target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None categorized_paths = categorize_files_by_type(orig_paths) @@ -343,8 +326,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if (options.copy_ec and not (tweaked_ecs_paths or options.from_pr)) or options.fix_deprecated_easyconfigs or \ - options.show_ec: + # only copy easyconfigs here if we're not using --try-* or --from-pr (that's are handled below) + copy_ec = options.copy_ec and not (tweaked_ecs_paths or options.from_pr) + if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: copy_ecs_to_target(determined_paths, target_path) @@ -373,10 +357,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s", paths) - # if we are only copying ec's for a specific PR we can do that now + # if we are only copying ec's from a specific PR we can do that now if options.copy_ec and options.from_pr: - determined_paths = [path[0] for path in paths] - copy_ecs_to_target(determined_paths, target_path) + copy_ecs_to_target([x for (x, _) in paths], target_path) clean_exit(logfile, eb_tmpdir, testing) # run regtest From b3615f41f6a14eb2daf7f911a5915204db427216 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 18:35:48 +0200 Subject: [PATCH 010/115] Fix linting --- easybuild/framework/easyconfig/tools.py | 1 + easybuild/main.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 51537b4b0d..b8bb7197f3 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -53,6 +53,7 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env +from easybuild.tools.filetools import copy_file, copy_files from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo from easybuild.tools.multidiff import multidiff diff --git a/easybuild/main.py b/easybuild/main.py index 823346b093..eea5c7d0d4 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -57,7 +57,7 @@ from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index +from easybuild.tools.filetools import adjust_permissions, cleanup, dump_index, load_index from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr From 2408626be636079f629c2127d47622594785a145 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 19 Oct 2020 18:36:46 +0200 Subject: [PATCH 011/115] Fix linting --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index b8bb7197f3..d0b353de94 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -746,4 +746,4 @@ def copy_ecs_to_target(determined_paths, target_path, prefix=False): copy_files(determined_paths, target_path) print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=prefix) else: - raise EasyBuildError("One of more files to copy should be specified!") \ No newline at end of file + raise EasyBuildError("One of more files to copy should be specified!") From 34cd4a5f451f02cd6da2b7629888ec7dffb0d78f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:08:06 +0200 Subject: [PATCH 012/115] Rework the approach and add additional tests --- easybuild/framework/easyconfig/tools.py | 5 +-- easybuild/main.py | 40 +++++++++++++++------- test/framework/options.py | 44 +++++++++++++++---------- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index d0b353de94..ea216f4288 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -731,15 +731,16 @@ def avail_easyblocks(): return easyblocks -def copy_ecs_to_target(determined_paths, target_path, prefix=False): +def copy_ecs_to_target(determined_paths, target_path, prefix=False, target_is_dir=False): """ Copy list of easyconfigs to specified path :param determined_paths: paths to ecs to copy :param target_path: target to copy files to :param prefix: include message prefix characters + :param target_is_dir: target is always a directory """ - if len(determined_paths) == 1: + if len(determined_paths) == 1 and not target_is_dir: copy_file(determined_paths[0], target_path) print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) elif len(determined_paths) > 1: diff --git a/easybuild/main.py b/easybuild/main.py index eea5c7d0d4..a19643f056 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -39,6 +39,7 @@ import os import stat import sys +import tempfile import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -59,8 +60,9 @@ from easybuild.tools.docs import list_software from easybuild.tools.filetools import adjust_permissions, cleanup, dump_index, load_index from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file -from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig -from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr +from easybuild.tools.github import check_github, close_pr, fetch_files_from_pr, find_easybuild_easyconfig +from easybuild.tools.github import install_github_token, list_prs, merge_pr, new_branch_github, new_pr +from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool @@ -310,7 +312,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list # use the absolute path as the --from-pr case drops us into a temporary directory - target_path = os.path.abspath(orig_paths.pop()) if options.copy_ec else None + target_path = orig_paths.pop() if options.copy_ec else None else: # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None @@ -326,9 +328,28 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - # only copy easyconfigs here if we're not using --try-* or --from-pr (that's are handled below) - copy_ec = options.copy_ec and not (tweaked_ecs_paths or options.from_pr) + # only copy easyconfigs here if we're not using --try-* (that's are handled below) + copy_ec = options.copy_ec and not tweaked_ecs_paths if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: + if options.from_pr: + # if we are using from pr we also want any (related) files that are not easyconfigs + pr_paths = fetch_files_from_pr(pr=options.from_pr, path=tempfile.mktemp()) + for pr_path in pr_paths: + # we assumed that the last argument from the command line was the target_path but if it appears in the + # PR file list then it was most likely intended to use the CWD and (also) copy that particular file + if target_path == os.path.basename(pr_path): + determined_paths.append(pr_path) + target_path = os.getcwd() + other_pr_paths = [] + for ec_path in determined_paths: + for pr_path in pr_paths: + if os.path.basename(ec_path) == os.path.basename(pr_path): + # Search for any associated patches (they would have the same dirname) + for patch_path in pr_paths: + if pr_path != patch_path and os.path.dirname(pr_path) == os.path.dirname(patch_path): + other_pr_paths.append(patch_path) + if other_pr_paths: + determined_paths += other_pr_paths if options.copy_ec: copy_ecs_to_target(determined_paths, target_path) @@ -357,16 +378,11 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s", paths) - # if we are only copying ec's from a specific PR we can do that now - if options.copy_ec and options.from_pr: - copy_ecs_to_target([x for (x, _) in paths], target_path) - clean_exit(logfile, eb_tmpdir, testing) - # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path - regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) + regtest_ok = regtest([x for (x, _) in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 @@ -434,7 +450,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) - copy_ecs_to_target(tweaked_ecs_in_all_ecs, target_path) + copy_ecs_to_target(tweaked_ecs_in_all_ecs, target_path, target_is_dir=True) clean_exit(logfile, eb_tmpdir, testing) # creating/updating PRs diff --git a/test/framework/options.py b/test/framework/options.py index 9ef1ab6035..b4cf7907b4 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertEqual(stderr, '') + # self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) @@ -1002,7 +1002,7 @@ def mocked_main(args): args = ['--copy-ec', 'toy-0.0.eb', target_fn] stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s/test.eb' % self.test_prefix) + self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') change_dir(cwd) @@ -1048,7 +1048,7 @@ def check_copied_files(): self.assertFalse(os.path.exists(args[-1])) stdout = mocked_main(args) - self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) + self.assertEqual(stdout, '2 file(s) copied to test_target_dir') check_copied_files() @@ -1085,27 +1085,37 @@ def check_copied_files(): change_dir(test_working_dir) args = ['--copy-ec', '--from-pr', '8007'] stdout = mocked_main(args) - self.assertEqual(stdout, '2 file(s) copied to %s' % test_working_dir) - # check that the two easyconfigs exist - self.assertTrue(os.path.exists(os.path.join(test_working_dir, all_ecs_pr8007[0]))) - remove_file(os.path.join(test_working_dir, all_ecs_pr8007[0])) - self.assertTrue(os.path.exists(os.path.join(test_working_dir, all_ecs_pr8007[2]))) - remove_file(os.path.join(test_working_dir, all_ecs_pr8007[2])) - # ...but the patch doesn't - self.assertFalse(os.path.exists(os.path.join(test_working_dir, all_ecs_pr8007[1]))) - remove_file(os.path.join(test_working_dir, all_ecs_pr8007[1])) + self.assertEqual(stdout, '3 file(s) copied to %s' % test_working_dir) + # check that the files exist + for pr_file in all_ecs_pr8007: + self.assertTrue(os.path.exists(os.path.join(test_working_dir, pr_file))) + remove_file(os.path.join(test_working_dir, pr_file)) # copying multiple easyconfig files to a non-existing target directory (which is created automatically) args = ['--copy-ec', '--from-pr', '8007', test_target_dir] stdout = mocked_main(args) + self.assertEqual(stdout, '3 file(s) copied to %s' % test_target_dir) + for pr_file in all_ecs_pr8007: + self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) + remove_dir(test_target_dir) + + # test where we select a single file from a PR but also has a patch file + args = ['--copy-ec', '--from-pr', '8007', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', test_target_dir] + stdout = mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) - # check that the two easyconfigs exist - self.assertTrue(os.path.exists(os.path.join(test_target_dir, all_ecs_pr8007[0]))) - self.assertTrue(os.path.exists(os.path.join(test_target_dir, all_ecs_pr8007[2]))) - # ...but the patch doesn't - self.assertFalse(os.path.exists(os.path.join(test_target_dir, all_ecs_pr8007[1]))) + for pr_file in ['bat-0.3.3-fix-pyspark.patch', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb']: + self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) remove_dir(test_target_dir) + # test the same thing but where we don't provide a target directory + args = ['--copy-ec', '--from-pr', '8007', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'] + stdout = mocked_main(args) + self.assertEqual(stdout, '2 file(s) copied to %s' % test_working_dir) + for pr_file in ['bat-0.3.3-fix-pyspark.patch', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb']: + path = os.path.join(test_working_dir, pr_file) + self.assertTrue(os.path.exists(path)) + remove_file(path) + # test with only one ec in the PR (final argument is taken as a filename) test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', '--from-pr', '11521', test_ec] From f821af19323e8ca33ec8cc1134d4389082ea86c5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:08:36 +0200 Subject: [PATCH 013/115] Rework the approach and add additional tests --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index b4cf7907b4..ec8e41f162 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - # self.assertEqual(stderr, '') + self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) From 33d8570b515c801e866644fa15f2c7b2b1f74a59 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:11:41 +0200 Subject: [PATCH 014/115] Fix comment --- easybuild/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index a19643f056..c616dd1573 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -311,7 +311,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): target_path = os.getcwd() elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list - # use the absolute path as the --from-pr case drops us into a temporary directory target_path = orig_paths.pop() if options.copy_ec else None else: # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory From 3980dc9c01d3911ba75518089f9496a89681ad22 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:19:07 +0200 Subject: [PATCH 015/115] Be a little more careful with patches --- easybuild/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index c616dd1573..df5396920b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -331,7 +331,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): copy_ec = options.copy_ec and not tweaked_ecs_paths if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: if options.from_pr: - # if we are using from pr we also want any (related) files that are not easyconfigs + # pull in the paths to all the changed files in the PR (need to do this in a new temp dir) pr_paths = fetch_files_from_pr(pr=options.from_pr, path=tempfile.mktemp()) for pr_path in pr_paths: # we assumed that the last argument from the command line was the target_path but if it appears in the @@ -346,9 +346,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # Search for any associated patches (they would have the same dirname) for patch_path in pr_paths: if pr_path != patch_path and os.path.dirname(pr_path) == os.path.dirname(patch_path): - other_pr_paths.append(patch_path) - if other_pr_paths: - determined_paths += other_pr_paths + # if it's an easyconfig, we already have it covered + if not patch_path.endswith('.eb'): + other_pr_paths.append(patch_path) + determined_paths += other_pr_paths if options.copy_ec: copy_ecs_to_target(determined_paths, target_path) From f7aa64619afd7bbefbd96e2d974b281b50128444 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:21:45 +0200 Subject: [PATCH 016/115] Retain old behaviour when using try-* --- easybuild/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index df5396920b..fd319107f9 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -451,7 +451,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) copy_ecs_to_target(tweaked_ecs_in_all_ecs, target_path, target_is_dir=True) - clean_exit(logfile, eb_tmpdir, testing) # creating/updating PRs if pr_options: From 11f5a48f754fab0d4107c010e56a912227c3c46b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:27:16 +0200 Subject: [PATCH 017/115] Tidy up the new tests --- test/framework/options.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index ec8e41f162..6eefe26313 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1073,6 +1073,14 @@ def check_copied_files(): self.assertEqual(read_file(copied_toy_cwd), toy_ec_txt) # Test --copy-ec coupled with --from-pr + + test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') + mkdir(test_working_dir) + change_dir(test_working_dir) + test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') + # Make sure the test target directory doesn't exist + remove_dir(test_target_dir) + all_ecs_pr8007 = [ 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', 'bat-0.3.3-fix-pyspark.patch', @@ -1080,9 +1088,6 @@ def check_copied_files(): ] # test use of `--copy-ec` with `--from-pr` to the cwd - test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') - mkdir(test_working_dir) - change_dir(test_working_dir) args = ['--copy-ec', '--from-pr', '8007'] stdout = mocked_main(args) self.assertEqual(stdout, '3 file(s) copied to %s' % test_working_dir) @@ -1125,7 +1130,7 @@ def check_copied_files(): self.assertTrue(os.path.exists(test_ec)) remove_file(test_ec) - # --copy-ec without arguments results in a proper error + # --copy-ec without arguments (and no --from-pr) results in a proper error args = ['--copy-ec'] error_pattern = "One of more files to copy should be specified!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) From 01412431cc0fd0c57cd0c69ce3940fbbf0313c4c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:32:17 +0200 Subject: [PATCH 018/115] Keep our patch list unique --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index fd319107f9..eaf46e3e6f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -347,7 +347,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): for patch_path in pr_paths: if pr_path != patch_path and os.path.dirname(pr_path) == os.path.dirname(patch_path): # if it's an easyconfig, we already have it covered - if not patch_path.endswith('.eb'): + if not patch_path.endswith('.eb') and patch_path not in other_pr_paths: other_pr_paths.append(patch_path) determined_paths += other_pr_paths From e61e72d9b9dc3baa582d67ffc8941811ee846cae Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 11:34:49 +0200 Subject: [PATCH 019/115] Fix copy_ecs_to_target for case where I want to copy a single file to a directory --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index ea216f4288..128d5eddcc 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -743,7 +743,7 @@ def copy_ecs_to_target(determined_paths, target_path, prefix=False, target_is_di if len(determined_paths) == 1 and not target_is_dir: copy_file(determined_paths[0], target_path) print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) - elif len(determined_paths) > 1: + elif determined_paths: copy_files(determined_paths, target_path) print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=prefix) else: From 30145e48b3ae5351420813cc62bae4b911d00071 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:03:48 +0200 Subject: [PATCH 020/115] Fix broken test --- test/framework/options.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 6eefe26313..69dd910c22 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2125,15 +2125,15 @@ def test_try_with_copy(self): self.mock_stdout(True) self.mock_stderr(True) - tweaked_ecs_dir = os.path.join(self.test_buildpath) - os.chdir(tweaked_ecs_dir) - self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a'], verbose=True, raise_error=True) + tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') + self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], + verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) self.assertTrue(r'foo-1.2.3-GCC-6.4.0-2.28.eb copied to ' + tweaked_ecs_dir in outtxt) self.assertFalse(errtxt) + self.mock_stdout(False) + self.mock_stderr(False) self.assertTrue( os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) ) From 49b4f475f7546ab54781c5116547ce8b11fbae0c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:04:49 +0200 Subject: [PATCH 021/115] Fix broken test --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 69dd910c22..c4a0c49203 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -2130,7 +2130,7 @@ def test_try_with_copy(self): verbose=True, raise_error=True) outtxt = self.get_stdout() errtxt = self.get_stderr() - self.assertTrue(r'foo-1.2.3-GCC-6.4.0-2.28.eb copied to ' + tweaked_ecs_dir in outtxt) + self.assertTrue(r'1 file(s) copied to ' + tweaked_ecs_dir in outtxt) self.assertFalse(errtxt) self.mock_stdout(False) self.mock_stderr(False) From 1a8eecf0625c42e871f440590b3da16edccc9917 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:17:41 +0200 Subject: [PATCH 022/115] Fix broken test --- easybuild/main.py | 4 +++- test/framework/options.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index eaf46e3e6f..8363a5a303 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -337,7 +337,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # we assumed that the last argument from the command line was the target_path but if it appears in the # PR file list then it was most likely intended to use the CWD and (also) copy that particular file if target_path == os.path.basename(pr_path): - determined_paths.append(pr_path) + if not orig_paths: + # It should have been the only easyconfig selected + determined_paths = [pr_path] target_path = os.getcwd() other_pr_paths = [] for ec_path in determined_paths: diff --git a/test/framework/options.py b/test/framework/options.py index c4a0c49203..9fb15b30b6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertEqual(stderr, '') + # self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) From 380387b4427efc4ea58435888724e88fbd900be0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:21:41 +0200 Subject: [PATCH 023/115] Fix broken test --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 9fb15b30b6..c4a0c49203 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - # self.assertEqual(stderr, '') + self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) From eedbfd10af99b8789bda68c6b6eb7551e237337f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:25:51 +0200 Subject: [PATCH 024/115] Fix broken test --- easybuild/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 8363a5a303..4aef13249f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -340,6 +340,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if not orig_paths: # It should have been the only easyconfig selected determined_paths = [pr_path] + else: + if os.path.basename(pr_path) not in [os.path.basename(path) for path in determined_paths]: + determined_paths.append(pr_path) target_path = os.getcwd() other_pr_paths = [] for ec_path in determined_paths: From ae9e0ea8b5f4dd41f4d78267ab5dee03a402fde7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:40:56 +0200 Subject: [PATCH 025/115] Only grab the current working directory once at the beginning. --- easybuild/main.py | 7 ++++--- test/framework/options.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 4aef13249f..14d5bf7425 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -307,14 +307,15 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # if only one easyconfig is specified, or if none are specified and we are using --from-pr, # use current directory as target directory + cwd = os.getcwd() if len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): - target_path = os.getcwd() + target_path = cwd elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list target_path = orig_paths.pop() if options.copy_ec else None else: # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory - target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None + target_path = cwd if (options.copy_ec and options.from_pr) else None categorized_paths = categorize_files_by_type(orig_paths) @@ -343,7 +344,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): else: if os.path.basename(pr_path) not in [os.path.basename(path) for path in determined_paths]: determined_paths.append(pr_path) - target_path = os.getcwd() + target_path = cwd other_pr_paths = [] for ec_path in determined_paths: for pr_path in pr_paths: diff --git a/test/framework/options.py b/test/framework/options.py index c4a0c49203..9fb15b30b6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertEqual(stderr, '') + # self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) From c5560c474b2031300ce2bdbc746fbe51d28360ec Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:51:40 +0200 Subject: [PATCH 026/115] Fix GitHub tools leaving you in temporary directory --- easybuild/main.py | 7 +++---- easybuild/tools/github.py | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 14d5bf7425..4aef13249f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -307,15 +307,14 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # if only one easyconfig is specified, or if none are specified and we are using --from-pr, # use current directory as target directory - cwd = os.getcwd() if len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): - target_path = cwd + target_path = os.getcwd() elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list target_path = orig_paths.pop() if options.copy_ec else None else: # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory - target_path = cwd if (options.copy_ec and options.from_pr) else None + target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None categorized_paths = categorize_files_by_type(orig_paths) @@ -344,7 +343,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): else: if os.path.basename(pr_path) not in [os.path.basename(path) for path in determined_paths]: determined_paths.append(pr_path) - target_path = cwd + target_path = os.getcwd() other_pr_paths = [] for ec_path in determined_paths: for pr_path in pr_paths: diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d1d9df5290..928c33fbe7 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -364,6 +364,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) base_dir = extract_file(target_path, path, forced=True, change_into_dir=False) + working_directory = os.getcwd() change_dir(base_dir) extracted_path = os.path.join(base_dir, extracted_dir_name) @@ -373,6 +374,9 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ write_file(latest_sha_path, latest_commit_sha, forced=True) + # go back to previous working directory + change_dir(working_directory) + _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) return extracted_path From 26d91152662a67ffe436b5558b3e5b8bf1f17ad1 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 12:51:53 +0200 Subject: [PATCH 027/115] Fix GitHub tools leaving you in temporary directory --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 9fb15b30b6..c4a0c49203 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - # self.assertEqual(stderr, '') + self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) From 7944c0a3e418b9a112b42390a1067a14c2569b11 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 13:07:56 +0200 Subject: [PATCH 028/115] See if starting in the right directory makes a difference --- test/framework/options.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index c4a0c49203..3bb150f21a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertEqual(stderr, '') + # self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) @@ -1113,6 +1113,7 @@ def check_copied_files(): remove_dir(test_target_dir) # test the same thing but where we don't provide a target directory + change_dir(test_working_dir) args = ['--copy-ec', '--from-pr', '8007', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'] stdout = mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to %s' % test_working_dir) From 6e5be8deac0b636edbc9eccb88f11c62c90db05b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Oct 2020 13:11:45 +0200 Subject: [PATCH 029/115] Final fix on broken test --- test/framework/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3bb150f21a..daaff28863 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -975,7 +975,7 @@ def mocked_main(args): stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - # self.assertEqual(stderr, '') + self.assertEqual(stderr, '') return stdout.strip() topdir = os.path.dirname(os.path.abspath(__file__)) @@ -1076,7 +1076,6 @@ def check_copied_files(): test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') mkdir(test_working_dir) - change_dir(test_working_dir) test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') # Make sure the test target directory doesn't exist remove_dir(test_target_dir) @@ -1088,6 +1087,7 @@ def check_copied_files(): ] # test use of `--copy-ec` with `--from-pr` to the cwd + change_dir(test_working_dir) args = ['--copy-ec', '--from-pr', '8007'] stdout = mocked_main(args) self.assertEqual(stdout, '3 file(s) copied to %s' % test_working_dir) From 1758308460ce7dfb50c19e6d45618121c8dc47d5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 11:16:23 +0200 Subject: [PATCH 030/115] use decorator to cache result of fetch_files_from_pr --- easybuild/tools/github.py | 46 +++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 928c33fbe7..8d7dac059a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -33,6 +33,7 @@ import copy import getpass import glob +import functools import os import random import re @@ -364,8 +365,7 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) base_dir = extract_file(target_path, path, forced=True, change_into_dir=False) - working_directory = os.getcwd() - change_dir(base_dir) + cwd = change_dir(base_dir) extracted_path = os.path.join(base_dir, extracted_dir_name) # check if extracted_path exists @@ -375,22 +375,40 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ write_file(latest_sha_path, latest_commit_sha, forced=True) # go back to previous working directory - change_dir(working_directory) + change_dir(cwd) _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) return extracted_path -def fetch_easyblocks_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" - return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) +def pr_files_cache(func): + """ + Decorator to cache result of fetch_files_from_pr. + """ + cache = {} + @functools.wraps(func) + def cache_aware_func(pr, *args, **kwargs): + """Retrieve cached resul, or fetch files from PR & cache result.""" + # cache key is combination of all function arguments (incl. optional ones) + key = tuple([pr] + [kwargs[key] for key in sorted(kwargs.keys())]) -def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" - return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) + if key in cache: + _log.debug("Using cached value for fetch_files_from_pr for PR #%s (%s)", pr, kwargs) + return cache[key] + else: + res = func(pr, *args, **kwargs) + cache[key] = res + return res + + # expose clear/update methods of cache to wrapped function + cache_aware_func.clear_cache = cache.clear + cache_aware_func.update_cache = cache.update + + return cache_aware_func +@pr_files_cache def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): """Fetch patched files for a particular PR.""" @@ -500,6 +518,16 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): return files +def fetch_easyblocks_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) + + +def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) + + def create_gist(txt, fn, descr=None, github_user=None, github_token=None): """Create a gist with the provided text.""" From 7bef831e27c473a3d098c93b9f7c68b6d00cefd3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 12:06:28 +0200 Subject: [PATCH 031/115] add locate_files function to easybuild.tools.filetools --- easybuild/tools/filetools.py | 43 +++++++++++++++++++++ test/framework/filetools.py | 73 ++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1c3dbf5f5b..1975ac780d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -792,6 +792,49 @@ def find_easyconfigs(path, ignore_dirs=None): return files +def locate_files(files, paths, ignore_subdirs=None): + """ + Determine full path for list of files, in given list of paths (directories). + """ + # determine which easyconfigs files need to be found, if any + files_to_find = [] + for idx, ec_file in enumerate(files): + if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): + files_to_find.append((idx, ec_file)) + _log.debug("List of files to find: %s" % files_to_find) + + # find missing easyconfigs by walking paths in robot search path + for path in paths: + _log.debug("Looking for missing files (%d left) in %s..." % (len(files_to_find), path)) + for (subpath, dirnames, filenames) in os.walk(path, topdown=True): + for idx, orig_path in files_to_find[:]: + if orig_path in filenames: + full_path = os.path.join(subpath, orig_path) + _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) + files[idx] = full_path + # if file was found, stop looking for it (first hit wins) + files_to_find.remove((idx, orig_path)) + + # stop os.walk insanity as soon as we have all we need (os.walk loop) + if not files_to_find: + break + + # ignore specified subdirectories + if ignore_subdirs: + dirnames[:] = [d for d in dirnames if d not in ignore_subdirs] + + # stop os.walk insanity as soon as we have all we need (outer loop) + if not files_to_find: + break + + if files_to_find: + filenames = ', '.join([f for (_, f) in files_to_find]) + paths = ', '.join(paths) + raise EasyBuildError("One or more files not found: %s (search paths: %s)", filenames, paths) + + return [os.path.abspath(f) for f in files] + + def find_glob_pattern(glob_pattern, fail_on_no_match=True): """Find unique file/dir matching glob_pattern (raises error if more than one match is found)""" if build_option('extended_dry_run'): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c5a572543..8081aef00c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2680,6 +2680,79 @@ def test_locks(self): self.assertFalse(os.path.exists(lock_path)) self.assertEqual(os.listdir(locks_dir), []) + def test_locate_files(self): + """Test locate_files function.""" + + # create some files to find + one = os.path.join(self.test_prefix, '1.txt') + ft.write_file(one, 'one') + two = os.path.join(self.test_prefix, 'subdirA', '2.txt') + ft.write_file(two, 'two') + three = os.path.join(self.test_prefix, 'subdirB', '3.txt') + ft.write_file(three, 'three') + ft.mkdir(os.path.join(self.test_prefix, 'empty_subdir')) + + # empty list of files yields empty result + self.assertEqual(ft.locate_files([], []), []) + self.assertEqual(ft.locate_files([], [self.test_prefix]), []) + + # error is raised if files could not be found + error_pattern = r"One or more files not found: nosuchfile.txt \(search paths: \)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['nosuchfile.txt'], []) + + # files specified via absolute path don't have to be found + res = ft.locate_files([one], []) + self.assertTrue(len(res) == 1) + self.assertTrue(os.path.samefile(res[0], one)) + + # note: don't compare file paths directly but use os.path.samefile instead, + # which is required to avoid failing tests in case temporary directory is a symbolic link (e.g. on macOS) + res = ft.locate_files(['1.txt'], [self.test_prefix]) + self.assertEqual(len(res), 1) + self.assertTrue(os.path.samefile(res[0], one)) + + res = ft.locate_files(['2.txt'], [self.test_prefix]) + self.assertEqual(len(res), 1) + self.assertTrue(os.path.samefile(res[0], two)) + + res = ft.locate_files(['1.txt', '3.txt'], [self.test_prefix]) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], one)) + self.assertTrue(os.path.samefile(res[1], three)) + + # search in multiple paths + files = ['2.txt', '3.txt'] + paths = [os.path.dirname(three), os.path.dirname(two)] + res = ft.locate_files(files, paths) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], two)) + self.assertTrue(os.path.samefile(res[1], three)) + + # same file specified multiple times works fine + files = ['1.txt', '2.txt', '1.txt', '3.txt', '2.txt'] + res = ft.locate_files(files, [self.test_prefix]) + self.assertEqual(len(res), 5) + for idx, expected in enumerate([one, two, one, three, two]): + self.assertTrue(os.path.samefile(res[idx], expected)) + + # only some files found yields correct warning + files = ['2.txt', '3.txt', '1.txt'] + error_pattern = r"One or more files not found: 3\.txt, 1.txt \(search paths: .*/subdirA\)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, files, [os.path.dirname(two)]) + + # check that relative paths are found in current working dir + ft.change_dir(self.test_prefix) + rel_paths = ['subdirA/2.txt', '1.txt'] + # result is still absolute paths to those files + res = ft.locate_files(rel_paths, []) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], two)) + self.assertTrue(os.path.samefile(res[1], one)) + + # no recursive search in current working dir (which would potentially be way too expensive) + error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def suite(): """ returns all the testcases in this module """ From aa470f109f8cd3bb4e772b2e5927804843cf471e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 13:35:25 +0200 Subject: [PATCH 032/115] check whether paths still exist before using cached result for fetch_files_from_pr --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8d7dac059a..2e0f0fc135 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -393,7 +393,7 @@ def cache_aware_func(pr, *args, **kwargs): # cache key is combination of all function arguments (incl. optional ones) key = tuple([pr] + [kwargs[key] for key in sorted(kwargs.keys())]) - if key in cache: + if key in cache and all(os.path.exists(x) for x in cache[key]): _log.debug("Using cached value for fetch_files_from_pr for PR #%s (%s)", pr, kwargs) return cache[key] else: From 5c18a8ef61517b05f3080f06622c66e8e2b6f152 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 13:38:51 +0200 Subject: [PATCH 033/115] use locate_files function in det_easyconfig_paths --- easybuild/framework/easyconfig/tools.py | 49 +++++-------------------- test/framework/robot.py | 3 +- 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 128d5eddcc..5ff5c7e9bd 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -53,8 +53,8 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import copy_file, copy_files -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file +from easybuild.tools.filetools import copy_file, copy_files, find_easyconfigs, is_patch_file, locate_files +from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict @@ -349,44 +349,13 @@ def det_easyconfig_paths(orig_paths): ec_files = [path for path in pr_files if path.endswith('.eb')] if ec_files and robot_path: - # look for easyconfigs with relative paths in robot search path, - # unless they were found at the given relative paths - - # determine which easyconfigs files need to be found, if any - ecs_to_find = [] - for idx, ec_file in enumerate(ec_files): - if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): - ecs_to_find.append((idx, ec_file)) - _log.debug("List of easyconfig files to find: %s" % ecs_to_find) - - # find missing easyconfigs by walking paths in robot search path - for path in robot_path: - _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) - for (subpath, dirnames, filenames) in os.walk(path, topdown=True): - for idx, orig_path in ecs_to_find[:]: - if orig_path in filenames: - full_path = os.path.join(subpath, orig_path) - _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) - ec_files[idx] = full_path - # if file was found, stop looking for it (first hit wins) - ecs_to_find.remove((idx, orig_path)) - - # stop os.walk insanity as soon as we have all we need (os.walk loop) - if not ecs_to_find: - break - - # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk - dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')] - - # ignore archived easyconfigs, unless specified otherwise - if not build_option('consider_archived_easyconfigs'): - dirnames[:] = [d for d in dirnames if d != EASYCONFIGS_ARCHIVE_DIR] - - # stop os.walk insanity as soon as we have all we need (outer loop) - if not ecs_to_find: - break - - return [os.path.abspath(ec_file) for ec_file in ec_files] + ignore_subdirs = build_option('ignore_dirs') + if not build_option('consider_archived_easyconfigs'): + ignore_subdirs.extend(EASYCONFIGS_ARCHIVE_DIR) + + ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs) + + return ec_files def parse_easyconfigs(paths, validate=True): diff --git a/test/framework/robot.py b/test/framework/robot.py index e50f570a37..6eca07b571 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -654,7 +654,8 @@ def test_det_easyconfig_paths(self): '--robot', '--unittest-file=%s' % self.logfile, ] - self.assertErrorRegex(EasyBuildError, "Can't find", self.eb_main, args, logfile=dummylogfn, raise_error=True) + error_pattern = "One or more files not found: intel-2012a.eb" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, logfile=dummylogfn, raise_error=True) args.append('--consider-archived-easyconfigs') outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True) From 756f3370f53230193068acc5b741963fa28625c0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 15:44:53 +0200 Subject: [PATCH 034/115] fix default for pr_target_account build option --- easybuild/tools/config.py | 4 ++++ easybuild/tools/options.py | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 10d5a19add..0340efd560 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -101,6 +101,7 @@ DEFAULT_PKG_TOOL = PKG_TOOL_FPM DEFAULT_PKG_TYPE = PKG_TYPE_RPM DEFAULT_PNS = 'EasyBuildPNS' +DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' DEFAULT_WAIT_ON_LOCK_INTERVAL = 60 @@ -307,6 +308,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_PKG_TYPE: [ 'package_type', ], + DEFAULT_PR_TARGET_ACCOUNT: [ + 'pr_target_account', + ], GENERAL_CLASS: [ 'suffix_modules_path', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 026ae528b1..c801fc3f6f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -64,11 +64,11 @@ from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY -from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT, EBROOT_ENV_VAR_ACTIONS -from easybuild.tools.config import ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR -from easybuild.tools.config import JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS, LOCAL_VAR_NAMING_CHECK_WARN -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECKS, WARN +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT +from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE +from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -78,7 +78,7 @@ from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc from easybuild.tools.filetools import move_file, which -from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED +from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token @@ -642,7 +642,7 @@ def github_options(self): str, 'store', None), 'pr-commit-msg': ("Commit message for new/updated pull request created with --new-pr", str, 'store', None), 'pr-descr': ("Description for new pull request created with --new-pr", str, 'store', None), - 'pr-target-account': ("Target account for new PRs", str, 'store', GITHUB_EB_MAIN), + 'pr-target-account': ("Target account for new PRs", str, 'store', DEFAULT_PR_TARGET_ACCOUNT), 'pr-target-branch': ("Target branch for new PRs", str, 'store', DEFAULT_BRANCH), 'pr-target-repo': ("Target repository for new/updating PRs (default: auto-detect based on provided files)", str, 'store', None), From 31261d2c848f7b5350fa2a8928e9a7faf5bbdad2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 15:45:25 +0200 Subject: [PATCH 035/115] add det_copy_ec_specs function to easybuild.framework.easyconfig.tools --- easybuild/framework/easyconfig/tools.py | 64 ++++++++++++++++++- test/framework/easyconfig.py | 81 ++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 5ff5c7e9bd..fa23383182 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -55,7 +55,7 @@ from easybuild.tools.environment import restore_env from easybuild.tools.filetools import copy_file, copy_files, find_easyconfigs, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo +from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_files_from_pr, download_repo from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -700,6 +700,68 @@ def avail_easyblocks(): return easyblocks +def det_copy_ec_specs(orig_paths, from_pr): + """Determine list of paths + target directory for --copy-ec.""" + + target_path, paths = None, [] + + # if only one argument is specified, use current directory as target directory + if len(orig_paths) == 1: + target_path = os.getcwd() + paths = orig_paths[:] + + # if multiple arguments are specified, assume that last argument is target location, + # and remove that from list of paths to copy + elif orig_paths: + target_path = orig_paths[-1] + paths = orig_paths[:-1] + + # if --from-pr was used in combination with --copy-ec, some extra care must be taken + if from_pr: + # pull in the paths to all the changed files in the PR, + # which includes easyconfigs but also patch files (& maybe more); + # do this in a dedicated subdirectory of the working tmpdir, + # to avoid potential trouble with already existing files in the working tmpdir + # (note: we use a fixed subdirectory in the working tmpdir here rather than a unique random subdirectory, + # to ensure that the caching for fetch_files_from_pr works across calls for the same PR) + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % from_pr) + pr_paths = fetch_files_from_pr(pr=from_pr, path=tmpdir) + + # assume that files need to be copied to current working directory for now + target_path = os.getcwd() + + if orig_paths: + last_path = orig_paths[-1] + + # check files touched by PR and see if the target directory for --copy-ec + # corresponds to the name of one of these files; + # if so we should copy the specified file(s) to the current working directory, + # since interpreting the last argument as target location is very unlikely incorrect in this case + pr_filenames = [os.path.basename(p) for p in pr_paths] + if last_path in pr_filenames: + paths = orig_paths[:] + else: + target_path = last_path + # exclude last argument that is used as target location + paths = orig_paths[:-1] + + # if list of files to copy is empty at this point, + # we simply copy *all* files touched by the PR + if not paths: + paths = pr_paths + + # replace path for files touched by PR (no need to worry about others) + for idx, path in enumerate(paths): + filename = os.path.basename(path) + pr_matches = [x for x in pr_paths if os.path.basename(x) == filename] + if len(pr_matches) == 1: + paths[idx] = pr_matches[0] + elif pr_matches: + raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches) + + return paths, target_path + + def copy_ecs_to_target(determined_paths, target_path, prefix=False, target_is_dir=False): """ Copy list of easyconfigs to specified path diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 3ebeb86f8d..d069f84007 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -56,7 +56,8 @@ from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str from easybuild.framework.easyconfig.style import check_easyconfigs_style from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph -from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs +from easybuild.framework.easyconfig.tools import det_copy_ec_specs, find_related_easyconfigs, get_paths_for +from easybuild.framework.easyconfig.tools import parse_easyconfigs from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain @@ -3975,6 +3976,84 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['configopts'], 'sm_42,sm_63') self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') + def test_det_copy_ec_specs(self): + """Test det_copy_ec_specs function.""" + + cwd = os.getcwd() + + # no problems on empty list as input + paths, target_path = det_copy_ec_specs([], None) + self.assertEqual(paths, []) + self.assertEqual(target_path, None) + + # single-element list, no --from-pr => use current directory as target location + paths, target_path = det_copy_ec_specs(['test.eb'], None) + self.assertEqual(paths, ['test.eb']) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # multi-element list, no --from-pr => last element is used as target location + for args in (['test.eb', 'dir'], ['test1.eb', 'test2.eb', 'dir']): + paths, target_path = det_copy_ec_specs(args, None) + self.assertEqual(paths, args[:-1]) + self.assertEqual(target_path, args[-1]) + + # use fixed PR (speeds up the test due to caching in fetch_files_from_pr; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007 + from_pr = 8007 + arrow_ec_fn = 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb' + bat_ec_fn = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb' + bat_patch_fn = 'bat-0.3.3-fix-pyspark.patch' + pr_files = [ + arrow_ec_fn, + bat_ec_fn, + bat_patch_fn, + ] + + # if no paths are specified, default is to copy all files touched by PR to current working directory + paths, target_path = det_copy_ec_specs([], from_pr) + self.assertEqual(len(paths), 3) + filenames = sorted([os.path.basename(x) for x in paths]) + self.assertEqual(filenames, sorted(pr_files)) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # last argument is used as target directory, + # unless it corresponds to a file touched by PR + args = [bat_ec_fn, 'target_dir'] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 1) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertEqual(target_path, 'target_dir') + + args = [bat_ec_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 1) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + args = [arrow_ec_fn, bat_ec_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 2) + self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn) + self.assertEqual(os.path.basename(paths[1]), bat_ec_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + args = [bat_ec_fn, bat_patch_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 2) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertEqual(os.path.basename(paths[1]), bat_patch_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # also test with combination of local files and files from PR + args = [arrow_ec_fn, 'test.eb', 'test.patch', bat_patch_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 4) + self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn) + self.assertEqual(paths[1], 'test.eb') + self.assertEqual(paths[2], 'test.patch') + self.assertEqual(os.path.basename(paths[3]), bat_patch_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + def suite(): """ returns all the testcases in this module """ From a761eab38c4ba01389ee818edb8ccbcf6e537c59 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 16:04:08 +0200 Subject: [PATCH 036/115] use det_copy_ec_specs in main.py --- easybuild/main.py | 58 ++++++++++++----------------------------------- 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 4aef13249f..ce12aed41b 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -39,7 +39,6 @@ import os import stat import sys -import tempfile import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -52,15 +51,16 @@ from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.framework.easyconfig.tools import categorize_files_by_type, copy_ecs_to_target, dep_graph -from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available +from easybuild.framework.easyconfig.tools import det_copy_ec_specs, det_easyconfig_paths, dump_env_script +from easybuild.framework.easyconfig.tools import get_paths_for, parse_easyconfigs, review_pr, run_contrib_checks +from easybuild.framework.easyconfig.tools import skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, dump_index, load_index +from easybuild.tools.filetools import adjust_permissions, cleanup, dump_index, load_index, locate_files from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file -from easybuild.tools.github import check_github, close_pr, fetch_files_from_pr, find_easybuild_easyconfig +from easybuild.tools.github import check_github, close_pr, find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr @@ -305,17 +305,6 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - # if only one easyconfig is specified, or if none are specified and we are using --from-pr, - # use current directory as target directory - if len(orig_paths) == 1 and not (options.copy_ec and options.from_pr): - target_path = os.getcwd() - elif orig_paths: - # last path is target when --copy-ec is used, so remove that from the list - target_path = orig_paths.pop() if options.copy_ec else None - else: - # if no easyconfig files are specified and we are using --from-pr, use current directory as target directory - target_path = os.getcwd() if (options.copy_ec and options.from_pr) else None - categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified @@ -327,37 +316,20 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - # only copy easyconfigs here if we're not using --try-* (that's are handled below) + # only copy easyconfigs here if we're not using --try-* (that's handled below) copy_ec = options.copy_ec and not tweaked_ecs_paths + if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: - if options.from_pr: - # pull in the paths to all the changed files in the PR (need to do this in a new temp dir) - pr_paths = fetch_files_from_pr(pr=options.from_pr, path=tempfile.mktemp()) - for pr_path in pr_paths: - # we assumed that the last argument from the command line was the target_path but if it appears in the - # PR file list then it was most likely intended to use the CWD and (also) copy that particular file - if target_path == os.path.basename(pr_path): - if not orig_paths: - # It should have been the only easyconfig selected - determined_paths = [pr_path] - else: - if os.path.basename(pr_path) not in [os.path.basename(path) for path in determined_paths]: - determined_paths.append(pr_path) - target_path = os.getcwd() - other_pr_paths = [] - for ec_path in determined_paths: - for pr_path in pr_paths: - if os.path.basename(ec_path) == os.path.basename(pr_path): - # Search for any associated patches (they would have the same dirname) - for patch_path in pr_paths: - if pr_path != patch_path and os.path.dirname(pr_path) == os.path.dirname(patch_path): - # if it's an easyconfig, we already have it covered - if not patch_path.endswith('.eb') and patch_path not in other_pr_paths: - other_pr_paths.append(patch_path) - determined_paths += other_pr_paths if options.copy_ec: - copy_ecs_to_target(determined_paths, target_path) + # figure out list of files to copy + target location (taking into account --from-pr) + paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) + + # at this point some paths may still just be filenames rather than absolute paths, + # so try to determine full path for those too via robot search path + paths = locate_files(paths, robot_path) + + copy_ecs_to_target(paths, target_path) elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) From 0ed52ec50e49a3ecd8b2cf9653a36819bd80861b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 16:04:39 +0200 Subject: [PATCH 037/115] clean up and enhance tests for combination of --copy-ec and --from-pr --- test/framework/options.py | 158 ++++++++++++++++++++++++++------------ 1 file changed, 109 insertions(+), 49 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index daaff28863..cc9cc23ee8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -50,8 +50,8 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, mkdir, read_file -from easybuild.tools.filetools import remove_dir, remove_file, which, write_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, is_patch_file, mkdir +from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token from easybuild.tools.modules import Lmod @@ -965,19 +965,21 @@ def test_show_ec(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def mocked_main(self, args): + """Run eb_main with mocked stdout/stderr.""" + #self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, raise_error=True) + #stderr, stdout = self.get_stderr(), self.get_stdout() + stdout = self.get_stdout() + #self.mock_stderr(False) + self.mock_stdout(False) + #self.assertEqual(stderr, '') + return stdout.strip() + def test_copy_ec(self): """Test --copy-ec.""" - def mocked_main(args): - self.mock_stderr(True) - self.mock_stdout(True) - self.eb_main(args, raise_error=True) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) - self.assertEqual(stderr, '') - return stdout.strip() - topdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs_dir = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -987,7 +989,7 @@ def mocked_main(args): # basic test: copying one easyconfig file to a non-existing absolute path test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', 'toy-0.0.eb', test_ec] - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_ec) self.assertTrue(os.path.exists(test_ec)) @@ -1001,7 +1003,7 @@ def mocked_main(args): self.assertFalse(os.path.exists(target_fn)) args = ['--copy-ec', 'toy-0.0.eb', target_fn] - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') change_dir(cwd) @@ -1013,7 +1015,7 @@ def mocked_main(args): test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') mkdir(test_target_dir) args = ['--copy-ec', 'toy-0.0.eb', test_target_dir] - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_target_dir) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') @@ -1035,7 +1037,7 @@ def check_copied_files(): # copying multiple easyconfig files to a non-existing target directory (which is created automatically) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', test_target_dir] - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) check_copied_files() @@ -1047,7 +1049,7 @@ def check_copied_files(): args[-1] = os.path.basename(test_target_dir) self.assertFalse(os.path.exists(args[-1])) - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to test_target_dir') check_copied_files() @@ -1065,14 +1067,23 @@ def check_copied_files(): change_dir(test_working_dir) self.assertEqual(len(os.listdir(os.getcwd())), 0) args = ['--copy-ec', 'toy-0.0.eb'] - stdout = mocked_main(args) + stdout = self.mocked_main(args) regex = re.compile('toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) self.assertTrue(regex.match(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_cwd = os.path.join(test_working_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_cwd)) self.assertEqual(read_file(copied_toy_cwd), toy_ec_txt) - # Test --copy-ec coupled with --from-pr + # --copy-ec without arguments results in a proper error + args = ['--copy-ec'] + error_pattern = "One of more files to copy should be specified!" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + + def test_copy_ec_from_pr(self): + """Test combination of --copy-ec with --from-pr.""" + if self.github_token is None: + print("Skipping test_copy_ec_from_pr, no GitHub token available?") + return test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') mkdir(test_working_dir) @@ -1080,62 +1091,111 @@ def check_copied_files(): # Make sure the test target directory doesn't exist remove_dir(test_target_dir) - all_ecs_pr8007 = [ + all_files_pr8007 = [ 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', 'bat-0.3.3-fix-pyspark.patch', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', ] - # test use of `--copy-ec` with `--from-pr` to the cwd - change_dir(test_working_dir) + # test use of --copy-ec with --from-pr to the current working directory + cwd = change_dir(test_working_dir) args = ['--copy-ec', '--from-pr', '8007'] - stdout = mocked_main(args) - self.assertEqual(stdout, '3 file(s) copied to %s' % test_working_dir) + stdout = self.mocked_main(args) + + regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_working_dir)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + # check that the files exist - for pr_file in all_ecs_pr8007: + for pr_file in all_files_pr8007: self.assertTrue(os.path.exists(os.path.join(test_working_dir, pr_file))) remove_file(os.path.join(test_working_dir, pr_file)) - # copying multiple easyconfig files to a non-existing target directory (which is created automatically) + # copying all files touched by PR to a non-existing target directory (which is created automatically) + self.assertFalse(os.path.exists(test_target_dir)) args = ['--copy-ec', '--from-pr', '8007', test_target_dir] - stdout = mocked_main(args) - self.assertEqual(stdout, '3 file(s) copied to %s' % test_target_dir) - for pr_file in all_ecs_pr8007: + stdout = self.mocked_main(args) + + regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_target_dir)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + for pr_file in all_files_pr8007: self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) remove_dir(test_target_dir) - # test where we select a single file from a PR but also has a patch file - args = ['--copy-ec', '--from-pr', '8007', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', test_target_dir] - stdout = mocked_main(args) - self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) - for pr_file in ['bat-0.3.3-fix-pyspark.patch', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb']: - self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) + # test where we select a single easyconfig file from a PR + mkdir(test_target_dir) + ec_filename = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb' + args = ['--copy-ec', '--from-pr', '8007', ec_filename, test_target_dir] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_target_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.listdir(test_target_dir), [ec_filename]) + self.assertTrue("name = 'bat'" in read_file(os.path.join(test_target_dir, ec_filename))) remove_dir(test_target_dir) - # test the same thing but where we don't provide a target directory + # test copying of a single easyconfig file from a PR to a non-existing path + bat_ec = os.path.join(self.test_prefix, 'bat.eb') + args[-1] = bat_ec + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/bat.eb" % ec_filename) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertTrue(os.path.exists(bat_ec)) + self.assertTrue("name = 'bat'" in read_file(bat_ec)) + + change_dir(cwd) + remove_dir(test_working_dir) + mkdir(test_working_dir) change_dir(test_working_dir) - args = ['--copy-ec', '--from-pr', '8007', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb'] - stdout = mocked_main(args) - self.assertEqual(stdout, '2 file(s) copied to %s' % test_working_dir) - for pr_file in ['bat-0.3.3-fix-pyspark.patch', 'bat-0.3.3-intel-2017b-Python-3.6.3.eb']: - path = os.path.join(test_working_dir, pr_file) - self.assertTrue(os.path.exists(path)) - remove_file(path) + + # test copying of a patch file from a PR via --copy-ec to current directory + patch_fn = 'bat-0.3.3-fix-pyspark.patch' + args = ['--copy-ec', '--from-pr', '8007', patch_fn, '.'] + stdout = self.mocked_main(args) + + self.assertEqual(os.listdir(test_working_dir), [patch_fn]) + patch_path = os.path.join(test_working_dir, patch_fn) + self.assertTrue(os.path.exists(patch_path)) + self.assertTrue(is_patch_file(patch_path)) + remove_file(patch_path) + + # test the same thing but where we don't provide a target location + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007', ec_filename] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_working_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.listdir(test_working_dir), [ec_filename]) + self.assertTrue("name = 'bat'" in read_file(ec_filename)) + + # also test copying of patch file to current directory (without specifying target location) + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007', patch_fn] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (patch_fn, os.path.basename(test_working_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(sorted(os.listdir(test_working_dir)), sorted([ec_filename, patch_fn])) + self.assertTrue(is_patch_file(patch_fn)) + + change_dir(cwd) + remove_dir(test_working_dir) # test with only one ec in the PR (final argument is taken as a filename) test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', '--from-pr', '11521', test_ec] ec_pr11521 = "ExifTool-12.00-GCCcore-9.3.0.eb" - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '%s copied to %s' % (ec_pr11521, test_ec)) self.assertTrue(os.path.exists(test_ec)) remove_file(test_ec) - # --copy-ec without arguments (and no --from-pr) results in a proper error - args = ['--copy-ec'] - error_pattern = "One of more files to copy should be specified!" - self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) - def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From 79f8edbbfb129d0010b5a24c7e2a17fee9aca36d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 16:30:49 +0200 Subject: [PATCH 038/115] fix changed error pattern in test_robot_path_check due to using locate_files in det_easyconfig_paths --- test/framework/options.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index cc9cc23ee8..afc6c50e76 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -967,14 +967,13 @@ def test_show_ec(self): def mocked_main(self, args): """Run eb_main with mocked stdout/stderr.""" - #self.mock_stderr(True) + self.mock_stderr(True) self.mock_stdout(True) self.eb_main(args, raise_error=True) - #stderr, stdout = self.get_stderr(), self.get_stdout() - stdout = self.get_stdout() - #self.mock_stderr(False) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) self.mock_stdout(False) - #self.assertEqual(stderr, '') + self.assertEqual(stderr, '') return stdout.strip() def test_copy_ec(self): @@ -2638,7 +2637,8 @@ def test_robot_path_check(self): # different error when a non-existing easyconfig file is specified to --robot args = ['--dry-run', '--robot', 'no_such_easyconfig_file_in_robot_search_path.eb'] - self.assertErrorRegex(EasyBuildError, "Can't find path", self.eb_main, args, raise_error=True) + error_pattern = "One or more files not found: no_such_easyconfig_file_in_robot_search_path.eb" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) for robot in ['-r%s' % self.test_prefix, '--robot=%s' % self.test_prefix]: args = ['toy-0.0.eb', '--dry-run', robot] From 2f6ac6e548c0f4e7ace59b6c4863a3b90dcc6b19 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 16:49:20 +0200 Subject: [PATCH 039/115] fix duplicate default value for pr_target_account build option --- easybuild/tools/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 0340efd560..da0e35ab4e 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -205,7 +205,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_branch_name', 'pr_commit_msg', 'pr_descr', - 'pr_target_account', 'pr_target_repo', 'pr_title', 'rpath_filter', From 173ca950667890b1f82ace63249991204946380d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 16:50:03 +0200 Subject: [PATCH 040/115] determine --copy-ec target path *before* categorizing eb arguments by file type --- easybuild/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index ce12aed41b..e2c85b0dec 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -305,6 +305,10 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) + if options.copy_ec: + # figure out list of files to copy + target location (taking into account --from-pr) + orig_paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) + categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified @@ -322,12 +326,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: - # figure out list of files to copy + target location (taking into account --from-pr) - paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) - # at this point some paths may still just be filenames rather than absolute paths, # so try to determine full path for those too via robot search path - paths = locate_files(paths, robot_path) + paths = locate_files(orig_paths, robot_path) copy_ecs_to_target(paths, target_path) From bb96d1a1e804435b0639953718639dfbce3e4488 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 18:04:16 +0200 Subject: [PATCH 041/115] don't make assumptions about current working directory when checking contents of copied file in test_copy_ec_from_pr --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index afc6c50e76..4d51759707 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1170,7 +1170,7 @@ def test_copy_ec_from_pr(self): self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) self.assertEqual(os.listdir(test_working_dir), [ec_filename]) - self.assertTrue("name = 'bat'" in read_file(ec_filename)) + self.assertTrue("name = 'bat'" in read_file(os.path.join(test_working_dir, ec_filename))) # also test copying of patch file to current directory (without specifying target location) change_dir(test_working_dir) From b6881eec71f941c58a7203834632210f36c090bc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 21:50:15 +0200 Subject: [PATCH 042/115] correctly add archive subdir to ignore_subdirs in det_easyconfig_paths + enhance test to catch the issue --- easybuild/framework/easyconfig/tools.py | 2 +- test/framework/robot.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index fa23383182..b8be79be36 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -351,7 +351,7 @@ def det_easyconfig_paths(orig_paths): if ec_files and robot_path: ignore_subdirs = build_option('ignore_dirs') if not build_option('consider_archived_easyconfigs'): - ignore_subdirs.extend(EASYCONFIGS_ARCHIVE_DIR) + ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR) ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs) diff --git a/test/framework/robot.py b/test/framework/robot.py index 6eca07b571..dc540674ec 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -620,12 +620,19 @@ def test_det_easyconfig_paths(self): test_ec = 'toy-0.0-deps.eb' shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + # copy hwloc easyconfig to h/hwloc subdir in robot search path, + # to trigger bug fixed in det_easyconfig_paths (.extend rather than .append for '__archive'__ to ignore_subdirs) + hwloc_ec = 'hwloc-1.11.8-GCC-6.4.0-2.28.eb' + subdir_hwloc = os.path.join(self.test_prefix, 'h', 'hwloc') + mkdir(subdir_hwloc, parents=True) + shutil.copy2(os.path.join(test_ecs_path, 'h', 'hwloc', hwloc_ec), subdir_hwloc) shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertFalse(os.path.exists(test_ec)) args = [ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), test_ec, # relative path, should be resolved via robot search path + hwloc_ec, '--dry-run', '--debug', '--robot', @@ -640,6 +647,7 @@ def test_det_easyconfig_paths(self): (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location (self.test_prefix, 'intel/2018a'), # dependency, found in robot search path (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + (self.test_prefix, 'hwloc/1.11.8-GCC-6.4.0-2.28'), # specified easyconfig, found in robot search path ] for path_prefix, module in modules: ec_fn = "%s.eb" % '-'.join(module.split('/')) From 1f7a0a7312e1be3bc47c602aaa81ae6e80736a3d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 24 Oct 2020 22:31:46 +0200 Subject: [PATCH 043/115] use full path when checking patch file in --copy-ec --from-pr test --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 4d51759707..0f3d0e6626 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1181,7 +1181,7 @@ def test_copy_ec_from_pr(self): self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) self.assertEqual(sorted(os.listdir(test_working_dir)), sorted([ec_filename, patch_fn])) - self.assertTrue(is_patch_file(patch_fn)) + self.assertTrue(is_patch_file(os.path.join(test_working_dir, patch_fn))) change_dir(cwd) remove_dir(test_working_dir) From aaa67373ebd8c82c4f9db11ddf9a04c719b3d81c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 10:06:10 +0100 Subject: [PATCH 044/115] take indices into account in locate_files --- easybuild/tools/filetools.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1975ac780d..372ea88893 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -806,24 +806,32 @@ def locate_files(files, paths, ignore_subdirs=None): # find missing easyconfigs by walking paths in robot search path for path in paths: _log.debug("Looking for missing files (%d left) in %s..." % (len(files_to_find), path)) - for (subpath, dirnames, filenames) in os.walk(path, topdown=True): - for idx, orig_path in files_to_find[:]: - if orig_path in filenames: - full_path = os.path.join(subpath, orig_path) - _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) + + # try to load index for current path, or create one + path_index = load_index(path, ignore_dirs=ignore_subdirs) + if path_index is None or build_option('ignore_index'): + if os.path.exists(path): + _log.info("No index found for %s, creating one...", path) + path_index = create_index(path, ignore_dirs=ignore_subdirs) + else: + path_index = [] + else: + _log.info("Index found for %s, so using it...", path) + + for filepath in path_index: + for idx, file_to_find in files_to_find[:]: + if os.path.basename(filepath) == file_to_find: + full_path = os.path.join(path, filepath) + _log.info("Found %s in %s: %s", file_to_find, path, full_path) files[idx] = full_path # if file was found, stop looking for it (first hit wins) - files_to_find.remove((idx, orig_path)) + files_to_find.remove((idx, file_to_find)) - # stop os.walk insanity as soon as we have all we need (os.walk loop) + # stop as soon as we have all we need (path index loop) if not files_to_find: break - # ignore specified subdirectories - if ignore_subdirs: - dirnames[:] = [d for d in dirnames if d not in ignore_subdirs] - - # stop os.walk insanity as soon as we have all we need (outer loop) + # stop as soon as we have all we need (paths loop) if not files_to_find: break From c989ce9574c137e25b55941ad9f21ed9c7830d3a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 11:21:49 +0100 Subject: [PATCH 045/115] enhance copy_files to avoid need for copy_ecs_to_target --- easybuild/framework/easyconfig/tools.py | 19 ------ easybuild/main.py | 15 ++--- easybuild/tools/filetools.py | 47 ++++++++++---- test/framework/filetools.py | 85 ++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 40 deletions(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index b8be79be36..5ca7fd6e6b 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -760,22 +760,3 @@ def det_copy_ec_specs(orig_paths, from_pr): raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches) return paths, target_path - - -def copy_ecs_to_target(determined_paths, target_path, prefix=False, target_is_dir=False): - """ - Copy list of easyconfigs to specified path - - :param determined_paths: paths to ecs to copy - :param target_path: target to copy files to - :param prefix: include message prefix characters - :param target_is_dir: target is always a directory - """ - if len(determined_paths) == 1 and not target_is_dir: - copy_file(determined_paths[0], target_path) - print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=prefix) - elif determined_paths: - copy_files(determined_paths, target_path) - print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=prefix) - else: - raise EasyBuildError("One of more files to copy should be specified!") diff --git a/easybuild/main.py b/easybuild/main.py index e2c85b0dec..b9f0031d4f 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -50,16 +50,15 @@ from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import categorize_files_by_type, copy_ecs_to_target, dep_graph -from easybuild.framework.easyconfig.tools import det_copy_ec_specs, det_easyconfig_paths, dump_env_script -from easybuild.framework.easyconfig.tools import get_paths_for, parse_easyconfigs, review_pr, run_contrib_checks -from easybuild.framework.easyconfig.tools import skip_available +from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph, det_copy_ec_specs +from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for +from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, dump_index, load_index, locate_files -from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file +from easybuild.tools.filetools import adjust_permissions, cleanup, copy_files, dump_index, load_index +from easybuild.tools.filetools import locate_files, read_file, register_lock_cleanup_signal_handlers, write_file from easybuild.tools.github import check_github, close_pr, find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch @@ -330,7 +329,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # so try to determine full path for those too via robot search path paths = locate_files(orig_paths, robot_path) - copy_ecs_to_target(paths, target_path) + copy_files(paths, target_path, target_single_file=True, allow_empty=False, verbose=True) elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) @@ -428,7 +427,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) - copy_ecs_to_target(tweaked_ecs_in_all_ecs, target_path, target_is_dir=True) + copy_files(tweaked_ecs_in_all_ecs, target_path, allow_empty=False, verbose=True) # creating/updating PRs if pr_options: diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 372ea88893..709ba0d0ec 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2085,26 +2085,49 @@ def copy_file(path, target_path, force_in_dry_run=False): raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) -def copy_files(paths, target_dir, force_in_dry_run=False): +def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=False, allow_empty=True, verbose=False): """ - Copy list of files to specified target directory (which is created if it doesn't exist yet). + Copy list of files to specified target path. + Target directory is created if it doesn't exist yet. - :param filepaths: list of files to copy - :param target_dir: target directory to copy files into + :param paths: list of filepaths to copy + :param target_path: path to copy files to :param force_in_dry_run: force copying of files during dry run + :param target_single_file: if there's only a single file to copy, copy to a file at target path (not a directory) + :param allow_empty: allow empty list of paths to copy as input (if False: raise error on empty input list) + :param verbose: print a message to report copying of files """ + # dry run: just report copying, don't actually copy if not force_in_dry_run and build_option('extended_dry_run'): - dry_run_msg("copied files %s to %s" % (paths, target_dir)) - else: - if os.path.exists(target_dir): - if os.path.isdir(target_dir): - _log.info("Copying easyconfigs into existing directory %s...", target_dir) + if len(paths) == 1: + dry_run_msg("copied %s to %s" % (paths[0], target_path)) + else: + dry_run_msg("copied %d files to %s" % (len(paths), target_path)) + + # special case: single file to copy and target_single_file is True => copy to file + elif len(paths) == 1 and target_single_file: + copy_file(paths[0], target_path) + if verbose: + print_msg("%s copied to %s" % (os.path.basename(paths[0]), target_path), prefix=False) + + elif paths: + # check target path: if it exists it should be a directory; if it doesn't exist, we create it + if os.path.exists(target_path): + if os.path.isdir(target_path): + _log.info("Copying easyconfigs into existing directory %s...", target_path) else: - raise EasyBuildError("%s exists but is not a directory", target_dir) + raise EasyBuildError("%s exists but is not a directory", target_path) else: - mkdir(target_dir, parents=True) + mkdir(target_path, parents=True) + for path in paths: - copy_file(path, target_dir) + copy_file(path, target_path) + + if verbose: + print_msg("%d file(s) copied to %s" % (len(paths), target_path), prefix=False) + + elif not allow_empty: + raise EasyBuildError("One of more files to copy should be specified!") def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8081aef00c..f4295d44f6 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1485,7 +1485,7 @@ def test_copy_file(self): else: # printing this message will make test suite fail in Travis/GitHub CI, # since we check for unexpected output produced by the tests - print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)", test_file_to_overwrite) + print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)" % test_file_to_overwrite) # also test behaviour of copy_file under --dry-run build_options = { @@ -1551,6 +1551,89 @@ def test_copy_files(self): error_pattern = "/toy-0.0.eb exists but is not a directory" self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [bzip2_ec], copied_toy_ec) + # by default copy_files allows empty input list, but if allow_empty=False then an error is raised + ft.copy_files([], self.test_prefix) + error_pattern = 'One of more files to copy should be specified!' + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [], self.test_prefix, allow_empty=False) + + # test special case: copying a single file to a file target via target_single_file=True + target = os.path.join(self.test_prefix, 'target') + self.assertFalse(os.path.exists(target)) + ft.copy_files([toy_ec], target, target_single_file=True) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isfile(target)) + self.assertEqual(toy_ec_txt, ft.read_file(target)) + + ft.remove_file(target) + + # default behaviour is to copy single file list to target *directory* + self.assertFalse(os.path.exists(target)) + ft.copy_files([toy_ec], target) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isdir(target)) + copied_toy_ec = os.path.join(target, 'toy-0.0.eb') + self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertEqual(toy_ec_txt, ft.read_file(copied_toy_ec)) + + ft.remove_dir(target) + + # test enabling verbose mode + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files([toy_ec], target, verbose=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + regex = re.compile(r"^1 file\(s\) copied to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + ft.remove_dir(target) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files([toy_ec], target, target_single_file=True, verbose=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + regex = re.compile(r"^toy-0\.0\.eb copied to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + ft.remove_file(target) + + # check behaviour under -x: only printing, no actual copying + init_config(build_options={'extended_dry_run': True}) + self.assertFalse(os.path.exists(target)) + self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files(['test.eb'], target) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + self.assertEqual(stderr, '') + + regex = re.compile("^copied test.eb to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files(['bar.eb', 'foo.eb'], target) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(os.path.exists(os.path.join(target, 'bar.eb'))) + self.assertFalse(os.path.exists(os.path.join(target, 'foo.eb'))) + self.assertEqual(stderr, '') + + regex = re.compile("^copied 2 files to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_copy_dir(self): """Test copy_dir function.""" testdir = os.path.dirname(os.path.abspath(__file__)) From 19402a667c811e451c44506616a0f07757667d43 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 11:28:29 +0100 Subject: [PATCH 046/115] clean up imports in easybuild/framework/easyconfig/tools.py --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 5ca7fd6e6b..8273d28265 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -53,7 +53,7 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import copy_file, copy_files, find_easyconfigs, is_patch_file, locate_files +from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files from easybuild.tools.filetools import read_file, resolve_path, which, write_file from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_files_from_pr, download_repo from easybuild.tools.multidiff import multidiff From e32d815a1f6d3f50937fa4aa84c5dd72ba881382 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 11:32:12 +0100 Subject: [PATCH 047/115] exit after copying when combining --copy-ec and --try-* --- easybuild/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index b9f0031d4f..616aca23cc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -429,6 +429,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): clean_up_easyconfigs(tweaked_ecs_in_all_ecs) copy_files(tweaked_ecs_in_all_ecs, target_path, allow_empty=False, verbose=True) + clean_exit(logfile, eb_tmpdir, testing) + # creating/updating PRs if pr_options: if options.new_pr: From 9bca899672aa28606f4e37ef4772521d92d95890 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 11:51:45 +0100 Subject: [PATCH 048/115] enhance copy_files: single file target, error on empty input list, verbose mode --- easybuild/tools/filetools.py | 47 +++++++++++++++----- test/framework/filetools.py | 85 +++++++++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 13 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1c3dbf5f5b..31976bf776 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2034,26 +2034,49 @@ def copy_file(path, target_path, force_in_dry_run=False): raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err) -def copy_files(paths, target_dir, force_in_dry_run=False): +def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=False, allow_empty=True, verbose=False): """ - Copy list of files to specified target directory (which is created if it doesn't exist yet). + Copy list of files to specified target path. + Target directory is created if it doesn't exist yet. - :param filepaths: list of files to copy - :param target_dir: target directory to copy files into + :param paths: list of filepaths to copy + :param target_path: path to copy files to :param force_in_dry_run: force copying of files during dry run + :param target_single_file: if there's only a single file to copy, copy to a file at target path (not a directory) + :param allow_empty: allow empty list of paths to copy as input (if False: raise error on empty input list) + :param verbose: print a message to report copying of files """ + # dry run: just report copying, don't actually copy if not force_in_dry_run and build_option('extended_dry_run'): - dry_run_msg("copied files %s to %s" % (paths, target_dir)) - else: - if os.path.exists(target_dir): - if os.path.isdir(target_dir): - _log.info("Copying easyconfigs into existing directory %s...", target_dir) + if len(paths) == 1: + dry_run_msg("copied %s to %s" % (paths[0], target_path)) + else: + dry_run_msg("copied %d files to %s" % (len(paths), target_path)) + + # special case: single file to copy and target_single_file is True => copy to file + elif len(paths) == 1 and target_single_file: + copy_file(paths[0], target_path) + if verbose: + print_msg("%s copied to %s" % (os.path.basename(paths[0]), target_path), prefix=False) + + elif paths: + # check target path: if it exists it should be a directory; if it doesn't exist, we create it + if os.path.exists(target_path): + if os.path.isdir(target_path): + _log.info("Copying easyconfigs into existing directory %s...", target_path) else: - raise EasyBuildError("%s exists but is not a directory", target_dir) + raise EasyBuildError("%s exists but is not a directory", target_path) else: - mkdir(target_dir, parents=True) + mkdir(target_path, parents=True) + for path in paths: - copy_file(path, target_dir) + copy_file(path, target_path) + + if verbose: + print_msg("%d file(s) copied to %s" % (len(paths), target_path), prefix=False) + + elif not allow_empty: + raise EasyBuildError("One of more files to copy should be specified!") def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c5a572543..a669b7a307 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1485,7 +1485,7 @@ def test_copy_file(self): else: # printing this message will make test suite fail in Travis/GitHub CI, # since we check for unexpected output produced by the tests - print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)", test_file_to_overwrite) + print("Skipping overwrite-file-owned-by-other-user copy_file test (%s is missing)" % test_file_to_overwrite) # also test behaviour of copy_file under --dry-run build_options = { @@ -1551,6 +1551,89 @@ def test_copy_files(self): error_pattern = "/toy-0.0.eb exists but is not a directory" self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [bzip2_ec], copied_toy_ec) + # by default copy_files allows empty input list, but if allow_empty=False then an error is raised + ft.copy_files([], self.test_prefix) + error_pattern = 'One of more files to copy should be specified!' + self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [], self.test_prefix, allow_empty=False) + + # test special case: copying a single file to a file target via target_single_file=True + target = os.path.join(self.test_prefix, 'target') + self.assertFalse(os.path.exists(target)) + ft.copy_files([toy_ec], target, target_single_file=True) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isfile(target)) + self.assertEqual(toy_ec_txt, ft.read_file(target)) + + ft.remove_file(target) + + # default behaviour is to copy single file list to target *directory* + self.assertFalse(os.path.exists(target)) + ft.copy_files([toy_ec], target) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isdir(target)) + copied_toy_ec = os.path.join(target, 'toy-0.0.eb') + self.assertTrue(os.path.exists(copied_toy_ec)) + self.assertEqual(toy_ec_txt, ft.read_file(copied_toy_ec)) + + ft.remove_dir(target) + + # test enabling verbose mode + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files([toy_ec], target, verbose=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + regex = re.compile(r"^1 file\(s\) copied to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + ft.remove_dir(target) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files([toy_ec], target, target_single_file=True, verbose=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + regex = re.compile(r"^toy-0\.0\.eb copied to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + ft.remove_file(target) + + # check behaviour under -x: only printing, no actual copying + init_config(build_options={'extended_dry_run': True}) + self.assertFalse(os.path.exists(target)) + self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files(['test.eb'], target) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) + self.assertEqual(stderr, '') + + regex = re.compile("^copied test.eb to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.mock_stderr(True) + self.mock_stdout(True) + ft.copy_files(['bar.eb', 'foo.eb'], target) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(os.path.exists(os.path.join(target, 'bar.eb'))) + self.assertFalse(os.path.exists(os.path.join(target, 'foo.eb'))) + self.assertEqual(stderr, '') + + regex = re.compile("^copied 2 files to .*/target") + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_copy_dir(self): """Test copy_dir function.""" testdir = os.path.dirname(os.path.abspath(__file__)) From f4cf87275c66e3660360a2578c013993d0c56e9d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 11:53:44 +0100 Subject: [PATCH 049/115] add locate_files function to filetools module --- easybuild/tools/filetools.py | 51 +++++++++++++++++++++++++ test/framework/filetools.py | 73 ++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1c3dbf5f5b..372ea88893 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -792,6 +792,57 @@ def find_easyconfigs(path, ignore_dirs=None): return files +def locate_files(files, paths, ignore_subdirs=None): + """ + Determine full path for list of files, in given list of paths (directories). + """ + # determine which easyconfigs files need to be found, if any + files_to_find = [] + for idx, ec_file in enumerate(files): + if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): + files_to_find.append((idx, ec_file)) + _log.debug("List of files to find: %s" % files_to_find) + + # find missing easyconfigs by walking paths in robot search path + for path in paths: + _log.debug("Looking for missing files (%d left) in %s..." % (len(files_to_find), path)) + + # try to load index for current path, or create one + path_index = load_index(path, ignore_dirs=ignore_subdirs) + if path_index is None or build_option('ignore_index'): + if os.path.exists(path): + _log.info("No index found for %s, creating one...", path) + path_index = create_index(path, ignore_dirs=ignore_subdirs) + else: + path_index = [] + else: + _log.info("Index found for %s, so using it...", path) + + for filepath in path_index: + for idx, file_to_find in files_to_find[:]: + if os.path.basename(filepath) == file_to_find: + full_path = os.path.join(path, filepath) + _log.info("Found %s in %s: %s", file_to_find, path, full_path) + files[idx] = full_path + # if file was found, stop looking for it (first hit wins) + files_to_find.remove((idx, file_to_find)) + + # stop as soon as we have all we need (path index loop) + if not files_to_find: + break + + # stop as soon as we have all we need (paths loop) + if not files_to_find: + break + + if files_to_find: + filenames = ', '.join([f for (_, f) in files_to_find]) + paths = ', '.join(paths) + raise EasyBuildError("One or more files not found: %s (search paths: %s)", filenames, paths) + + return [os.path.abspath(f) for f in files] + + def find_glob_pattern(glob_pattern, fail_on_no_match=True): """Find unique file/dir matching glob_pattern (raises error if more than one match is found)""" if build_option('extended_dry_run'): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c5a572543..8081aef00c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2680,6 +2680,79 @@ def test_locks(self): self.assertFalse(os.path.exists(lock_path)) self.assertEqual(os.listdir(locks_dir), []) + def test_locate_files(self): + """Test locate_files function.""" + + # create some files to find + one = os.path.join(self.test_prefix, '1.txt') + ft.write_file(one, 'one') + two = os.path.join(self.test_prefix, 'subdirA', '2.txt') + ft.write_file(two, 'two') + three = os.path.join(self.test_prefix, 'subdirB', '3.txt') + ft.write_file(three, 'three') + ft.mkdir(os.path.join(self.test_prefix, 'empty_subdir')) + + # empty list of files yields empty result + self.assertEqual(ft.locate_files([], []), []) + self.assertEqual(ft.locate_files([], [self.test_prefix]), []) + + # error is raised if files could not be found + error_pattern = r"One or more files not found: nosuchfile.txt \(search paths: \)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['nosuchfile.txt'], []) + + # files specified via absolute path don't have to be found + res = ft.locate_files([one], []) + self.assertTrue(len(res) == 1) + self.assertTrue(os.path.samefile(res[0], one)) + + # note: don't compare file paths directly but use os.path.samefile instead, + # which is required to avoid failing tests in case temporary directory is a symbolic link (e.g. on macOS) + res = ft.locate_files(['1.txt'], [self.test_prefix]) + self.assertEqual(len(res), 1) + self.assertTrue(os.path.samefile(res[0], one)) + + res = ft.locate_files(['2.txt'], [self.test_prefix]) + self.assertEqual(len(res), 1) + self.assertTrue(os.path.samefile(res[0], two)) + + res = ft.locate_files(['1.txt', '3.txt'], [self.test_prefix]) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], one)) + self.assertTrue(os.path.samefile(res[1], three)) + + # search in multiple paths + files = ['2.txt', '3.txt'] + paths = [os.path.dirname(three), os.path.dirname(two)] + res = ft.locate_files(files, paths) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], two)) + self.assertTrue(os.path.samefile(res[1], three)) + + # same file specified multiple times works fine + files = ['1.txt', '2.txt', '1.txt', '3.txt', '2.txt'] + res = ft.locate_files(files, [self.test_prefix]) + self.assertEqual(len(res), 5) + for idx, expected in enumerate([one, two, one, three, two]): + self.assertTrue(os.path.samefile(res[idx], expected)) + + # only some files found yields correct warning + files = ['2.txt', '3.txt', '1.txt'] + error_pattern = r"One or more files not found: 3\.txt, 1.txt \(search paths: .*/subdirA\)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, files, [os.path.dirname(two)]) + + # check that relative paths are found in current working dir + ft.change_dir(self.test_prefix) + rel_paths = ['subdirA/2.txt', '1.txt'] + # result is still absolute paths to those files + res = ft.locate_files(rel_paths, []) + self.assertEqual(len(res), 2) + self.assertTrue(os.path.samefile(res[0], two)) + self.assertTrue(os.path.samefile(res[1], one)) + + # no recursive search in current working dir (which would potentially be way too expensive) + error_pattern = r"One or more files not found: 2\.txt \(search paths: \)" + self.assertErrorRegex(EasyBuildError, error_pattern, ft.locate_files, ['2.txt'], []) + def suite(): """ returns all the testcases in this module """ From 20faaaf9fb3fc6a9557575d8f5ed70d5cd2d73ca Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 12:04:12 +0100 Subject: [PATCH 050/115] move definition of fetch_easyblocks_from_pr and fetch_easyconfigs_from_pr below fetch_files_from_pr definition --- easybuild/tools/github.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index d1d9df5290..c13775f632 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -377,16 +377,6 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ return extracted_path -def fetch_easyblocks_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" - return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) - - -def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): - """Fetch patched easyconfig files for a particular PR.""" - return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) - - def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): """Fetch patched files for a particular PR.""" @@ -496,6 +486,16 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): return files +def fetch_easyblocks_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYBLOCKS_REPO) + + +def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): + """Fetch patched easyconfig files for a particular PR.""" + return fetch_files_from_pr(pr, path, github_user, github_repo=GITHUB_EASYCONFIGS_REPO) + + def create_gist(txt, fn, descr=None, github_user=None, github_token=None): """Create a gist with the provided text.""" From 4028afe210f24e49b469ae24edb12fcd6289583c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 25 Oct 2020 12:05:44 +0100 Subject: [PATCH 051/115] cache result of fetch_files_from_pr (mainly to speed up tests involving --from-pr) --- easybuild/tools/github.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index c13775f632..1492f8a248 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -33,6 +33,7 @@ import copy import getpass import glob +import functools import os import random import re @@ -377,6 +378,34 @@ def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_ return extracted_path +def pr_files_cache(func): + """ + Decorator to cache result of fetch_files_from_pr. + """ + cache = {} + + @functools.wraps(func) + def cache_aware_func(pr, *args, **kwargs): + """Retrieve cached resul, or fetch files from PR & cache result.""" + # cache key is combination of all function arguments (incl. optional ones) + key = tuple([pr] + [kwargs[key] for key in sorted(kwargs.keys())]) + + if key in cache and all(os.path.exists(x) for x in cache[key]): + _log.debug("Using cached value for fetch_files_from_pr for PR #%s (%s)", pr, kwargs) + return cache[key] + else: + res = func(pr, *args, **kwargs) + cache[key] = res + return res + + # expose clear/update methods of cache to wrapped function + cache_aware_func.clear_cache = cache.clear + cache_aware_func.update_cache = cache.update + + return cache_aware_func + + +@pr_files_cache def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): """Fetch patched files for a particular PR.""" From c6091f7bd536314f263d9a16ca10585efb641546 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 29 Oct 2020 10:25:58 +0100 Subject: [PATCH 052/115] also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace --- easybuild/tools/filetools.py | 5 +++-- test/framework/filetools.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 1c3dbf5f5b..0fab9cae9b 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2275,10 +2275,11 @@ def install_fake_vsc(): ' filename, lineno = cand_filename, cand_lineno', ' break', '', - '# ignore imports from pkgutil.py (part of Python standard library),', + '# ignore imports from pkgutil.py (part of Python standard library)', + '# or from pkg_resources/__init__.py (setuptools),', '# which may happen due to a system-wide installation of vsc-base', '# even if it is not actually actively used...', - 'if os.path.basename(filename) != "pkgutil.py":', + 'if os.path.basename(filename) != "pkgutil.py" and not filename.endswith("pkg_resources/__init__.py"):', ' error_msg = "\\nERROR: Detected import from \'vsc\' namespace in %s (line %s)\\n" % (filename, lineno)', ' error_msg += "vsc-base & vsc-install were ingested into the EasyBuild framework in EasyBuild v4.0\\n"', ' error_msg += "The functionality you need may be available in the \'easybuild.base.*\' namespace.\\n"', diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 8c5a572543..214824f250 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2433,7 +2433,7 @@ def test_fake_vsc(self): regex = re.compile(r"^\nERROR: %s" % error_pattern) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) - # no error if import was detected from pkgutil.py, + # no error if import was detected from pkgutil.py or pkg_resources/__init__.py, # since that may be triggered by a system-wide vsc-base installation # (even though no code is doing 'import vsc'...) ft.move_file(test_python_mod, os.path.join(os.path.dirname(test_python_mod), 'pkgutil.py')) @@ -2441,6 +2441,15 @@ def test_fake_vsc(self): from test_fake_vsc import pkgutil self.assertTrue(pkgutil.__file__.endswith('/test_fake_vsc/pkgutil.py')) + pkg_resources_init = os.path.join(os.path.dirname(test_python_mod), 'pkg_resources', '__init__.py') + ft.write_file(pkg_resources_init, 'import vsc') + + # cleanup to force new import of 'vsc', avoid using cached import from previous attempt + del sys.modules['vsc'] + + from test_fake_vsc import pkg_resources + self.assertTrue(pkg_resources.__file__.endswith('/test_fake_vsc/pkg_resources/__init__.py')) + def test_is_generic_easyblock(self): """Test for is_generic_easyblock function.""" From ec1916d8961d78433746330a8b2a803f3c00c8cd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 30 Oct 2020 11:15:19 +0100 Subject: [PATCH 053/115] bump version to 4.3.2dev --- easybuild/tools/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index 1c15089e5a..bd156b8d13 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.1') +VERSION = LooseVersion('4.3.2.dev0') UNKNOWN = 'UNKNOWN' From baad8f75717e8a854c6172ce84a052782493b387 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 10:17:02 +0100 Subject: [PATCH 054/115] fix cache key in pr_files_cache + expose cache itself via _cache --- easybuild/tools/github.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 1492f8a248..8f44b586c3 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -385,20 +385,22 @@ def pr_files_cache(func): cache = {} @functools.wraps(func) - def cache_aware_func(pr, *args, **kwargs): + def cache_aware_func(pr, path=None, github_user=None, github_account=None, github_repo=None): """Retrieve cached resul, or fetch files from PR & cache result.""" # cache key is combination of all function arguments (incl. optional ones) - key = tuple([pr] + [kwargs[key] for key in sorted(kwargs.keys())]) + key = (pr, github_account, github_repo, path) if key in cache and all(os.path.exists(x) for x in cache[key]): - _log.debug("Using cached value for fetch_files_from_pr for PR #%s (%s)", pr, kwargs) + _log.info("Using cached value for fetch_files_from_pr for PR #%s (account=%s, repo=%s, path=%s)", + pr, github_account, github_repo, path) return cache[key] else: - res = func(pr, *args, **kwargs) + res = func(pr, path=path, github_user=github_user, github_account=github_account, github_repo=github_repo) cache[key] = res return res - # expose clear/update methods of cache to wrapped function + # expose clear/update methods of cache + cache itself to wrapped function + cache_aware_func._cache = cache # useful in tests cache_aware_func.clear_cache = cache.clear cache_aware_func.update_cache = cache.update @@ -406,7 +408,7 @@ def cache_aware_func(pr, *args, **kwargs): @pr_files_cache -def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): +def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, github_repo=None): """Fetch patched files for a particular PR.""" if github_user is None: @@ -429,7 +431,8 @@ def fetch_files_from_pr(pr, path=None, github_user=None, github_repo=None): # make sure path exists, create it if necessary mkdir(path, parents=True) - github_account = build_option('pr_target_account') + if github_account is None: + github_account = build_option('pr_target_account') if github_repo == GITHUB_EASYCONFIGS_REPO: easyfiles = 'easyconfigs' @@ -1312,7 +1315,7 @@ def merge_pr(pr): pr_target_account = build_option('pr_target_account') pr_target_repo = build_option('pr_target_repo') or GITHUB_EASYCONFIGS_REPO - pr_data, pr_url = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) + pr_data, _ = fetch_pr_data(pr, pr_target_account, pr_target_repo, github_user, full=True) msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login']) msg += "you are using GitHub account '%s'\n" % github_user From 9a9b657be89a2244370594529a7887202033a9d9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 10:17:10 +0100 Subject: [PATCH 055/115] add test for pr_files_cache --- test/framework/github.py | 64 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/test/framework/github.py b/test/framework/github.py index 46cb79c514..65e4337c26 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -126,13 +126,13 @@ def test_fetch_pr_data(self): print("Skipping test_fetch_pr_data, no GitHub token available?") return - pr_data, pr_url = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT) + pr_data, _ = gh.fetch_pr_data(1, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT) self.assertEqual(pr_data['number'], 1) self.assertEqual(pr_data['title'], "a pr") self.assertFalse(any(key in pr_data for key in ['issue_comments', 'review', 'status_last_commit'])) - pr_data, pr_url = gh.fetch_pr_data(2, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT, full=True) + pr_data, _ = gh.fetch_pr_data(2, GITHUB_USER, GITHUB_REPO, GITHUB_TEST_ACCOUNT, full=True) self.assertEqual(pr_data['number'], 2) self.assertEqual(pr_data['title'], "an open pr (do not close this please)") self.assertTrue(pr_data['issue_comments']) @@ -327,6 +327,66 @@ def test_fetch_easyconfigs_from_pr(self): except URLError as err: print("Ignoring URLError '%s' in test_fetch_easyconfigs_from_pr" % err) + def test_fetch_files_from_pr_cache(self): + """Test caching for fetch_files_from_pr.""" + + init_config(build_options={ + 'pr_target_account': gh.GITHUB_EB_MAIN, + }) + + # clear cache first, to make sure we start with a clean slate + gh.fetch_files_from_pr.clear_cache() + self.assertFalse(gh.fetch_files_from_pr._cache) + + pr7159_filenames = [ + 'DOLFIN-2018.1.0.post1-foss-2018a-Python-3.6.4.eb', + 'OpenFOAM-5.0-20180108-foss-2018a.eb', + 'OpenFOAM-5.0-20180108-intel-2018a.eb', + 'OpenFOAM-6-foss-2018b.eb', + 'OpenFOAM-6-intel-2018a.eb', + 'OpenFOAM-v1806-foss-2018b.eb', + 'PETSc-3.9.3-foss-2018a.eb', + 'SCOTCH-6.0.6-foss-2018a.eb', + 'SCOTCH-6.0.6-foss-2018b.eb', + 'SCOTCH-6.0.6-intel-2018a.eb', + 'Trilinos-12.12.1-foss-2018a-Python-3.6.4.eb' + ] + pr7159_files = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) + self.assertEqual(sorted(pr7159_filenames), sorted(os.path.basename(f) for f in pr7159_files)) + + # check that cache has been populated for PR 7159 + self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1) + + print(gh.fetch_files_from_pr._cache.keys()) + + # github_account value is None (results in using default 'easybuilders') + cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix) + self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys()) + + cache_entry = gh.fetch_files_from_pr._cache[cache_key] + self.assertEqual(sorted([os.path.basename(f) for f in cache_entry]), sorted(pr7159_filenames)) + + # same query should return result from cache entry + res = gh.fetch_easyconfigs_from_pr(7159, path=self.test_prefix, github_user=GITHUB_TEST_ACCOUNT) + self.assertEqual(res, pr7159_files) + + # inject entry in cache and check result of matching query + pr_id = 12345 + tmpdir = os.path.join(self.test_prefix, 'easyblocks-pr-12345') + pr12345_files = [ + os.path.join(tmpdir, 'foo.py'), + os.path.join(tmpdir, 'bar.py'), + ] + for fp in pr12345_files: + write_file(fp, '') + + # github_account value is None (results in using default 'easybuilders') + cache_key = (pr_id, None, 'easybuild-easyblocks', tmpdir) + gh.fetch_files_from_pr.update_cache({cache_key: pr12345_files}) + + res = gh.fetch_easyblocks_from_pr(12345, tmpdir) + self.assertEqual(sorted(pr12345_files), sorted(res)) + def test_fetch_latest_commit_sha(self): """Test fetch_latest_commit_sha function.""" if self.skip_github_tests: From 00d3f8afe459dd15c29a92a3dbad17a63f1c1b83 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 10:19:39 +0100 Subject: [PATCH 056/115] fix typo in docstring, fix overindented line + drop print in fetch_files_from_pr cache test --- easybuild/tools/github.py | 4 ++-- test/framework/github.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8f44b586c3..5a361baa89 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -386,13 +386,13 @@ def pr_files_cache(func): @functools.wraps(func) def cache_aware_func(pr, path=None, github_user=None, github_account=None, github_repo=None): - """Retrieve cached resul, or fetch files from PR & cache result.""" + """Retrieve cached result, or fetch files from PR & cache result.""" # cache key is combination of all function arguments (incl. optional ones) key = (pr, github_account, github_repo, path) if key in cache and all(os.path.exists(x) for x in cache[key]): _log.info("Using cached value for fetch_files_from_pr for PR #%s (account=%s, repo=%s, path=%s)", - pr, github_account, github_repo, path) + pr, github_account, github_repo, path) return cache[key] else: res = func(pr, path=path, github_user=github_user, github_account=github_account, github_repo=github_repo) diff --git a/test/framework/github.py b/test/framework/github.py index 65e4337c26..2f25e10434 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -357,8 +357,6 @@ def test_fetch_files_from_pr_cache(self): # check that cache has been populated for PR 7159 self.assertEqual(len(gh.fetch_files_from_pr._cache.keys()), 1) - print(gh.fetch_files_from_pr._cache.keys()) - # github_account value is None (results in using default 'easybuilders') cache_key = (7159, None, 'easybuild-easyconfigs', self.test_prefix) self.assertTrue(cache_key in gh.fetch_files_from_pr._cache.keys()) From 8922459ba8a4b9d44c43e7d4ef89a2c49776a739 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 10:45:30 +0100 Subject: [PATCH 057/115] don't pass username in github_api_get_request when no GitHub token is available (fixes #2921) --- easybuild/tools/github.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 4d80344d29..8773f2c947 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -249,6 +249,12 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): if token is None: token = fetch_github_token(github_user) + # if we don't have a GitHub token, don't pass username either; + # this maeks sense for read-only actions like fetching files from PRs + if token is None: + _log.info("Not specifying username since no GitHub token is available for %s", github_user) + github_user = None + url = request_f(RestClient(GITHUB_API_URL, username=github_user, token=token)) try: From df6a8641432a7daf45f11998fe85554c0d105128 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 10:46:28 +0100 Subject: [PATCH 058/115] skip test_fetch_files_from_pr_cache when no GitHub token is available --- test/framework/github.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 2f25e10434..9a13584d82 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -329,6 +329,9 @@ def test_fetch_easyconfigs_from_pr(self): def test_fetch_files_from_pr_cache(self): """Test caching for fetch_files_from_pr.""" + if self.skip_github_tests: + print("Skipping test_fetch_files_from_pr_cache, no GitHub token available?") + return init_config(build_options={ 'pr_target_account': gh.GITHUB_EB_MAIN, From 638e94e675399ea1bbf7def7856a50420a5cebea Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 16:43:42 +0100 Subject: [PATCH 059/115] also inject -rpath options for all entries in $LIBRARY_PATH in RPATH wrappers --- easybuild/scripts/rpath_args.py | 9 ++++- test/framework/toolchain.py | 62 +++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 66b52b8408..b7609c2297 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -81,7 +81,6 @@ add_rpath_args = False cmd_args.append(arg) - # FIXME: also consider $LIBRARY_PATH? # FIXME: support to hard inject additional library paths? # FIXME: support to specify list of path prefixes that should not be RPATH'ed into account? # FIXME skip paths in /tmp, build dir, etc.? @@ -119,6 +118,14 @@ idx += 1 +# also inject -rpath options for all entries in $LIBRARY_PATH, +# unless they are there already +for lib_path in os.getenv('LIBRARY_PATH', '').split(os.pathsep): + if lib_path and (rpath_filter is None or not rpath_filter.match(lib_path)): + rpath_arg = flag_prefix + '-rpath=%s' % lib_path + if rpath_arg not in cmd_args_rpath: + cmd_args_rpath.append(rpath_arg) + if add_rpath_args: # try to make sure that RUNPATH is not used by always injecting --disable-new-dtags cmd_args_rpath.insert(0, flag_prefix + '--disable-new-dtags') diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 5e33ad3608..43ee6e6c66 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -1729,6 +1729,11 @@ def test_compiler_cache(self): def test_rpath_args_script(self): """Test rpath_args.py script""" + + # $LIBRARY_PATH affects result of rpath_args.py, so make sure it's not set + if 'LIBRARY_PATH' in os.environ: + del os.environ['LIBRARY_PATH'] + script = find_eb_script('rpath_args.py') rpath_inc = ','.join([ @@ -2013,6 +2018,63 @@ def test_rpath_args_script(self): cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')] self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + # check whether $LIBRARY_PATH is taken into account + test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc) + pre_cmd_args_gcc = [ + "'-Wl,-rpath=%s/lib'" % self.test_prefix, + "'-Wl,-rpath=%s/lib64'" % self.test_prefix, + "'-Wl,-rpath=$ORIGIN'", + "'-Wl,-rpath=$ORIGIN/../lib'", + "'-Wl,-rpath=$ORIGIN/../lib64'", + "'-Wl,--disable-new-dtags'", + ] + post_cmd_args_gcc = [ + "'-c'", + "'foo.c'", + ] + + test_cmd_ld = "%s ld '' '%s' -L/foo foo.o -L/lib64 -lfoo -lbar -L/usr/lib -L/bar" % (script, rpath_inc) + pre_cmd_args_ld = [ + "'-rpath=%s/lib'" % self.test_prefix, + "'-rpath=%s/lib64'" % self.test_prefix, + "'-rpath=$ORIGIN'", + "'-rpath=$ORIGIN/../lib'", + "'-rpath=$ORIGIN/../lib64'", + "'--disable-new-dtags'", + "'-rpath=/foo'", + "'-rpath=/lib64'", + "'-rpath=/usr/lib'", + "'-rpath=/bar'", + ] + post_cmd_args_ld = [ + "'-L/foo'", + "'foo.o'", + "'-L/lib64'", + "'-lfoo'", + "'-lbar'", + "'-L/usr/lib'", + "'-L/bar'", + ] + + library_paths = [ + ('',), # special case: empty value + ('/path/to/lib',), + ('/path/to/lib', '/another/path/to/lib64'), + ('/path/to/lib', '/another/path/to/lib64', '/yet-another/path/to/libraries'), + ] + for library_path in library_paths: + os.environ['LIBRARY_PATH'] = ':'.join(library_path) + + out, ec = run_cmd(test_cmd_gcc, simple=False) + self.assertEqual(ec, 0) + cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc + self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + + out, ec = run_cmd(test_cmd_ld, simple=False) + self.assertEqual(ec, 0) + cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld + self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" From c35647f5ea352af3af6343b15658194f32975694 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 17:47:07 +0100 Subject: [PATCH 060/115] retain library path for RPATH linking with same condition for paths specified via -L and $LIBRARY_PATH --- easybuild/scripts/rpath_args.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index b7609c2297..150cdc3c3b 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -95,14 +95,15 @@ else: lib_path = arg[2:] - if os.path.isabs(lib_path) and (rpath_filter is None or not rpath_filter.match(lib_path)): + if lib_path and os.path.isabs(lib_path) and (rpath_filter is None or not rpath_filter.match(lib_path)): # inject -rpath flag in front for every -L with an absolute path, # also retain the -L flag (without reordering!) cmd_args_rpath.append(flag_prefix + '-rpath=%s' % lib_path) cmd_args.append('-L%s' % lib_path) else: - # don't RPATH in relative paths; - # it doesn't make much sense, and it can also break the build because it may result in reordering lib paths + # don't RPATH in empty or relative paths, or paths that are filtered out; + # linking relative paths via RPATH doesn't make much sense, + # and it can also break the build because it may result in reordering lib paths cmd_args.append('-L%s' % lib_path) # replace --enable-new-dtags with --disable-new-dtags if it's used; @@ -121,7 +122,7 @@ # also inject -rpath options for all entries in $LIBRARY_PATH, # unless they are there already for lib_path in os.getenv('LIBRARY_PATH', '').split(os.pathsep): - if lib_path and (rpath_filter is None or not rpath_filter.match(lib_path)): + if lib_path and os.path.isabs(lib_path) and (rpath_filter is None or not rpath_filter.match(lib_path)): rpath_arg = flag_prefix + '-rpath=%s' % lib_path if rpath_arg not in cmd_args_rpath: cmd_args_rpath.append(rpath_arg) From c5f48c5236c5e37e1c1e8fe501b235ffe01a2e3c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 2 Nov 2020 20:11:16 +0100 Subject: [PATCH 061/115] exclude test configuration with Lmod 7 and Python 3, except for Python 3.6 --- .github/workflows/unit_tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 803e5aa65f..7c54b83fec 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -43,6 +43,14 @@ jobs: python: 3.8 - modules_tool: modules-4.1.4 python: 3.9 + - modules_tool: Lmod-7.8.22 + python: 3.5 + - modules_tool: Lmod-7.8.22 + python: 3.7 + - modules_tool: Lmod-7.8.22 + python: 3.8 + - modules_tool: Lmod-7.8.22 + python: 3.9 fail-fast: false steps: - uses: actions/checkout@v2 From 1952311a26e42d537f7d7d916bdeb83c3a3767f2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Nov 2020 18:25:06 +0100 Subject: [PATCH 062/115] add support for %(mod_name)s template value --- easybuild/framework/easyconfig/templates.py | 7 ++++++- test/framework/easyconfig.py | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 47f5c3175c..58bb1d78b8 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -47,6 +47,7 @@ # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('mod_name', "Module name"), ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), ('toolchain_version', "Toolchain version"), @@ -72,8 +73,8 @@ ] # values taken from the EasyBlock before each step TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [ - ('installdir', "Installation directory"), ('builddir', "Build directory"), + ('installdir', "Installation directory"), ] # software names for which to define ver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ @@ -208,6 +209,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] + + elif name[0] == 'mod_name': + template_values['mod_name'] = getattr(config, 'short_mod_name', None) + else: raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7a6c9af46b..8150cc87c5 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -994,6 +994,7 @@ def test_templating(self): 'Perl: %%(perlver)s, %%(perlmajver)s, %%(perlminver)s, %%(perlshortver)s', 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), + 'modextrapaths = {"PI_MOD_NAME": "%%(mod_name)s"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -1028,6 +1029,7 @@ def test_templating(self): "Perl: 5.22.0, 5, 22, 5.22; " "R: 3.2.3, 3, 2, 3.2") self.assertEqual(eb['modloadmsg'], expected) + self.assertEqual(eb['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) self.assertEqual(eb['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) From 68a105c2ea9cbe23c81fc1900e15e9d1160a0b52 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 5 Nov 2020 19:11:15 +0100 Subject: [PATCH 063/115] fix test_template_constant_dict after introducing %(mod_name)s template value --- test/framework/easyconfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 8150cc87c5..7af6190bf4 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -2870,6 +2870,7 @@ def test_template_constant_dict(self): expected = { 'bitbucket_account': 'gzip', 'github_account': 'gzip', + 'mod_name': 'gzip/1.5-foss-2018a', 'name': 'gzip', 'namelower': 'gzip', 'nameletter': 'g', @@ -2938,6 +2939,7 @@ def test_template_constant_dict(self): 'javaminver': '8', 'javashortver': '1.8', 'javaver': '1.8.0_221', + 'mod_name': None, 'name': 'toy', 'namelower': 'toy', 'nameletter': 't', @@ -2978,6 +2980,7 @@ def test_template_constant_dict(self): self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) expected = { + 'mod_name': None, 'name': 'foo', 'namelower': 'foo', 'nameletter': 'f', From d86ced28068db8fd79ef715c493e1f19c7593f5e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 6 Nov 2020 14:29:22 +0100 Subject: [PATCH 064/115] rename %(mod_name)s template to %(module_name)s --- easybuild/framework/easyconfig/templates.py | 6 +++--- test/framework/easyconfig.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 58bb1d78b8..9af16131d1 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -47,7 +47,7 @@ # derived from easyconfig, but not from ._config directly TEMPLATE_NAMES_EASYCONFIG = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), - ('mod_name', "Module name"), + ('module_name', "Module name"), ('nameletter', "First letter of software name"), ('toolchain_name', "Toolchain name"), ('toolchain_version', "Toolchain version"), @@ -210,8 +210,8 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) if softname is not None: template_values['nameletter'] = softname[0] - elif name[0] == 'mod_name': - template_values['mod_name'] = getattr(config, 'short_mod_name', None) + elif name[0] == 'module_name': + template_values['module_name'] = getattr(config, 'short_mod_name', None) else: raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 7af6190bf4..c36ea1ee64 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -994,7 +994,7 @@ def test_templating(self): 'Perl: %%(perlver)s, %%(perlmajver)s, %%(perlminver)s, %%(perlshortver)s', 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), - 'modextrapaths = {"PI_MOD_NAME": "%%(mod_name)s"}', + 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -2870,7 +2870,7 @@ def test_template_constant_dict(self): expected = { 'bitbucket_account': 'gzip', 'github_account': 'gzip', - 'mod_name': 'gzip/1.5-foss-2018a', + 'module_name': 'gzip/1.5-foss-2018a', 'name': 'gzip', 'namelower': 'gzip', 'nameletter': 'g', @@ -2939,7 +2939,7 @@ def test_template_constant_dict(self): 'javaminver': '8', 'javashortver': '1.8', 'javaver': '1.8.0_221', - 'mod_name': None, + 'module_name': None, 'name': 'toy', 'namelower': 'toy', 'nameletter': 't', @@ -2980,7 +2980,7 @@ def test_template_constant_dict(self): self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern)) expected = { - 'mod_name': None, + 'module_name': None, 'name': 'foo', 'namelower': 'foo', 'nameletter': 'f', From 7b3801ec411154cb05e6c1b3c802138e12752124 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Nov 2020 19:55:32 +0100 Subject: [PATCH 065/115] significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed --- easybuild/framework/easyconfig/format/format.py | 7 ++++++- easybuild/framework/easyconfig/format/one.py | 15 +++++++++++++-- easybuild/framework/easyconfig/parser.py | 2 -- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index 89bf9ecc3e..ff9acac3d6 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -612,12 +612,17 @@ def __init__(self): raise EasyBuildError('Invalid version number %s (incorrect length)', self.VERSION) self.rawtext = None # text version of the easyconfig - self.comments = {} # comments in easyconfig file + self._comments = {} # comments in easyconfig file self.header = None # easyconfig header (e.g., format version, license, ...) self.docstring = None # easyconfig docstring (e.g., author, maintainer, ...) self.specs = {} + @property + def comments(self): + """Return comments in easyconfig file""" + return self._comments + def set_specifications(self, specs): """Set specifications.""" self.log.debug('Set copy of specs %s' % specs) diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index e28b94bd50..6d789a4c6a 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -135,7 +135,8 @@ def parse(self, txt): """ Pre-process txt to extract header, docstring and pyheader, with non-indented section markers enforced. """ - super(FormatOneZero, self).parse(txt, strict_section_markers=True) + self.rawcontent = txt + super(FormatOneZero, self).parse(self.rawcontent, strict_section_markers=True) def _reformat_line(self, param_name, param_val, outer=False, addlen=0): """ @@ -356,6 +357,16 @@ def dump(self, ecfg, default_values, templ_const, templ_val, toolchain_hierarchy return '\n'.join(dump) + @property + def comments(self): + """ + Return comments (and extract them first if needed). + """ + if not self._comments: + self.extract_comments(self.rawcontent) + + return self._comments + def extract_comments(self, rawtxt): """ Extract comments from raw content. @@ -363,7 +374,7 @@ def extract_comments(self, rawtxt): Discriminates between comment header, comments above a line (parameter definition), and inline comments. Inline comments on items of iterable values are also extracted. """ - self.comments = { + self._comments = { 'above': {}, # comments above a parameter definition 'header': [], # header comment lines 'inline': {}, # inline comments diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index e1294cf545..e291a2d356 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -110,8 +110,6 @@ def __init__(self, filename=None, format_version=None, rawcontent=None, else: raise EasyBuildError("Neither filename nor rawcontent provided to EasyConfigParser") - self._formatter.extract_comments(self.rawcontent) - def process(self, filename=None): """Create an instance""" self._read(filename=filename) From 69ab582d21acc70dc759ed0cd511abdde5d94ca5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 9 Nov 2020 16:41:28 +0100 Subject: [PATCH 066/115] avoid TypeError being raised by list_toolchains --- easybuild/tools/docs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 9624d43983..37563c3122 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -727,13 +727,17 @@ def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') all_tcs_names = [x.NAME for x in all_tcs] - tclist = sorted(zip(all_tcs_names, all_tcs)) - tcs = dict() - for (tcname, tcc) in tclist: + # start with dict that maps toolchain name to corresponding subclass of Toolchain + tcs = dict(zip(all_tcs_names, all_tcs)) + + for tcname in sorted(tcs.keys()): + + tcc = tcs[tcname] # filter deprecated 'dummy' toolchain if tcname == DUMMY_TOOLCHAIN_NAME: + del tcs[tcname] continue tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there From ab4aa6fd2047300fcf9d0cc61a4ce6e84eb961af Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 9 Nov 2020 21:38:25 +0100 Subject: [PATCH 067/115] clean up implementation in list_toolchain functions --- easybuild/tools/docs.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 37563c3122..1fd8d119e9 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -726,20 +726,16 @@ def list_software_txt(software, detailed=False): def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') + + # filter deprecated 'dummy' toolchain + all_tcs = [x for x in all_tcs if x.NAME != DUMMY_TOOLCHAIN_NAME] all_tcs_names = [x.NAME for x in all_tcs] # start with dict that maps toolchain name to corresponding subclass of Toolchain tcs = dict(zip(all_tcs_names, all_tcs)) - for tcname in sorted(tcs.keys()): - + for tcname in sorted(tcs): tcc = tcs[tcname] - - # filter deprecated 'dummy' toolchain - if tcname == DUMMY_TOOLCHAIN_NAME: - del tcs[tcname] - continue - tc = tcc(version='1.2.3') # version doesn't matter here, but something needs to be there tcs[tcname] = tc.definition() From ad8a2b586a43ff9c11891b4deeb21e2c49c77a21 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 18 Apr 2017 16:10:02 +0000 Subject: [PATCH 068/115] Added toolchain options to specify extra compiler arguments --- easybuild/tools/toolchain/compiler.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 27f255f6ec..bb3ae32616 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,6 +93,8 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), + 'extra_cflags':(None,"Specify extra CFLAGS options. Do not specify a leading dash as one is prepended already."), + 'extra_fflags':(None,"Specify extra FFLAGS options. Do not specify a leading dash as one is prepended already."), } COMPILER_UNIQUE_OPTION_MAP = None @@ -110,6 +112,8 @@ class Compiler(Toolchain): 'static': 'static', 'unroll': 'unroll', 'verbose': 'v', + 'extra_cflags':'%(value)s', + 'extra_fflags':'%(value)s', } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = None @@ -121,13 +125,13 @@ class Compiler(Toolchain): COMPILER_CC = None COMPILER_CXX = None - COMPILER_C_FLAGS = ['cstd'] + COMPILER_C_FLAGS = ['cstd','extra_cflags'] COMPILER_C_UNIQUE_FLAGS = [] COMPILER_F77 = None COMPILER_F90 = None COMPILER_FC = None - COMPILER_F_FLAGS = ['i8', 'r8'] + COMPILER_F_FLAGS = ['i8', 'r8','extra_fflags'] COMPILER_F_UNIQUE_FLAGS = [] LINKER_TOGGLE_STATIC_DYNAMIC = None From af6d09209b38d546e4c04622af626951dfe16a17 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 19 Apr 2017 13:07:33 +0000 Subject: [PATCH 069/115] cosmetic changes --- easybuild/tools/toolchain/compiler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index bb3ae32616..70a0b88f00 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,8 +93,8 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), - 'extra_cflags':(None,"Specify extra CFLAGS options. Do not specify a leading dash as one is prepended already."), - 'extra_fflags':(None,"Specify extra FFLAGS options. Do not specify a leading dash as one is prepended already."), + 'extra_cflags': (None,"Specify extra CFLAGS options. Do not specify a leading dash, one is prepended already."), + 'extra_fflags': (None,"Specify extra FFLAGS options. Do not specify a leading dash, one is prepended already."), } COMPILER_UNIQUE_OPTION_MAP = None @@ -112,8 +112,8 @@ class Compiler(Toolchain): 'static': 'static', 'unroll': 'unroll', 'verbose': 'v', - 'extra_cflags':'%(value)s', - 'extra_fflags':'%(value)s', + 'extra_cflags': '%(value)s', + 'extra_fflags': '%(value)s', } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = None @@ -125,13 +125,13 @@ class Compiler(Toolchain): COMPILER_CC = None COMPILER_CXX = None - COMPILER_C_FLAGS = ['cstd','extra_cflags'] + COMPILER_C_FLAGS = ['cstd', 'extra_cflags'] COMPILER_C_UNIQUE_FLAGS = [] COMPILER_F77 = None COMPILER_F90 = None COMPILER_FC = None - COMPILER_F_FLAGS = ['i8', 'r8','extra_fflags'] + COMPILER_F_FLAGS = ['i8', 'r8', 'extra_fflags'] COMPILER_F_UNIQUE_FLAGS = [] LINKER_TOGGLE_STATIC_DYNAMIC = None From 2ca2236f7ff81c20ae32e16d9d8fc0078d381c86 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 10 Nov 2020 01:06:14 +0000 Subject: [PATCH 070/115] Appease the Hound --- easybuild/tools/toolchain/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 70a0b88f00..250d71b164 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,8 +93,8 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), - 'extra_cflags': (None,"Specify extra CFLAGS options. Do not specify a leading dash, one is prepended already."), - 'extra_fflags': (None,"Specify extra FFLAGS options. Do not specify a leading dash, one is prepended already."), + 'extra_cflags': (None, "Specify extra CFLAGS options. Do not specify a leading dash, one is prepended already."), + 'extra_fflags': (None, "Specify extra FFLAGS options. Do not specify a leading dash, one is prepended already."), } COMPILER_UNIQUE_OPTION_MAP = None From 4756e49ea4665b69e4261afdf9b666842ed87fb9 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 10 Nov 2020 01:46:37 +0000 Subject: [PATCH 071/115] Introduce extra_cxxflags/fcflags/f90flags. Also lets them start with a '-' and streamlines their handling. --- easybuild/tools/toolchain/compiler.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 250d71b164..4de4ca0f32 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -93,8 +93,11 @@ class Compiler(Toolchain): 'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"), 'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly 'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"), - 'extra_cflags': (None, "Specify extra CFLAGS options. Do not specify a leading dash, one is prepended already."), - 'extra_fflags': (None, "Specify extra FFLAGS options. Do not specify a leading dash, one is prepended already."), + 'extra_cflags': (None, "Specify extra CFLAGS options."), + 'extra_cxxflags': (None, "Specify extra CXXFLAGS options."), + 'extra_fflags': (None, "Specify extra FFLAGS options."), + 'extra_fcflags': (None, "Specify extra FCFLAGS options."), + 'extra_f90flags': (None, "Specify extra F90FLAGS options."), } COMPILER_UNIQUE_OPTION_MAP = None @@ -281,15 +284,19 @@ def _set_compiler_flags(self): self.variables.nextend('PRECFLAGS', precflags[:1]) # precflags last - for var in ['CFLAGS', 'CXXFLAGS']: + for var in ['CFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'F90FLAGS']: self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS') self.variables.nextend(var, flags) - self.variables.nextend(var, cflags) - - for var in ['FCFLAGS', 'FFLAGS', 'F90FLAGS']: - self.variables.join(var, 'OPTFLAGS', 'PRECFLAGS') - self.variables.nextend(var, flags) - self.variables.nextend(var, fflags) + if var.startswith('C'): + self.variables.nextend(var, cflags) + else: + self.variables.nextend(var, fflags) + extra = 'extra_' + var.lower() + if self.options.get(extra): + flags = self.options.option(extra) + if not flags or flags[0] != '-': + raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, flags)) + self.variables.nextend(var, self.options.option(extra[1:])) def _set_optimal_architecture(self, default_optarch=None): """ From cb4d07dee1f3349cb3160c37d700754f47bf0a4f Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Tue, 10 Nov 2020 02:05:38 +0000 Subject: [PATCH 072/115] Fix logic and add test. --- easybuild/tools/toolchain/compiler.py | 9 ++++++--- test/framework/toolchain.py | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 4de4ca0f32..885872e81c 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -116,7 +116,10 @@ class Compiler(Toolchain): 'unroll': 'unroll', 'verbose': 'v', 'extra_cflags': '%(value)s', + 'extra_cxxflags': '%(value)s', 'extra_fflags': '%(value)s', + 'extra_fcflags': '%(value)s', + 'extra_f90flags': '%(value)s', } COMPILER_OPTIMAL_ARCHITECTURE_OPTION = None @@ -128,13 +131,13 @@ class Compiler(Toolchain): COMPILER_CC = None COMPILER_CXX = None - COMPILER_C_FLAGS = ['cstd', 'extra_cflags'] + COMPILER_C_FLAGS = ['cstd'] COMPILER_C_UNIQUE_FLAGS = [] COMPILER_F77 = None COMPILER_F90 = None COMPILER_FC = None - COMPILER_F_FLAGS = ['i8', 'r8', 'extra_fflags'] + COMPILER_F_FLAGS = ['i8', 'r8'] COMPILER_F_UNIQUE_FLAGS = [] LINKER_TOGGLE_STATIC_DYNAMIC = None @@ -296,7 +299,7 @@ def _set_compiler_flags(self): flags = self.options.option(extra) if not flags or flags[0] != '-': raise EasyBuildError("toolchainopts %s: '%s' must start with a '-'." % (extra, flags)) - self.variables.nextend(var, self.options.option(extra[1:])) + self.variables.nappend_el(var, flags[1:]) def _set_optimal_architecture(self, default_optarch=None): """ diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 43ee6e6c66..e9ce1d3b14 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -621,6 +621,15 @@ def test_misc_flags_shared(self): self.assertTrue(flag not in flags, "%s: False means no %s in %s" % (opt, flag, flags)) self.modtool.purge() + value = '--see-if-this-propagates' + for var in flag_vars: + opt = 'extra_' + var.lower() + tc = self.get_toolchain('foss', version='2018a') + tc.set_options({opt: value}) + tc.prepare() + self.assertTrue(tc.get_variable(var).endswith(' ' + value)) + self.modtool.purge() + def test_misc_flags_unique(self): """Test whether unique compiler flags are set correctly.""" From 831fe94612a8a1719ea72331360031eeb4a9cb01 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Nov 2020 12:26:24 +0100 Subject: [PATCH 073/115] fix typo (maeks -> makes) --- easybuild/tools/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 8773f2c947..e70109e19d 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -250,7 +250,7 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): token = fetch_github_token(github_user) # if we don't have a GitHub token, don't pass username either; - # this maeks sense for read-only actions like fetching files from PRs + # this makes sense for read-only actions like fetching files from PRs if token is None: _log.info("Not specifying username since no GitHub token is available for %s", github_user) github_user = None From 2e8949ce6a404c8158030364a3ba988496a78747 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 16 Nov 2020 12:31:06 +0800 Subject: [PATCH 074/115] check if PR is already merged in --merge-pr --- easybuild/tools/github.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index abed829172..e03609edab 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1325,7 +1325,9 @@ def merge_pr(pr): msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login']) msg += "you are using GitHub account '%s'\n" % github_user print_msg(msg, prefix=False) - if pr_data['user']['login'] == github_user: + if pr_data['merged']: + raise EasyBuildError("This PR is already merged.") + elif pr_data['user']['login'] == github_user: raise EasyBuildError("Please do not merge your own PRs!") force = build_option('force') From 90a613979d34e8fe2b4bb3e97942f21170ecc48d Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Mon, 16 Nov 2020 12:50:04 +0800 Subject: [PATCH 075/115] only raise already merged PR error in --merge-pr when not using dry-run --- easybuild/tools/github.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index e03609edab..4375edfbf3 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1325,14 +1325,15 @@ def merge_pr(pr): msg = "\n%s/%s PR #%s was submitted by %s, " % (pr_target_account, pr_target_repo, pr, pr_data['user']['login']) msg += "you are using GitHub account '%s'\n" % github_user print_msg(msg, prefix=False) - if pr_data['merged']: - raise EasyBuildError("This PR is already merged.") - elif pr_data['user']['login'] == github_user: + if pr_data['user']['login'] == github_user: raise EasyBuildError("Please do not merge your own PRs!") force = build_option('force') dry_run = build_option('dry_run') or build_option('extended_dry_run') + if pr_data['merged'] and not dry_run: + raise EasyBuildError("This PR is already merged.") + def merge_url(gh): """Utility function to fetch merge URL for a specific PR.""" return gh.repos[pr_target_account][pr_target_repo].pulls[pr].merge From 8107a75b7fee6992099358bd778e1f2aa42846af Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Nov 2020 09:20:18 +0100 Subject: [PATCH 076/115] fix remarks --- easybuild/tools/filetools.py | 4 ++-- test/framework/filetools.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 31976bf776..1dbed72354 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -2057,7 +2057,7 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa elif len(paths) == 1 and target_single_file: copy_file(paths[0], target_path) if verbose: - print_msg("%s copied to %s" % (os.path.basename(paths[0]), target_path), prefix=False) + print_msg("%s copied to %s" % (paths[0], target_path), prefix=False) elif paths: # check target path: if it exists it should be a directory; if it doesn't exist, we create it @@ -2076,7 +2076,7 @@ def copy_files(paths, target_path, force_in_dry_run=False, target_single_file=Fa print_msg("%d file(s) copied to %s" % (len(paths), target_path), prefix=False) elif not allow_empty: - raise EasyBuildError("One of more files to copy should be specified!") + raise EasyBuildError("One or more files to copy should be specified!") def copy_dir(path, target_path, force_in_dry_run=False, dirs_exist_ok=False, **kwargs): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index a669b7a307..df19ad361b 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1553,7 +1553,7 @@ def test_copy_files(self): # by default copy_files allows empty input list, but if allow_empty=False then an error is raised ft.copy_files([], self.test_prefix) - error_pattern = 'One of more files to copy should be specified!' + error_pattern = 'One or more files to copy should be specified!' self.assertErrorRegex(EasyBuildError, error_pattern, ft.copy_files, [], self.test_prefix, allow_empty=False) # test special case: copying a single file to a file target via target_single_file=True @@ -1566,6 +1566,17 @@ def test_copy_files(self): ft.remove_file(target) + # also test target_single_file=True with path including a missing subdirectory + target = os.path.join(self.test_prefix, 'target_parent', 'target_subdir', 'target.txt') + self.assertFalse(os.path.exists(target)) + self.assertFalse(os.path.exists(os.path.dirname(target))) + ft.copy_files([toy_ec], target, target_single_file=True) + self.assertTrue(os.path.exists(target)) + self.assertTrue(os.path.isfile(target)) + self.assertEqual(toy_ec_txt, ft.read_file(target)) + + ft.remove_file(target) + # default behaviour is to copy single file list to target *directory* self.assertFalse(os.path.exists(target)) ft.copy_files([toy_ec], target) @@ -1597,14 +1608,13 @@ def test_copy_files(self): self.mock_stderr(False) self.mock_stdout(False) self.assertEqual(stderr, '') - regex = re.compile(r"^toy-0\.0\.eb copied to .*/target") + regex = re.compile(r"/.*/toy-0\.0\.eb copied to .*/target") self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) ft.remove_file(target) # check behaviour under -x: only printing, no actual copying init_config(build_options={'extended_dry_run': True}) - self.assertFalse(os.path.exists(target)) self.assertFalse(os.path.exists(os.path.join(target, 'test.eb'))) self.mock_stderr(True) From bb18de4cf84ba109120b6300ded900b133e8229e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Nov 2020 10:25:19 +0100 Subject: [PATCH 077/115] tackle remarks --- easybuild/tools/filetools.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 372ea88893..07853362f8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -796,25 +796,28 @@ def locate_files(files, paths, ignore_subdirs=None): """ Determine full path for list of files, in given list of paths (directories). """ - # determine which easyconfigs files need to be found, if any + # determine which files need to be found, if any files_to_find = [] - for idx, ec_file in enumerate(files): - if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): - files_to_find.append((idx, ec_file)) - _log.debug("List of files to find: %s" % files_to_find) + for idx, filepath in enumerate(files): + if filepath == os.path.basename(filepath) and not os.path.exists(filepath): + files_to_find.append((idx, filepath)) + _log.debug("List of files to find: %s", files_to_find) # find missing easyconfigs by walking paths in robot search path for path in paths: - _log.debug("Looking for missing files (%d left) in %s..." % (len(files_to_find), path)) + + # skip non-existing paths + if not os.path.exists(path): + _log.debug("%s does not exist, skipping it...", path) + continue + + _log.debug("Looking for missing files (%d left) in %s...", len(files_to_find), path) # try to load index for current path, or create one path_index = load_index(path, ignore_dirs=ignore_subdirs) if path_index is None or build_option('ignore_index'): - if os.path.exists(path): - _log.info("No index found for %s, creating one...", path) - path_index = create_index(path, ignore_dirs=ignore_subdirs) - else: - path_index = [] + _log.info("No index found for %s, creating one (in memory)...", path) + path_index = create_index(path, ignore_dirs=ignore_subdirs) else: _log.info("Index found for %s, so using it...", path) From 731b01aafbceed50ed949c337280d20e11bf3f4a Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 19 Nov 2020 15:46:00 +0800 Subject: [PATCH 078/115] graciously handle wrong PR # in fetch_pr_data --- easybuild/tools/github.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index abed829172..05ec3ae192 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -2167,7 +2167,13 @@ def pr_url(gh): else: return gh.repos[pr_target_account][pr_target_repo].pulls[pr] - status, pr_data = github_api_get_request(pr_url, github_user, **parameters) + try: + status, pr_data = github_api_get_request(pr_url, github_user, **parameters) + except HTTPError as err: + raise EasyBuildError("Failed to get data for PR #%d from %s/%s (%s)\n" + "Please check PR #, account and repo.", + pr, pr_target_account, pr_target_repo, err) + if status != HTTP_STATUS_OK: raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", pr, pr_target_account, pr_target_repo, status, pr_data) From efb67a4ed78c67a5ae345b7e822cda1596a66d29 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Wed, 25 Nov 2020 10:33:50 +0000 Subject: [PATCH 079/115] gobff --- easybuild/toolchains/gobff.py | 41 ++++++++++++++++++++++++++++ easybuild/toolchains/linalg/blis.py | 14 ++++++++++ easybuild/toolchains/linalg/flame.py | 5 ++-- 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 easybuild/toolchains/gobff.py diff --git a/easybuild/toolchains/gobff.py b/easybuild/toolchains/gobff.py new file mode 100644 index 0000000000..be396c60ed --- /dev/null +++ b/easybuild/toolchains/gobff.py @@ -0,0 +1,41 @@ +## +# Copyright 2013-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for gobff compiler toolchain (includes GCC, OpenMPI, BLIS, libFLAME, ScaLAPACK and FFTW). + +:author: Sebastian Achilles (Forschungszentrum Juelich GmbH) +""" + +from easybuild.toolchains.gompi import Gompi +from easybuild.toolchains.linalg.blis import Blis +from easybuild.toolchains.linalg.flame import Flame +from easybuild.toolchains.linalg.scalapack import ScaLAPACK +from easybuild.toolchains.fft.fftw import Fftw + + +class Gobff(Gompi, Blis, Flame, ScaLAPACK, Fftw): + """Compiler toolchain with GCC, OpenMPI, BLIS, libFLAME, ScaLAPACK and FFTW.""" + NAME = 'gobff' + SUBTOOLCHAIN = Gompi.NAME diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index 95b37769a4..5e5a5aa288 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -27,7 +27,9 @@ :author: Kenneth Hoste (Ghent University) :author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada) +:author: Sebastian Achilles (Forschungszentrum Juelich GmbH) """ +from distutils.version import LooseVersion from easybuild.tools.toolchain.linalg import LinAlg @@ -42,3 +44,15 @@ class Blis(LinAlg): BLAS_MODULE_NAME = ['BLIS'] BLAS_LIB = ['blis'] BLAS_FAMILY = TC_CONSTANT_BLIS + + def _set_blas_variables(self): + """AMD's fork with version number > 2.1 names the MT library blis-mt, while vanilla BLIS doesn't.""" + + # This assumes that AMD's BLIS has ver > 2.1 and vanilla BLIS < 2.1 + + found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] + ver = LooseVersion(found_version) + if ver > LooseVersion('2.1'): + self.BLAS_LIB_MT = ['blis-mt'] + + super(Blis, self)._set_blas_variables() diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 8d67bf2558..95869892da 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -27,6 +27,7 @@ :author: Stijn De Weirdt (Ghent University) :author: Kenneth Hoste (Ghent University) +:author: Sebastian Achilles (Forschungszentrum Juelich GmbH) """ from easybuild.toolchains.linalg.lapack import Lapack @@ -37,6 +38,6 @@ class Flame(Lapack): """Less trivial module, provides FLAME support.""" - LAPACK_MODULE_NAME = ['FLAME'] + Lapack.LAPACK_MODULE_NAME # no super() - LAPACK_LIB = ['lapack2flame', 'flame'] + Lapack.LAPACK_LIB # no super() + LAPACK_MODULE_NAME = ['libFLAME'] # + Lapack.LAPACK_MODULE_NAME # no super() + LAPACK_LIB = ['flame'] # + Lapack.LAPACK_LIB # no super() LAPACK_FAMILY = TC_CONSTANT_FLAME From 03ce47644adb1c3fb537c879f741d36d031c33d0 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Wed, 25 Nov 2020 10:43:48 +0000 Subject: [PATCH 080/115] gobff --- easybuild/toolchains/linalg/blis.py | 8 ++++---- easybuild/toolchains/linalg/flame.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index 5e5a5aa288..20d8ee0a4f 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -44,15 +44,15 @@ class Blis(LinAlg): BLAS_MODULE_NAME = ['BLIS'] BLAS_LIB = ['blis'] BLAS_FAMILY = TC_CONSTANT_BLIS - + def _set_blas_variables(self): """AMD's fork with version number > 2.1 names the MT library blis-mt, while vanilla BLIS doesn't.""" - + # This assumes that AMD's BLIS has ver > 2.1 and vanilla BLIS < 2.1 - + found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] ver = LooseVersion(found_version) if ver > LooseVersion('2.1'): self.BLAS_LIB_MT = ['blis-mt'] - + super(Blis, self)._set_blas_variables() diff --git a/easybuild/toolchains/linalg/flame.py b/easybuild/toolchains/linalg/flame.py index 95869892da..ce99050a65 100644 --- a/easybuild/toolchains/linalg/flame.py +++ b/easybuild/toolchains/linalg/flame.py @@ -38,6 +38,6 @@ class Flame(Lapack): """Less trivial module, provides FLAME support.""" - LAPACK_MODULE_NAME = ['libFLAME'] # + Lapack.LAPACK_MODULE_NAME # no super() - LAPACK_LIB = ['flame'] # + Lapack.LAPACK_LIB # no super() + LAPACK_MODULE_NAME = ['libFLAME'] # + Lapack.LAPACK_MODULE_NAME # no super() + LAPACK_LIB = ['flame'] # + Lapack.LAPACK_LIB # no super() LAPACK_FAMILY = TC_CONSTANT_FLAME From bc2d08eed709a6096a2f8ddfd933e7805bb64210 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Nov 2020 14:17:13 +0100 Subject: [PATCH 081/115] fix broken tests --- test/framework/options.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index f47bbc2f37..27b0fd0888 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -989,7 +989,8 @@ def test_copy_ec(self): test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', 'toy-0.0.eb', test_ec] stdout = self.mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_ec) + regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_ec) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertTrue(os.path.exists(test_ec)) self.assertEqual(toy_ec_txt, read_file(test_ec)) @@ -1003,7 +1004,8 @@ def test_copy_ec(self): args = ['--copy-ec', 'toy-0.0.eb', target_fn] stdout = self.mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') + regex = re.compile(r'.*/toy-0.0.eb copied to test.eb') + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) change_dir(cwd) @@ -1015,7 +1017,8 @@ def test_copy_ec(self): mkdir(test_target_dir) args = ['--copy-ec', 'toy-0.0.eb', test_target_dir] stdout = self.mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_target_dir) + regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_target_dir) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_ec)) @@ -1067,7 +1070,7 @@ def check_copied_files(): self.assertEqual(len(os.listdir(os.getcwd())), 0) args = ['--copy-ec', 'toy-0.0.eb'] stdout = self.mocked_main(args) - regex = re.compile('toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) + regex = re.compile('.*/toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) self.assertTrue(regex.match(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_cwd = os.path.join(test_working_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_cwd)) @@ -1075,7 +1078,7 @@ def check_copied_files(): # --copy-ec without arguments results in a proper error args = ['--copy-ec'] - error_pattern = "One of more files to copy should be specified!" + error_pattern = "One or more files to copy should be specified!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) def test_copy_ec_from_pr(self): @@ -1191,7 +1194,8 @@ def test_copy_ec_from_pr(self): args = ['--copy-ec', '--from-pr', '11521', test_ec] ec_pr11521 = "ExifTool-12.00-GCCcore-9.3.0.eb" stdout = self.mocked_main(args) - self.assertEqual(stdout, '%s copied to %s' % (ec_pr11521, test_ec)) + regex = re.compile(r'.*/%s copied to %s' % (ec_pr11521, test_ec)) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertTrue(os.path.exists(test_ec)) remove_file(test_ec) From c37f6e692b729a2c23662cfe2241295c78a3223c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Nov 2020 17:46:23 +0100 Subject: [PATCH 082/115] fix double return in cache_aware_func that was introduced in merge with develop --- easybuild/tools/github.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 95acda641c..abed829172 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -411,8 +411,6 @@ def cache_aware_func(pr, path=None, github_user=None, github_account=None, githu return cache_aware_func - return cache_aware_func - @pr_files_cache def fetch_files_from_pr(pr, path=None, github_user=None, github_account=None, github_repo=None): From d7236c4bc1a0db4bd1602ff92ab5163010b76925 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 27 Nov 2020 10:23:39 +0800 Subject: [PATCH 083/115] also check if PR is closed in --merge-pr --- easybuild/tools/github.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 4375edfbf3..f76b60e1fe 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -1331,8 +1331,11 @@ def merge_pr(pr): force = build_option('force') dry_run = build_option('dry_run') or build_option('extended_dry_run') - if pr_data['merged'] and not dry_run: - raise EasyBuildError("This PR is already merged.") + if not dry_run: + if pr_data['merged']: + raise EasyBuildError("This PR is already merged.") + elif pr_data['state'] == GITHUB_STATE_CLOSED: + raise EasyBuildError("This PR is closed.") def merge_url(gh): """Utility function to fetch merge URL for a specific PR.""" From 6eb53047f0a1419a9a03c2f3d9797d8c582dd23c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Fri, 27 Nov 2020 16:43:57 +0800 Subject: [PATCH 084/115] update test for --merge-pr to check that it errors out for closed/merged PRs --- test/framework/options.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 2727baa392..4ddec8198a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -3985,6 +3985,29 @@ def test_merge_pr(self): print("Skipping test_merge_pr, no GitHub token available?") return + # start by making sure --merge-pr without dry-run errors out for a closed PR + args = [ + '--merge-pr', + '11753', # closed PR + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + ] + error_msg = r"This PR is closed." + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + self.mock_stdout(False) + + # and also for an already merged PR + args = [ + '--merge-pr', + '11769', # already merged PR + '--github-user=%s' % GITHUB_TEST_ACCOUNT, + ] + error_msg = r"This PR is already merged." + self.mock_stdout(True) + self.assertErrorRegex(EasyBuildError, error_msg, self.eb_main, args, raise_error=True) + self.mock_stdout(False) + + # merged PR for EasyBuild-3.3.0.eb, is missing approved review args = [ '--merge-pr', '4781', # PR for easyconfig for EasyBuild-3.3.0.eb @@ -3993,7 +4016,6 @@ def test_merge_pr(self): '--pr-target-branch=some_branch', ] - # merged PR for EasyBuild-3.3.0.eb, is missing approved review stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) expected_stdout = '\n'.join([ From 66ce1954498e7ac09460f1de79b67d7dda790ce8 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Fri, 27 Nov 2020 10:37:43 +0000 Subject: [PATCH 085/115] gobff: Update blis.py --- easybuild/toolchains/linalg/blis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/easybuild/toolchains/linalg/blis.py b/easybuild/toolchains/linalg/blis.py index 20d8ee0a4f..74c916a5c9 100644 --- a/easybuild/toolchains/linalg/blis.py +++ b/easybuild/toolchains/linalg/blis.py @@ -51,8 +51,7 @@ def _set_blas_variables(self): # This assumes that AMD's BLIS has ver > 2.1 and vanilla BLIS < 2.1 found_version = self.get_software_version(self.BLAS_MODULE_NAME)[0] - ver = LooseVersion(found_version) - if ver > LooseVersion('2.1'): + if LooseVersion(found_version) > LooseVersion('2.1'): self.BLAS_LIB_MT = ['blis-mt'] super(Blis, self)._set_blas_variables() From 7bb0ecfafe6938fe7dc94b8a021540830d374f56 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Nov 2020 13:43:21 +0100 Subject: [PATCH 086/115] fix typo in comment --- easybuild/framework/easyconfig/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 8273d28265..56a64c5588 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -736,7 +736,7 @@ def det_copy_ec_specs(orig_paths, from_pr): # check files touched by PR and see if the target directory for --copy-ec # corresponds to the name of one of these files; # if so we should copy the specified file(s) to the current working directory, - # since interpreting the last argument as target location is very unlikely incorrect in this case + # since interpreting the last argument as target location is very unlikely to be correct in this case pr_filenames = [os.path.basename(p) for p in pr_paths] if last_path in pr_filenames: paths = orig_paths[:] From fb34ca96e8870b7352036371fdf54073ae24a8d2 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Nov 2020 15:45:46 +0100 Subject: [PATCH 087/115] fix regression in apply_regex_substitutions introduced in #3450: also accept list of paths to patch (fixes #3493) --- easybuild/tools/filetools.py | 64 ++++++++++++++++++++---------------- test/framework/filetools.py | 13 ++++++++ 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 087dada195..31670bc43c 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1333,22 +1333,26 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git_am=Fa return True -def apply_regex_substitutions(path, regex_subs, backup='.orig.eb'): +def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): """ Apply specified list of regex substitutions. - :param path: path to file to patch + :param paths: list of paths to files to patch (or just a single filepath) :param regex_subs: list of substitutions to apply, specified as (, ) :param backup: create backup of original file with specified suffix (no backup if value evaluates to False) """ + if isinstance(paths, string_type): + paths = [paths] + # only report when in 'dry run' mode if build_option('extended_dry_run'): - dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent')) - for regex, subtxt in regex_subs: - dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt)) + for path in paths: + dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent')) + for regex, subtxt in regex_subs: + dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt)) else: - _log.info("Applying following regex substitutions to %s: %s", path, regex_subs) + _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs) for i, (regex, subtxt) in enumerate(regex_subs): regex_subs[i] = (re.compile(regex), subtxt) @@ -1359,30 +1363,32 @@ def apply_regex_substitutions(path, regex_subs, backup='.orig.eb'): # no (persistent) backup file is created if empty string value is passed to 'backup' in fileinput.input backup_ext = '' - try: - # make sure that file can be opened in text mode; - # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 + for path in paths: try: - with open(path, 'r') as fp: - _ = fp.read() - except UnicodeDecodeError as err: - _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) - path_backup = back_up_file(path) - _log.info("Editing %s to strip out non-UTF-8 characters (backup at %s)", path, path_backup) - txt = read_file(path, mode='rb') - txt_utf8 = txt.decode(encoding='utf-8', errors='replace') - write_file(path, txt_utf8) - - for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): - for regex, subtxt in regex_subs: - match = regex.search(line) - if match: - _log.info("Replacing line %d in %s: '%s' -> '%s'", (line_id + 1), path, match.group(0), subtxt) - line = regex.sub(subtxt, line) - sys.stdout.write(line) - - except (IOError, OSError) as err: - raise EasyBuildError("Failed to patch %s: %s", path, err) + # make sure that file can be opened in text mode; + # it's possible this fails with UnicodeDecodeError when running EasyBuild with Python 3 + try: + with open(path, 'r') as fp: + _ = fp.read() + except UnicodeDecodeError as err: + _log.info("Encountered UnicodeDecodeError when opening %s in text mode: %s", path, err) + path_backup = back_up_file(path) + _log.info("Editing %s to strip out non-UTF-8 characters (backup at %s)", path, path_backup) + txt = read_file(path, mode='rb') + txt_utf8 = txt.decode(encoding='utf-8', errors='replace') + write_file(path, txt_utf8) + + for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): + for regex, subtxt in regex_subs: + match = regex.search(line) + if match: + origtxt = match.group(0) + _log.info("Replacing line %d in %s: '%s' -> '%s'", (line_id + 1), path, origtxt, subtxt) + line = regex.sub(subtxt, line) + sys.stdout.write(line) + + except (IOError, OSError) as err: + raise EasyBuildError("Failed to patch %s: %s", path, err) def modify_env(old, new): diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 3fed4ec0e2..2ae207f793 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1219,6 +1219,19 @@ def test_apply_regex_substitutions(self): self.assertTrue(txt.startswith('FOO ')) self.assertTrue(txt.endswith(' bar')) + # also test apply_regex_substitutions with a *list* of paths + # cfr. https://github.com/easybuilders/easybuild-framework/issues/3493 + test_dir = os.path.join(self.test_prefix, 'test_dir') + test_file1 = os.path.join(test_dir, 'one.txt') + test_file2 = os.path.join(test_dir, 'two.txt') + ft.write_file(test_file1, "Donald is an elephant") + ft.write_file(test_file2, "2 + 2 = 5") + regexs = [ + ('Donald', 'Dumbo'), + ('= 5', '= 4'), + ] + ft.apply_regex_substitutions([test_file1, test_file2], regexs) + def test_find_flexlm_license(self): """Test find_flexlm_license function.""" lic_file1 = os.path.join(self.test_prefix, 'one.lic') From f6ce140dce3d398c30d7a4b37eb356de73f13817 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Nov 2020 16:12:58 +0100 Subject: [PATCH 088/115] better dry run output for apply_regex_substitutions + don't compile regular expressions in list that is passed in --- easybuild/tools/filetools.py | 15 ++++++++------- test/framework/filetools.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 31670bc43c..dc44bfee3d 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -1346,16 +1346,17 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): # only report when in 'dry run' mode if build_option('extended_dry_run'): - for path in paths: - dry_run_msg("applying regex substitutions to file %s" % path, silent=build_option('silent')) - for regex, subtxt in regex_subs: - dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt)) + paths_str = ', '.join(paths) + dry_run_msg("applying regex substitutions to file(s): %s" % paths_str, silent=build_option('silent')) + for regex, subtxt in regex_subs: + dry_run_msg(" * regex pattern '%s', replacement string '%s'" % (regex, subtxt)) else: _log.info("Applying following regex substitutions to %s: %s", paths, regex_subs) - for i, (regex, subtxt) in enumerate(regex_subs): - regex_subs[i] = (re.compile(regex), subtxt) + compiled_regex_subs = [] + for regex, subtxt in regex_subs: + compiled_regex_subs.append((re.compile(regex), subtxt)) if backup: backup_ext = backup @@ -1379,7 +1380,7 @@ def apply_regex_substitutions(paths, regex_subs, backup='.orig.eb'): write_file(path, txt_utf8) for line_id, line in enumerate(fileinput.input(path, inplace=1, backup=backup_ext)): - for regex, subtxt in regex_subs: + for regex, subtxt in compiled_regex_subs: match = regex.search(line) if match: origtxt = match.group(0) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 2ae207f793..44ae181c8a 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -1232,6 +1232,24 @@ def test_apply_regex_substitutions(self): ] ft.apply_regex_substitutions([test_file1, test_file2], regexs) + # also check dry run mode + init_config(build_options={'extended_dry_run': True}) + self.mock_stderr(True) + self.mock_stdout(True) + ft.apply_regex_substitutions([test_file1, test_file2], regexs) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + regex = re.compile('\n'.join([ + r"applying regex substitutions to file\(s\): .*/test_dir/one.txt, .*/test_dir/two.txt", + r" \* regex pattern 'Donald', replacement string 'Dumbo'", + r" \* regex pattern '= 5', replacement string '= 4'", + '', + ])) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_find_flexlm_license(self): """Test find_flexlm_license function.""" lic_file1 = os.path.join(self.test_prefix, 'one.lic') From bd4a8858bb9541959568b6dc8b4e963a56569310 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 28 Nov 2020 17:39:16 +0100 Subject: [PATCH 089/115] don't include file/ldd/readelf commands run during RPATH sanity check in --trace output --- easybuild/framework/easyblock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 14489e27b9..7e9a1d984b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2460,7 +2460,7 @@ def sanity_check_rpath(self, rpath_dirs=None): for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: self.log.debug("Sanity checking RPATH for %s", path) - out, ec = run_cmd("file %s" % path, simple=False) + out, ec = run_cmd("file %s" % path, simple=False, trace=False) if ec: fails.append("Failed to run 'file %s': %s" % (path, out)) @@ -2470,7 +2470,7 @@ def sanity_check_rpath(self, rpath_dirs=None): # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped if "dynamically linked" in out: # check whether all required libraries are found via 'ldd' - out, ec = run_cmd("ldd %s" % path, simple=False) + out, ec = run_cmd("ldd %s" % path, simple=False, trace=False) if ec: fail_msg = "Failed to run 'ldd %s': %s" % (path, out) self.log.warning(fail_msg) @@ -2483,7 +2483,7 @@ def sanity_check_rpath(self, rpath_dirs=None): self.log.debug("Output of 'ldd %s' checked, looks OK", path) # check whether RPATH section in 'readelf -d' output is there - out, ec = run_cmd("readelf -d %s" % path, simple=False) + out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False) if ec: fail_msg = "Failed to run 'readelf %s': %s" % (path, out) self.log.warning(fail_msg) From 3ce315a646dcf0693a7f9014bea956941f0b935e Mon Sep 17 00:00:00 2001 From: ocaisa Date: Mon, 30 Nov 2020 14:17:51 +0100 Subject: [PATCH 090/115] also check file contents in test for --copy-ec --from-pr --- test/framework/options.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/options.py b/test/framework/options.py index 27b0fd0888..620be17c34 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1197,6 +1197,7 @@ def test_copy_ec_from_pr(self): regex = re.compile(r'.*/%s copied to %s' % (ec_pr11521, test_ec)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertTrue(os.path.exists(test_ec)) + self.assertTrue("name = 'ExifTool'" in read_file(test_ec)) remove_file(test_ec) def test_dry_run(self): From 17986bbd731351e8119528032143db7d6ff1e067 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Nov 2020 15:07:01 +0100 Subject: [PATCH 091/115] clarify input format for --cuda-compute-capabilities (fixes #3500) --- easybuild/tools/options.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index c800b7de5d..e208e914a4 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -356,8 +356,9 @@ def override_options(self): 'consider-archived-easyconfigs': ("Also consider archived easyconfigs", None, 'store_true', False), 'containerize': ("Generate container recipe/image", None, 'store_true', False, 'C'), 'copy-ec': ("Copy specified easyconfig(s) to specified location", None, 'store_true', False), - 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software", - 'strlist', 'extend', None), + 'cuda-compute-capabilities': ("List of CUDA compute capabilities to use when building GPU software; " + "values should be specified as digits separated by a dot, " + "for example: 3.5,5.0,7.2", 'strlist', 'extend', None), 'debug-lmod': ("Run Lmod modules tool commands in debug module", None, 'store_true', False), 'default-opt-level': ("Specify default optimisation level", 'choice', 'store', DEFAULT_OPT_LEVEL, Compiler.COMPILER_OPT_FLAGS), From 9e4f49e7debcbc43a346a7aedf68cdb7b49ff2fe Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Nov 2020 15:16:09 +0100 Subject: [PATCH 092/115] update installation procedure for EasyBuild in generated Singularity container recipes (fixes #3121) --- easybuild/tools/containers/singularity.py | 7 ------- test/framework/containers.py | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 4ecbed6990..9583f25d32 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -314,13 +314,6 @@ def resolve_template_data(self): "# install EasyBuild using pip", # upgrade pip "pip install -U pip", - "pip install wheel", - # EasyBuild 3.x requires setuptools as runtime dependency - "pip install -U setuptools", - # stick to previous version of vsc-install to avoid requiring mock (which causes installation problems) - # stick to previous version of vsc-base to avoid requiring 'future' (irrelevant for EasyBuild) - # this is just a temporary measure, since vsc-install & vsc-base have been ingested for EasyBuild 4.x - "pip install 'vsc-install<0.11.4' 'vsc-base<2.9.0'", "pip install easybuild", ]) diff --git a/test/framework/containers.py b/test/framework/containers.py index 3a3447e8fe..c642728b52 100644 --- a/test/framework/containers.py +++ b/test/framework/containers.py @@ -141,9 +141,7 @@ def test_end2end_singularity_recipe_config(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) pip_patterns = [ - # EasyBuild and dependencies are installed with pip by default - "pip install -U setuptools", - "pip install.*vsc-base", + # EasyBuild is installed with pip by default "pip install easybuild", ] post_commands_patterns = [ From ff3d33f4c592d01655f992a09c54260c4b74201d Mon Sep 17 00:00:00 2001 From: deniskristak <35582739+deniskristak@users.noreply.github.com> Date: Mon, 30 Nov 2020 20:31:13 +0100 Subject: [PATCH 093/115] add (experimental) support for specifying easyconfig files via "easystack" file (#3479) Co-authored-by: deniskristak Co-authored-by: Kenneth Hoste --- easybuild/framework/easystack.py | 234 ++++++++++++++++++ easybuild/main.py | 8 + easybuild/tools/options.py | 2 + easybuild/tools/utilities.py | 2 +- .../easystacks/test_easystack_asterisk.yaml | 6 + .../easystacks/test_easystack_basic.yaml | 13 + .../easystacks/test_easystack_labels.yaml | 7 + .../test_easystack_wrong_structure.yaml | 6 + test/framework/general.py | 2 +- test/framework/options.py | 67 +++++ .../t/toy/extensions/bar-0.0-local.tar.gz | Bin 0 -> 277 bytes .../sandbox/sources/t/toy/exts-git.tar.gz | Bin 0 -> 395 bytes 12 files changed, 345 insertions(+), 2 deletions(-) create mode 100644 easybuild/framework/easystack.py create mode 100644 test/framework/easystacks/test_easystack_asterisk.yaml create mode 100644 test/framework/easystacks/test_easystack_basic.yaml create mode 100644 test/framework/easystacks/test_easystack_labels.yaml create mode 100644 test/framework/easystacks/test_easystack_wrong_structure.yaml create mode 100644 test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz create mode 100644 test/framework/sandbox/sources/t/toy/exts-git.tar.gz diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py new file mode 100644 index 0000000000..982bc41132 --- /dev/null +++ b/easybuild/framework/easystack.py @@ -0,0 +1,234 @@ +# Copyright 2020-2020 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +# +""" +Support for easybuild-ing from multiple easyconfigs based on +information obtained from provided file (easystack) with build specifications. + +:author: Denis Kristak (Inuits) +:author: Pavel Grochal (Inuits) +""" + +from easybuild.base import fancylogger +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import read_file +from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.utilities import only_if_module_is_available +try: + import yaml +except ImportError: + pass +_log = fancylogger.getLogger('easystack', fname=False) + + +class EasyStack(object): + """One class instance per easystack. General options + list of all SoftwareSpecs instances""" + + def __init__(self): + self.easybuild_version = None + self.robot = False + self.software_list = [] + + def compose_ec_filenames(self): + """Returns a list of all easyconfig names""" + ec_filenames = [] + for sw in self.software_list: + full_ec_version = det_full_ec_version({ + 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, + 'version': sw.version, + 'versionsuffix': sw.versionsuffix, + }) + ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) + ec_filenames.append(ec_filename) + return ec_filenames + + # flags applicable to all sw (i.e. robot) + def get_general_options(self): + """Returns general options (flags applicable to all sw (i.e. --robot))""" + general_options = {} + # TODO add support for general_options + # general_options['robot'] = self.robot + # general_options['easybuild_version'] = self.easybuild_version + return general_options + + +class SoftwareSpecs(object): + """Contains information about every software that should be installed""" + + def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name): + self.name = name + self.version = version + self.toolchain_version = toolchain_version + self.toolchain_name = toolchain_name + self.versionsuffix = versionsuffix + + +class EasyStackParser(object): + """Parser for easystack files (in YAML syntax).""" + + @only_if_module_is_available('yaml', pkgname='PyYAML') + @staticmethod + def parse(filepath): + """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" + yaml_txt = read_file(filepath) + easystack_raw = yaml.safe_load(yaml_txt) + easystack = EasyStack() + + try: + software = easystack_raw["software"] + except KeyError: + wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found" + raise EasyBuildError(wrong_structure_file) + + # assign software-specific easystack attributes + for name in software: + # ensure we have a string value (YAML parser returns type = dict + # if levels under the current attribute are present) + name = str(name) + try: + toolchains = software[name]['toolchains'] + except KeyError: + raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) + for toolchain in toolchains: + toolchain = str(toolchain) + toolchain_parts = toolchain.split('-', 1) + if len(toolchain_parts) == 2: + toolchain_name, toolchain_version = toolchain_parts + elif len(toolchain_parts) == 1: + toolchain_name, toolchain_version = toolchain, '' + else: + raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", + name, filepath, toolchain_parts) + + try: + # if version string containts asterisk or labels, raise error (asterisks not supported) + versions = toolchains[toolchain]['versions'] + except TypeError as err: + wrong_structure_err = "An error occurred when interpreting " + wrong_structure_err += "the data for software %s: %s" % (name, err) + raise EasyBuildError(wrong_structure_err) + if '*' in str(versions): + asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. " + asterisk_err += "Wildcard feature is not supported yet." + raise EasyBuildError(asterisk_err, name, filepath) + + # yaml versions can be in different formats in yaml file + # firstly, check if versions in yaml file are read as a dictionary. + # Example of yaml structure: + # ======================================================================== + # versions: + # 2.25: + # 2.23: + # versionsuffix: '-R-4.0.0' + # ======================================================================== + if isinstance(versions, dict): + for version in versions: + if versions[version] is not None: + version_spec = versions[version] + if 'versionsuffix' in version_spec: + versionsuffix = str(version_spec['versionsuffix']) + else: + versionsuffix = '' + if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec): + lab_err = "EasyStack specifications of '%s' in %s " + lab_err += "contain labels. Labels aren't supported yet." + raise EasyBuildError(lab_err, name, filepath) + else: + versionsuffix = '' + + specs = { + 'name': name, + 'toolchain_name': toolchain_name, + 'toolchain_version': toolchain_version, + 'version': version, + 'versionsuffix': versionsuffix, + } + sw = SoftwareSpecs(**specs) + + # append newly created class instance to the list in instance of EasyStack class + easystack.software_list.append(sw) + continue + + # is format read as a list of versions? + # ======================================================================== + # versions: + # [2.24, 2.51] + # ======================================================================== + elif isinstance(versions, list): + versions_list = versions + + # format = multiple lines without ':' (read as a string)? + # ======================================================================== + # versions: + # 2.24 + # 2.51 + # ======================================================================== + elif isinstance(versions, str): + versions_list = str(versions).split() + + # format read as float (containing one version only)? + # ======================================================================== + # versions: + # 2.24 + # ======================================================================== + elif isinstance(versions, float): + versions_list = [str(versions)] + + # if no version is a dictionary, versionsuffix isn't specified + versionsuffix = '' + + for version in versions_list: + sw = SoftwareSpecs( + name=name, version=version, versionsuffix=versionsuffix, + toolchain_name=toolchain_name, toolchain_version=toolchain_version) + # append newly created class instance to the list in instance of EasyStack class + easystack.software_list.append(sw) + + # assign general easystack attributes + easystack.easybuild_version = easystack_raw.get('easybuild_version', None) + easystack.robot = easystack_raw.get('robot', False) + + return easystack + + +def parse_easystack(filepath): + """Parses through easystack file, returns what EC are to be installed together with their options.""" + log_msg = "Support for easybuild-ing from multiple easyconfigs based on " + log_msg += "information obtained from provided file (easystack) with build specifications." + _log.experimental(log_msg) + _log.info("Building from easystack: '%s'" % filepath) + + # class instance which contains all info about planned build + easystack = EasyStackParser.parse(filepath) + + easyconfig_names = easystack.compose_ec_filenames() + + general_options = easystack.get_general_options() + + _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: \n'%s'" % "',\n'".join(easyconfig_names)) + if len(general_options) != 0: + _log.debug("General options for installation are: \n%s" % str(general_options)) + else: + _log.debug("No general options were specified in easystack") + + return easyconfig_names, general_options diff --git a/easybuild/main.py b/easybuild/main.py index 616aca23cc..c2030a7d68 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -47,6 +47,7 @@ from easybuild.framework.easyblock import build_and_install_one, inject_checksums from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check @@ -224,6 +225,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) + # if easystack is provided with the command, commands with arguments from it will be executed + if options.easystack: + # TODO add general_options (i.e. robot) to build options + orig_paths, general_options = parse_easystack(options.easystack) + if general_options: + raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.") + # check whether packaging is supported when it's being used if options.package: check_pkg_support() diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e208e914a4..89663340e3 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -605,6 +605,8 @@ def informative_options(self): 'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False), 'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False), 'terse': ("Terse output (machine-readable)", None, 'store_true', False), + 'easystack': ("Path to easystack file in YAML format, specifying details of a software stack", + None, 'store', None), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index f4e6dafe0f..e878489043 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -173,7 +173,7 @@ def wrap(orig): pass if imported is None: - raise ImportError("None of the specified modules %s is available" % ', '.join(modnames)) + raise ImportError("None of the specified modules (%s) is available" % ', '.join(modnames)) else: return orig diff --git a/test/framework/easystacks/test_easystack_asterisk.yaml b/test/framework/easystacks/test_easystack_asterisk.yaml new file mode 100644 index 0000000000..7f440636cd --- /dev/null +++ b/test/framework/easystacks/test_easystack_asterisk.yaml @@ -0,0 +1,6 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + "2.11.*" \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml new file mode 100644 index 0000000000..4272aaf0fc --- /dev/null +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -0,0 +1,13 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + 2.25: + 2.26: + toy: + toolchains: + gompi-2018a: + versions: + 0.0: + versionsuffix: '-test' \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml new file mode 100644 index 0000000000..51a113523f --- /dev/null +++ b/test/framework/easystacks/test_easystack_labels.yaml @@ -0,0 +1,7 @@ +software: + binutils: + toolchains: + GCCcore-4.9.3: + versions: + 3.11: + exclude-labels: arch:aarch64 diff --git a/test/framework/easystacks/test_easystack_wrong_structure.yaml b/test/framework/easystacks/test_easystack_wrong_structure.yaml new file mode 100644 index 0000000000..a328b5413b --- /dev/null +++ b/test/framework/easystacks/test_easystack_wrong_structure.yaml @@ -0,0 +1,6 @@ +software: + Bioconductor: + toolchains: + # foss-2020a: + versions: + 3.11 \ No newline at end of file diff --git a/test/framework/general.py b/test/framework/general.py index 07e8c36011..fd43213be6 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -88,7 +88,7 @@ def bar(): def bar2(): pass - err_pat = "ImportError: None of the specified modules nosuchmodule, anothernosuchmodule is available" + err_pat = r"ImportError: None of the specified modules \(nosuchmodule, anothernosuchmodule\) is available" self.assertErrorRegex(EasyBuildError, err_pat, bar2) class Foo(): diff --git a/test/framework/options.py b/test/framework/options.py index 30946915d0..b4f6e4d90a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -42,6 +42,7 @@ import easybuild.tools.toolchain from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock +from easybuild.framework.easystack import parse_easystack from easybuild.framework.easyconfig import BUILD, CUSTOM, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT, LICENSE from easybuild.framework.easyconfig import MANDATORY, MODULES, OTHER, TOOLCHAIN from easybuild.framework.easyconfig.easyconfig import EasyConfig, get_easyblock_class, robot_find_easyconfig @@ -107,11 +108,14 @@ def setUp(self): self.orig_terminal_supports_colors = easybuild.tools.options.terminal_supports_colors self.orig_os_getuid = easybuild.main.os.getuid + self.orig_experimental = easybuild.tools.build_log.EXPERIMENTAL def tearDown(self): """Clean up after test.""" easybuild.main.os.getuid = self.orig_os_getuid easybuild.tools.options.terminal_supports_colors = self.orig_terminal_supports_colors + easybuild.tools.build_log.EXPERIMENTAL = self.orig_experimental + super(CommandLineOptionsTest, self).tearDown() def purge_environment(self): @@ -5600,6 +5604,69 @@ def test_sysroot(self): os.environ['EASYBUILD_SYSROOT'] = doesnotexist self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, ['--show-config'], raise_error=True) + # end-to-end testing of unknown filename + def test_easystack_wrong_read(self): + """Test for --easystack when wrong name is provided""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_nonexistent.yaml') + args = ['--easystack', toy_easystack, '--experimental'] + expected_err = "No such file or directory: '%s'" % toy_easystack + self.assertErrorRegex(EasyBuildError, expected_err, self.eb_main, args, raise_error=True) + + # testing basics - end-to-end + # expecting successful build + def test_easystack_basic(self): + """Test for --easystack -> success case""" + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') + + args = ['--easystack', toy_easystack, '--stop', '--debug', '--experimental'] + stdout, err = self.eb_main(args, do_build=True, return_error=True) + patterns = [ + r"[\S\s]*INFO Building from easystack:[\S\s]*", + r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs:.*?[\n]" + r"[\S\s]*INFO building and installing binutils/2\.25-GCCcore-4\.9\.3[\S\s]*", + r"[\S\s]*INFO building and installing binutils/2\.26-GCCcore-4\.9\.3[\S\s]*", + r"[\S\s]*INFO building and installing toy/0\.0-gompi-2018a-test[\S\s]*", + r"[\S\s]*INFO COMPLETED: Installation STOPPED successfully[\S\s]*", + r"[\S\s]*INFO Build succeeded for 3 out of 3[\S\s]*" + ] + for pattern in patterns: + regex = re.compile(pattern) + self.assertTrue(regex.match(stdout) is not None) + + def test_easystack_wrong_structure(self): + """Test for --easystack when yaml easystack has wrong structure""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') + + expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" + expected_err += r" 'float' object is not subscriptable[\S\s]*" + expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*" + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + + def test_easystack_asterisk(self): + """Test for --easystack when yaml easystack contains asterisk (wildcard)""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_asterisk.yaml') + + expected_err = "EasyStack specifications of 'binutils' in .*/test_easystack_asterisk.yaml contain asterisk. " + expected_err += "Wildcard feature is not supported yet." + + self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + + def test_easystack_labels(self): + """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" + easybuild.tools.build_log.EXPERIMENTAL = True + topdir = os.path.dirname(os.path.abspath(__file__)) + toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') + + error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " + error_msg += "Labels aren't supported yet." + self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz b/test/framework/sandbox/sources/t/toy/extensions/bar-0.0-local.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..98f527167838a9ac205a305c6a1b951b69b78591 GIT binary patch literal 277 zcmb2|=3q!#u{D^1`R&F1euoVN*gl-A>w$?$@SNHjh!_&L2g=D{g(wEduZR zF0+Y=u(`B0yx%@2b^e;o`+Lil%Y451reTI_q?xIzyCPJhPiw9nj@{hhd>*$6H@3ut* zUd$*rS|h(cZ`%1lmxImR&%Y+sRqj#sy=S-9W42Mw?R_fI!LJ`@8^^CJKfCVJUgIOL ZU#R|iY0k`m2L6bycMtovf_y8jBDk`y$}q z*ANP*G@qY6KYbe3Yp7r*g)7Ze7ySSlk?>CJd_2?!xslE6nycagAo7xw^-vN(wt zODO|F6FYMR$wjj1&>H00rZN|mGAMfI*=c2YBJqJk+003#r9y*l*!{{j-Qh zUVj&K^naN=K0QyuMKReb}%v4isi@<=bz%TrtGUEN;1^xQ-X(?+F;u`jJBMoH%V-wC{d zS~dby2`JVWTst3+TaNVqd}`W!h~I&;{hzTYqTc^qa8&<+3czXq&$|B4GUol?1-NbS p?Ztf=xJiEQ|G&3`^ZUOzBGjLM7x@0q Date: Tue, 1 Dec 2020 09:23:02 +0100 Subject: [PATCH 094/115] tweak test_easystack_basic to get better error reporting if it fails --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index b4f6e4d90a..3b4c53bab7 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5633,7 +5633,7 @@ def test_easystack_basic(self): ] for pattern in patterns: regex = re.compile(pattern) - self.assertTrue(regex.match(stdout) is not None) + self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) def test_easystack_wrong_structure(self): """Test for --easystack when yaml easystack has wrong structure""" From 54c8ee73bb834dc8b4960944969f6ac1ea2f5f49 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Dec 2020 10:15:45 +0100 Subject: [PATCH 095/115] enable raising of errors in test_easystack_basic --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3b4c53bab7..3d69766fc6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5621,7 +5621,7 @@ def test_easystack_basic(self): toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') args = ['--easystack', toy_easystack, '--stop', '--debug', '--experimental'] - stdout, err = self.eb_main(args, do_build=True, return_error=True) + stdout = self.eb_main(args, do_build=True, raise_error=True) patterns = [ r"[\S\s]*INFO Building from easystack:[\S\s]*", r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs:.*?[\n]" From 17e4c5e1d993cc9a9a0172b8784f3df7137817f7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Dec 2020 13:09:51 +0100 Subject: [PATCH 096/115] fix error when --easystack is used without having PyYAML installed --- easybuild/framework/easystack.py | 2 +- easybuild/tools/utilities.py | 10 +++++----- test/framework/general.py | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 982bc41132..5daf6fbef7 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -86,7 +86,6 @@ def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_na class EasyStackParser(object): """Parser for easystack files (in YAML syntax).""" - @only_if_module_is_available('yaml', pkgname='PyYAML') @staticmethod def parse(filepath): """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" @@ -211,6 +210,7 @@ def parse(filepath): return easystack +@only_if_module_is_available('yaml', pkgname='PyYAML') def parse_easystack(filepath): """Parses through easystack file, returns what EC are to be installed together with their options.""" log_msg = "Support for easybuild-ing from multiple easyconfigs based on " diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index e878489043..886c39d055 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -173,18 +173,18 @@ def wrap(orig): pass if imported is None: - raise ImportError("None of the specified modules (%s) is available" % ', '.join(modnames)) + raise ImportError else: return orig - except ImportError as err: - # need to pass down 'err' via named argument to ensure it's in scope when using Python 3.x - def error(err=err, *args, **kwargs): - msg = "%s; required module '%s' is not available" % (err, modname) + except ImportError: + def error(*args, **kwargs): + msg = "None of the specified modules (%s) is available" % ', '.join(modnames) if pkgname: msg += " (provided by Python package %s, available from %s)" % (pkgname, url) elif url: msg += " (available from %s)" % url + msg += ", yet one of them is required!" raise EasyBuildError("ImportError: %s", msg) return error diff --git a/test/framework/general.py b/test/framework/general.py index fd43213be6..ecc0798146 100644 --- a/test/framework/general.py +++ b/test/framework/general.py @@ -81,7 +81,8 @@ def foo2(): def bar(): pass - err_pat = "required module 'nosuchmoduleoutthere' is not available.*package nosuchpkg.*pypi/nosuchpkg" + err_pat = r"None of the specified modules \(nosuchmoduleoutthere\) is available.*" + err_pat += r"package nosuchpkg.*pypi/nosuchpkg" self.assertErrorRegex(EasyBuildError, err_pat, bar) @only_if_module_is_available(('nosuchmodule', 'anothernosuchmodule')) @@ -96,7 +97,8 @@ class Foo(): def foobar(self): pass - err_pat = r"required module 'thisdoesnotexist' is not available \(available from http://example.com\)" + err_pat = r"None of the specified modules \(thisdoesnotexist\) is available " + err_pat += r"\(available from http://example.com\)" self.assertErrorRegex(EasyBuildError, err_pat, Foo().foobar) def test_docstrings(self): From c0d12f8d4b74bc5fbb33e85bf235da77ec63352a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 4 Dec 2020 15:26:46 +0100 Subject: [PATCH 097/115] also include *.yaml files from test/ dir in package --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0e89e87364..5a739b43a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include optcomplete.bash recursive-include etc * recursive-include easybuild *py recursive-include easybuild/scripts * -recursive-include test *py *eb +recursive-include test *py *eb *yaml recursive-include test/framework/modules * recursive-include test/framework/sandbox/sources * include CONTRIBUTING.md From 10c869890a674579cd3522850ac2f70216fbf854 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 4 Dec 2020 15:31:22 +0100 Subject: [PATCH 098/115] run tests outside of checked out repository dir (so we test installed version of easybuild-framework) --- .github/workflows/unit_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7c54b83fec..3288472673 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -130,6 +130,9 @@ jobs: EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} run: | + # run tests *outside* of checked out easybuild-framework directory, + # to ensure we're testing installed version (see previous step) + cd $HOME # initialize environment for modules tool if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi source $(cat $HOME/mod_init); type module From 2aafb1757ce9bfb4bf9a15edae1d41f1dae029e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 4 Dec 2020 15:32:11 +0100 Subject: [PATCH 099/115] also test bootstrap script with Python 3.9 (supported since EasyBuild v4.3.1) --- .github/workflows/unit_tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3288472673..30407b2268 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -170,9 +170,6 @@ jobs: test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) - name: test bootstrap script - # skip testing of bootstrap script with Python 3.9, - # until an EasyBuild release that is compatible with Python 3.9 is available - if: ${{ matrix.python != 3.9 }} run: | # (re)initialize environment for modules tool if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi From 3fc575ff84124cd461865c0ac048ac56938c1930 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 4 Dec 2020 20:43:09 +0100 Subject: [PATCH 100/115] also take easystacks subdir into account in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6c3fe46069..c2dc739e7c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ def find_rel_test(): current = os.getcwd() os.chdir(basedir) res = [] - for subdir in ["sandbox", "easyconfigs", "modules"]: + for subdir in ["easyconfigs", "easystacks", "modules", "sandbox"]: res.extend([os.path.join(root, filename) for root, dirnames, filenames in os.walk(subdir) for filename in filenames if os.path.isfile(os.path.join(root, filename))]) From b45f5194104474328cd3c30ceb0041efdcc37553 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Dec 2020 22:58:51 +0000 Subject: [PATCH 101/115] Bump cryptography from 2.9.2 to 3.2 Bumps [cryptography](https://github.com/pyca/cryptography) from 2.9.2 to 3.2. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/2.9.2...3.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ebbcbaaf29..0c623ae756 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,5 +55,5 @@ archspec; python_version >= '2.7' # cryptography 3.0 deprecates Python 2.7 # and cryptography is not needed at all for Python 2.6 -cryptography==2.9.2; python_version == '2.7' +cryptography==3.2; python_version == '2.7' cryptography; python_version >= '3.5' From 41c3dec8219b872783c7b50cee099dca314983ef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Dec 2020 12:25:23 +0100 Subject: [PATCH 102/115] ignore CryptographyDeprecationWarning when using Python 2 --- .github/workflows/unit_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 30407b2268..2cae3138f3 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -164,7 +164,7 @@ jobs: # run test suite python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend" + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.5|from cryptography.*default_backend|CryptographyDeprecationWarning: Python 2" # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite\n${PRINTED_MSG}" && exit 1) From a8863d4950a2eb30216be4051aab61c9d1236342 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Dec 2020 12:27:38 +0100 Subject: [PATCH 103/115] bump to cryptography 3.2.1 when using Python 2 --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0c623ae756..0defb13c7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ requests archspec; python_version >= '2.7' -# cryptography 3.0 deprecates Python 2.7 -# and cryptography is not needed at all for Python 2.6 -cryptography==3.2; python_version == '2.7' +# cryptography 3.0 deprecates Python 2.7 (but v3.2.1 still works with Python 2.7); +# cryptography is not needed at all for Python 2.6 +cryptography==3.2.1; python_version == '2.7' cryptography; python_version >= '3.5' From 04bca8f90e5fd93fe3fd97a2e3ae4a09c23181d4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 6 Dec 2020 15:46:39 +0100 Subject: [PATCH 104/115] take into account slightly different error with Python 2.6 in test_easystack_wrong_structure --- test/framework/options.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3d69766fc6..7b7d50605a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5642,8 +5642,9 @@ def test_easystack_wrong_structure(self): toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_wrong_structure.yaml') expected_err = r"[\S\s]*An error occurred when interpreting the data for software Bioconductor:" - expected_err += r" 'float' object is not subscriptable[\S\s]*" - expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*" + expected_err += r"( 'float' object is not subscriptable[\S\s]*" + expected_err += r"| 'float' object is unsubscriptable" + expected_err += r"| 'float' object has no attribute '__getitem__'[\S\s]*)" self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) def test_easystack_asterisk(self): From dc91f0c78f25b006c74270b81c606fa983758cde Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 7 Dec 2020 09:03:27 +0100 Subject: [PATCH 105/115] use --dry-run in test_easystack_basic (to avoid downloading of binutils sources) --- easybuild/framework/easystack.py | 2 +- test/framework/options.py | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 5daf6fbef7..6a6649bd59 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -225,7 +225,7 @@ def parse_easystack(filepath): general_options = easystack.get_general_options() - _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: \n'%s'" % "',\n'".join(easyconfig_names)) + _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ", ".join(easyconfig_names)) if len(general_options) != 0: _log.debug("General options for installation are: \n%s" % str(general_options)) else: diff --git a/test/framework/options.py b/test/framework/options.py index 7b7d50605a..7a30ea3bd5 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5620,20 +5620,19 @@ def test_easystack_basic(self): topdir = os.path.dirname(os.path.abspath(__file__)) toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_basic.yaml') - args = ['--easystack', toy_easystack, '--stop', '--debug', '--experimental'] + args = ['--easystack', toy_easystack, '--debug', '--experimental', '--dry-run'] stdout = self.eb_main(args, do_build=True, raise_error=True) patterns = [ r"[\S\s]*INFO Building from easystack:[\S\s]*", - r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs:.*?[\n]" - r"[\S\s]*INFO building and installing binutils/2\.25-GCCcore-4\.9\.3[\S\s]*", - r"[\S\s]*INFO building and installing binutils/2\.26-GCCcore-4\.9\.3[\S\s]*", - r"[\S\s]*INFO building and installing toy/0\.0-gompi-2018a-test[\S\s]*", - r"[\S\s]*INFO COMPLETED: Installation STOPPED successfully[\S\s]*", - r"[\S\s]*INFO Build succeeded for 3 out of 3[\S\s]*" + r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs: " + r"binutils-2.25-GCCcore-4.9.3.eb, binutils-2.26-GCCcore-4.9.3.eb, toy-0.0-gompi-2018a-test.eb", + r"\* \[ \] .*/test_ecs/b/binutils/binutils-2.25-GCCcore-4.9.3.eb \(module: binutils/2.25-GCCcore-4.9.3\)", + r"\* \[ \] .*/test_ecs/b/binutils/binutils-2.26-GCCcore-4.9.3.eb \(module: binutils/2.26-GCCcore-4.9.3\)", + r"\* \[ \] .*/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb \(module: toy/0.0-gompi-2018a-test\)", ] for pattern in patterns: regex = re.compile(pattern) - self.assertTrue(regex.match(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) def test_easystack_wrong_structure(self): """Test for --easystack when yaml easystack has wrong structure""" From 84911f88903b38668a031e2061c8250c1cfb6e32 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 7 Dec 2020 11:39:44 +0100 Subject: [PATCH 106/115] sort easyconfig filenames in log message for parsed easystack to fix failing test --- easybuild/framework/easystack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 6a6649bd59..8d71866c65 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -225,7 +225,7 @@ def parse_easystack(filepath): general_options = easystack.get_general_options() - _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ", ".join(easyconfig_names)) + _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names))) if len(general_options) != 0: _log.debug("General options for installation are: \n%s" % str(general_options)) else: From cf850e7a86412af8763708bddbf739bac6c0502b Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 9 Dec 2020 11:12:07 +0800 Subject: [PATCH 107/115] prepare release notes for EasyBuild v4.3.2 + bump version to 4.3.2 --- RELEASE_NOTES | 37 +++++++++++++++++++++++++++++++++++++ easybuild/tools/version.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index bd91b3a56c..c9ab2ab28d 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,6 +3,43 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.3.2 (December 10th 2020) +--------------------------- + +update/bugfix release + +- add (experimental) support for specifying easyconfig files via "easystack" file (#3479) +- add definition for new gobff toolchain using BLIS and LibFLAME (#3505) +- various enhancements, including: + - add support for toolchain options like 'extra_cxxflags' to specify extra compiler options (#2193) + - fix combination of --copy-ec and --from-pr (#3482) + - enhance copy_files: single file target, error on empty input list, verbose mode (#3483) + - cache result of fetch_files_from_pr function (#3484) + - add locate_files function to filetools module (#3485) + - add support for %(module_name)s template value (#3497) + - clarify input format for --cuda-compute-capabilities (#3509) +- various bug fixes, including: + - also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace (#3491) + - don't pass username in github_api_get_request when no GitHub token is available (#3494) + - also inject -rpath options for all entries in $LIBRARY_PATH in RPATH wrappers (#3495) + - avoid TypeError being raised by list_toolchains (#3499) + - check if PR is already merged in --merge-pr (#3502) + - graciously handle wrong PR # in fetch_pr_data (#3503) + - fix regression in apply_regex_substitutions: also accept list of paths to patch (#3507) + - update installation procedure for EasyBuild in generated Singularity container recipes (#3510) + - tweak test_easystack_basic to get better error reporting if it fails (#3511) + - fix error when --easystack is used without having PyYAML installed (#3515) + - also include *.yaml files from test/ dir in package (#3517) + - fix GitHub Actions workflow for test suite: run outside of repo checkout + also test bootstrap script with Python 3.9 (#3518) + - bump cryptography from 2.9.2 to 3.2 (#3519) + - take into account slightly different error with Python 2.6 in test_easystack_wrong_structure (#3520) + - use --dry-run in test_easystack_basic to avoid downloading of binutils sources (#3521) +- other changes: + - exclude test configuration with Lmod 7 and Python 3, except for Python 3.6 (#3496) + - significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed (#3498) + - don't include file/ldd/readelf commands run during RPATH sanity check in --trace output (#3508) + + v4.3.1 (October 29th 2020) -------------------------- diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index bd156b8d13..ccf5fd3bfd 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.2.dev0') +VERSION = LooseVersion('4.3.2') UNKNOWN = 'UNKNOWN' From c744a1d27032bdac46c6adb62650225557c5f93d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Dec 2020 11:29:56 +0100 Subject: [PATCH 108/115] Fix missing command output in test_step --- easybuild/framework/easyblock.py | 11 ++++++++--- easybuild/tools/config.py | 1 + easybuild/tools/options.py | 1 + test/framework/options.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 7e9a1d984b..5d43b050c4 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -2139,10 +2139,11 @@ def build_step(self): def test_step(self): """Run unit tests provided by software (if any).""" - if self.cfg['runtest']: + unit_test_cmd = self.cfg['runtest'] + if unit_test_cmd: - self.log.debug("Trying to execute %s as a command for running unit tests...") - (out, _) = run_cmd(self.cfg['runtest'], log_all=True, simple=False) + self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd) + (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False) return out @@ -3310,6 +3311,10 @@ def build_and_install_one(ecdict, init_env): _log.debug("Skip set to %s" % skip) app.cfg['skip'] = skip + if build_option('skip_test_step'): + _log.debug('Adding test_step to skipped steps') + app.cfg.update('skipsteps', TEST_STEP, allow_duplicate=False) + # build easyconfig errormsg = '(no error)' # timing info diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index da0e35ab4e..170040576e 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -253,6 +253,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'sequential', 'set_gid_bit', 'skip_test_cases', + 'skip_test_step', 'generate_devel_module', 'sticky_bit', 'trace', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 89663340e3..13dbe41a91 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -439,6 +439,7 @@ def override_options(self): 'silence-deprecation-warnings': ("Silence specified deprecation warnings", 'strlist', 'extend', None), 'sticky-bit': ("Set sticky bit on newly created directories", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), + 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), 'generate-devel-module': ("Generate a develop module file, implies --force if disabled", None, 'store_true', True), 'sysroot': ("Location root directory of system, prefix for standard paths like /usr/lib and /usr/include", diff --git a/test/framework/options.py b/test/framework/options.py index 7a30ea3bd5..81f773e527 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -340,6 +340,36 @@ def test_skip(self): self.assertEqual(len(glob.glob(toy_mod_glob)), 1) + def test_skip_test_step(self): + """Test skipping testing the build (--skip-test-step).""" + + topdir = os.path.abspath(os.path.dirname(__file__)) + toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + # check log message without --skip-test-step + args = [ + toy_ec, + '--extended-dry-run', + '--force', + '--debug', + ] + outtxt = self.eb_main(args, do_build=True) + found_msg = "Running method test_step part of step test" + found = re.search(found_msg, outtxt) + test_run_msg = "execute make_test dummy_cmd as a command for running unit tests" + self.assertTrue(found, "Message about test step being run is present, outtxt: %s" % outtxt) + found = re.search(test_run_msg, outtxt) + self.assertTrue(found, "Test execution command is present, outtxt: %s" % outtxt) + + # And now with the argument + args.append('--skip-test-step') + outtxt = self.eb_main(args, do_build=True) + found_msg = "Skipping test step" + found = re.search(found_msg, outtxt) + self.assertTrue(found, "Message about test step being skipped is present, outtxt: %s" % outtxt) + found = re.search(test_run_msg, outtxt) + self.assertFalse(found, "Test execution command is NOT present, outtxt: %s" % outtxt) + def test_job(self): """Test submitting build as a job.""" From b5b391e36a2efe18b94cecd6e989949da954414b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Dec 2020 11:30:24 +0100 Subject: [PATCH 109/115] Add --skip-test-step to skip the test_step --- .../test_ecs/t/toy/toy-0.0-test.eb | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb new file mode 100644 index 0000000000..90cc7429d3 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-test.eb @@ -0,0 +1,35 @@ +name = 'toy' +version = '0.0' +versionsuffix = '-test' + +homepage = 'https://easybuilders.github.io/easybuild' +description = "Toy C program, 100% toy." + +toolchain = SYSTEM + +sources = [SOURCE_TAR_GZ] +checksums = [[ + 'be662daa971a640e40be5c804d9d7d10', # default (MD5) + '44332000aa33b99ad1e00cbd1a7da769220d74647060a10e807b916d73ea27bc', # default (SHA256) + ('adler32', '0x998410035'), + ('crc32', '0x1553842328'), + ('md5', 'be662daa971a640e40be5c804d9d7d10'), + ('sha1', 'f618096c52244539d0e89867405f573fdb0b55b0'), + ('size', 273), +]] +patches = [ + 'toy-0.0_fix-silly-typo-in-printf-statement.patch', + ('toy-extra.txt', 'toy-0.0'), +] + +sanity_check_paths = { + 'files': [('bin/yot', 'bin/toy')], + 'dirs': ['bin'], +} + +runtest = "make_test dummy_cmd" # Provide some value which is unique enough to be checked for + +postinstallcmds = ["echo TOY > %(installdir)s/README"] + +moduleclass = 'tools' +# trailing comment, leave this here, it may trigger bugs with extract_comments() From b29968f8a5ed5fbded6deb695e942e4c4c1600c0 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Dec 2020 11:53:50 +0100 Subject: [PATCH 110/115] Avoid stdout when running eb_main --- test/framework/options.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 81f773e527..fc4500e282 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -353,7 +353,9 @@ def test_skip_test_step(self): '--force', '--debug', ] + self.mock_stdout(True) outtxt = self.eb_main(args, do_build=True) + self.mock_stdout(False) found_msg = "Running method test_step part of step test" found = re.search(found_msg, outtxt) test_run_msg = "execute make_test dummy_cmd as a command for running unit tests" @@ -363,7 +365,9 @@ def test_skip_test_step(self): # And now with the argument args.append('--skip-test-step') + self.mock_stdout(True) outtxt = self.eb_main(args, do_build=True) + self.mock_stdout(False) found_msg = "Skipping test step" found = re.search(found_msg, outtxt) self.assertTrue(found, "Message about test step being skipped is present, outtxt: %s" % outtxt) From bcafacd44dd7b802406432ba28a2189495f0ca05 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 9 Dec 2020 12:26:47 +0100 Subject: [PATCH 111/115] Update other tests for new test EC --- test/framework/docs.py | 4 ++-- test/framework/filetools.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/framework/docs.py b/test/framework/docs.py index f6716ea171..d99344f4c5 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -256,7 +256,7 @@ def test_list_software(self): " * toy v0.0 (versionsuffix: '-deps'): system", " * toy v0.0 (versionsuffix: '-iter'): system", " * toy v0.0 (versionsuffix: '-multiple'): system", - " * toy v0.0 (versionsuffix: '-test'): gompi/2018a", + " * toy v0.0 (versionsuffix: '-test'): gompi/2018a, system", ] txt = list_software(output_format='txt', detailed=True) lines = txt.split('\n') @@ -278,7 +278,7 @@ def test_list_software(self): '``0.0`` ``-deps`` ``system`` ', '``0.0`` ``-iter`` ``system`` ', '``0.0`` ``-multiple`` ``system`` ', - '``0.0`` ``-test`` ``gompi/2018a`` ', + '``0.0`` ``-test`` ``gompi/2018a``, ``system``', '======= ============= ===========================', ] txt = list_software(output_format='rst', detailed=True) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 44ae181c8a..7aabac47cd 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2013,7 +2013,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 82) + self.assertEqual(len(index), 83) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), From 33b38e596aed8b4468203cc92a0e2210ea187b98 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Dec 2020 13:01:45 +0100 Subject: [PATCH 112/115] minor tweaks to EasyBuild v4.3.2 release notes --- RELEASE_NOTES | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c9ab2ab28d..7e54d8e0ee 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -8,34 +8,30 @@ v4.3.2 (December 10th 2020) update/bugfix release -- add (experimental) support for specifying easyconfig files via "easystack" file (#3479) -- add definition for new gobff toolchain using BLIS and LibFLAME (#3505) +- add (experimental) support for specifying easyconfig files via an "easystack" file (#3479, #3511, #3515, #3517, #3520, #3521) + - see also https://easybuild.readthedocs.io/en/latest/Easystack-files.html +- add definition for new 'gobff' toolchain using BLIS and LibFLAME (#3505) - various enhancements, including: - add support for toolchain options like 'extra_cxxflags' to specify extra compiler options (#2193) - fix combination of --copy-ec and --from-pr (#3482) - - enhance copy_files: single file target, error on empty input list, verbose mode (#3483) - - cache result of fetch_files_from_pr function (#3484) + - enhance copy_files function: support single file target, error on empty input list, support verbose mode (#3483) + - cache result of fetch_files_from_pr function (mainly to speed up framework test suite) (#3484) - add locate_files function to filetools module (#3485) - add support for %(module_name)s template value (#3497) - - clarify input format for --cuda-compute-capabilities (#3509) + - clarify input format for --cuda-compute-capabilities in 'eb --help' output (#3509) - various bug fixes, including: - also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace (#3491) - don't pass username in github_api_get_request when no GitHub token is available (#3494) - also inject -rpath options for all entries in $LIBRARY_PATH in RPATH wrappers (#3495) - avoid TypeError being raised by list_toolchains (#3499) - check if PR is already merged in --merge-pr (#3502) - - graciously handle wrong PR # in fetch_pr_data (#3503) + - graciously handle wrong PR id in fetch_pr_data (#3503) - fix regression in apply_regex_substitutions: also accept list of paths to patch (#3507) - update installation procedure for EasyBuild in generated Singularity container recipes (#3510) - - tweak test_easystack_basic to get better error reporting if it fails (#3511) - - fix error when --easystack is used without having PyYAML installed (#3515) - - also include *.yaml files from test/ dir in package (#3517) - fix GitHub Actions workflow for test suite: run outside of repo checkout + also test bootstrap script with Python 3.9 (#3518) - - bump cryptography from 2.9.2 to 3.2 (#3519) - - take into account slightly different error with Python 2.6 in test_easystack_wrong_structure (#3520) - - use --dry-run in test_easystack_basic to avoid downloading of binutils sources (#3521) + - bump cryptography from 2.9.2 to 3.2 for Python 2 in requirements.txt (#3519) - other changes: - - exclude test configuration with Lmod 7 and Python 3, except for Python 3.6 (#3496) + - exclude test configurations with Lmod 7 and Python 3, except for Python 3.6 (#3496) - significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed (#3498) - don't include file/ldd/readelf commands run during RPATH sanity check in --trace output (#3508) From d1fb2dad5a1e88614ab07f7d4dda3618af9b08a4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Dec 2020 13:09:19 +0100 Subject: [PATCH 113/115] include #3524 in release notes for EasyBuild v4.3.2 --- RELEASE_NOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 7e54d8e0ee..d2351046b8 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -19,6 +19,7 @@ update/bugfix release - add locate_files function to filetools module (#3485) - add support for %(module_name)s template value (#3497) - clarify input format for --cuda-compute-capabilities in 'eb --help' output (#3509) + - add support for skiping unit tests (test step) via --skip-test-step (#3524) - various bug fixes, including: - also ignore vsc.* imports coming from from pkg_resources/__init__.py (setuptools) in fake vsc namespace (#3491) - don't pass username in github_api_get_request when no GitHub token is available (#3494) From 0639ea42fa0d981bb1cee20f001594d7a5eb961e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Dec 2020 15:27:43 +0100 Subject: [PATCH 114/115] fix 'eb --help=rst' when running with Python 3 --- easybuild/tools/utilities.py | 4 ++++ test/framework/options.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 886c39d055..931a6e1eac 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -258,6 +258,10 @@ def mk_rst_table(titles, columns): """ Returns an rst table with given titles and columns (a nested list of string columns for each column) """ + # take into account that passed values may be iterators produced via 'map' + titles = list(titles) + columns = list(columns) + title_cnt, col_cnt = len(titles), len(columns) if title_cnt != col_cnt: msg = "Number of titles/columns should be equal, found %d titles and %d columns" % (title_cnt, col_cnt) diff --git a/test/framework/options.py b/test/framework/options.py index fc4500e282..c3bbd35813 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -174,6 +174,26 @@ def test_help_long(self): regex = re.compile("default: True; disable with --disable-cleanup-builddir", re.M) self.assertTrue(regex.search(outtxt), "Pattern '%s' found in: %s" % (regex.pattern, outtxt)) + def test_help_rst(self): + """Test generating --help in RST output format.""" + + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(['--help=rst'], raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertFalse(stderr) + + patterns = [ + r"^Basic options\n-------------", + r"^``--fetch``[ ]*Allow downloading sources", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_no_args(self): """Test using no arguments.""" From 5764650e86d38ba67149db0b96f6ccb1369aee89 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 9 Dec 2020 15:29:52 +0100 Subject: [PATCH 115/115] also include #3525 in release notes for EasyBuild v4.3.2 --- RELEASE_NOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index d2351046b8..10cf597707 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -31,6 +31,7 @@ update/bugfix release - update installation procedure for EasyBuild in generated Singularity container recipes (#3510) - fix GitHub Actions workflow for test suite: run outside of repo checkout + also test bootstrap script with Python 3.9 (#3518) - bump cryptography from 2.9.2 to 3.2 for Python 2 in requirements.txt (#3519) + - fix 'eb --help=rst' when running with Python 3 (#3525) - other changes: - exclude test configurations with Lmod 7 and Python 3, except for Python 3.6 (#3496) - significantly speed up parsing of easyconfig files by only extracting comments from an easyconfig file when they're actually needed (#3498)