From 399bd25929950eb27d22f16badd097e94e49837e Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 19 Jul 2024 15:28:55 -0700 Subject: [PATCH 01/28] Replaced report_results.py with run_ats.py which runs ATS, checks for failures, reruns it if necessary, and reports the results --- .gitlab/scripts.yml | 3 +- scripts/gitlab/report_results.py | 12 ---- scripts/gitlab/run_ats.py | 104 +++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 14 deletions(-) delete mode 100755 scripts/gitlab/report_results.py create mode 100755 scripts/gitlab/run_ats.py diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 7dea9688e..6d2b63335 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -33,9 +33,8 @@ - cd $CI_BUILD_DIR && cat job-name.txt - cat build_gitlab/install/spheral-lcatstest - - $TEST_ALLOC ./build_gitlab/install/spheral-lcatstest --logs test-logs build_gitlab/install/$ATS_FILE --timelimit="45m" + - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc $TEST_ALLOC --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR - cp -r test-logs $CI_PROJECT_DIR - - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/report_results.py artifacts: when: always paths: diff --git a/scripts/gitlab/report_results.py b/scripts/gitlab/report_results.py deleted file mode 100755 index b7cb463e2..000000000 --- a/scripts/gitlab/report_results.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys - -exec(compile(open("test-logs/atsr.py").read(), "test-logs/atsr.py", 'exec')) -failed_tests = [t for t in state['testlist'] if t['status'] in [FAILED,TIMEDOUT] ] -if len(failed_tests) > 0: - print(("ATS failed {0} tests.".format(len(failed_tests)))) - for t in failed_tests: - print(t['name']) - sys.exit(1) -else: - print("ATS passed all tests.") -sys.exit(0) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py new file mode 100755 index 000000000..bad1783e1 --- /dev/null +++ b/scripts/gitlab/run_ats.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +import sys, subprocess, argparse + +# If the number of failed tests exceeds this value, ATS is not rerun +max_test_failures = 10 +# Number of times to rerun the ATS tests +max_reruns = 2 + +# Helper function for executing commands stolen from uberenv +def sexe(cmd,ret_output=False,echo=True): + """ Helper for executing shell commands. """ + if echo: + print("[exe: {0}]".format(cmd)) + p = subprocess.run(cmd, shell=True, + capture_output=ret_output, + check=True, text=True) + if ret_output: + if echo: + print(p.stdout) + return p.stdout + +#------------------------------------------------------------------------------ + +def parse_args(): + parser = argparse.ArgumentParser() + + # Spec args + parser.add_argument('--test-alloc', type=str, + help='Allocation command for the machine.') + parser.add_argument('--ats-file', type=str, + help='ATS test file to run.') + parser.add_argument('--ci-build-dir', type=str, + help='CI build directory.') + return parser.parse_args() + +#------------------------------------------------------------------------------ + +# Run ats.py to check results and return the number of failed tests +def report_results(output_dir): + ats_py = os.path.join(output_dir, "atsr.py") + if (not os.path.exists(ats_py)): + print(f"{ats_py} does not exists") + sys.exit(1) + exec(compile(open(ats_py).read(), ats_py, 'exec')) + failed_tests = [t for t in state['testlist'] if t['status'] in [FAILED,TIMEDOUT] ] + if len(failed_tests) > 0: + print(f"ATS failed {len(failed_tests)} tests.") + for t in failed_tests: + print(t['name']) + return len(failed_tests) + else: + print("ATS passed all tests.") + return 0 + +#------------------------------------------------------------------------------ + +def run_and_report(run_command, ci_output, num_runs=0): + if (num_runs > max_reruns): + print("Exceeded number of ATS reruns") + sys.exit(1) + sexe(run_command) + tests_passed = report_results(ci_output) + if (tests_passed == 0): + sys.exit(0) + elif (test_passed >= max_test_failures): + print("Too many test failures, not rerunning ATS") + sys.exit(1) + else: + rerun_command = run_command + if (num_runs == 0): + ats_cont_file = os.path.join(ci_output, "continue.ats") + if (not os.path.exists(ats_cont_file)): + print(f"{ats_cont_file} not found, ATS cannot be rerun") + sys.exit(1) + rerun_command = f"{run_command} {ats_cont_file}" + run_and_report(rerun_command, ci_output, num_runs + 1) + +#------------------------------------------------------------------------------ + +def run_ats_test(args): + build_gl_dir = os.path.join(args.ci_build_dir, "build_gitlab", "install") + ats_file = os.path.join(build_gl_dir, args.ats_file) + if (not os.path.exists(ats_file)): + print(f"{ats_file} does not exists") + sys.exit(1) + lcats_test = os.path.join(build_gl_dir, "spheral-lcatstest") + if (not os.path.exists(lcats_test)): + print(f"{lcats_test} does not exists") + sys.exit(1) + ats_configs = ' --timelimit="45m"' + run_command = f"{args.test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_config}" + ci_output = os.path.join(args.ci_build_dir, "test-logs") + run_and_report(run_command, ci_output) + +#------------------------------------------------------------------------------ + +def main(): + args = parse_args() + run_ats_test(args) + + +if __name__ == "__main__": + main() From b13da8cc857f31d6c9f0869b0febc6d38d50caf6 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 19 Jul 2024 16:16:53 -0700 Subject: [PATCH 02/28] Fix bug with test-alloc input to run_ats.py --- scripts/gitlab/run_ats.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index bad1783e1..84435f6fe 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -26,7 +26,7 @@ def parse_args(): parser = argparse.ArgumentParser() # Spec args - parser.add_argument('--test-alloc', type=str, + parser.add_argument('--test-alloc', type=str, nargs="+", help='Allocation command for the machine.') parser.add_argument('--ats-file', type=str, help='ATS test file to run.') @@ -55,6 +55,7 @@ def report_results(output_dir): #------------------------------------------------------------------------------ +# Run the tests and check if any failed def run_and_report(run_command, ci_output, num_runs=0): if (num_runs > max_reruns): print("Exceeded number of ATS reruns") @@ -89,7 +90,8 @@ def run_ats_test(args): print(f"{lcats_test} does not exists") sys.exit(1) ats_configs = ' --timelimit="45m"' - run_command = f"{args.test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_config}" + test_alloc = " ".join(args.test_alloc) + run_command = f"{test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_config}" ci_output = os.path.join(args.ci_build_dir, "test-logs") run_and_report(run_command, ci_output) From 8c16f5a0a09f12d68f97d0ac0dee9eed6663bf71 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 19 Jul 2024 16:45:52 -0700 Subject: [PATCH 03/28] Added quotes around TEST_ALLOC input --- .gitlab/scripts.yml | 2 +- scripts/gitlab/run_ats.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 6d2b63335..6154e8a2a 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -33,7 +33,7 @@ - cd $CI_BUILD_DIR && cat job-name.txt - cat build_gitlab/install/spheral-lcatstest - - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc $TEST_ALLOC --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR + - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR - cp -r test-logs $CI_PROJECT_DIR artifacts: when: always diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 84435f6fe..49d454edd 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -5,7 +5,7 @@ # If the number of failed tests exceeds this value, ATS is not rerun max_test_failures = 10 # Number of times to rerun the ATS tests -max_reruns = 2 +max_reruns = 1 # Helper function for executing commands stolen from uberenv def sexe(cmd,ret_output=False,echo=True): From 626ccd68ab16a6e6bb740721dd2d8b26e42e54aa Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 22 Jul 2024 07:57:08 -0700 Subject: [PATCH 04/28] Bring in os module --- scripts/gitlab/run_ats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 49d454edd..4a05b710c 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import sys, subprocess, argparse +import sys, subprocess, argparse, os # If the number of failed tests exceeds this value, ATS is not rerun max_test_failures = 10 From d0d005b1dc68903d4dac4f24c61384ced9f650ec Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 22 Jul 2024 09:29:25 -0700 Subject: [PATCH 05/28] Fix variable name in run_ats --- scripts/gitlab/run_ats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 4a05b710c..95ab88946 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -56,7 +56,7 @@ def report_results(output_dir): #------------------------------------------------------------------------------ # Run the tests and check if any failed -def run_and_report(run_command, ci_output, num_runs=0): +def run_and_report(run_command, ci_output, num_runs): if (num_runs > max_reruns): print("Exceeded number of ATS reruns") sys.exit(1) @@ -81,7 +81,7 @@ def run_and_report(run_command, ci_output, num_runs=0): def run_ats_test(args): build_gl_dir = os.path.join(args.ci_build_dir, "build_gitlab", "install") - ats_file = os.path.join(build_gl_dir, args.ats_file) + ats_file = os.path.join(build_gl_dir, args.ats_file) if (not os.path.exists(ats_file)): print(f"{ats_file} does not exists") sys.exit(1) @@ -91,9 +91,9 @@ def run_ats_test(args): sys.exit(1) ats_configs = ' --timelimit="45m"' test_alloc = " ".join(args.test_alloc) - run_command = f"{test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_config}" + run_command = f"{test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_configs}" ci_output = os.path.join(args.ci_build_dir, "test-logs") - run_and_report(run_command, ci_output) + run_and_report(run_command, ci_output, 0) #------------------------------------------------------------------------------ From c98979658fb87d88fbeefadd26a34d47fb8f740d Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 22 Jul 2024 12:46:14 -0700 Subject: [PATCH 06/28] Fix issue that prevented code from accessing global variables from atsr.py script execution --- scripts/gitlab/run_ats.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 95ab88946..3e60657ab 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -32,6 +32,10 @@ def parse_args(): help='ATS test file to run.') parser.add_argument('--ci-build-dir', type=str, help='CI build directory.') + parser.add_argument('--ci-install-dir', type=str, + default="build_gitlab/install", + help="Location of Spheral installation "+\ + "relative to --ci-build-dir") return parser.parse_args() #------------------------------------------------------------------------------ @@ -42,7 +46,8 @@ def report_results(output_dir): if (not os.path.exists(ats_py)): print(f"{ats_py} does not exists") sys.exit(1) - exec(compile(open(ats_py).read(), ats_py, 'exec')) + exec(compile(open(ats_py).read(), ats_py, 'exec'), globals()) + state = globals()["state"] failed_tests = [t for t in state['testlist'] if t['status'] in [FAILED,TIMEDOUT] ] if len(failed_tests) > 0: print(f"ATS failed {len(failed_tests)} tests.") @@ -80,7 +85,7 @@ def run_and_report(run_command, ci_output, num_runs): #------------------------------------------------------------------------------ def run_ats_test(args): - build_gl_dir = os.path.join(args.ci_build_dir, "build_gitlab", "install") + build_gl_dir = os.path.join(args.ci_build_dir, args.ci_install_dir) ats_file = os.path.join(build_gl_dir, args.ats_file) if (not os.path.exists(ats_file)): print(f"{ats_file} does not exists") From 7313119d49d0323c1ec482e829ec34d3770b0501 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 22 Jul 2024 14:04:41 -0700 Subject: [PATCH 07/28] Fixed dumb typo --- scripts/gitlab/run_ats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 3e60657ab..ea1054895 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -69,7 +69,7 @@ def run_and_report(run_command, ci_output, num_runs): tests_passed = report_results(ci_output) if (tests_passed == 0): sys.exit(0) - elif (test_passed >= max_test_failures): + elif (tests_passed >= max_test_failures): print("Too many test failures, not rerunning ATS") sys.exit(1) else: From f219637891b85ec77ae77b3264b20f941c84892a Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Mon, 22 Jul 2024 15:30:56 -0700 Subject: [PATCH 08/28] Add exit codes for warning when tests are rerun --- .gitlab/scripts.yml | 3 +++ scripts/gitlab/run_ats.py | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 6154e8a2a..dd937b51c 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -40,6 +40,9 @@ paths: - ci-dir.txt - test-logs/ + allow_failure: + exit_codes: + - 80 # ------------------------------------------------------------------------------ # Shared TPL scripts. diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index ea1054895..3fa107ab4 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -68,7 +68,10 @@ def run_and_report(run_command, ci_output, num_runs): sexe(run_command) tests_passed = report_results(ci_output) if (tests_passed == 0): - sys.exit(0) + if (num_runs == 0): + sys.exit(0) + else: + sys.exit(80) elif (tests_passed >= max_test_failures): print("Too many test failures, not rerunning ATS") sys.exit(1) @@ -80,6 +83,7 @@ def run_and_report(run_command, ci_output, num_runs): print(f"{ats_cont_file} not found, ATS cannot be rerun") sys.exit(1) rerun_command = f"{run_command} {ats_cont_file}" + print("Rerunning ATS") run_and_report(rerun_command, ci_output, num_runs + 1) #------------------------------------------------------------------------------ @@ -93,13 +97,13 @@ def run_ats_test(args): lcats_test = os.path.join(build_gl_dir, "spheral-lcatstest") if (not os.path.exists(lcats_test)): print(f"{lcats_test} does not exists") - sys.exit(1) + sys.exit(1) ats_configs = ' --timelimit="45m"' test_alloc = " ".join(args.test_alloc) run_command = f"{test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_configs}" ci_output = os.path.join(args.ci_build_dir, "test-logs") run_and_report(run_command, ci_output, 0) - + #------------------------------------------------------------------------------ def main(): From 153f8811000671743410ad6908af58051a4321bd Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 23 Jul 2024 09:24:16 -0700 Subject: [PATCH 09/28] Fix for returning exit codes properly --- .gitlab/scripts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 57390bcd6..90eab7ecc 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -46,7 +46,9 @@ - cat build_gitlab/install/spheral-lcatstest - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR + - exit_code=$? - cp -r test-logs $CI_PROJECT_DIR + - exit $exit_code artifacts: when: always paths: From 72af6bb56de62e8abbc50c80cccfa7cac2c5cdbe Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Tue, 23 Jul 2024 16:54:35 -0700 Subject: [PATCH 10/28] Updated release notes --- RELEASE_NOTES.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index fcf107d28..e15615d14 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -13,12 +13,13 @@ Notable changes include: * RAJA & Umpire added as first level dependencies. * Axom updated to v0.9.0. * TPL builds have been split off into a separate Gitlab CI stage to help with timeouts on allocations. + * Failed ATS runs are automatically retested once in the Gitlab CI. * Build changes / improvements: - * Distributed source directory must always be built now + * Distributed source directory must always be built now. * Bug Fixes / improvements: - * Wrappers for MPI calls are simplified and improved + * Wrappers for MPI calls are simplified and improved. Version v2024.06.1 -- Release date 2024-07-09 ============================================== From aa89eb91f3c35bf684a848e823e56fe3398047eb Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 09:12:44 -0700 Subject: [PATCH 11/28] Simplified gitlab ci temporarily for faster testing, cleaned up lcats to reduce amount of unnecessary printed information --- .gitlab-ci.yml | 14 ++-- .gitlab/jobs-mpi.yml | 38 +++++----- .gitlab/jobs-prod.yml | 56 +++++++-------- .gitlab/jobs-seq.yml | 52 +++++++------- .gitlab/scripts.yml | 4 +- scripts/gitlab/run_ats.py | 5 +- scripts/lc/lcats | 143 +------------------------------------- 7 files changed, 87 insertions(+), 225 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f1810f0f..2204d748d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,14 +16,14 @@ variables: - SPHERAL_REV_STR="$SPHERAL_REV" stages: - - tpls - - build_and_install +# - tpls +# - build_and_install - run_ats - - update_tpls - - generate_buildcache - - install_production - - update_permissions - - cleanup +# - update_tpls +# - generate_buildcache +# - install_production +# - update_permissions +# - cleanup include: - project: lc-templates/id_tokens diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index d187c0953..7355581c4 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -1,38 +1,38 @@ # ------------------------------------------------------------------------------ # BUILD JOBS -toss_gcc_mvapich2_cxxonly_tpls: - extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] +# toss_gcc_mvapich2_cxxonly_tpls: +# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] -toss_gcc_mvapich2_cxxonly_build: - extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] - needs: [toss_gcc_mvapich2_cxxonly_tpls] +# toss_gcc_mvapich2_cxxonly_build: +# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] +# needs: [toss_gcc_mvapich2_cxxonly_tpls] -toss_gcc_mvapich2_tpls: - extends: [.toss_resource2, .gcc_mvapich2, .tpls] +# toss_gcc_mvapich2_tpls: +# extends: [.toss_resource2, .gcc_mvapich2, .tpls] -toss_gcc_mvapich2_build: - extends: [.toss_resource2, .gcc_mvapich2, .build_and_test] - needs: [toss_gcc_mvapich2_tpls] +# toss_gcc_mvapich2_build: +# extends: [.toss_resource2, .gcc_mvapich2, .build_and_test] +# needs: [toss_gcc_mvapich2_tpls] toss_gcc_mvapich2_test: extends: [.toss_resource2, .gcc_mvapich2, .run_ats] - needs: [toss_gcc_mvapich2_build] + #needs: [toss_gcc_mvapich2_build] -toss_clang_mvapich2_tpls: - extends: [.toss_resource1, .clang_mvapich2, .tpls] +# toss_clang_mvapich2_tpls: +# extends: [.toss_resource1, .clang_mvapich2, .tpls] -toss_clang_mvapich2_build: - extends: [.toss_resource1, .clang_mvapich2, .build_and_test] - needs: [toss_clang_mvapich2_tpls] +# toss_clang_mvapich2_build: +# extends: [.toss_resource1, .clang_mvapich2, .build_and_test] +# needs: [toss_clang_mvapich2_tpls] -toss_clang_mvapich2_test: - extends: [.toss_resource2, .clang_mvapich2, .run_ats] - needs: [toss_clang_mvapich2_build] +# toss_clang_mvapich2_test: +# extends: [.toss_resource2, .clang_mvapich2, .run_ats] +# needs: [toss_clang_mvapich2_build] diff --git a/.gitlab/jobs-prod.yml b/.gitlab/jobs-prod.yml index 488ac824e..4443d4dee 100644 --- a/.gitlab/jobs-prod.yml +++ b/.gitlab/jobs-prod.yml @@ -1,39 +1,39 @@ -# ------------------------------------------------------------------------------ -# UPDATE TPL JOBS +# # ------------------------------------------------------------------------------ +# # UPDATE TPL JOBS -toss_update_tpls: - extends: [.toss_resource2, .update_tpls, .merge_pr_rule] +# toss_update_tpls: +# extends: [.toss_resource2, .update_tpls, .merge_pr_rule] -blueos_update_tpls: - extends: [.blueos_resource2, .update_tpls, .merge_pr_rule] - needs: [toss_update_tpls] +# blueos_update_tpls: +# extends: [.blueos_resource2, .update_tpls, .merge_pr_rule] +# needs: [toss_update_tpls] -# ------------------------------------------------------------------------------ -# UPDATE PERM JOBS +# # ------------------------------------------------------------------------------ +# # UPDATE PERM JOBS -shared_tpls_update_permissions: - extends: [.toss_resource_general, .toss_update_permissions, .merge_pr_rule] +# shared_tpls_update_permissions: +# extends: [.toss_resource_general, .toss_update_permissions, .merge_pr_rule] -# ------------------------------------------------------------------------------ -# PROD BUILD JOBS +# # ------------------------------------------------------------------------------ +# # PROD BUILD JOBS -### TAG RELEASE ### -toss_build_dev_pkg_release: - extends: [.toss_resource_general, .gcc_mvapich2, .build_dev_pkg, .tag_release_rule] +# ### TAG RELEASE ### +# toss_build_dev_pkg_release: +# extends: [.toss_resource_general, .gcc_mvapich2, .build_dev_pkg, .tag_release_rule] -toss_install_dev_pkg_release: - extends: [.toss_resource_general, .gcc_mvapich2, .install_dev_pkg, .tag_release_rule] - needs: [toss_build_dev_pkg_release] +# toss_install_dev_pkg_release: +# extends: [.toss_resource_general, .gcc_mvapich2, .install_dev_pkg, .tag_release_rule] +# needs: [toss_build_dev_pkg_release] -toss_release_permissions: - variables: - ALIAS: $CI_COMMIT_TAG - extends: [.toss_resource_general, .prod_permissions, .tag_release_rule] - needs: [toss_install_dev_pkg_release] +# toss_release_permissions: +# variables: +# ALIAS: $CI_COMMIT_TAG +# extends: [.toss_resource_general, .prod_permissions, .tag_release_rule] +# needs: [toss_install_dev_pkg_release] -# ------------------------------------------------------------------------------ -# CLEAN OLD BUILD DIRS +# # ------------------------------------------------------------------------------ +# # CLEAN OLD BUILD DIRS -cleanup_build_dirs: - extends: [.clean_dirs] +# cleanup_build_dirs: +# extends: [.clean_dirs] diff --git a/.gitlab/jobs-seq.yml b/.gitlab/jobs-seq.yml index 34847a7b9..78cd3b787 100644 --- a/.gitlab/jobs-seq.yml +++ b/.gitlab/jobs-seq.yml @@ -1,37 +1,37 @@ -# ------------------------------------------------------------------------------ -# BUILD JOBS +# # ------------------------------------------------------------------------------ +# # BUILD JOBS -toss_gcc_~mpi_tpls: - extends: [.gcc_~mpi, .tpls, .toss_resource1] +# toss_gcc_~mpi_tpls: +# extends: [.gcc_~mpi, .tpls, .toss_resource1] -toss_gcc_~mpi_build: - extends: [.gcc_~mpi, .build_and_test, .toss_resource1] - needs: [toss_gcc_~mpi_tpls] +# toss_gcc_~mpi_build: +# extends: [.gcc_~mpi, .build_and_test, .toss_resource1] +# needs: [toss_gcc_~mpi_tpls] -toss_gcc_~mpi_test: - extends: [.gcc_~mpi, .run_ats, .toss_resource1] - needs: [toss_gcc_~mpi_build] +# toss_gcc_~mpi_test: +# extends: [.gcc_~mpi, .run_ats, .toss_resource1] +# needs: [toss_gcc_~mpi_build] -blueos_cuda_11_gcc_~mpi_tpls: - extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls] +# blueos_cuda_11_gcc_~mpi_tpls: +# extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls] -blueos_cuda_11_gcc_~mpi_build: - extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test] - needs: [blueos_cuda_11_gcc_~mpi_tpls] +# blueos_cuda_11_gcc_~mpi_build: +# extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test] +# needs: [blueos_cuda_11_gcc_~mpi_tpls] -blueos_cuda_11_gcc_~mpi_test: - extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats] - needs: [blueos_cuda_11_gcc_~mpi_build] +# blueos_cuda_11_gcc_~mpi_test: +# extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats] +# needs: [blueos_cuda_11_gcc_~mpi_build] -blueos_gcc_~mpi_Debug_tpls: - extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls] +# blueos_gcc_~mpi_Debug_tpls: +# extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls] -blueos_gcc_~mpi_Debug_build: - extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test] - needs: [blueos_gcc_~mpi_Debug_tpls] +# blueos_gcc_~mpi_Debug_build: +# extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test] +# needs: [blueos_gcc_~mpi_Debug_tpls] -blueos_gcc_~mpi_Debug_test: - extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats] - needs: [blueos_gcc_~mpi_Debug_build] +# blueos_gcc_~mpi_Debug_test: +# extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats] +# needs: [blueos_gcc_~mpi_Debug_build] diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 90eab7ecc..331e56a40 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -41,12 +41,12 @@ .run_ats: stage: run_ats script: + - set +e - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt - cat build_gitlab/install/spheral-lcatstest - - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR - - exit_code=$? + - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code = $? - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code artifacts: diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 3fa107ab4..f605c5a6b 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -107,8 +107,9 @@ def run_ats_test(args): #------------------------------------------------------------------------------ def main(): - args = parse_args() - run_ats_test(args) + sys.exit(80) + args = parse_args() + run_ats_test(args) if __name__ == "__main__": diff --git a/scripts/lc/lcats b/scripts/lc/lcats index f6d445732..64fdd06fa 100755 --- a/scripts/lc/lcats +++ b/scripts/lc/lcats @@ -69,7 +69,6 @@ def createBsubFile(inCommand, inAllOptions): FILE.write("\n\n") FILE.write("setenv MACHINE_TYPE " + machineSettings.options.machineType + '\n') FILE.write("setenv SYS_TYPE " + SYS_TYPE + '\n') - FILE.write("setenv UNIQUE_KULL_TEST_SUBDIR " + uniqueKullSubdir + '\n') FILE.write(""+ '\n') FILE.write("date"+ '\n') @@ -119,7 +118,6 @@ def createMsubFile(inCommand, inAllOptions): FILE.write("setenv MACHINE_TYPE " + machineSettings.options.machineType + '\n') FILE.write("setenv SYS_TYPE " + SYS_TYPE + '\n') - FILE.write("setenv UNIQUE_KULL_TEST_SUBDIR " + uniqueKullSubdir + '\n') FILE.write(""+ '\n') FILE.write("date"+ '\n') @@ -166,7 +164,6 @@ def createSbatchFile(inCommand, inAllOptions): # LLNL specific FILE.write("setenv MACHINE_TYPE " + machineSettings.options.machineType + '\n') FILE.write("setenv SYS_TYPE " + SYS_TYPE + '\n') - FILE.write("setenv UNIQUE_KULL_TEST_SUBDIR " + uniqueKullSubdir + '\n') FILE.write(""+ '\n') FILE.write("date"+ '\n') @@ -619,17 +616,6 @@ msubFilenameDefault= "tmpAts." + ezatsStartTime + ".job" bsubFilenameDefault= "tmpAts." + ezatsStartTime + ".job" -# All test output should be placed in the KullTest -kullTestSubDir = "KullTest" - -# unique test string combines system, time and uuid -# the whole uuid4 is overkill, just take first 8 characters -from uuid import uuid4 -uniqueSubDir = SYS_TYPE + "_" + ezatsStartTime + "_" + str(uuid4())[0:8] - -uniqueKullSubdir = os.path.join(kullTestSubDir, uniqueSubDir) - - #--------------------------------------------------------------------------- # options affecting machine settings #--------------------------------------------------------------------------- @@ -698,7 +684,6 @@ parser.add_option( "--testpath", action="store", type="string", dest="testpath", parser.add_option( "--debug-build", action="store_true", dest="debugbuild", default=False, help="assume we are testing a debug build and should skip expensive (level>=100) tests.") -#parser.disable_interspersed_args() # doesn't work for this "atsWrap -b -e bin/kull test.ats " (options, args) = parser.parse_args(sys.argv[:]) # If running in SLURM, use defaults of less nodes and pdebug partition @@ -757,20 +742,8 @@ if "--help" in atsArgs or "-h" in atsArgs or "-help" in atsArgs: sys.exit(0) -#atsArgs= string.join(atsArgs) # note: lose user-intended grouping here -tmpstring=" " # moving to python3 I had to work around the oneliner above. -print(type(tmpstring)) # normally you don't need a tmp for things like this... -print("type of atsArgs var") -print(type(atsArgs)) -atsArgsStr= tmpstring.join(atsArgs) # note: lose user-intended grouping here -print(atsArgsStr) #Should be the joined list of args (space with args) -print("value of atsArgs var before assignment of atsArgsStr") -print(atsArgs) # should be the unmodified list -atsArgs=atsArgsStr -print("type of atsArgs var before assignment of atsArgsStr") -print(type(atsArgs)) -print("value of atsArgs var after assignment of atsArgsStr") -print(atsArgs) # should be the unmodified list as string +# Convert array of strings to a string with spaces for delimiters +atsArgs = " ".join(str(x) for x in atsArgs) #--------------------------------------------------------------------------- # Added this section to allow ezats to determine an appropriate filesystem @@ -826,92 +799,6 @@ def checkFileSystem(path, timeout=30): return timeoutFunction( timeout, False, canWriteToFileSystem, path ) - -def getUniqueTestPath(): - - # Helper function that verifies a temporary file can be created on file system. - # Will also catch if file system hung. - - filesystems = {} - - # SCF LLNL machines - # BG/Q -- seq entry must be before the rzuseq entry, or rzuseq will match 'seq' and try to use 'lscratch1'. - for name in ['seq']: - filesystems[name] = ['/p/lscratch1'] - - for name in ['zin', 'max']: - filesystems[name] = ['/p/lscratch2', '/p/lustre1', '/p/lscratch1'] - - for name in ['jade', 'agate', 'mica', 'magma', 'ruby']: - filesystems[name] = ['/p/lustre2', '/p/lustre1'] - - for name in ['sierra', 'tron']: - filesystems[name] = [ '/p/gpfs1' ] - - # RZ LLNL machines - for name in ['rzgenie']: - filesystems[name] = [ '/p/lustre1' ] - - for name in ['rztopaz']: - filesystems[name] = [ '/p/lustre1' ] - - for name in ['rzansel', 'lassen']: - filesystems[name] = [ '/p/gpfs1' ] - - for name in ['rzwhippet']: - filesystems[name] = [ '/p/lustre1' ] - - # Don't use NFS filesystems with BG/Q, terrible latency issues. - - # Sandia machines - filesystems['chama'] = ['/fscratch','/gscratch'] - filesystems['glory'] = ['/fscratch','/gscratch'] - filesystems['uno'] = ['/fscratch','/gscratch'] - - # Cielo - filesystems['nid'] = ['/scratch5', '/scratch4'] - - # Check which file system to use for this machine. - from platform import node - nodeName = node() - - assert 'USER' in os.environ, "ezats: No environment variable 'USER' found." - - filesystem = None - for name in list(filesystems.keys()): - if name in nodeName: - filesystem = filesystems[name] - assert filesystem, "ezats: Could not find file system entry for '%s'" % nodeName - - print("ezats: Searching for a filesystem to use...") - print("ezats: If this process hangs, you need to manually specify a file system to use via: ezats --testpath=\"/my/output/path\"") - - # Try to access file system, if no response within 4 seconds, try next file system. - fileSystemOK = False - - for entry in filesystem: - thePath = os.path.join( entry, os.environ['USER'], kullTestSubDir, uniqueSubDir ) - print("Checking filesystem path ", thePath) - fileSystemOK = checkFileSystem( thePath ) - if not fileSystemOK: - print("ezats: Could not write to filesystem location '%s', trying next file system." % thePath) - else: - fileSystemOK = True - filesystem = thePath - break - - assert fileSystemOK, "ezats: Could not locate a write accessable file system, aborting..." - - print("Found and verified file system location: %s" % filesystem) - - return filesystem - -if options.testpath: - options.testpath = os.path.abspath(options.testpath) - assert checkFileSystem(options.testpath), "ezats: Unable to create/access test path location: %s" % options.testpath -else: - options.testpath = getUniqueTestPath() - #--------------------------------------------------------------------------- #---------------------------------------------------------- @@ -920,16 +807,6 @@ else: print("Note: the srun message 'error: ioctl(TIOCGWINSZ)' can be ignored. \n[It means the process is trying to do something that requires a tty \nbut it's not doing either a read or write.]\n") -#---------------------------------------------------------- -# if batch, add allInteractive flag -#----------------------------------------------------------- - -#if os.environ.has_key('MACHINE_TYPE'): -# print "Note: setenv MACHINE_TYPE ", os.environ['MACHINE_TYPE'] -#if os.environ.has_key('BATCH_TYPE'): -# print "Note: setenv BATCH_TYPE ", os.environ['BATCH_TYPE'] -#print - #---------------------------------------------------------- # get args to add - added threaded option to the ezatsArgs or it would be passed to ats #---------------------------------------------------------- @@ -1079,12 +956,9 @@ else: if machineSettings.options.name in ['rzwhippet_flux']: os.environ["MACHINE_TYPE"] = "flux00" os.environ["BATCH_TYPE"] = "None" - os.environ["UNIQUE_KULL_TEST_SUBDIR"] = uniqueKullSubdir if platform.processor() == 'ppc64': numProcsLine = "" - # informs srun-wrapper.sh to not use kull's forkserver as ATS will run it - os.environ["KULLINATS"] = "yes" else: numProcsLine = " -n %d" % ( machineSettings.options.numNodes* cpu_count() ) @@ -1100,8 +974,6 @@ else: if machineSettings.options.name in ['chama', 'glory']: HERT_WC_ID = ' --account=' + machineSettings.options.wcid - print(finalCommandToRun) - if machineSettings.options.name in ['rzwhippet_flux']: finalCommandToRun= "flux alloc --exclusive " \ + " " + allocTime \ @@ -1111,7 +983,6 @@ else: + numProcsLine + " " \ + finalCommandToRun # + " -p " + machineSettings.options.partition + " " - print (finalCommandToRun) # Threaded tests under ats should NOT use salloc elif not options.threaded and 'blue' not in os.environ['SYS_TYPE']: finalCommandToRun= "salloc --exclusive " \ @@ -1124,21 +995,12 @@ else: + finalCommandToRun else: finalCommandToRun += " --numNodes="+ str(machineSettings.options.numNodes) - - print(finalCommandToRun) #sys.exit() if (d_debug==1): print("whichAts= ", whichAts) print("finalCommandToRun after= ", finalCommandToRun) - -#---------------------------------------------------------- - -if (d_debug==1): - print("whichAts= ", whichAts) - print("finalCommandToRun after= ", finalCommandToRun) - #---------------------------------------------------------- # Find filter part and keep whole # @@ -1170,7 +1032,6 @@ if (d_debug==1): #os.system(finalCommandToRun) # [SD] Problem was execv & the way it took the args split up was breaking the filter, and cmd failed. # Tested on rzgenie (pdebug), rztopaz (pbatch), and bgq (pdebug) from subprocess import check_call -print("Running: ", finalCommandToRun) try: check_call( finalCommandToRun,shell=True ) except Exception as e: From 7f724e679b4ae0e35d2710551a324e3bbfa47716 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 09:19:06 -0700 Subject: [PATCH 12/28] Trying to fix mistake in yml file --- .gitlab-ci.yml | 4 ++-- .gitlab/jobs-mpi.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2204d748d..7d8e42d00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,5 +33,5 @@ include: - local: .gitlab/scripts.yml - local: .gitlab/specs.yml - local: .gitlab/jobs-mpi.yml - - local: .gitlab/jobs-seq.yml - - local: .gitlab/jobs-prod.yml + # - local: .gitlab/jobs-seq.yml + # - local: .gitlab/jobs-prod.yml diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index 7355581c4..b39ca18af 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -19,7 +19,7 @@ toss_gcc_mvapich2_test: extends: [.toss_resource2, .gcc_mvapich2, .run_ats] - #needs: [toss_gcc_mvapich2_build] + # needs: [toss_gcc_mvapich2_build] From 34e7f7d6dd02ea12c9e688b17f65829de868c0f7 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 09:27:04 -0700 Subject: [PATCH 13/28] Reverting last changes but leaving exit code in run_ats --- .gitlab-ci.yml | 18 +++++++------- .gitlab/jobs-mpi.yml | 38 ++++++++++++++--------------- .gitlab/jobs-prod.yml | 56 +++++++++++++++++++++---------------------- .gitlab/jobs-seq.yml | 52 ++++++++++++++++++++-------------------- .gitlab/scripts.yml | 2 +- 5 files changed, 83 insertions(+), 83 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7d8e42d00..8f1810f0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,14 +16,14 @@ variables: - SPHERAL_REV_STR="$SPHERAL_REV" stages: -# - tpls -# - build_and_install + - tpls + - build_and_install - run_ats -# - update_tpls -# - generate_buildcache -# - install_production -# - update_permissions -# - cleanup + - update_tpls + - generate_buildcache + - install_production + - update_permissions + - cleanup include: - project: lc-templates/id_tokens @@ -33,5 +33,5 @@ include: - local: .gitlab/scripts.yml - local: .gitlab/specs.yml - local: .gitlab/jobs-mpi.yml - # - local: .gitlab/jobs-seq.yml - # - local: .gitlab/jobs-prod.yml + - local: .gitlab/jobs-seq.yml + - local: .gitlab/jobs-prod.yml diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index b39ca18af..d187c0953 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -1,38 +1,38 @@ # ------------------------------------------------------------------------------ # BUILD JOBS -# toss_gcc_mvapich2_cxxonly_tpls: -# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] +toss_gcc_mvapich2_cxxonly_tpls: + extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] -# toss_gcc_mvapich2_cxxonly_build: -# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] -# needs: [toss_gcc_mvapich2_cxxonly_tpls] +toss_gcc_mvapich2_cxxonly_build: + extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] + needs: [toss_gcc_mvapich2_cxxonly_tpls] -# toss_gcc_mvapich2_tpls: -# extends: [.toss_resource2, .gcc_mvapich2, .tpls] +toss_gcc_mvapich2_tpls: + extends: [.toss_resource2, .gcc_mvapich2, .tpls] -# toss_gcc_mvapich2_build: -# extends: [.toss_resource2, .gcc_mvapich2, .build_and_test] -# needs: [toss_gcc_mvapich2_tpls] +toss_gcc_mvapich2_build: + extends: [.toss_resource2, .gcc_mvapich2, .build_and_test] + needs: [toss_gcc_mvapich2_tpls] toss_gcc_mvapich2_test: extends: [.toss_resource2, .gcc_mvapich2, .run_ats] - # needs: [toss_gcc_mvapich2_build] + needs: [toss_gcc_mvapich2_build] -# toss_clang_mvapich2_tpls: -# extends: [.toss_resource1, .clang_mvapich2, .tpls] +toss_clang_mvapich2_tpls: + extends: [.toss_resource1, .clang_mvapich2, .tpls] -# toss_clang_mvapich2_build: -# extends: [.toss_resource1, .clang_mvapich2, .build_and_test] -# needs: [toss_clang_mvapich2_tpls] +toss_clang_mvapich2_build: + extends: [.toss_resource1, .clang_mvapich2, .build_and_test] + needs: [toss_clang_mvapich2_tpls] -# toss_clang_mvapich2_test: -# extends: [.toss_resource2, .clang_mvapich2, .run_ats] -# needs: [toss_clang_mvapich2_build] +toss_clang_mvapich2_test: + extends: [.toss_resource2, .clang_mvapich2, .run_ats] + needs: [toss_clang_mvapich2_build] diff --git a/.gitlab/jobs-prod.yml b/.gitlab/jobs-prod.yml index 4443d4dee..488ac824e 100644 --- a/.gitlab/jobs-prod.yml +++ b/.gitlab/jobs-prod.yml @@ -1,39 +1,39 @@ -# # ------------------------------------------------------------------------------ -# # UPDATE TPL JOBS +# ------------------------------------------------------------------------------ +# UPDATE TPL JOBS -# toss_update_tpls: -# extends: [.toss_resource2, .update_tpls, .merge_pr_rule] +toss_update_tpls: + extends: [.toss_resource2, .update_tpls, .merge_pr_rule] -# blueos_update_tpls: -# extends: [.blueos_resource2, .update_tpls, .merge_pr_rule] -# needs: [toss_update_tpls] +blueos_update_tpls: + extends: [.blueos_resource2, .update_tpls, .merge_pr_rule] + needs: [toss_update_tpls] -# # ------------------------------------------------------------------------------ -# # UPDATE PERM JOBS +# ------------------------------------------------------------------------------ +# UPDATE PERM JOBS -# shared_tpls_update_permissions: -# extends: [.toss_resource_general, .toss_update_permissions, .merge_pr_rule] +shared_tpls_update_permissions: + extends: [.toss_resource_general, .toss_update_permissions, .merge_pr_rule] -# # ------------------------------------------------------------------------------ -# # PROD BUILD JOBS +# ------------------------------------------------------------------------------ +# PROD BUILD JOBS -# ### TAG RELEASE ### -# toss_build_dev_pkg_release: -# extends: [.toss_resource_general, .gcc_mvapich2, .build_dev_pkg, .tag_release_rule] +### TAG RELEASE ### +toss_build_dev_pkg_release: + extends: [.toss_resource_general, .gcc_mvapich2, .build_dev_pkg, .tag_release_rule] -# toss_install_dev_pkg_release: -# extends: [.toss_resource_general, .gcc_mvapich2, .install_dev_pkg, .tag_release_rule] -# needs: [toss_build_dev_pkg_release] +toss_install_dev_pkg_release: + extends: [.toss_resource_general, .gcc_mvapich2, .install_dev_pkg, .tag_release_rule] + needs: [toss_build_dev_pkg_release] -# toss_release_permissions: -# variables: -# ALIAS: $CI_COMMIT_TAG -# extends: [.toss_resource_general, .prod_permissions, .tag_release_rule] -# needs: [toss_install_dev_pkg_release] +toss_release_permissions: + variables: + ALIAS: $CI_COMMIT_TAG + extends: [.toss_resource_general, .prod_permissions, .tag_release_rule] + needs: [toss_install_dev_pkg_release] -# # ------------------------------------------------------------------------------ -# # CLEAN OLD BUILD DIRS +# ------------------------------------------------------------------------------ +# CLEAN OLD BUILD DIRS -# cleanup_build_dirs: -# extends: [.clean_dirs] +cleanup_build_dirs: + extends: [.clean_dirs] diff --git a/.gitlab/jobs-seq.yml b/.gitlab/jobs-seq.yml index 78cd3b787..34847a7b9 100644 --- a/.gitlab/jobs-seq.yml +++ b/.gitlab/jobs-seq.yml @@ -1,37 +1,37 @@ -# # ------------------------------------------------------------------------------ -# # BUILD JOBS +# ------------------------------------------------------------------------------ +# BUILD JOBS -# toss_gcc_~mpi_tpls: -# extends: [.gcc_~mpi, .tpls, .toss_resource1] +toss_gcc_~mpi_tpls: + extends: [.gcc_~mpi, .tpls, .toss_resource1] -# toss_gcc_~mpi_build: -# extends: [.gcc_~mpi, .build_and_test, .toss_resource1] -# needs: [toss_gcc_~mpi_tpls] +toss_gcc_~mpi_build: + extends: [.gcc_~mpi, .build_and_test, .toss_resource1] + needs: [toss_gcc_~mpi_tpls] -# toss_gcc_~mpi_test: -# extends: [.gcc_~mpi, .run_ats, .toss_resource1] -# needs: [toss_gcc_~mpi_build] +toss_gcc_~mpi_test: + extends: [.gcc_~mpi, .run_ats, .toss_resource1] + needs: [toss_gcc_~mpi_build] -# blueos_cuda_11_gcc_~mpi_tpls: -# extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls] +blueos_cuda_11_gcc_~mpi_tpls: + extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls] -# blueos_cuda_11_gcc_~mpi_build: -# extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test] -# needs: [blueos_cuda_11_gcc_~mpi_tpls] +blueos_cuda_11_gcc_~mpi_build: + extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test] + needs: [blueos_cuda_11_gcc_~mpi_tpls] -# blueos_cuda_11_gcc_~mpi_test: -# extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats] -# needs: [blueos_cuda_11_gcc_~mpi_build] +blueos_cuda_11_gcc_~mpi_test: + extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats] + needs: [blueos_cuda_11_gcc_~mpi_build] -# blueos_gcc_~mpi_Debug_tpls: -# extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls] +blueos_gcc_~mpi_Debug_tpls: + extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls] -# blueos_gcc_~mpi_Debug_build: -# extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test] -# needs: [blueos_gcc_~mpi_Debug_tpls] +blueos_gcc_~mpi_Debug_build: + extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test] + needs: [blueos_gcc_~mpi_Debug_tpls] -# blueos_gcc_~mpi_Debug_test: -# extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats] -# needs: [blueos_gcc_~mpi_Debug_build] +blueos_gcc_~mpi_Debug_test: + extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats] + needs: [blueos_gcc_~mpi_Debug_build] diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 331e56a40..b96f6c046 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -46,7 +46,7 @@ - cd $CI_BUILD_DIR && cat job-name.txt - cat build_gitlab/install/spheral-lcatstest - - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code = $? + - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code artifacts: From 51c24449b3751195d21b1fdd68e25da49f4d4faa Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 10:33:25 -0700 Subject: [PATCH 14/28] Removed test exit from run_ats --- scripts/gitlab/run_ats.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index f605c5a6b..43c166501 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -107,7 +107,6 @@ def run_ats_test(args): #------------------------------------------------------------------------------ def main(): - sys.exit(80) args = parse_args() run_ats_test(args) From 4b2736e7d25138a80e7bce1f1df9955e1b5b45d4 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 12:50:15 -0700 Subject: [PATCH 15/28] Set Gitlab feature flag in run_ats to allow proper exit code detection --- .gitlab/scripts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index b96f6c046..92e679dfd 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -40,6 +40,8 @@ .run_ats: stage: run_ats + variables: + FF_USE_NEW_BASH_EVAL_STRATEGY: 1 script: - set +e - CI_BUILD_DIR=$(cat ci-dir.txt) From a7a5616c8e17dc668f98f6ab14521c52fb08e6aa Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 15:40:24 -0700 Subject: [PATCH 16/28] Adding more bash commands to try getting exit codes to work with gitlab runner --- .gitlab/scripts.yml | 1 + scripts/lc/lcats | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 92e679dfd..d098f1123 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -41,6 +41,7 @@ .run_ats: stage: run_ats variables: + FF_ENABLE_BASH_EXIT_CODE_CHECK: 1 FF_USE_NEW_BASH_EVAL_STRATEGY: 1 script: - set +e diff --git a/scripts/lc/lcats b/scripts/lc/lcats index 64fdd06fa..0555d6796 100755 --- a/scripts/lc/lcats +++ b/scripts/lc/lcats @@ -1031,6 +1031,7 @@ if (d_debug==1): #os.system(finalCommandToRun) # [SD] Problem was execv & the way it took the args split up was breaking the filter, and cmd failed. # Tested on rzgenie (pdebug), rztopaz (pbatch), and bgq (pdebug) + from subprocess import check_call try: check_call( finalCommandToRun,shell=True ) From 3aa12091540240f3fe70f14ce977c2dcc5c04908 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 15:59:56 -0700 Subject: [PATCH 17/28] Trying quicker runs again --- .gitlab/scripts.yml | 15 +++++++-------- scripts/gitlab/run_ats.py | 1 + 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index d098f1123..7a52901a1 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -13,7 +13,7 @@ - cd $CI_BUILD_DIR - echo $SPEC - - $BUILD_ALLOC ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --tpls-only +# - $BUILD_ALLOC ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --tpls-only artifacts: paths: - ci-dir.txt @@ -24,7 +24,7 @@ script: - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt - - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/host-config-build.py --host-config gitlab.cmake --build $EXTRA_CMAKE_ARGS +# - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/host-config-build.py --host-config gitlab.cmake --build $EXTRA_CMAKE_ARGS .build_and_test: extends: [.build] @@ -32,7 +32,7 @@ - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt - - ./build_gitlab/install/spheral -c "import $SPHERAL_MODULE" +# - ./build_gitlab/install/spheral -c "import $SPHERAL_MODULE" artifacts: paths: - ci-dir.txt @@ -44,13 +44,12 @@ FF_ENABLE_BASH_EXIT_CODE_CHECK: 1 FF_USE_NEW_BASH_EVAL_STRATEGY: 1 script: - - set +e - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt - - cat build_gitlab/install/spheral-lcatstest - - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? - - cp -r test-logs $CI_PROJECT_DIR +# - cat build_gitlab/install/spheral-lcatstest + - ./build_gitlab/install/.venv/bin/python -B $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? +# - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code artifacts: when: always @@ -68,7 +67,7 @@ .update_tpls: stage: update_tpls script: - - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --spec-list="$SCRIPT_DIR/devtools/spec-list.json" --spheral-spack-dir=$UPSTREAM_DIR +# - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --spec-list="$SCRIPT_DIR/devtools/spec-list.json" --spheral-spack-dir=$UPSTREAM_DIR .toss_update_permissions: stage: update_permissions diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 43c166501..f605c5a6b 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -107,6 +107,7 @@ def run_ats_test(args): #------------------------------------------------------------------------------ def main(): + sys.exit(80) args = parse_args() run_ats_test(args) From a80fc9935c70c55645f9e55aa84dcd53007ac18a Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 16:06:24 -0700 Subject: [PATCH 18/28] Added filler command to update_tpls --- .gitlab/scripts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 7a52901a1..f5c1b3df3 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -67,6 +67,7 @@ .update_tpls: stage: update_tpls script: + - echo "TEST" # - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --spec-list="$SCRIPT_DIR/devtools/spec-list.json" --spheral-spack-dir=$UPSTREAM_DIR .toss_update_permissions: From ceee9bb902473993e86a51f83760af9fe7024cd4 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Wed, 24 Jul 2024 17:00:03 -0700 Subject: [PATCH 19/28] Must actually build spheral, changed run_ats to use command directly instead of .sh scripts in the install --- .gitlab-ci.yml | 2 +- .gitlab/jobs-mpi.yml | 60 +++++++++++++++++++-------------------- .gitlab/scripts.yml | 11 ++++--- scripts/gitlab/run_ats.py | 19 ++++++++++--- 4 files changed, 51 insertions(+), 41 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f1810f0f..a789eb162 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,5 +33,5 @@ include: - local: .gitlab/scripts.yml - local: .gitlab/specs.yml - local: .gitlab/jobs-mpi.yml - - local: .gitlab/jobs-seq.yml +# - local: .gitlab/jobs-seq.yml - local: .gitlab/jobs-prod.yml diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index d187c0953..ee393fee7 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -1,12 +1,12 @@ # ------------------------------------------------------------------------------ # BUILD JOBS -toss_gcc_mvapich2_cxxonly_tpls: - extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] +# toss_gcc_mvapich2_cxxonly_tpls: +# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] -toss_gcc_mvapich2_cxxonly_build: - extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] - needs: [toss_gcc_mvapich2_cxxonly_tpls] +# toss_gcc_mvapich2_cxxonly_build: +# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] +# needs: [toss_gcc_mvapich2_cxxonly_tpls] @@ -23,16 +23,16 @@ toss_gcc_mvapich2_test: -toss_clang_mvapich2_tpls: - extends: [.toss_resource1, .clang_mvapich2, .tpls] +# toss_clang_mvapich2_tpls: +# extends: [.toss_resource1, .clang_mvapich2, .tpls] -toss_clang_mvapich2_build: - extends: [.toss_resource1, .clang_mvapich2, .build_and_test] - needs: [toss_clang_mvapich2_tpls] +# toss_clang_mvapich2_build: +# extends: [.toss_resource1, .clang_mvapich2, .build_and_test] +# needs: [toss_clang_mvapich2_tpls] -toss_clang_mvapich2_test: - extends: [.toss_resource2, .clang_mvapich2, .run_ats] - needs: [toss_clang_mvapich2_build] +# toss_clang_mvapich2_test: +# extends: [.toss_resource2, .clang_mvapich2, .run_ats] +# needs: [toss_clang_mvapich2_build] @@ -41,30 +41,30 @@ toss_clang_mvapich2_test: -blueos_gcc_spectrum_tpls: - extends: [.blueos_resource1, .gcc_spectrum, .tpls] +# blueos_gcc_spectrum_tpls: +# extends: [.blueos_resource1, .gcc_spectrum, .tpls] -blueos_gcc_spectrum_build: - extends: [.blueos_resource1, .gcc_spectrum, .build_and_test] - needs: [blueos_gcc_spectrum_tpls] +# blueos_gcc_spectrum_build: +# extends: [.blueos_resource1, .gcc_spectrum, .build_and_test] +# needs: [blueos_gcc_spectrum_tpls] -blueos_gcc_spectrum_test: - extends: [.blueos_resource1, .gcc_spectrum, .run_ats] - needs: [blueos_gcc_spectrum_build] +# blueos_gcc_spectrum_test: +# extends: [.blueos_resource1, .gcc_spectrum, .run_ats] +# needs: [blueos_gcc_spectrum_build] -blueos_cuda_11_gcc_spectrum_tpls: - extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls] +# blueos_cuda_11_gcc_spectrum_tpls: +# extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls] -blueos_cuda_11_gcc_spectrum_build: - extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test] - needs: [blueos_cuda_11_gcc_spectrum_tpls] +# blueos_cuda_11_gcc_spectrum_build: +# extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test] +# needs: [blueos_cuda_11_gcc_spectrum_tpls] -blueos_cuda_11_gcc_spectrum_test: - extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats] - needs: [blueos_cuda_11_gcc_spectrum_build] - allow_failure: true +# blueos_cuda_11_gcc_spectrum_test: +# extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats] +# needs: [blueos_cuda_11_gcc_spectrum_build] +# allow_failure: true diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index f5c1b3df3..606c867a5 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -13,7 +13,7 @@ - cd $CI_BUILD_DIR - echo $SPEC -# - $BUILD_ALLOC ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --tpls-only + - $BUILD_ALLOC ./$SCRIPT_DIR/gitlab/build_and_install.py --spec="$SPEC" --tpls-only artifacts: paths: - ci-dir.txt @@ -24,7 +24,7 @@ script: - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt -# - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/host-config-build.py --host-config gitlab.cmake --build $EXTRA_CMAKE_ARGS + - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/host-config-build.py --host-config gitlab.cmake --build $EXTRA_CMAKE_ARGS .build_and_test: extends: [.build] @@ -32,7 +32,7 @@ - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt -# - ./build_gitlab/install/spheral -c "import $SPHERAL_MODULE" + - ./build_gitlab/install/spheral -c "import $SPHERAL_MODULE" artifacts: paths: - ci-dir.txt @@ -47,7 +47,7 @@ - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt -# - cat build_gitlab/install/spheral-lcatstest + - cat build_gitlab/install/spheral-lcatstest - ./build_gitlab/install/.venv/bin/python -B $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? # - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code @@ -67,8 +67,7 @@ .update_tpls: stage: update_tpls script: - - echo "TEST" -# - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --spec-list="$SCRIPT_DIR/devtools/spec-list.json" --spheral-spack-dir=$UPSTREAM_DIR + - $BUILD_ALLOC ./$SCRIPT_DIR/devtools/tpl-manager.py --spec-list="$SCRIPT_DIR/devtools/spec-list.json" --spheral-spack-dir=$UPSTREAM_DIR .toss_update_permissions: stage: update_permissions diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index f605c5a6b..600f59a96 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -94,13 +94,24 @@ def run_ats_test(args): if (not os.path.exists(ats_file)): print(f"{ats_file} does not exists") sys.exit(1) - lcats_test = os.path.join(build_gl_dir, "spheral-lcatstest") - if (not os.path.exists(lcats_test)): - print(f"{lcats_test} does not exists") + venv_bin = os.path.join(build_gl_dir, ".venv", "bin") + python_exec = os.path.join(venv_bin, "python") + if (not os.path.exists(python_exec)): + print(f"{python_exec} does not exists}") sys.exit(1) + lcats_exec = os.path.join(build_gl_dir, "scripts", "lcats") + if (not os.path.exists(lcats_exec)): + print(f"{lcats_exec} does not exists") + sys.exit(1) + ats_exec = os.path.join(venv_bin, "ats") + if (not os.path.exists(ats_exec)): + print(f"{ats_exec} does not exists") + sys.exit(1) + python_command = python_exec + " -B" + lcats_command = python_command + " " + lcats_exec + " --atsExe " + ats_exec + " -e " + python_command ats_configs = ' --timelimit="45m"' test_alloc = " ".join(args.test_alloc) - run_command = f"{test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_configs}" + run_command = f"{test_alloc} {lcats_command} --logs test-logs {ats_file} {ats_configs}" ci_output = os.path.join(args.ci_build_dir, "test-logs") run_and_report(run_command, ci_output, 0) From 4defdea6ef0df6caacf065de313243619de057f5 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 09:00:22 -0700 Subject: [PATCH 20/28] Another dumb typo --- scripts/gitlab/run_ats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 600f59a96..e5a7a0201 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -97,7 +97,7 @@ def run_ats_test(args): venv_bin = os.path.join(build_gl_dir, ".venv", "bin") python_exec = os.path.join(venv_bin, "python") if (not os.path.exists(python_exec)): - print(f"{python_exec} does not exists}") + print(f"{python_exec} does not exists") sys.exit(1) lcats_exec = os.path.join(build_gl_dir, "scripts", "lcats") if (not os.path.exists(lcats_exec)): From 46930b4886dfdaaf98bcb9760695a5e348835094 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 10:12:24 -0700 Subject: [PATCH 21/28] Try setting build_failure_exit_code variable --- .gitlab/scripts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 606c867a5..f29ec6001 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -49,6 +49,7 @@ - cat build_gitlab/install/spheral-lcatstest - ./build_gitlab/install/.venv/bin/python -B $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? + - export BUILD_FAILURE_EXIT_CODE=$exit_code # - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code artifacts: From a5f53679f4d7717a536e4b07ecd00d01e4fc02cc Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 11:22:09 -0700 Subject: [PATCH 22/28] Admitting defeat for getting exit codes to work properly with Jacamar, trying the none GIT_STRATEGY with run_ats --- .gitlab-ci.yml | 2 +- .gitlab/jobs-mpi.yml | 60 +++++++++++++++++++-------------------- .gitlab/scripts.yml | 5 ++-- scripts/gitlab/run_ats.py | 11 +++---- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a789eb162..8f1810f0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,5 +33,5 @@ include: - local: .gitlab/scripts.yml - local: .gitlab/specs.yml - local: .gitlab/jobs-mpi.yml -# - local: .gitlab/jobs-seq.yml + - local: .gitlab/jobs-seq.yml - local: .gitlab/jobs-prod.yml diff --git a/.gitlab/jobs-mpi.yml b/.gitlab/jobs-mpi.yml index ee393fee7..d187c0953 100644 --- a/.gitlab/jobs-mpi.yml +++ b/.gitlab/jobs-mpi.yml @@ -1,12 +1,12 @@ # ------------------------------------------------------------------------------ # BUILD JOBS -# toss_gcc_mvapich2_cxxonly_tpls: -# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] +toss_gcc_mvapich2_cxxonly_tpls: + extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .tpls] -# toss_gcc_mvapich2_cxxonly_build: -# extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] -# needs: [toss_gcc_mvapich2_cxxonly_tpls] +toss_gcc_mvapich2_cxxonly_build: + extends: [.toss_resource1, .gcc_mvapich2_cxxonly, .build] + needs: [toss_gcc_mvapich2_cxxonly_tpls] @@ -23,16 +23,16 @@ toss_gcc_mvapich2_test: -# toss_clang_mvapich2_tpls: -# extends: [.toss_resource1, .clang_mvapich2, .tpls] +toss_clang_mvapich2_tpls: + extends: [.toss_resource1, .clang_mvapich2, .tpls] -# toss_clang_mvapich2_build: -# extends: [.toss_resource1, .clang_mvapich2, .build_and_test] -# needs: [toss_clang_mvapich2_tpls] +toss_clang_mvapich2_build: + extends: [.toss_resource1, .clang_mvapich2, .build_and_test] + needs: [toss_clang_mvapich2_tpls] -# toss_clang_mvapich2_test: -# extends: [.toss_resource2, .clang_mvapich2, .run_ats] -# needs: [toss_clang_mvapich2_build] +toss_clang_mvapich2_test: + extends: [.toss_resource2, .clang_mvapich2, .run_ats] + needs: [toss_clang_mvapich2_build] @@ -41,30 +41,30 @@ toss_gcc_mvapich2_test: -# blueos_gcc_spectrum_tpls: -# extends: [.blueos_resource1, .gcc_spectrum, .tpls] +blueos_gcc_spectrum_tpls: + extends: [.blueos_resource1, .gcc_spectrum, .tpls] -# blueos_gcc_spectrum_build: -# extends: [.blueos_resource1, .gcc_spectrum, .build_and_test] -# needs: [blueos_gcc_spectrum_tpls] +blueos_gcc_spectrum_build: + extends: [.blueos_resource1, .gcc_spectrum, .build_and_test] + needs: [blueos_gcc_spectrum_tpls] -# blueos_gcc_spectrum_test: -# extends: [.blueos_resource1, .gcc_spectrum, .run_ats] -# needs: [blueos_gcc_spectrum_build] +blueos_gcc_spectrum_test: + extends: [.blueos_resource1, .gcc_spectrum, .run_ats] + needs: [blueos_gcc_spectrum_build] -# blueos_cuda_11_gcc_spectrum_tpls: -# extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls] +blueos_cuda_11_gcc_spectrum_tpls: + extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls] -# blueos_cuda_11_gcc_spectrum_build: -# extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test] -# needs: [blueos_cuda_11_gcc_spectrum_tpls] +blueos_cuda_11_gcc_spectrum_build: + extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test] + needs: [blueos_cuda_11_gcc_spectrum_tpls] -# blueos_cuda_11_gcc_spectrum_test: -# extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats] -# needs: [blueos_cuda_11_gcc_spectrum_build] -# allow_failure: true +blueos_cuda_11_gcc_spectrum_test: + extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats] + needs: [blueos_cuda_11_gcc_spectrum_build] + allow_failure: true diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index f29ec6001..12698d5cc 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -41,16 +41,15 @@ .run_ats: stage: run_ats variables: + GIT_STRATEGY: none FF_ENABLE_BASH_EXIT_CODE_CHECK: 1 FF_USE_NEW_BASH_EVAL_STRATEGY: 1 script: - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt - - cat build_gitlab/install/spheral-lcatstest - ./build_gitlab/install/.venv/bin/python -B $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? - - export BUILD_FAILURE_EXIT_CODE=$exit_code -# - cp -r test-logs $CI_PROJECT_DIR + - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code artifacts: when: always diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index e5a7a0201..7536aec17 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -68,10 +68,12 @@ def run_and_report(run_command, ci_output, num_runs): sexe(run_command) tests_passed = report_results(ci_output) if (tests_passed == 0): - if (num_runs == 0): - sys.exit(0) - else: - sys.exit(80) + sys.exit(0) + # This should be added back in once Jacamar can handle exit codes properly + # if (num_runs == 0): + # sys.exit(0) + # else: + # sys.exit(80) elif (tests_passed >= max_test_failures): print("Too many test failures, not rerunning ATS") sys.exit(1) @@ -118,7 +120,6 @@ def run_ats_test(args): #------------------------------------------------------------------------------ def main(): - sys.exit(80) args = parse_args() run_ats_test(args) From 43de83c0f767830f1e3391337ccbab2fd05cbfc5 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 13:21:31 -0700 Subject: [PATCH 23/28] Going back to using bash scripts --- .gitlab/scripts.yml | 2 +- scripts/gitlab/run_ats.py | 20 ++++---------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 12698d5cc..9ad748645 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -48,7 +48,7 @@ - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt - - ./build_gitlab/install/.venv/bin/python -B $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? + - ./build_gitlab/install/spheral $SCRIPT_DIR/gitlab/run_ats.py --test-alloc "$TEST_ALLOC" --ats-file $ATS_FILE --ci-build-dir $CI_BUILD_DIR || exit_code=$? - cp -r test-logs $CI_PROJECT_DIR - exit $exit_code artifacts: diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index 7536aec17..ce6b28132 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -96,24 +96,12 @@ def run_ats_test(args): if (not os.path.exists(ats_file)): print(f"{ats_file} does not exists") sys.exit(1) - venv_bin = os.path.join(build_gl_dir, ".venv", "bin") - python_exec = os.path.join(venv_bin, "python") - if (not os.path.exists(python_exec)): - print(f"{python_exec} does not exists") - sys.exit(1) - lcats_exec = os.path.join(build_gl_dir, "scripts", "lcats") - if (not os.path.exists(lcats_exec)): - print(f"{lcats_exec} does not exists") - sys.exit(1) - ats_exec = os.path.join(venv_bin, "ats") - if (not os.path.exists(ats_exec)): - print(f"{ats_exec} does not exists") - sys.exit(1) - python_command = python_exec + " -B" - lcats_command = python_command + " " + lcats_exec + " --atsExe " + ats_exec + " -e " + python_command + lcats_test = os.path.join(build_gl_dir, "spheral-lcatstest") + if (not os.path.exists(lcats_test)): + print(f"{lcats_test} does not exists") ats_configs = ' --timelimit="45m"' test_alloc = " ".join(args.test_alloc) - run_command = f"{test_alloc} {lcats_command} --logs test-logs {ats_file} {ats_configs}" + run_command = f"{test_alloc} {lcats_test} --logs test-logs {ats_file} {ats_configs}" ci_output = os.path.join(args.ci_build_dir, "test-logs") run_and_report(run_command, ci_output, 0) From e9bd30067ba8824718c81b1f397a91b98b68b159 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 15:07:01 -0700 Subject: [PATCH 24/28] Removed some comments from lcats, changed GIT_STRATEGY to none for all jobs except tpl, created a spheralutils.py in scripts so all python scripts use the same sexe routine --- .gitlab/scripts.yml | 8 +++++ scripts/devtools/host-config-build.py | 22 ++---------- scripts/devtools/tpl-manager.py | 44 ++---------------------- scripts/gitlab/build_and_install.py | 18 ++-------- scripts/gitlab/run_ats.py | 49 +++++++++++---------------- scripts/lc/lcats | 19 ----------- scripts/spheralutils.py | 42 +++++++++++++++++++++++ 7 files changed, 79 insertions(+), 123 deletions(-) create mode 100644 scripts/spheralutils.py diff --git a/.gitlab/scripts.yml b/.gitlab/scripts.yml index 9ad748645..f78314f5b 100644 --- a/.gitlab/scripts.yml +++ b/.gitlab/scripts.yml @@ -21,6 +21,8 @@ .build: stage: build_and_install + variables: + GIT_STRATEGY: none script: - CI_BUILD_DIR=$(cat ci-dir.txt) - cd $CI_BUILD_DIR && cat job-name.txt @@ -71,6 +73,8 @@ .toss_update_permissions: stage: update_permissions + variables: + GIT_STRATEGY: none script: - ml load mpifileutils - srun -N 1 -p $PARTITION -n 20 -t 10 dchmod --mode go+rx $UPSTREAM_DIR @@ -129,6 +133,8 @@ .prod_permissions: stage: update_permissions + variables: + GIT_STRATEGY: none script: - INSTALL_DIR=$(cat install-dir.txt) @@ -149,6 +155,8 @@ # and never fill the sphapp workspace storage. .clean_dirs: stage: cleanup + variables: + GIT_STRATEGY: none script: - ml load mpifileutils - cd $SPHERAL_BUILDS_DIR diff --git a/scripts/devtools/host-config-build.py b/scripts/devtools/host-config-build.py index abc2d0e0e..11cdbfb75 100755 --- a/scripts/devtools/host-config-build.py +++ b/scripts/devtools/host-config-build.py @@ -3,7 +3,9 @@ import os import sys import argparse -import subprocess + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from spheralutils import sexe source_dir=os.getcwd() @@ -36,24 +38,6 @@ def parse_args(): return parser.parse_args() - -# Helper function for executing commands stolen from uberenv -def sexe(cmd,ret_output=False,echo=False): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {0}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out = p.communicate()[0] - out = out.decode('utf8') - return p.returncode,out - else: - return subprocess.call(cmd,shell=True) - - def main(): args = parse_args() print(args) diff --git a/scripts/devtools/tpl-manager.py b/scripts/devtools/tpl-manager.py index 79bb01e30..bc36f144f 100755 --- a/scripts/devtools/tpl-manager.py +++ b/scripts/devtools/tpl-manager.py @@ -3,9 +3,11 @@ import argparse import os import sys -import subprocess import json +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from spheralutils import sexe + #------------------------------------------------------------------------------ project_dir=os.path.abspath(os.path.join(os.path.realpath(__file__), "../../../")) @@ -64,46 +66,6 @@ def parse_args(): return parser.parse_args() - -# Helper function for executing commands stolen from uberenv -def sexe(cmd,ret_output=False,echo=True): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {0}]".format(cmd)) - - # If we want to return the output as string a print to stdout - # in real-time we need to let subprocess print as normal to - # PIPE and STDOUT. We then need to read it back ourselves and - # append to an ouput string of our own making. There is no way - # to do this with subprocess currently. - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding='utf8') - out = ""; - while True: - realtime_output = p.stdout.readline() - - if realtime_output == '' and p.poll() is not None: - break - - if realtime_output: - print(realtime_output.strip(), flush=True) - out += realtime_output - - if echo: - print(out) - return out - - # If we do not need to return the output as a string, run() - # will suffice. - else: - p = subprocess.run(cmd, shell=True, - check=True, text=True) - - # Parse the json formatted spec list... def parse_spec_list(file_path): with open(file_path) as f: diff --git a/scripts/gitlab/build_and_install.py b/scripts/gitlab/build_and_install.py index 6d0a958fc..65d991c07 100755 --- a/scripts/gitlab/build_and_install.py +++ b/scripts/gitlab/build_and_install.py @@ -4,7 +4,9 @@ import os import sys import argparse -import subprocess + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from spheralutils import sexe #------------------------------------------------------------------------------ @@ -41,20 +43,6 @@ def parse_args(): return parser.parse_args() - -# Helper function for executing commands stolen from uberenv -def sexe(cmd,ret_output=False,echo=True): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {0}]".format(cmd)) - p = subprocess.run(cmd, shell=True, - capture_output=ret_output, - check=True, text=True) - if ret_output: - if echo: - print(p.stdout) - return p.stdout - #------------------------------------------------------------------------------ def main(): diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index ce6b28132..bd8feadc2 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -2,41 +2,31 @@ import sys, subprocess, argparse, os +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) +from spheralutils import sexe + # If the number of failed tests exceeds this value, ATS is not rerun max_test_failures = 10 # Number of times to rerun the ATS tests max_reruns = 1 -# Helper function for executing commands stolen from uberenv -def sexe(cmd,ret_output=False,echo=True): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {0}]".format(cmd)) - p = subprocess.run(cmd, shell=True, - capture_output=ret_output, - check=True, text=True) - if ret_output: - if echo: - print(p.stdout) - return p.stdout - #------------------------------------------------------------------------------ def parse_args(): - parser = argparse.ArgumentParser() - - # Spec args - parser.add_argument('--test-alloc', type=str, nargs="+", - help='Allocation command for the machine.') - parser.add_argument('--ats-file', type=str, - help='ATS test file to run.') - parser.add_argument('--ci-build-dir', type=str, - help='CI build directory.') - parser.add_argument('--ci-install-dir', type=str, - default="build_gitlab/install", - help="Location of Spheral installation "+\ - "relative to --ci-build-dir") - return parser.parse_args() + parser = argparse.ArgumentParser() + + # Spec args + parser.add_argument('--test-alloc', type=str, nargs="+", + help='Allocation command for the machine.') + parser.add_argument('--ats-file', type=str, + help='ATS test file to run.') + parser.add_argument('--ci-build-dir', type=str, + help='CI build directory.') + parser.add_argument('--ci-install-dir', type=str, + default="build_gitlab/install", + help="Location of Spheral installation "+\ + "relative to --ci-build-dir") + return parser.parse_args() #------------------------------------------------------------------------------ @@ -68,6 +58,8 @@ def run_and_report(run_command, ci_output, num_runs): sexe(run_command) tests_passed = report_results(ci_output) if (tests_passed == 0): + if (run_nums > 0): + print("WARNING: Some tests were run multiple times") sys.exit(0) # This should be added back in once Jacamar can handle exit codes properly # if (num_runs == 0): @@ -85,7 +77,7 @@ def run_and_report(run_command, ci_output, num_runs): print(f"{ats_cont_file} not found, ATS cannot be rerun") sys.exit(1) rerun_command = f"{run_command} {ats_cont_file}" - print("Rerunning ATS") + print("WARNING: Test failure, rerunning ATS") run_and_report(rerun_command, ci_output, num_runs + 1) #------------------------------------------------------------------------------ @@ -111,6 +103,5 @@ def main(): args = parse_args() run_ats_test(args) - if __name__ == "__main__": main() diff --git a/scripts/lc/lcats b/scripts/lc/lcats index 0555d6796..83ea7145b 100755 --- a/scripts/lc/lcats +++ b/scripts/lc/lcats @@ -668,9 +668,7 @@ parser.add_option( '--timelimit', dest='timelimit', default=30, help='Set the default time limit on each test. The value may be given as a digit followed by an s, m, or h to give the time in seconds, minutes (the default), or hours.') # The P2 version is a sym-link to the latest python 2 version of ATS. There's a P3 when we're ready for Python3 -#parser.add_option( "--atsExe", action="store", type="string", dest="atsExe", default="/usr/apps/ats/7.0.P2/bin/ats", help="Sets which ats to use.") parser.add_option( "--atsExe", action="store", type="string", dest="atsExe", default="/usr/apps/ats/7.0.P3/bin/ats", help="Sets which ats to use.") -#parser.add_option( "--atsExe", action="store", type="string", dest="atsExe", default="/usr/apps/ats/7.0.111/bin/ats", help="Sets which ats to use.") parser.add_option( "--addOp", action="store", type="string", dest="extraEzatsArgs", default='', help="Adds extra job scheduler option to ezats.") @@ -1029,9 +1027,6 @@ if (d_debug==1): print("atsExe= ", options.atsExe) print("atsArgs= ", argsToUse) -#os.system(finalCommandToRun) # [SD] Problem was execv & the way it took the args split up was breaking the filter, and cmd failed. -# Tested on rzgenie (pdebug), rztopaz (pbatch), and bgq (pdebug) - from subprocess import check_call try: check_call( finalCommandToRun,shell=True ) @@ -1039,18 +1034,4 @@ except Exception as e: print("Caught - non-zero exit status 3 - thrown by final command", e) print("Tests appear to execute correctly...but this output is here to keep an eye on this.") -# [SD] This was the former call before adding threaded option -#import os -#if not machineSettings.options.batch: -# os.execv("/usr/bin/" + argsToUse[0], argsToUse) # (path,args) -#else: -# os.execv(argsToUse[0], argsToUse) # (path,args) - - -# Tried RunCmd & that isn't working either - it's resulting in salloc controlling terminal errors. -#from Shell import RunCmd -#def run(cmd): -# print 'running %s'%cmd -# print 'Set echoOutput to false to avoid seeing the queue contents repeatedly' -# return RunCmd(cmd, echoOutput=0, echoCmd=True)[1] diff --git a/scripts/spheralutils.py b/scripts/spheralutils.py new file mode 100644 index 000000000..c2f53af53 --- /dev/null +++ b/scripts/spheralutils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import subprocess + +# Helper function for executing commands stolen from uberenv +def sexe(cmd,ret_output=False,echo=True): + """ Helper for executing shell commands. """ + if echo: + print("[exe: {0}]".format(cmd)) + + # If we want to return the output as string a print to stdout + # in real-time we need to let subprocess print as normal to + # PIPE and STDOUT. We then need to read it back ourselves and + # append to an ouput string of our own making. There is no way + # to do this with subprocess currently. + if ret_output: + p = subprocess.Popen(cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding='utf8') + out = ""; + while True: + realtime_output = p.stdout.readline() + + if realtime_output == '' and p.poll() is not None: + break + + if realtime_output: + print(realtime_output.strip(), flush=True) + out += realtime_output + + if echo: + print(out) + return out + + # If we do not need to return the output as a string, run() + # will suffice. + else: + p = subprocess.run(cmd, shell=True, + check=True, text=True) + From 3431ecafbb4ffe239e64dd8deb523eb851f7fdbf Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 15:17:54 -0700 Subject: [PATCH 25/28] Fix indentation in spheralutils.py --- RELEASE_NOTES.md | 2 ++ scripts/spheralutils.py | 32 +++++++++++++++----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e15615d14..cdb57b0bc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -14,9 +14,11 @@ Notable changes include: * Axom updated to v0.9.0. * TPL builds have been split off into a separate Gitlab CI stage to help with timeouts on allocations. * Failed ATS runs are automatically retested once in the Gitlab CI. + * Python execute command is centralized in scripts/spheralutils.py now. * Build changes / improvements: * Distributed source directory must always be built now. + * Git strategies in the Gitlab CI are fixed so a clone only occurs on the first stage for each job, instead of for all stages for each job. * Bug Fixes / improvements: * Wrappers for MPI calls are simplified and improved. diff --git a/scripts/spheralutils.py b/scripts/spheralutils.py index c2f53af53..0f86674ef 100644 --- a/scripts/spheralutils.py +++ b/scripts/spheralutils.py @@ -9,8 +9,8 @@ def sexe(cmd,ret_output=False,echo=True): print("[exe: {0}]".format(cmd)) # If we want to return the output as string a print to stdout - # in real-time we need to let subprocess print as normal to - # PIPE and STDOUT. We then need to read it back ourselves and + # in real-time we need to let subprocess print as normal to + # PIPE and STDOUT. We then need to read it back ourselves and # append to an ouput string of our own making. There is no way # to do this with subprocess currently. if ret_output: @@ -19,24 +19,22 @@ def sexe(cmd,ret_output=False,echo=True): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf8') - out = ""; - while True: - realtime_output = p.stdout.readline() + out = ""; + while True: + realtime_output = p.stdout.readline() - if realtime_output == '' and p.poll() is not None: - break + if realtime_output == '' and p.poll() is not None: + break - if realtime_output: - print(realtime_output.strip(), flush=True) - out += realtime_output + if realtime_output: + print(realtime_output.strip(), flush=True) + out += realtime_output - if echo: - print(out) - return out - - # If we do not need to return the output as a string, run() - # will suffice. + if echo: + print(out) + return out else: + # If we do not need to return the output as a string, run() + # will suffice. p = subprocess.run(cmd, shell=True, check=True, text=True) - From 4bf2fb39346ff57230228bb66b6701bc7d5dae46 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 16:10:55 -0700 Subject: [PATCH 26/28] Fix host-config-build.py with changes to sexe routine --- scripts/devtools/host-config-build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/devtools/host-config-build.py b/scripts/devtools/host-config-build.py index 11cdbfb75..86219e88e 100755 --- a/scripts/devtools/host-config-build.py +++ b/scripts/devtools/host-config-build.py @@ -60,7 +60,7 @@ def main(): install_dir=args.install_dir build_dir=build_dir+"/build" # Pull the cmake command to use out of our host config. - cmake_cmd=sexe("grep 'CMake executable' \"{0}\"".format(hostconfig_path), ret_output=True, echo=True)[1].split()[-1] + cmake_cmd=sexe("grep 'CMake executable' \"{0}\"".format(hostconfig_path), ret_output=True, echo=True).split()[-1] cmake_extra_args="" if args.D and args.D != ['']: @@ -107,7 +107,7 @@ def main(): print("~~~~~ Building Spheral") print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - build_result = sexe("{0} {1} --build . -j 48 --target install".format(ml_cmd, cmake_cmd), echo=True) + build_result = sexe("{0} {1} --build . -j 48 --target install".format(ml_cmd, cmake_cmd), echo=True, ret_output=True) # If our build or install failed, run again to get our first error. if build_result != 0: From dd85ac91cdde971ee630bb80aef367940d39c2b8 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Thu, 25 Jul 2024 17:11:29 -0700 Subject: [PATCH 27/28] More changes based on sexe changes --- scripts/devtools/host-config-build.py | 14 +++----------- scripts/gitlab/build_and_install.py | 4 ++-- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/scripts/devtools/host-config-build.py b/scripts/devtools/host-config-build.py index 86219e88e..0c90c45e8 100755 --- a/scripts/devtools/host-config-build.py +++ b/scripts/devtools/host-config-build.py @@ -47,7 +47,7 @@ def main(): hostconfig_path=args.host_config else: hostconfig_path=os.path.abspath(args.host_config) - + # Set up our directory structure paths. if not args.build_dir: @@ -60,7 +60,7 @@ def main(): install_dir=args.install_dir build_dir=build_dir+"/build" # Pull the cmake command to use out of our host config. - cmake_cmd=sexe("grep 'CMake executable' \"{0}\"".format(hostconfig_path), ret_output=True, echo=True).split()[-1] + cmake_cmd=sexe("grep 'CMake executable' \"{0}\"".format(hostconfig_path), ret_output=True, echo=False).split()[-1] cmake_extra_args="" if args.D and args.D != ['']: @@ -107,15 +107,7 @@ def main(): print("~~~~~ Building Spheral") print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - build_result = sexe("{0} {1} --build . -j 48 --target install".format(ml_cmd, cmake_cmd), echo=True, ret_output=True) - - # If our build or install failed, run again to get our first error. - if build_result != 0: - print(build_result) - print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - print("Compilation failed") - print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - sys.exit(1) + sexe("{0} {1} --build . -j 48 --target install".format(ml_cmd, cmake_cmd), echo=True, ret_output=False) if __name__ == "__main__": main() diff --git a/scripts/gitlab/build_and_install.py b/scripts/gitlab/build_and_install.py index 65d991c07..84f706619 100755 --- a/scripts/gitlab/build_and_install.py +++ b/scripts/gitlab/build_and_install.py @@ -56,7 +56,7 @@ def main(): # Get the host-config name and path. if not args.build_only and not args.host_config: - hostconfig="{1}-{2}.cmake".format(host, sys_type, (args.spec).replace(" ","_")) + hostconfig="{0}-{1}.cmake".format(sys_type, (args.spec).replace(" ","_")) sexe("cp {0} gitlab.cmake".format(hostconfig)) hostconfig_path=os.path.join(os.getcwd(), "gitlab.cmake") else: @@ -65,7 +65,7 @@ def main(): print(hostconfig) if not args.tpls_only: - if sexe("{0} --host-config=\"{1}\" --lc-modules=\"{2}\" --build {3}".format(host_congfig_build_cmd, hostconfig_path, args.lc_modules, args.extra_cmake_args)) : sys.exit(1) + sexe("{0} --host-config=\"{1}\" --lc-modules=\"{2}\" --build {3}".format(host_congfig_build_cmd, hostconfig_path, args.lc_modules, args.extra_cmake_args)) if __name__ == "__main__": main() From d39a768b271fbdecd66c8c7ea6170e44d03107d5 Mon Sep 17 00:00:00 2001 From: Landon Owen Date: Fri, 26 Jul 2024 07:49:29 -0700 Subject: [PATCH 28/28] Fixed another dumb typo --- scripts/gitlab/run_ats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/run_ats.py b/scripts/gitlab/run_ats.py index bd8feadc2..0e86c701b 100755 --- a/scripts/gitlab/run_ats.py +++ b/scripts/gitlab/run_ats.py @@ -58,7 +58,7 @@ def run_and_report(run_command, ci_output, num_runs): sexe(run_command) tests_passed = report_results(ci_output) if (tests_passed == 0): - if (run_nums > 0): + if (num_runs > 0): print("WARNING: Some tests were run multiple times") sys.exit(0) # This should be added back in once Jacamar can handle exit codes properly