Merge branch '5.0.x' into feature-run_shell_cmd_timeout

easybuilders · Oct 2, 2024 · ebba4d8 · ebba4d8
2 parents a6a6430 + 3f5edf5
commit ebba4d8
Show file tree

Hide file tree

Showing 82 changed files with 4,696 additions and 2,198 deletions.
diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml
@@ -73,7 +73,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
           pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test

diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml
@@ -73,7 +73,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
           pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz
 
     - name: run test

diff --git a/.github/workflows/eb_command.yml b/.github/workflows/eb_command.yml
@@ -72,7 +72,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
 
     - name: run tests for 'eb' command
       env:

diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -17,6 +17,7 @@ jobs:
       fail-fast: false
     container:
       image: ghcr.io/easybuilders/${{ matrix.container }}-amd64
+    env: {ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} # Allow using Node16 actions
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3

diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -14,7 +14,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
-
     steps:
     - uses: actions/checkout@v3
 

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -14,7 +14,8 @@ jobs:
     runs-on: ubuntu-20.04
     outputs:
         lmod8: Lmod-8.7.6
-        modules4: modules-4.1.4
+        modules4: modules-4.5.3
+        modules5: modules-5.3.1
     steps:
       - run: "true"
   build:
@@ -28,6 +29,7 @@ jobs:
           # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
           - ${{needs.setup.outputs.lmod8}}
           - ${{needs.setup.outputs.modules4}}
+          - ${{needs.setup.outputs.modules5}}
         lc_all: [""]
         include:
           # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
@@ -101,12 +103,12 @@ jobs:
         # and are only run after the PR gets merged
         GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
       run: |
-        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit;
+        # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit
         # tests that require a GitHub token are skipped automatically when no GitHub token is available
         if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then
           if [ ! -z $GITHUB_TOKEN ]; then
-            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
-            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
+            SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"
+            python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"
           fi
           echo "GitHub token installed!"
         else
@@ -138,7 +140,7 @@ jobs:
           python setup.py sdist
           ls dist
           export PREFIX=/tmp/$USER/$GITHUB_SHA
-          pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
+          pip install --prefix $PREFIX dist/easybuild[-_]framework*tar.gz
 
     - name: run test suite
       env:
@@ -150,15 +152,17 @@ jobs:
           cd $HOME
           # initialize environment for modules tool
           if [ -f $HOME/moduleshome ]; then export MODULESHOME=$(cat $HOME/moduleshome); fi
-          source $(cat $HOME/mod_init); type module
+          source $(cat $HOME/mod_init)
+          type module
+          module --version
           # make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that);
           # also pick up changes to $PATH set by sourcing $MOD_INIT
           export PREFIX=/tmp/$USER/$GITHUB_SHA
           export PATH=$PREFIX/bin:$(cat $HOME/path)
           export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH
           eb --version
           # tell EasyBuild which modules tool is available
-          if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then
+          if [[ ${{matrix.modules_tool}} =~ ^modules- ]]; then
             export EASYBUILD_MODULES_TOOL=EnvironmentModules
           else
             export EASYBUILD_MODULES_TOOL=Lmod
@@ -187,7 +191,17 @@ jobs:
             # run test suite
             python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
             # try and make sure output of running tests is clean (no printed messages/warnings)
-            IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS="no GitHub token available"
+            IGNORE_PATTERNS+="|skipping SvnRepository test"
+            IGNORE_PATTERNS+="|requires Lmod as modules tool"
+            IGNORE_PATTERNS+="|stty: 'standard input': Inappropriate ioctl for device"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 3.[56]"
+            IGNORE_PATTERNS+="|from cryptography.* import "
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: Python 2"
+            IGNORE_PATTERNS+="|Blowfish"
+            IGNORE_PATTERNS+="|GC3Pie not available, skipping test"
+            IGNORE_PATTERNS+="|CryptographyDeprecationWarning: TripleDES has been moved"
+            IGNORE_PATTERNS+="|algorithms.TripleDES"
             # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
             PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
             test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)

diff --git a/RELEASE_NOTES b/RELEASE_NOTES
@@ -4,6 +4,77 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.4 (22 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - set $LMOD_TERSE_DECORATIONS to 'no' to avoid additional info in output produced by 'ml --terse avail' (#4648)
+- various bug fixes, including:
+  - implement workaround for permission error when copying read-only files that have extended attributes set and using Python 3.6 (#4642)
+  - take into account alternate sysroot for /bin/bash used by run_cmd (#4646)
+
+
+v4.9.3 (14 September 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for `--extra-source-urls` to fetch sources from additional URLs (#4079)
+  - add definition for gmpflf toolchain (#4566, #4571)
+  - reuse pre-computed checksums (#4569)
+  - add `cuda_cc_space_sep` variant that does not have periods (#4583)
+  - add `--skip-sanity-check` option (#4590)
+  - add `GNU_FTP_SOURCE` template constant (#4597)
+  - improve error messages for empty easyconfigs (#4603)
+  - improve help string for `--dep-graph` (#4610)
+  - only call `_sanity_check_step_extensions` if `--skip-extensions` is not set (#4620)
+  - add support for `--software-commit` and an associated template `%(software_commit)s` (#4628)
+- various bug fixes, including:
+  - correctly evaluate result for `--dep-graph` (#4554)
+  - fix fetch progress bar showing to many files (#4568)
+  - resolve internal for imkl>=2021 version subdir via "latest" symlink (#4570)
+  - fix typo in message about including an easyblock from a commit (#4575)
+  - don't use special flags for `strict`, `precise`, `loose`, `veryloose` toolchain options on RISC-V (#4576)
+  - fix help text for `cuda_compute_capabilities` template (#4589)
+  - fix help message for `--http-headers-fields-urlpat` configuration option (#4594)
+  - fix `test_compiler_cache` in case `gcc` is available multiple times (#4599)
+  - handle post-install patches in check_checksums_for (#4605)
+  - fix `copy_file` with a folder as the target (#4609)
+  - allow for case where `homepage = None` when generating the docs (#4626)
+  - fix test_github_det_commit_status by using more recent commits (#4636)
+- other changes:
+  - clean up code that was only there to support Python 2.6 + avoid syntax warnings when parsing py2vs3/py.p2 with Python 3.x (#3788)
+  - use Intel's oneAPI Fortran compiler by default for version 2024.0.0 and newer (`oneapi_fortran` toolchain option set to `True`) (#4567)
+  - allow using Node 16 actions in CI (#4574)
+  - remove a superflous check in `EasyBlock.run_all_steps` (#4623)
+  - remove trailing dots from backup message produced by --inject-checksums (#4632)
+
+
+v4.9.2 (12 June 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - improve behavior when using extension which has 'nosource' enabled (#4506)
+  - enhance 'get_software_libdir' to return 'lib' or 'lib64' if only one of them contains library files (#4513)
+  - implement versions checks to avoid mixing major versions across the EasyBuild components (#4520, #4553)
+  - add support for easyconfig parameter 'module_only' (#4537)
+- various bug fixes, including:
+  - fix typo in patch_step logging (#4505)
+  - consider both 'easybuild-framework*.tar.gz' and 'easybuild_framework*.tar.gz' in CI workflows (#4507)
+  - don't delete existing environment module files when using '--dump-env-script' with '--force' or '--rebuild' (#4512)
+  - fix resolved (template) values in case of failure  (#4532)
+  - also consider '$CRAY_PE_LIBSCI_PREFIX_DIR' to determine installation prefix for cray-libsci (#4551)
+  - symlink downloaded repo at specified commit when using '--from-commit' so easyconfigs for dependencies are found (#4552)
+- other changes:
+  - code cleanup in 'easyblock.py' (#4519)
+  - stop running unit tests on Python 3.5 (#4530)
+
+
 v4.9.1 (5 April 2024)
 ---------------------
-Original file line number
+Diff line change
@@ Expand Up / @@ -14,7 +14,6 @@ jobs: @@
         strategy:
           matrix:
             python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
         steps:
         - uses: actions/checkout@v3
@@ Expand Down @@