Merge branch 'development' into python_use_valid_for_fields_getitem

.clang-tidy

@@ -1,19 +1,23 @@
-Checks: '-*,
-    bugprone-*
+Checks: '
+    -*,
+    bugprone-*,
         -bugprone-easily-swappable-parameters,
         -bugprone-implicit-widening-of-multiplication-result,
         -bugprone-misplaced-widening-cast,
         -bugprone-unchecked-optional-access,
-    cert-*
+    cert-*,
         -cert-err58-cpp,
-    cppcoreguidelines-avoid-goto,
-    cppcoreguidelines-interfaces-global-init,
+    cppcoreguidelines-*,
+        -cppcoreguidelines-avoid-c-arrays,
+        -cppcoreguidelines-avoid-magic-numbers,
         -cppcoreguidelines-avoid-non-const-global-variables,
         -cppcoreguidelines-init-variables,
         -cppcoreguidelines-macro-usage,
-        -cppcoreguidelines-narrowing-conversions,
         -cppcoreguidelines-no-malloc,
+        -cppcoreguidelines-narrowing-conversions,
+        -cppcoreguidelines-non-private-member-variables-in-classes,
         -cppcoreguidelines-owning-memory,
+        -cppcoreguidelines-pro-*,
     google-build-explicit-make-pair,
     google-build-namespaces,
     google-global-names-in-headers,
@@ -74,9 +78,12 @@ Checks: '-*,
     '
 
 CheckOptions:
-- key:          modernize-pass-by-value.ValuesOnly
-  value:        'true'
+- key:          bugprone-narrowing-conversions.WarnOnIntegerToFloatingPointNarrowingConversion
+  value:        "false"
 - key:          misc-definitions-in-headers.HeaderFileExtensions
   value:        "H,"
+- key:          modernize-pass-by-value.ValuesOnly
+  value:        "true"
+
 
 HeaderFilterRegex: 'Source[a-z_A-Z0-9\/]+\.H$'

.github/workflows/cuda.yml

@@ -27,7 +27,7 @@ jobs:
     - uses: actions/setup-python@v4
       name: Install Python
       with:
-        python-version: '3.x'
+        python-version: '3.10'
     - name: install dependencies
       run: |
         .github/workflows/dependencies/nvcc11-3.sh
@@ -111,7 +111,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 23.10 && cd -
+        cd ../amrex && git checkout --detach be6c6415467d09da6109d27cfa218868abc1f9db && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
   build_nvhpc21-11-nvcc:

.github/workflows/windows.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.x'
+        python-version: '3.10'
     - name: CCache Cache
       uses: actions/cache@v3
       # - once stored under a key, they become immutable (even if local cache path content changes)
@@ -64,7 +64,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.x'
+        python-version: '3.8'
     - uses: seanmiddleditch/gha-setup-ninja@master
     - name: CCache Cache
       uses: actions/cache@v3

.pre-commit-config.yaml

@@ -68,7 +68,7 @@ repos:
 
 # Autoremoves unused Python imports
 - repo: https://github.com/hadialqattan/pycln
-  rev: v2.2.2
+  rev: v2.3.0
   hooks:
   - id: pycln
     name: pycln (python)

CMakeLists.txt

@@ -646,7 +646,7 @@ if(WarpX_PYTHON)
         ${CMAKE_COMMAND} -E rm -f -r warpx-whl
         COMMAND
             ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$<TARGET_FILE_DIR:pyWarpX_${WarpX_DIMS_LAST}>
-                python3 -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR}
+                ${Python_EXECUTABLE} -m pip wheel -v --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR}
         WORKING_DIRECTORY
             ${WarpX_BINARY_DIR}
         DEPENDS
@@ -660,7 +660,7 @@ if(WarpX_PYTHON)
         set(pyWarpX_REQUIREMENT_FILE "requirements.txt")
     endif()
     add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements
-        python3 -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}"
+        ${Python_EXECUTABLE} -m pip install ${PYINSTALLOPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}"
         WORKING_DIRECTORY
             ${WarpX_BINARY_DIR}
     )
@@ -677,7 +677,7 @@ if(WarpX_PYTHON)
     # because otherwise pip would also force reinstall all dependencies.
     add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install
         ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
-            python3 -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx
+            ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx
         WORKING_DIRECTORY
             ${WarpX_BINARY_DIR}
         DEPENDS
@@ -686,6 +686,17 @@ if(WarpX_PYTHON)
             ${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements
             ${_EXTRA_INSTALL_DEPENDS}
     )
+
+    # this is for package managers only
+    add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps
+        ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
+            ${Python_EXECUTABLE} -m pip install --force-reinstall --no-index --no-deps ${PYINSTALLOPTIONS} --find-links=warpx-whl pywarpx
+        WORKING_DIRECTORY
+            ${WarpX_BINARY_DIR}
+        DEPENDS
+            pyWarpX_${WarpX_DIMS_LAST}
+            ${WarpX_CUSTOM_TARGET_PREFIX}pip_wheel
+    )
 endif()
 
 

Docs/source/acknowledge_us.rst

@@ -55,9 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t
 
 - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
   **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
-  14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023.
-  `preprint <https://www.ipac23.org/preproc/pdf/WEPA101.pdf>`__,
-  `DOI:10.18429/JACoW-IPAC-23-WEPA101 <https://doi.org/10.18429/JACoW-IPAC-23-WEPA101>`__
+  14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023.
+  `DOI:10.18429/JACoW-IPAC2023-WEPA101 <https://doi.org/10.18429/JACoW-IPAC2023-WEPA101>`__
 
 - Huebl A, Lehe R, Zoni E, Shapoval O, Sandberg R T, Garten M, Formenti A, Jambunathan R, Kumar P, Gott K, Myers A, Zhang W, Almgren A, Mitchell C E, Qiang J, Sinn A, Diederichs S, Thevenet M, Grote D, Fedeli L, Clark T, Zaim N, Vincenti H, Vay JL.
   **From Compact Plasma Particle Sources to Advanced Accelerators with Modeling at Exascale**.
@@ -75,6 +74,11 @@ If your project uses a specific algorithm or component, please consider citing t
   *New Journal of Physics* **24** 025009, 2022.
   `DOI:10.1088/1367-2630/ac4ef1 <https://doi.org/10.1088/1367-2630/ac4ef1>`__
 
+- Lehe R, Blelly A, Giacomel L, Jambunathan R, Vay JL.
+  **Absorption of charged particles in perfectly matched layers by optimal damping of the deposited current**.
+  *Physical Review E* **106** 045306, 2022.
+  `DOI:10.1103/PhysRevE.106.045306 <https://doi.org/10.1103/PhysRevE.106.045306>`__
+
 - Zoni E, Lehe R, Shapoval O, Belkin D, Zaim N, Fedeli L, Vincenti H, Vay JL.
   **A hybrid nodal-staggered pseudo-spectral electromagnetic particle-in-cell method with finite-order centering**. *Computer Physics Communications* **279**, 2022.
   `DOI:10.1016/j.cpc.2022.108457 <https://doi.org/10.1016/j.cpc.2022.108457>`__

Docs/source/highlights.rst

@@ -24,11 +24,10 @@ Scientific works in laser-plasma and beam-plasma acceleration.
    Phys. Rev. Research **5**, 033112, 2023
    `DOI:10.1103/PhysRevResearch.5.033112 <https://doi.org/10.1103/PhysRevResearch.5.033112>`__
 
-#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A.
+#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
    **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
-   14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023.
-   `preprint <https://www.ipac23.org/preproc/pdf/WEPA101.pdf>`__,
-   `DOI:10.18429/JACoW-IPAC-23-WEPA101 <https://doi.org/10.18429/JACoW-IPAC-23-WEPA101>`__
+   14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023.
+   `DOI:10.18429/JACoW-IPAC2023-WEPA101 <https://doi.org/10.18429/JACoW-IPAC2023-WEPA101>`__
 
 #. Wang J, Zeng M, Li D, Wang X, Gao J.
    **High quality beam produced by tightly focused laser driven wakefield accelerators**.

Docs/source/install/hpc.rst

@@ -42,6 +42,7 @@ This section documents quick-start guides for a selection of supercomputers that
    hpc/karolina
    hpc/lassen
    hpc/lawrencium
+   hpc/leonardo
    hpc/lumi
    hpc/lxplus
    hpc/ookami

Docs/source/install/hpc/lawrencium.rst

@@ -81,6 +81,7 @@ Optionally, download and install Python packages for :ref:`PICMI <usage-picmi>`
    source $HOME/sw/v100/venvs/warpx/bin/activate
    python3 -m pip install --upgrade pip
    python3 -m pip install --upgrade wheel
+   python3 -m pip install --upgrade setuptools
    python3 -m pip install --upgrade cython
    python3 -m pip install --upgrade numpy
    python3 -m pip install --upgrade pandas

Docs/source/install/hpc/leonardo.rst

@@ -0,0 +1,172 @@
+.. _building-leonardo:
+
+Leonardo (CINECA)
+=================
+
+The `Leonardo cluster <https://leonardo-supercomputer.cineca.eu/>`_ is hosted at `CINECA <https://www.cineca.it/en>`_.
+
+On Leonardo, each one of the 3456 compute nodes features a custom Atos Bull Sequana XH21355 "Da Vinci" blade, composed of:
+
+* 1 x CPU Intel Ice Lake Xeon 8358 32 cores 2.60 GHz
+* 512 (8 x 64) GB RAM DDR4 3200 MHz
+* 4 x NVidia custom Ampere A100 GPU 64GB HBM2
+* 2 x NVidia HDR 2×100 GB/s cards
+
+Introduction
+------------
+
+If you are new to this system, **please see the following resources**:
+
+* `Leonardo website <https://leonardo-supercomputer.cineca.eu/>`_
+* `Leonardo user guide <https://wiki.u-gov.it/confluence/display/SCAIUS/UG3.2%3A+LEONARDO+UserGuide>`_
+
+Storage organization:
+
+* ``$HOME``: permanent, backed up, user specific (50 GB quota)
+* ``$CINECA_SCRATCH``: temporary, user specific, no backup, a large disk for the storage of run time data and files, automatic cleaning procedure of data older than 40 days
+* ``$PUBLIC``: permanent, no backup (50 GB quota)
+* ``$WORK``: permanent, project specific, no backup
+
+.. _building-leonardo-preparation:
+
+Preparation
+-----------
+
+Use the following commands to download the WarpX source code:
+
+.. code-block:: bash
+
+   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/leonardo_gpu_warpx.profile``.
+Create it now:
+
+.. code-block:: bash
+
+   cp $HOME/src/warpx/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example $HOME/leonardo_gpu_warpx.profile
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example
+      :language: bash
+
+.. important::
+
+   Now, and as the first step on future logins to Leonardo, activate these environment settings:
+
+   .. code-block:: bash
+
+      source $HOME/leonardo_gpu_warpx.profile
+
+Finally, since Leonardo does not yet provide software modules for some of our dependencies, install them once:
+
+.. code-block:: bash
+
+   bash $HOME/src/warpx/Tools/machines/leonardo_cineca/install_gpu_dependencies.sh
+   source $HOME/sw/venvs/warpx/bin/activate
+
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
+
+   .. literalinclude:: ../../../../Tools/machines/leonardo-cineca/install_gpu_dependencies.sh
+      :language: bash
+
+
+.. _building-leonardo-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands <building-cmake>` to compile the application executable:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+   rm -rf build_gpu
+
+   cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_gpu -j 16
+
+The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``.
+Additionally, the following commands will install WarpX as a Python module:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+   rm -rf build_gpu_py
+
+   cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3"
+   cmake --build build_gpu_py -j 16 --target pip_install
+
+Now, you can :ref:`submit Leonardo compute jobs <running-cpp-leonardo>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
+Or, you can use the WarpX executables to submit Leonardo jobs (:ref:`example inputs <usage-examples>`).
+For executables, you can reference their location in your :ref:`job script <running-cpp-leonardo>` or copy them to a location in ``$CINECA_SCRATCH``.
+
+.. _building-leonardo-update:
+
+Update WarpX & Dependencies
+---------------------------
+
+If you already installed WarpX in the past and want to update it, start by getting the latest source code:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+
+   # read the output of this command - does it look ok?
+   git status
+
+   # get the latest WarpX source code
+   git fetch
+   git pull
+
+   # read the output of these commands - do they look ok?
+   git status
+   git log     # press q to exit
+
+And, if needed,
+
+- :ref:`update the leonardo_gpu_warpx.profile file <building-leonardo-preparation>`,
+- log out and into the system, activate the now updated environment profile as usual,
+- :ref:`execute the dependency install scripts <building-leonardo-preparation>`.
+
+As a last step, clean the build directories ``rm -rf $HOME/src/warpx/build_gpu*`` and rebuild WarpX.
+
+
+.. _running-leonardo:
+
+Running
+-------
+The batch script below can be used to run a WarpX simulation on multiple nodes on Leonardo.
+Replace descriptions between chevrons ``<>`` by relevant values.
+Note that we run one MPI rank per GPU.
+
+.. literalinclude:: ../../../../Tools/machines/leonardo-cineca/job.sh
+   :language: bash
+   :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/leonardo-cineca/job.sh``.
+
+To run a simulation, copy the lines above to a file ``job.sh`` and run
+
+.. code-block:: bash
+
+   sbatch job.sh
+
+to submit the job.
+
+.. _post-processing-leonardo:
+
+Post-Processing
+---------------
+
+For post-processing, activate the environment settings:
+
+.. code-block:: bash
+
+  source $HOME/leonardo_gpu_warpx.profile
+
+and run python scripts.

Docs/source/maintenance/release.rst

@@ -13,6 +13,7 @@ The following scripts automate this workflow, in case one needs a newer commit o
 .. code-block:: sh
 
    ./Tools/Release/updateAMReX.py
+   ./Tools/Release/updatepyAMReX.py
    ./Tools/Release/updatePICSAR.py
 
 
@@ -32,6 +33,7 @@ In order to create a GitHub release, you need to:
     .. code-block:: sh
 
        ./Tools/Release/updateAMReX.py
+       ./Tools/Release/updatepyAMReX.py
        ./Tools/Release/updatePICSAR.py
 
        ./Tools/Release/newVersion.sh

Docs/source/refs.bib

@@ -483,3 +483,16 @@ @article{Stanier2020
 author = {A. Stanier and L. Chacón and A. Le},
 keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather},
 }
+
+@book{Stix1992,
+    author = {Stix, T.H.},
+    date-added = {2023-06-29 13:51:16 -0700},
+    date-modified = {2023-06-29 13:51:16 -0700},
+    isbn = {978-0-88318-859-0},
+    lccn = {lc91033341},
+    publisher = {American Inst. of Physics},
+    title = {Waves in {Plasmas}},
+    url = {https://books.google.com/books?id=OsOWJ8iHpmMC},
+    year = {1992},
+    bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC}
+}