Merge branch 'development' into multiple_injection_sources

.github/workflows/cuda.yml

@@ -18,16 +18,12 @@ jobs:
     env:
       CXXFLAGS: "-Werror"
       CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       name: Install Python
       with:
-        python-version: '3.10'
+        python-version: '3.x'
     - name: install dependencies
       run: |
         .github/workflows/dependencies/nvcc11-3.sh
@@ -75,7 +71,8 @@ jobs:
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build_sp -j 2
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
         export WARPX_MPI=ON
         export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/
         python3 -m pip wheel .
@@ -111,20 +108,16 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach da79aff8053058371a78d4bf85488384242368ee && cd -
+        cd ../amrex && git checkout --detach 15a0bb9a8c1b34136632b16c5511375e9d56b184 && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
   build_nvhpc21-11-nvcc:
     name: [email protected] NVCC/NVC++ Release [tests]
     runs-on: ubuntu-20.04
     if: github.event.pull_request.draft == false
-    env:
-      # For NVHPC, Ninja is slower than the default:
-      #CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
+    #env:
+    #  # For NVHPC, Ninja is slower than the default:
+    #  CMAKE_GENERATOR: Ninja
     steps:
     - uses: actions/checkout@v3
     - name: Dependencies
@@ -175,7 +168,8 @@ jobs:
         #   https://github.com/mpi4py/mpi4py/issues/114
         #export CFLAGS="-noswitcherror"
 
-        #python3 -m pip install --upgrade pip setuptools wheel
+        #python3 -m pip install --upgrade pip
+        #python3 -m pip install --upgrade build packaging setuptools wheel
         #export WARPX_MPI=ON
         #export PYWARPX_LIB_DIR=$PWD/build/lib/site-packages/pywarpx/
         #python3 -m pip wheel .

.github/workflows/intel.yml

@@ -41,7 +41,8 @@ jobs:
         export CXX=$(which icpc)
         export CC=$(which icc)
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         cmake -S . -B build_dp \
           -DCMAKE_VERBOSE_MAKEFILE=ON \
@@ -104,7 +105,7 @@ jobs:
         export CC=$(which icx)
 
         python3 -m pip install --upgrade pip
-        python3 -m pip install --upgrade setuptools wheel
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         cmake -S . -B build_sp         \
           -DCMAKE_CXX_FLAGS_RELEASE="-O1 -DNDEBUG" \
@@ -178,6 +179,6 @@ jobs:
 
      # Skip this as it will copy the binary artifacts and we are tight on disk space
      #   python3 -m pip install --upgrade pip
-     #   python3 -m pip install --upgrade setuptools wheel
+     #   python3 -m pip install --upgrade build packaging setuptools wheel
      #   PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel .
      #   python3 -m pip install *.whl

.github/workflows/macos.yml

@@ -16,10 +16,6 @@ jobs:
       HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: TRUE
       # For macOS, Ninja is slower than the default:
       #CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
     steps:
     - uses: actions/checkout@v3
     - name: Brew Cache
@@ -52,6 +48,15 @@ jobs:
         set -e
         brew tap openpmd/openpmd
         brew install openpmd-api
+
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade virtualenv
+
+        python3 -m venv py-venv
+        source py-venv/bin/activate
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
+        python3 -m pip install --upgrade mpi4py
     - name: CCache Cache
       uses: actions/cache@v3
       # - once stored under a key, they become immutable (even if local cache path content changes)
@@ -64,8 +69,7 @@ jobs:
           ccache-macos-appleclang-
     - name: build WarpX
       run: |
-        python3 -m pip install --upgrade pip
-        python3 -m pip install --upgrade pip setuptools wheel
+        source py-venv/bin/activate
 
         cmake -S . -B build_dp         \
           -DCMAKE_VERBOSE_MAKEFILE=ON  \
@@ -87,5 +91,7 @@ jobs:
 
     - name: run pywarpx
       run: |
+        source py-venv/bin/activate
         export OMP_NUM_THREADS=1
+
         mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py

.github/workflows/ubuntu.yml

@@ -167,10 +167,6 @@ jobs:
       CXX: clang++
       # On CI for this test, Ninja is slower than the default:
       #CMAKE_GENERATOR: Ninja
-      # setuptools/mp4py work-around, see
-      #   https://github.com/mpi4py/mpi4py/pull/159
-      #   https://github.com/mpi4py/mpi4py/issues/157#issuecomment-1001022274
-      SETUPTOOLS_USE_DISTUTILS: stdlib
     steps:
     - uses: actions/checkout@v3
     - name: install dependencies
@@ -190,7 +186,8 @@ jobs:
           ccache-openmp-pyfull-
     - name: build WarpX
       run: |
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        python3 -m pip install --upgrade build packaging setuptools wheel
 
         export CXXFLAGS="-Werror -Wno-error=pass-failed"
 

.github/workflows/windows.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.x'
     - name: CCache Cache
       uses: actions/cache@v3
       # - once stored under a key, they become immutable (even if local cache path content changes)
@@ -41,7 +41,9 @@ jobs:
         cmake --build build --config Debug --parallel 2
         if(!$?) { Exit $LASTEXITCODE }
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        if(!$?) { Exit $LASTEXITCODE }
+        python3 -m pip install --upgrade build packaging setuptools wheel
         if(!$?) { Exit $LASTEXITCODE }
         cmake --build build --config Debug --target install
         if(!$?) { Exit $LASTEXITCODE }
@@ -100,7 +102,9 @@ jobs:
         cmake --build build --config Release --target install
         if errorlevel 1 exit 1
 
-        python3 -m pip install --upgrade pip setuptools wheel
+        python3 -m pip install --upgrade pip
+        if errorlevel 1 exit 1
+        python3 -m pip install --upgrade build packaging setuptools wheel
         if errorlevel 1 exit 1
         python3 -m pip install --upgrade -r requirements.txt
         if errorlevel 1 exit 1

CMakeLists.txt

@@ -1,7 +1,7 @@
 # Preamble ####################################################################
 #
 cmake_minimum_required(VERSION 3.20.0)
-project(WarpX VERSION 23.10)
+project(WarpX VERSION 23.11)
 
 include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake)
 
@@ -127,7 +127,7 @@ option(WarpX_PYTHON_IPO
 )
 
 set(pyWarpX_VERSION_INFO "" CACHE STRING
-    "PEP-440 conformant version (set by distutils)")
+    "PEP-440 conformant version (set by setup.py)")
 
 # enforce consistency of dependent options
 if(WarpX_APP OR WarpX_PYTHON)

Docs/source/acknowledge_us.rst

@@ -55,9 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t
 
 - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
   **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
-  14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023.
-  `preprint <https://www.ipac23.org/preproc/pdf/WEPA101.pdf>`__,
-  `DOI:10.18429/JACoW-IPAC-23-WEPA101 <https://doi.org/10.18429/JACoW-IPAC-23-WEPA101>`__
+  14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023.
+  `DOI:10.18429/JACoW-IPAC2023-WEPA101 <https://doi.org/10.18429/JACoW-IPAC2023-WEPA101>`__
 
 - Huebl A, Lehe R, Zoni E, Shapoval O, Sandberg R T, Garten M, Formenti A, Jambunathan R, Kumar P, Gott K, Myers A, Zhang W, Almgren A, Mitchell C E, Qiang J, Sinn A, Diederichs S, Thevenet M, Grote D, Fedeli L, Clark T, Zaim N, Vincenti H, Vay JL.
   **From Compact Plasma Particle Sources to Advanced Accelerators with Modeling at Exascale**.
@@ -75,6 +74,11 @@ If your project uses a specific algorithm or component, please consider citing t
   *New Journal of Physics* **24** 025009, 2022.
   `DOI:10.1088/1367-2630/ac4ef1 <https://doi.org/10.1088/1367-2630/ac4ef1>`__
 
+- Lehe R, Blelly A, Giacomel L, Jambunathan R, Vay JL.
+  **Absorption of charged particles in perfectly matched layers by optimal damping of the deposited current**.
+  *Physical Review E* **106** 045306, 2022.
+  `DOI:10.1103/PhysRevE.106.045306 <https://doi.org/10.1103/PhysRevE.106.045306>`__
+
 - Zoni E, Lehe R, Shapoval O, Belkin D, Zaim N, Fedeli L, Vincenti H, Vay JL.
   **A hybrid nodal-staggered pseudo-spectral electromagnetic particle-in-cell method with finite-order centering**. *Computer Physics Communications* **279**, 2022.
   `DOI:10.1016/j.cpc.2022.108457 <https://doi.org/10.1016/j.cpc.2022.108457>`__

Docs/source/conf.py

@@ -79,9 +79,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'23.10'
+version = u'23.11'
 # The full version, including alpha/beta/rc tags.
-release = u'23.10'
+release = u'23.11'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

Docs/source/dataanalysis/formats.rst

@@ -19,7 +19,7 @@ When using the AMReX `plotfile` format, users can set the ``amrex.async_out=1``
 option to perform the IO in a non-blocking fashion, meaning that the simulation
 will continue to run while an IO thread controls writing the data to disk.
 This can significantly reduce the overall time spent in IO. This is primarily intended for
-large runs on supercomputers such as Summit and Cori; depending on the MPI
+large runs on supercomputers (e.g. at OLCF or NERSC); depending on the MPI
 implementation you are using, you may not see a benefit on your workstation.
 
 When writing plotfiles, each rank will write to a separate file, up to some maximum number

Docs/source/developers/fields.rst

@@ -40,7 +40,7 @@ By default, the ``MultiFab`` are set to ``0`` at initialization. They can be ass
 Field solver
 ------------
 
-The field solver is performed in ``WarpX::EvolveE`` for the electric field and ``WarpX::EvolveB`` for the magnetic field, called from ``WarpX::OneStep_nosub`` in ``WarpX::EvolveEM``. This section describes the FDTD field push. It is implemented in ``Source/FieldSolver/FiniteDifferenceSolver/``.
+The field solver is performed in ``WarpX::EvolveE`` for the electric field and ``WarpX::EvolveB`` for the magnetic field, called from ``WarpX::OneStep_nosub`` in ``WarpX::Evolve``. This section describes the FDTD field push. It is implemented in ``Source/FieldSolver/FiniteDifferenceSolver/``.
 
 As all cell-wise operation, the field push is done as follows (this is split in multiple functions in the actual implementation to avoid code duplication)
 :

Docs/source/developers/repo_organization.rst

@@ -8,7 +8,7 @@ Repo Organization
 
 All the WarpX source code is located in ``Source/``.
 All sub-directories have a pretty straightforward name.
-The PIC loop is part of the WarpX class, in function ``WarpX::EvolveEM`` implemented in ``Source/WarpXEvolveEM.cpp``.
+The PIC loop is part of the WarpX class, in function ``WarpX::Evolve`` implemented in ``Source/WarpXEvolve.cpp``.
 The core of the PIC loop (i.e., without diagnostics etc.) is in ``WarpX::OneStep_nosub`` (when subcycling is OFF) or ``WarpX::OneStep_sub1`` (when subcycling is ON, with method 1).
 Here is a `visual representation <https://mango-dune-07a8b7110.1.azurestaticapps.net/?repo=ECP-WarpX%2FWarpX>`__ of the repository structure.
 

Docs/source/glossary.rst

@@ -34,6 +34,7 @@ Abbreviations
 * **GPU:** originally graphics processing unit, now used for fast `general purpose computing (GPGPU) <https://en.wikipedia.org/wiki/Graphics_processing_unit#Stream_processing_and_general_purpose_GPUs_(GPGPU)>`__; also called (hardware) accelerator
 * **IO:** input/output, usually files and/or data
 * **IPO:** `interprocedural optimization <https://en.wikipedia.org/wiki/Interprocedural_optimization>`__, a collection of compiler optimization techniques that analyze the whole code to avoid duplicate calculations and optimize performance
+* **ISI:** Induced Spectral Incoherence (a laser pulse manipulation technique)
 * **LDRD:** Laboratory Directed Research and Development, a :ref:`funding program in U.S. DOE laboratories <funding>` that kick-started ABLASTR development
 * **LPA:** laser-plasma acceleration, historically used for laser-electron acceleration
 * **LPI:** laser-plasma interaction (often for laser-solid physics) *or* laser-plasma instability (often in fusion physics), depending on context
@@ -56,9 +57,11 @@ Abbreviations
 * **PWFA:** plasma-wakefield acceleration
 * **QED:** `quantum electrodynamics <https://en.wikipedia.org/wiki/Quantum_electrodynamics>`__
 * **RPA:** radiation-pressure acceleration (of protons/ions), e.g. hole-boring (HB) or light-sail (LS) acceleration
+* **RPP:** Random Phase Plate (a laser pulse manipulation technique)
 * **RZ:** for the coordinate system ``r-z`` in cylindrical geometry; we use "RZ" when we refer to quasi-cylindrical geometry, decomposed in azimuthal modes (see details `here <https://fbpic.github.io/overview/pic_algorithm.html#cylindrical-grid-with-azimuthal-decomposition>`__)
 * **SENSEI:** `Scalable in situ analysis and visualization <https://sensei-insitu.org/>`__, light weight framework for in situ data analysis offering access to multiple visualization and analysis backends
 * **SEE:** secondary electron emission
+* **SSD:** Smoothing by Spectral Dispersion (a laser pulse manipulation technique)
 * **TNSA:** target-normal sheet acceleration (of protons/ions)
 
 Terms
@@ -69,7 +72,7 @@ Terms
 * **Ascent:** `many-core capable flyweight in situ visualization and analysis infrastructure <https://alpine-dav.github.io/ascent/>`__, a visualization backend usable with WarpX data
 * **boosted frame:** a :ref:`Lorentz-boosted frame of reference <theory-boostedframe>` for a simulation
 * **evolve:** this is a generic term to advance a quantity (same nomenclature in AMReX).
-              For instance, ``WarpX::EvolveE(dt)`` advances the electric field for duration ``dt``, ``PhysicalParticleContainer::Evolve(...)`` does field gather + particle push + current deposition for all particles in ``PhysicalParticleContainer``, and ``WarpX::EvolveEM`` is the central ``WarpX`` function that performs 1 PIC iteration.
+              For instance, ``WarpX::EvolveE(dt)`` advances the electric field for duration ``dt``, ``PhysicalParticleContainer::Evolve(...)`` does field gather + particle push + current deposition for all particles in ``PhysicalParticleContainer``, and ``WarpX::Evolve`` is the central ``WarpX`` function that performs 1 PIC iteration.
 * **Frontier:** an `Exascale supercomputer at OLCF <https://www.olcf.ornl.gov/frontier/>`__
 * **hybrid-PIC:** a plasma simulation scheme that combines fluid and kinetic approaches, with (usually) the electrons treated as a fluid and the ions as kinetic particles (see :ref:`theory-kinetic-fluid-hybrid-model`)
 * **laser:** most of the time, we mean a `laser pulse <https://en.wikipedia.org/wiki/Ultrashort_pulse>`__

Docs/source/highlights.rst

@@ -24,11 +24,10 @@ Scientific works in laser-plasma and beam-plasma acceleration.
    Phys. Rev. Research **5**, 033112, 2023
    `DOI:10.1103/PhysRevResearch.5.033112 <https://doi.org/10.1103/PhysRevResearch.5.033112>`__
 
-#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A.
+#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
    **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.
-   14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023.
-   `preprint <https://www.ipac23.org/preproc/pdf/WEPA101.pdf>`__,
-   `DOI:10.18429/JACoW-IPAC-23-WEPA101 <https://doi.org/10.18429/JACoW-IPAC-23-WEPA101>`__
+   14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023.
+   `DOI:10.18429/JACoW-IPAC2023-WEPA101 <https://doi.org/10.18429/JACoW-IPAC2023-WEPA101>`__
 
 #. Wang J, Zeng M, Li D, Wang X, Gao J.
    **High quality beam produced by tightly focused laser driven wakefield accelerators**.
@@ -141,3 +140,11 @@ High-Performance Computing and Numerics
 Scientific works in High-Performance Computing, applied mathematics and numerics.
 
 Please see :ref:`this section <acknowledge_warpx_all_refs>`.
+
+Nuclear Fusion - Magnetically Confined Plasmas
+**********************************************
+
+#. Nicks, B. S., Putvinski, S. and Tajima, T.
+   **Stabilization of the Alfvén-ion cyclotron instability through short plasmas: Fully kinetic simulations in a high-beta regime**.
+   Physics of Plasmas **30**, 102108, 2023.
+   `DOI:10.1063/5.0163889 <https://doi.org/10.1063/5.0163889>`__

Docs/source/install/cmake.rst

@@ -211,7 +211,8 @@ PICMI Python Bindings
 
    .. code-block:: bash
 
-      python3 -m pip install -U pip setuptools wheel
+      python3 -m pip install -U pip
+      python3 -m pip install -U build packaging setuptools wheel
       python3 -m pip install -U cmake
       python3 -m pip install -r requirements.txt
 

Docs/source/install/hpc.rst

@@ -33,7 +33,6 @@ This section documents quick-start guides for a selection of supercomputers that
    :maxdepth: 1
 
    hpc/adastra
-   hpc/cori
    hpc/crusher
    hpc/frontier
    hpc/fugaku