Merge branch 'development' into add_StrangImplicitSpectralEM_redo

dpgrote · Jul 26, 2024 · bde4f7f · bde4f7f
2 parents 16a5e49 + 675369d
commit bde4f7f
Show file tree

Hide file tree

Showing 29 changed files with 329 additions and 446 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -115,7 +115,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd -
+        cd ../amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s

diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst
@@ -24,27 +24,9 @@ If you are new to this system, **please see the following resources**:
 Login
 -----
 
-.. tab-set::
-
-   .. tab-item:: TOSS4 (RHEL8)
-
-      Lassen is currently transitioning to RHEL8.
-      During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes.
-
-      .. code-block:: bash
-
-         ssh lassen.llnl.gov
-         ssh eatoss4
-
-      Approximately October/November 2023, the new software environment on these nodes will be the new default.
-
-   .. tab-item:: TOSS3 (RHEL7)
-
-      .. code-block:: bash
-
-         ssh lassen.llnl.gov
+.. code-block:: bash
 
-      Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well.
+   ssh lassen.llnl.gov
 
 
 .. _building-lassen-preparation:
@@ -58,135 +40,54 @@ Use the following commands to download the WarpX source code:
 
    git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx
 
-.. tab-set::
-
-   .. tab-item:: TOSS4 (RHEL8)
-
-      We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``.
-      Create it now:
-
-      .. code-block:: bash
-
-         cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile
-
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
-
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example
-            :language: bash
-
-      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
-      For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``.
-      Enter the edit mode by typing ``i`` and edit line 2 to read:
-
-      .. code-block:: bash
-
-         export proj="nsldt"
-
-      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
-
-      .. important::
-
-         Now, and as the first step on future logins to lassen, activate these environment settings:
-
-         .. code-block:: bash
+We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``.
+Create it now:
 
-            source $HOME/lassen_v100_warpx.profile
-
-   .. tab-item:: TOSS3 (RHEL7)
-
-      We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``.
-      Create it now:
-
-      .. code-block:: bash
+.. code-block:: bash
 
-         cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile
+   cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile
 
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
 
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example
-            :language: bash
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example
+      :language: bash
 
-      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
-      For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``.
-      Enter the edit mode by typing ``i`` and edit line 2 to read:
+Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``.
+Enter the edit mode by typing ``i`` and edit line 2 to read:
 
-      .. code-block:: bash
+.. code-block:: bash
 
-         export proj="nsldt"
+   export proj="nsldt"
 
-      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
 
-      .. important::
+.. important::
 
-         Now, and as the first step on future logins to lassen, activate these environment settings:
+   Now, and as the first step on future logins to lassen, activate these environment settings:
 
-         .. code-block:: bash
+   .. code-block:: bash
 
-            source $HOME/lassen_v100_warpx_toss3.profile
+      source $HOME/lassen_v100_warpx_toss3.profile
 
 Finally, since lassen does not yet provide software modules for some of our dependencies, install them once:
 
-.. tab-set::
-
-   .. tab-item:: TOSS4 (RHEL8)
-
-      .. code-block:: bash
-
-         bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh
-         source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate
-
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
-
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh
-            :language: bash
-
-      .. dropdown:: AI/ML Dependencies (Optional)
-         :animate: fade-in-slide-down
-
-         If you plan to run AI/ML workflows depending on pyTorch, run the next step as well.
-         This will take a while and should be skipped if not needed.
-
-         .. code-block:: bash
-
-            runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh
-
-         .. dropdown:: Script Details
-            :color: light
-            :icon: info
-            :animate: fade-in-slide-down
-
-            .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh
-               :language: bash
-
-         For `optimas dependencies <https://github.com/optimas-org/optimas>`__ (incl. scikit-learn), plan another hour of build time:
-
-         .. code-block:: bash
-
-            python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt
-
-   .. tab-item:: TOSS3 (RHEL7)
+.. code-block:: bash
 
-      .. code-block:: bash
+   bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
+   source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate
 
-         bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
-         source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate
+.. dropdown:: Script Details
+   :color: light
+   :icon: info
+   :animate: fade-in-slide-down
 
-      .. dropdown:: Script Details
-         :color: light
-         :icon: info
-         :animate: fade-in-slide-down
+   .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
+      :language: bash
 
-         .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh
-            :language: bash
 
 .. _building-lassen-compilation:
 

diff --git a/Examples/Tests/collision/analysis_collision_2d.py b/Examples/Tests/collision/analysis_collision_2d.py
@@ -114,4 +114,4 @@
                                           dim, species_name)
 
 test_name = os.path.split(os.getcwd())[1]
-checksumAPI.evaluate_checksum(test_name, fn, do_particles=False)
+checksumAPI.evaluate_checksum(test_name, fn)
diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py
@@ -59,8 +59,6 @@ def attrlist(self):
         for attr, value in self.argvattrs.items():
             if value is None:
                 continue
-            # --- repr is applied to value so that for floats, all of the digits are included.
-            # --- The strip of "'" is then needed when value is a string.
             if isinstance(value, str):
                 if value.find('=') > -1:
                     # --- Expressions with temporary variables need to be inside quotes
@@ -73,11 +71,11 @@ def attrlist(self):
                     continue
                 # --- For lists, tuples, and arrays make a space delimited string of the values.
                 # --- The lambda is needed in case this is a list of strings.
-                rhs = ' '.join(map(lambda s : repr(s).strip("'"), value))
+                rhs = ' '.join(map(lambda s : f'{s}', value))
             elif isinstance(value, bool):
                 rhs = 1 if value else 0
             else:
                 rhs = value
-            attrstring = '{0}.{1} = {2}'.format(self.instancename, attr, repr(rhs).strip("'"))
+            attrstring = f'{self.instancename}.{attr} = {rhs}'
             result += [attrstring]
         return result
diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json
@@ -6,25 +6,25 @@
     "Ex": 0.0,
     "Ey": 0.0,
     "Ez": 0.0,
-    "T_electron": 353604.6247926339,
-    "T_ion": 347976.6168136309
+    "T_electron": 381488.6528070591,
+    "T_ion": 320091.2785835478
   },
   "electron": {
-    "particle_momentum_x": 8.370755929299189e-19,
-    "particle_momentum_y": 8.228112213603589e-19,
-    "particle_momentum_z": 8.204295817378347e-19,
-    "particle_position_x": 21284971.94721422,
-    "particle_position_y": 21212829.42991966,
-    "particle_position_z": 21214774.536558084,
+    "particle_momentum_x": 8.667025573698235e-19,
+    "particle_momentum_y": 8.457499789250831e-19,
+    "particle_momentum_z": 8.482438182280524e-19,
+    "particle_position_x": 21262567.138872623,
+    "particle_position_y": 21245135.070665065,
+    "particle_position_z": 21232644.283726066,
     "particle_weight": 7.168263344048695e+28
   },
   "ion": {
-    "particle_momentum_x": 2.0074097598289766e-18,
-    "particle_momentum_y": 1.8203553942782305e-18,
-    "particle_momentum_z": 1.823420185235695e-18,
-    "particle_position_x": 21227192.857240494,
-    "particle_position_y": 21286501.692027714,
-    "particle_position_z": 21245587.6706009,
+    "particle_momentum_x": 1.9300495097720012e-18,
+    "particle_momentum_y": 1.747257416857836e-18,
+    "particle_momentum_z": 1.7510296287537058e-18,
+    "particle_position_x": 21217348.883301035,
+    "particle_position_y": 21300859.0630925,
+    "particle_position_z": 21237901.246521123,
     "particle_weight": 7.168263344048695e+28
   }
 }
diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json
@@ -6,5 +6,21 @@
     "Ex": 0.0,
     "Ey": 0.0,
     "Ez": 0.0
+  },
+  "ion": {
+    "particle_momentum_x": 2.458306853810186e-19,
+    "particle_momentum_y": 2.272685285153902e-19,
+    "particle_momentum_z": 2.281205462681013e-19,
+    "particle_position_x": 2645436.647039526,
+    "particle_position_y": 2672571.48688055,
+    "particle_weight": 1.7256099431746894e+26
+  },
+  "electron": {
+    "particle_momentum_x": 1.0454942263455085e-19,
+    "particle_momentum_y": 1.0323735347957779e-19,
+    "particle_momentum_z": 1.0199134968670343e-19,
+    "particle_position_x": 2681776.3648108337,
+    "particle_position_y": 2663907.8843079703,
+    "particle_weight": 1.7256099431746894e+26
   }
 }
diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini
@@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more
 
 [AMReX]
 dir = /home/regtester/git/amrex/
-branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf
+branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc
 
 [source]
 dir = /home/regtester/git/WarpX

diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini
@@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det
 
 [AMReX]
 dir = /home/regtester/AMReX_RegTesting/amrex/
-branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf
+branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc
 
 [source]
 dir = /home/regtester/AMReX_RegTesting/warpx

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H
@@ -38,20 +38,26 @@ public:
     void ReadParameters ();
 
     /**
-     * \brief Initialize multifabs storing macroscopic multifabs
+     * \brief Allocate multifabs storing macroscopic multifabs
      *
      * @param[in] ba the box array associated to the multifabs E and B
      * @param[in] dmap the distribution mapping
      * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B
+     */
+    void AllocateLevelMFs (
+        const amrex::BoxArray& ba,
+        const amrex::DistributionMapping& dm,
+        const amrex::IntVect& ng_EB_alloc );
+
+    /**
+     * \brief Initialize multifabs storing macroscopic multifabs
+     *
      * @param[in] geom the geometry
      * @param[in] Ex_stag staggering of the Ex field
      * @param[in] Ey_stag staggering of the Ey field
      * @param[in] Ez_stag staggering of the Ez field
      */
     void InitData (
-        const amrex::BoxArray& ba,
-        const amrex::DistributionMapping& dmap,
-        const amrex::IntVect& ng_EB_alloc,
         const amrex::Geometry& geom,
         const amrex::IntVect& Ex_stag,
         const amrex::IntVect& Ey_stag,

diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp
@@ -119,24 +119,26 @@ MacroscopicProperties::ReadParameters ()
 }
 
 void
-MacroscopicProperties::InitData (
+MacroscopicProperties::AllocateLevelMFs (
     const amrex::BoxArray& ba,
     const amrex::DistributionMapping& dmap,
-    const amrex::IntVect& ng_EB_alloc,
-    const amrex::Geometry& geom,
-    const amrex::IntVect& Ex_stag,
-    const amrex::IntVect& Ey_stag,
-    const amrex::IntVect& Ez_stag)
+    const amrex::IntVect& ng_EB_alloc )
 {
-    amrex::Print() << Utils::TextMsg::Info("we are in init data of macro");
-
-    // Define material property multifabs using ba and dmap from WarpX instance
     // sigma is cell-centered MultiFab
     m_sigma_mf = std::make_unique<amrex::MultiFab>(ba, dmap, 1, ng_EB_alloc);
     // epsilon is cell-centered MultiFab
     m_eps_mf = std::make_unique<amrex::MultiFab>(ba, dmap, 1, ng_EB_alloc);
     // mu is cell-centered MultiFab
     m_mu_mf = std::make_unique<amrex::MultiFab>(ba, dmap, 1, ng_EB_alloc);
+}
+
+void
+MacroscopicProperties::InitData (
+    const amrex::Geometry& geom,
+    const amrex::IntVect& Ex_stag,
+    const amrex::IntVect& Ey_stag,
+    const amrex::IntVect& Ez_stag)
+{
     // Initialize sigma
     if (m_sigma_s == "constant") {
 

diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp
@@ -474,12 +474,9 @@ WarpX::InitData ()
 
     BuildBufferMasks();
 
-    if (WarpX::em_solver_medium==1) {
+    if (WarpX::em_solver_medium == MediumForEM::Macroscopic) {
         const int lev_zero = 0;
         m_macroscopic_properties->InitData(
-            boxArray(lev_zero),
-            DistributionMap(lev_zero),
-            getngEB(),
             Geom(lev_zero),
             getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(),
             getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(),