diff --git a/.github/workflows/cmake_ubuntu.yml b/.github/workflows/cmake_ubuntu.yml index c718e7ab2..58ac44f3a 100644 --- a/.github/workflows/cmake_ubuntu.yml +++ b/.github/workflows/cmake_ubuntu.yml @@ -78,11 +78,11 @@ jobs: make -j2 && sudo make install && cd ../.. && rm -rf samrai cd ${{runner.workspace}}/build && rm -rf * cmake $GITHUB_WORKSPACE -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON --fresh \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -Dasan=OFF \ + -DCMAKE_BUILD_TYPE=Debug -Dasan=OFF \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DlowResourceTests=ON -DdevMode=ON -Dbench=ON \ - -DCMAKE_CXX_FLAGS="-DPHARE_DIAG_DOUBLES=1 " -Dphare_configurator=ON + -DCMAKE_CXX_FLAGS="-O3 -DPHARE_DIAG_DOUBLES=1 " -Dphare_configurator=ON - name: Build working-directory: ${{runner.workspace}}/build diff --git a/src/amr/data/field/refine/electric_field_refiner.hpp b/src/amr/data/field/refine/electric_field_refiner.hpp index 57698b350..bd849659d 100644 --- a/src/amr/data/field/refine/electric_field_refiner.hpp +++ b/src/amr/data/field/refine/electric_field_refiner.hpp @@ -4,13 +4,15 @@ #include "core/def/phare_mpi.hpp" -#include - +#include "amr/amr_constants.hpp" #include "amr/resources_manager/amr_utils.hpp" + #include "core/utilities/constants.hpp" #include "core/data/grid/gridlayoutdefs.hpp" #include "core/utilities/point/point.hpp" +#include + #include namespace PHARE::amr @@ -43,8 +45,8 @@ class ElectricFieldRefiner { TBOX_ASSERT(coarseField.physicalQuantity() == fineField.physicalQuantity()); - auto const locFineIdx = AMRToLocal(fineIndex, fineBox_); - auto constexpr refinementRatio = 2; + auto const locFineIdx = AMRToLocal(fineIndex, fineBox_); + auto coarseIdx{fineIndex}; for (auto& idx : coarseIdx) idx = idx / refinementRatio; diff --git a/src/amr/data/particles/refine/particles_data_split.hpp b/src/amr/data/particles/refine/particles_data_split.hpp index 90c62cc9c..9b0fd443e 100644 --- a/src/amr/data/particles/refine/particles_data_split.hpp +++ b/src/amr/data/particles/refine/particles_data_split.hpp @@ -2,30 +2,30 @@ #define PHARE_PARTICLES_DATA_SPLIT_HPP +#include "core/def.hpp" +#include "phare_core.hpp" #include "core/def/phare_mpi.hpp" +#include "core/utilities/constants.hpp" -#include "core/def.hpp" -#include "amr/data/particles/particles_data.hpp" -#include "amr/resources_manager/amr_utils.hpp" #include "split.hpp" -#include "core/utilities/constants.hpp" -#include "phare_core.hpp" #include "amr/amr_constants.hpp" +#include "amr/utilities/box/amr_box.hpp" +#include "amr/resources_manager/amr_utils.hpp" +#include "amr/data/particles/particles_data.hpp" #include #include #include #include -#include namespace PHARE { namespace amr { - enum class ParticlesDataSplitType { - coarseBoundary, + enum class ParticlesDataSplitType : std::uint8_t { + coarseBoundary = 0, interior, coarseBoundaryOld, coarseBoundaryNew @@ -53,6 +53,12 @@ namespace amr template class ParticlesRefineOperator : public SAMRAI::hier::RefineOperator { + using Particle_t = typename ParticleArray::value_type; + static constexpr bool putParticlesInCoarseBoundary + = splitType == ParticlesDataSplitType::coarseBoundary + || splitType == ParticlesDataSplitType::coarseBoundaryOld + || splitType == ParticlesDataSplitType::coarseBoundaryNew; + public: static constexpr auto dim = Splitter::dimension; static constexpr auto interpOrder = Splitter::interp_order; @@ -99,7 +105,7 @@ namespace amr = std::dynamic_pointer_cast>( source.getPatchData(sourceComponent)); - // Finnaly we need the cartesion geometry of both patch. + // Finaly we need the cartesion geometry of both patch. auto patchGeomDestination = std::dynamic_pointer_cast( destination.getPatchGeometry()); @@ -133,9 +139,15 @@ namespace amr } } + template + static void _reserve(ParticleArray& particles, Fn&& counter) + { // only reserve for regular refinement of domain to start with + if constexpr (!putParticlesInCoarseBoundary) + particles.reserve(counter() * nbRefinedPart); + } /** @brief given two ParticlesData (destination and source), - * an overlap , a ratio and the geometry of both patches, perform the + * an overlap, a ratio and the geometry of both patches, perform the * splitting of coarse particles onto the destination patch */ void refine_(ParticlesData& destParticlesData, @@ -166,6 +178,22 @@ namespace amr // same for destinationGhostBox and destinationDomainBox the later will allow to get an // index relative to the interior + std::array const particlesArrays{&srcInteriorParticles, &srcGhostParticles}; + + auto const count_expected = [&]() { + std::size_t incoming_estimate = 0; + for (auto const& destinationBox : destBoxes) + { + auto const coarseSplitBox + = coarsen(phare_box_from(getSplitBox(destinationBox))); + for (auto const& sourceParticlesArray : particlesArrays) + incoming_estimate += sourceParticlesArray->nbr_particles_in(coarseSplitBox); + } + return incoming_estimate; + }; + + _reserve(destDomainParticles, count_expected); + Splitter split; // The PatchLevelFillPattern had compute boxes that correspond to the expected filling. @@ -173,36 +201,25 @@ namespace amr // in case of interior, this will be just one box usually for (auto const& destinationBox : destBoxes) { - std::array particlesArrays{&srcInteriorParticles, &srcGhostParticles}; - auto splitBox = getSplitBox(destinationBox); + auto const splitBox = getSplitBox(destinationBox); - auto isInDest = [&destinationBox](auto const& particle) // + auto const isInDest = [&destinationBox](auto const& particle) // { return isInBox(destinationBox, particle); }; - for (auto const& sourceParticlesArray : particlesArrays) { for (auto const& particle : *sourceParticlesArray) { - std::array - refinedParticles; - auto particleRefinedPos = toFineGrid(particle); + auto const particleRefinedPos = toFineGrid(particle); if (isInBox(splitBox, particleRefinedPos)) { + std::array refinedParticles; split(particleRefinedPos, refinedParticles); - // we need to know in which of interior or levelGhostParticlesXXXX // arrays we must put particles - bool constexpr putParticlesInCoarseBoundary - = splitType == ParticlesDataSplitType::coarseBoundary - || splitType == ParticlesDataSplitType::coarseBoundaryOld - || splitType == ParticlesDataSplitType::coarseBoundaryNew; - - - if constexpr (putParticlesInCoarseBoundary) { if constexpr (splitType == ParticlesDataSplitType::coarseBoundary) @@ -243,9 +260,9 @@ namespace amr std::back_inserter(destDomainParticles), isInDest); } } // end is candidate for split - } // end loop on particles - } // end loop on source particle arrays - } // loop on destination box + } // end loop on particles + } // end loop on source particle arrays + } // loop on destination box } diff --git a/src/amr/utilities/box/amr_box.hpp b/src/amr/utilities/box/amr_box.hpp index 4b615acd9..8004a897c 100644 --- a/src/amr/utilities/box/amr_box.hpp +++ b/src/amr/utilities/box/amr_box.hpp @@ -2,13 +2,13 @@ #define PHARE_AMR_UTILITIES_BOX_BOX_HPP +#include "core/def.hpp" #include "core/def/phare_mpi.hpp" +#include "core/utilities/box/box.hpp" +#include "amr/amr_constants.hpp" #include "SAMRAI/hier/Box.h" -#include "core/utilities/box/box.hpp" -#include "core/def.hpp" - namespace PHARE::amr { @@ -85,6 +85,22 @@ struct Box : public PHARE::core::Box } }; +template +auto refine(core::Box box) +{ + for (std::uint8_t di = 0; di < dim; ++di) + box.lower[di] *= refinementRatio, box.upper[di] *= refinementRatio; + return box; +} +template +auto coarsen(core::Box box) +{ + for (std::uint8_t di = 0; di < dim; ++di) + box.lower[di] /= refinementRatio, box.upper[di] /= refinementRatio; + + return box; +} + } // namespace PHARE::amr #endif diff --git a/src/core/data/ions/particle_initializers/maxwellian_particle_initializer.hpp b/src/core/data/ions/particle_initializers/maxwellian_particle_initializer.hpp index 49098af06..ff30af972 100644 --- a/src/core/data/ions/particle_initializers/maxwellian_particle_initializer.hpp +++ b/src/core/data/ions/particle_initializers/maxwellian_particle_initializer.hpp @@ -44,7 +44,7 @@ class MaxwellianParticleInitializer : public ParticleInitializer seed = {}, Basis const basis = Basis::Cartesian, std::array const& magneticField = {nullptr, nullptr, nullptr}, - double densityCutOff = 1e-5) + double const densityCutOff = 1e-5, double const over_alloc_factor = .1) : density_{density} , bulkVelocity_{bulkVelocity} , thermalVelocity_{thermalVelocity} @@ -54,6 +54,7 @@ class MaxwellianParticleInitializer : public ParticleInitializer rngSeed_; + double over_alloc_factor_ = .1; }; @@ -178,6 +180,17 @@ void MaxwellianParticleInitializer::loadParticles( auto randGen = getRNG(rngSeed_); ParticleDeltaDistribution deltaDistrib; + auto const expected_size = std::accumulate(n, n + ndCellIndices.size(), std::size_t{0}, + [&](auto const& sum, auto const& density_value) { + return (density_value > densityCutOff_) + ? sum + nbrParticlePerCell_ + : sum; + }); + + auto const incoming_estimate + = (layout.AMRBox().surface_cell_count() * (nbrParticlePerCell_ * over_alloc_factor_)); + particles.reserve(expected_size + incoming_estimate); + for (std::size_t flatCellIdx = 0; flatCellIdx < ndCellIndices.size(); ++flatCellIdx) { if (n[flatCellIdx] < densityCutOff_) diff --git a/src/core/data/particles/particle.hpp b/src/core/data/particles/particle.hpp index 0457865c2..bc81289b0 100644 --- a/src/core/data/particles/particle.hpp +++ b/src/core/data/particles/particle.hpp @@ -39,6 +39,7 @@ NO_DISCARD auto cellAsPoint(Particle const& particle) template struct Particle { + std::array constexpr static sizes = {52, 64, 76}; static_assert(dim > 0 and dim < 4, "Only dimensions 1,2,3 are supported."); static const size_t dimension = dim; @@ -54,12 +55,11 @@ struct Particle Particle() = default; - double weight = 0; - double charge = 0; - - std::array iCell = ConstArray(); - std::array delta = ConstArray(); - std::array v = ConstArray(); + double weight = 0; // 8 + double charge = 0; // 8 + std::array iCell = ConstArray(); // 4 * dim + std::array delta = ConstArray(); // 8 * dim + std::array v = ConstArray(); // 8 * 3 NO_DISCARD bool operator==(Particle const& that) const { @@ -121,9 +121,9 @@ inline constexpr auto is_phare_particle_type template typename ParticleA, template typename ParticleB> -NO_DISCARD typename std::enable_if_t< - is_phare_particle_type> and is_phare_particle_type>, - bool> +NO_DISCARD typename std::enable_if_t> + and is_phare_particle_type>, + bool> operator==(ParticleA const& particleA, ParticleB const& particleB) { return particleA.weight == particleB.weight and // diff --git a/src/core/utilities/box/box.hpp b/src/core/utilities/box/box.hpp index 0ff5d1a8a..ec0832609 100644 --- a/src/core/utilities/box/box.hpp +++ b/src/core/utilities/box/box.hpp @@ -26,7 +26,7 @@ template struct Box { static const size_t dimension = dim; - + using value_type = Type; Point lower; Point upper; @@ -130,9 +130,16 @@ struct Box return iterator{this, {upper[0] + 1, upper[1] + 1, upper[2] + 1}}; } } - using value_type = Type; + NO_DISCARD auto surface_cell_count() const + { + // assumes box never smaller than 3 in any direction + auto const shape_ = shape(); + auto const nested = shape_ - 2; + return core::product(shape_) - core::product(nested); + } + NO_DISCARD constexpr static std::size_t nbrRemainBoxes() { if constexpr (dim == 1) diff --git a/tests/amr/data/particles/refine/CMakeLists.txt b/tests/amr/data/particles/refine/CMakeLists.txt index 70fddf6bb..162e4057d 100644 --- a/tests/amr/data/particles/refine/CMakeLists.txt +++ b/tests/amr/data/particles/refine/CMakeLists.txt @@ -29,6 +29,4 @@ target_link_libraries(${PROJECT_NAME} PRIVATE ${GTEST_LIBS}) -add_phare_test(${PROJECT_NAME} ${CMAKE_CURRENT_BINARY_DIR}) - - +add_no_mpi_phare_test(${PROJECT_NAME} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/amr/data/particles/refine/test_particle_data_refine_tag_strategy.hpp b/tests/amr/data/particles/refine/test_particle_data_refine_tag_strategy.hpp index 7808db27d..3ab2f60da 100644 --- a/tests/amr/data/particles/refine/test_particle_data_refine_tag_strategy.hpp +++ b/tests/amr/data/particles/refine/test_particle_data_refine_tag_strategy.hpp @@ -42,15 +42,14 @@ std::array boxBoundsUpper(Box const& box) template -ParticleArray loadCell(int iCellX, int iCellY, int iCellZ) +void loadCell(ParticleArray& particles, int iCellX, int iCellY, int iCellZ) { - std::array _3diCell = {iCellX, iCellY, iCellZ}; + std::array const _3diCell = {iCellX, iCellY, iCellZ}; - float middle = 0.5; - float delta = 0.30f; + double const middle = 0.5; + double const delta = 0.30; Particle particle; - ParticleArray particles; particle.weight = 1.; particle.charge = 1.; @@ -79,8 +78,6 @@ ParticleArray loadCell(int iCellX, int iCellY, int iCellZ) particle.delta[dirX] = middle + delta / 3; particles.push_back(particle); - - return particles; } @@ -156,17 +153,9 @@ class TagStrategy : public SAMRAI::mesh::StandardTagAndInitStrategy auto const upper = boxBoundsUpper(particlesBox); for (auto iCellX = lower[dirX]; iCellX <= upper[dirX]; ++iCellX) - { for (auto iCellY = lower[dirY]; iCellY <= upper[dirY]; ++iCellY) - { for (auto iCellZ = lower[dirZ]; iCellZ <= upper[dirZ]; ++iCellZ) - { - auto const particles = loadCell(iCellX, iCellY, iCellZ); - interior.insert(std::end(interior), std::begin(particles), - std::end(particles)); - } - } - } + loadCell(interior, iCellX, iCellY, iCellZ); } } } @@ -210,7 +199,19 @@ class TagStrategy : public SAMRAI::mesh::StandardTagAndInitStrategy // do nothing } - + auto domainParticlesForLevel(std::shared_ptr const& hierarchy, + int levelNumber) + { + std::vector*> particle_arrays; + auto level = hierarchy->getPatchLevel(levelNumber); + for (auto& patch : *level) + for (auto const& [name, dataId] : dataToAllocate_) + particle_arrays.emplace_back( + &std::dynamic_pointer_cast>>( + patch->getPatchData(dataId)) + ->domainParticles); + return particle_arrays; + } private: std::map dataToAllocate_; diff --git a/tests/amr/data/particles/refine/test_particledata_refine_basic_hierarchy.hpp b/tests/amr/data/particles/refine/test_particledata_refine_basic_hierarchy.hpp index edd6d9c9f..38c7f0c50 100644 --- a/tests/amr/data/particles/refine/test_particledata_refine_basic_hierarchy.hpp +++ b/tests/amr/data/particles/refine/test_particledata_refine_basic_hierarchy.hpp @@ -48,6 +48,10 @@ template, interpOrder>; + using Splitter_t = Splitter, InterpConst, + RefinedParticlesConst>; + using ParticlesRefineOperator_t + = ParticlesRefineOperator, splitType, Splitter_t>; public: /** @@ -74,7 +78,7 @@ class BasicHierarchy } - explicit BasicHierarchy(int _ratio) + explicit BasicHierarchy(int _ratio = 2) : ratio{SAMRAI::tbox::Dimension{dimension}, _ratio} , inputDatabase_{SAMRAI::tbox::InputManager::getManager()->parseInputFile( inputString(_ratio))} @@ -97,10 +101,7 @@ class BasicHierarchy dimension_, "ChopAndPackLoadBalancer", inputDatabase_->getDatabase("ChopAndPackLoadBalancer"))} - , refineOperator_{std::make_shared< - ParticlesRefineOperator, splitType, - Splitter, InterpConst, - RefinedParticlesConst>>>()} + , refineOperator_{std::make_shared()} , tagStrategy_{std::make_shared>(variablesIds_, refineOperator_, @@ -167,6 +168,11 @@ class BasicHierarchy SAMRAI::hier::IntVector const ratio; + auto domainParticlesForLevel(int levelNumber) const + { + return tagStrategy_->domainParticlesForLevel(hierarchy_, levelNumber); + } + private: std::map getVariablesIds_() { diff --git a/tests/amr/data/particles/refine/test_split.cpp b/tests/amr/data/particles/refine/test_split.cpp index 8027c35a2..c376fdcfb 100644 --- a/tests/amr/data/particles/refine/test_split.cpp +++ b/tests/amr/data/particles/refine/test_split.cpp @@ -5,6 +5,8 @@ #include "core/utilities/types.hpp" #include "amr/data/particles/refine/split.hpp" +#include "test_particledata_refine_basic_hierarchy.hpp" + #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -12,7 +14,7 @@ namespace { template -using Splitter +using Splitter_t = PHARE::amr::Splitter, PHARE::core::InterpConst, PHARE::core::RefinedParticlesConst>; @@ -22,7 +24,7 @@ struct SplitterTest : public ::testing::Test SplitterTest() { Splitter splitter; } }; -using Splitters = testing::Types, Splitter<2, 1, 8> /*, Splitter<3, 1, 27>*/>; +using Splitters = testing::Types, Splitter_t<2, 1, 8> /*, Splitter<3, 1, 27>*/>; TYPED_TEST_SUITE(SplitterTest, Splitters); @@ -31,4 +33,20 @@ TYPED_TEST(SplitterTest, constexpr_init) constexpr TypeParam param{}; } + + } // namespace + + +using BasicHierarchy_t = BasicHierarchy<2, 3, ParticlesDataSplitType::interior, 4>; +struct Hierarchy_t : public ::testing::Test, public BasicHierarchy_t +{ +}; + +TEST_F(Hierarchy_t, checkCapacityPostRefinement) +{ // without reserve the capacity is 4096 and will likely have multiple allocations + auto domainParticles = domainParticlesForLevel(1); + ASSERT_EQ(domainParticles.size(), 1); + EXPECT_EQ(domainParticles[0]->size(), 2800); + EXPECT_EQ(domainParticles[0]->capacity(), 4032); +} diff --git a/tests/core/data/maxwellian_particle_initializer/test_maxwellian_particle_initializer.cpp b/tests/core/data/maxwellian_particle_initializer/test_maxwellian_particle_initializer.cpp index f3f4899cb..09c0013e6 100644 --- a/tests/core/data/maxwellian_particle_initializer/test_maxwellian_particle_initializer.cpp +++ b/tests/core/data/maxwellian_particle_initializer/test_maxwellian_particle_initializer.cpp @@ -9,22 +9,23 @@ #include "core/utilities/box/box.hpp" #include "core/utilities/point/point.hpp" -#include "gmock/gmock.h" #include "gtest/gtest.h" #include "tests/initializer/init_functions.hpp" -using namespace PHARE::initializer::test_fn::func_1d; // density/etc are here +#include "tests/core/data/gridlayout/test_gridlayout.hpp" using namespace PHARE::core; -using namespace PHARE::initializer; + +namespace PHARE::initializer::test_fn::func_1d +{ class AMaxwellianParticleInitializer1D : public ::testing::Test { private: - using GridLayoutT = GridLayout>; using ParticleArrayT = ParticleArray<1>; using InitFunctionArray = std::array, 3>; + using GridLayoutT = GridLayout>; public: AMaxwellianParticleInitializer1D() @@ -34,10 +35,8 @@ class AMaxwellianParticleInitializer1D : public ::testing::Test density, InitFunctionArray{vx, vy, vz}, InitFunctionArray{vthx, vthy, vthz}, 1., nbrParticlesPerCell)} { - // } - GridLayoutT layout; ParticleArrayT particles; std::uint32_t nbrParticlesPerCell{10000}; @@ -46,7 +45,6 @@ class AMaxwellianParticleInitializer1D : public ::testing::Test - TEST_F(AMaxwellianParticleInitializer1D, loadsTheCorrectNbrOfParticles) { auto nbrCells = layout.nbrCells(); @@ -57,7 +55,6 @@ TEST_F(AMaxwellianParticleInitializer1D, loadsTheCorrectNbrOfParticles) - TEST_F(AMaxwellianParticleInitializer1D, loadsParticlesInTheDomain) { initializer->loadParticles(particles, layout); @@ -73,8 +70,56 @@ TEST_F(AMaxwellianParticleInitializer1D, loadsParticlesInTheDomain) } } +} // namespace PHARE::initializer::test_fn::func_1d + + +namespace PHARE::initializer::test_fn::func_2d +{ + +class AMaxwellianParticleInitializer2D : public ::testing::Test +{ +private: + using ParticleArrayT = ParticleArray<2>; + using InitFunctionArray = std::array, 3>; + using GridLayoutT = GridLayout>; + + + +public: + AMaxwellianParticleInitializer2D() + : layout{50} + , initializer{std::make_unique>( + density, InitFunctionArray{vx, vy, vz}, InitFunctionArray{vthx, vthy, vthz}, 1., + nbrParticlesPerCell)} + { + } + + TestGridLayout layout; + ParticleArrayT particles{layout.AMRBox()}; + std::uint32_t nbrParticlesPerCell{600}; + std::unique_ptr> initializer; +}; + +TEST_F(AMaxwellianParticleInitializer2D, loadsTheCorrectNbrOfParticles) +{ + // vector push back allocation observations + // 100 ppc = 262144 - 250000 = 12144 == 12144 * 64 / 1e6 == .7MB overallocated + // 600 ppc = 2097152 - 1500000 = 597152 * 64 / 1e6 == 38MB overallocated + + auto const expectedNbrParticles = nbrParticlesPerCell * product(layout.AMRBox().shape()); + initializer->loadParticles(particles, layout); + EXPECT_EQ(expectedNbrParticles, particles.size()); + auto outer_cell_count = std::pow(50, 2) - std::pow(48, 2); + EXPECT_EQ(particles.capacity(), + particles.size() + (outer_cell_count * nbrParticlesPerCell * .1)); + + // new method + // 100 ppc = (1511760 - 1500000) * 64 / 1e6 == 0.12544 overallocated + // 600 ppc = (1511760 - 1500000) * 64 / 1e6 == 0.75264 overallocated +} +} // namespace PHARE::initializer::test_fn::func_2d int main(int argc, char** argv) {