From 8421deb91c072a35feea00de0e7f893bbddaa4e3 Mon Sep 17 00:00:00 2001 From: deegan Date: Sun, 23 Jun 2024 17:20:50 +0200 Subject: [PATCH] env handling --- pyphare/pyphare/cpp/__init__.py | 13 +++ res/cmake/test.cmake | 1 + src/core/env.hpp | 114 ++++++++++++++++++++++++ src/core/utilities/mpi_utils.cpp | 1 + src/core/utilities/mpi_utils.hpp | 16 ++++ src/core/utilities/types.hpp | 27 ++++++ src/hdf5/detail/h5/h5_file.hpp | 15 +--- src/phare/phare.hpp | 4 +- src/python3/cpp_etc.cpp | 28 +++++- src/simulator/simulator.hpp | 30 +------ tests/core/utilities/env/CMakeLists.txt | 21 +++++ tests/core/utilities/env/test_env.cpp | 41 +++++++++ 12 files changed, 269 insertions(+), 42 deletions(-) create mode 100644 src/core/env.hpp create mode 100644 tests/core/utilities/env/CMakeLists.txt create mode 100644 tests/core/utilities/env/test_env.cpp diff --git a/pyphare/pyphare/cpp/__init__.py b/pyphare/pyphare/cpp/__init__.py index 79d0c8cf5..f3e1463b4 100644 --- a/pyphare/pyphare/cpp/__init__.py +++ b/pyphare/pyphare/cpp/__init__.py @@ -36,6 +36,19 @@ def build_config(): return cpp_etc_lib().phare_build_config() +def env_vars(): + return cpp_etc_lib().phare_env_vars() + + +def print_env_vars_info(): + for k, v in cpp_etc_lib().phare_env_vars().items(): + print(f"{k}: {v.desc}") + print("Options:") + for k, v in v.options: + print(f" {k}: {v}") + print("") + + def build_config_as_json(): return json.dumps(build_config()) diff --git a/res/cmake/test.cmake b/res/cmake/test.cmake index 1f7331b1d..ea7355bd9 100644 --- a/res/cmake/test.cmake +++ b/res/cmake/test.cmake @@ -18,6 +18,7 @@ if (test AND ${PHARE_EXEC_LEVEL_MIN} GREATER 0) # 0 = no tests add_subdirectory(tests/core/data/maxwellian_particle_initializer) add_subdirectory(tests/core/data/particle_initializer) add_subdirectory(tests/core/utilities/box) + add_subdirectory(tests/core/utilities/env) add_subdirectory(tests/core/utilities/range) add_subdirectory(tests/core/utilities/index) add_subdirectory(tests/core/utilities/indexer) diff --git a/src/core/env.hpp b/src/core/env.hpp new file mode 100644 index 000000000..5e83c10c4 --- /dev/null +++ b/src/core/env.hpp @@ -0,0 +1,114 @@ +#ifndef PHARE_CORE_ENV_HPP +#define PHARE_CORE_ENV_HPP + +// Single source for handling env vars + +#include +#include +#include +#include +#include +#include "core/utilities/types.hpp" +#include "core/utilities/mpi_utils.hpp" + +namespace PHARE::env +{ + +struct Var +{ + using value_type = std::string; + using results_type = std::unordered_map; + auto constexpr static noop_results = [](Var const&) { return results_type{}; }; + + std::string_view const id; + std::string_view const desc; + std::vector> const options; + + std::optional const _default = std::nullopt; + std::function const _results = noop_results; + std::optional const v = get(); + results_type const results_ = _results(*this); + + std::optional get() const + { + std::string _id{id}; + if (_default) + return core::get_env(_id, *_default); + return core::get_env(_id); + } + auto& operator()() const { return v; } + auto& operator()(std::string const& s) const { return results_.at(s); } + // bool exists() const { return v != std::nullopt; } + // operator bool() const { return exists(); } +}; + +} // namespace PHARE::env + +namespace PHARE +{ +class Env +{ +public: + template + using map_t = std::unordered_map; + + static Env& INSTANCE() + { + if (!self) + self = std::make_unique(); + return *self; + } + static auto& reinit() { return *(self = std::make_unique()); } + + env::Var const PHARE_LOG{ + "PHARE_LOG", + "Write logs to $CWD/.log", + {{{"RANK_FILES", "Write logs $CWD/.log, a file per rank"}, + {"DATETIME_FILES", "Write logs $CWD/.log, filename per rank and datetime"}, + {"NONE", "print normally to std::cout"}}}, + std::nullopt, + [](auto const& self) { + std::string_view constexpr file_key = "PHARE_LOG_FILE"; + typename env::Var::results_type map; + if (auto const& opt = self()) + { + auto const& val = *opt; + if (val == "RANK_FILES") + map.emplace(std::string(file_key), + ".log/" + std::to_string(core::mpi::rank()) + ".out"); + else if (val == "DATETIME_FILES") + { + auto date_time = core::mpi::date_time(); + auto rank = std::to_string(core::mpi::rank()); + auto size = std::to_string(core::mpi::size()); + map.emplace(std::string(file_key), + ".log/" + date_time + "_" + rank + "_of_" + size + ".out"); + } + else if (val != "NONE") + throw std::runtime_error("PHARE_LOG invalid type, valid keys are " + "RANK_FILES/DATETIME_FILES/NONE"); + } + return map; + } // + }; + env::Var const PHARE_SCOPE_TIMING{ + "PHARE_SCOPE_TIMING", "Enable function scope timing", {{{"1", "ON"}, {"0", "OFF"}}}, "0"}; + + map_t const vars = {{ + {"PHARE_LOG", &PHARE_LOG}, + {"PHARE_SCOPE_TIMING", &PHARE_SCOPE_TIMING}, + }}; + + auto& operator()(std::string const& s) const + { + assert(vars.count(s)); + return *vars.at(s); + } + +private: + static inline std::unique_ptr self = nullptr; +}; + +} // namespace PHARE + +#endif /* PHARE_CORE_ERRORS_H */ diff --git a/src/core/utilities/mpi_utils.cpp b/src/core/utilities/mpi_utils.cpp index dae6f2b49..22b46b5cd 100644 --- a/src/core/utilities/mpi_utils.cpp +++ b/src/core/utilities/mpi_utils.cpp @@ -27,6 +27,7 @@ std::size_t max(std::size_t const local, int mpi_size) + bool any(bool b) { int global_sum, local_sum = static_cast(b); diff --git a/src/core/utilities/mpi_utils.hpp b/src/core/utilities/mpi_utils.hpp index 9f1f5a0c1..f3d85a371 100644 --- a/src/core/utilities/mpi_utils.hpp +++ b/src/core/utilities/mpi_utils.hpp @@ -40,6 +40,22 @@ inline bool is_init() return flag > 0; } + +struct Lifecycle +{ // mostly used in core tests without samrai + Lifecycle(int argc, char** argv) + { + if (MPI_Init(&argc, &argv) != MPI_SUCCESS) + throw std::runtime_error("MPI Initialization failed"); + } + ~Lifecycle() + { + if (MPI_Finalize() != MPI_SUCCESS) + std::cerr << "MPI Finalization failed" << std::endl; + } +}; + + template NO_DISCARD auto mpi_type_for() { diff --git a/src/core/utilities/types.hpp b/src/core/utilities/types.hpp index 347d44c33..8f5b664b1 100644 --- a/src/core/utilities/types.hpp +++ b/src/core/utilities/types.hpp @@ -195,6 +195,15 @@ namespace core T var; }; + template + NO_DISCARD T from_string(std::string const& s) + { + T t; + std::stringstream ss(s); + ss >> t; + return t; + } + template NO_DISCARD std::string to_string_with_precision(T const& a_value, std::size_t const len) { @@ -246,6 +255,24 @@ namespace core } + template + NO_DISCARD inline std::optional get_env_as(std::string const& key) + { + if (auto e = get_env(key)) + return from_string(*e); + return std::nullopt; + } + + + template + NO_DISCARD inline T get_env_as(std::string const& key, T const& t) + { + if (auto e = get_env(key)) + return from_string(*e); + return t; + } + + } // namespace core } // namespace PHARE diff --git a/src/hdf5/detail/h5/h5_file.hpp b/src/hdf5/detail/h5/h5_file.hpp index ad4371c8e..ee56b751e 100644 --- a/src/hdf5/detail/h5/h5_file.hpp +++ b/src/hdf5/detail/h5/h5_file.hpp @@ -69,10 +69,7 @@ class HighFiveFile ~HighFiveFile() {} - NO_DISCARD HiFile& file() - { - return h5file_; - } + NO_DISCARD HiFile& file() { return h5file_; } template @@ -146,15 +143,11 @@ class HighFiveFile void write_attribute(std::string const& keyPath, std::string const& key, Data const& data) { // assumes all keyPaths and values are identical, and no null patches - // clang-format off - PHARE_DEBUG_DO( + PHARE_DEBUG_DO( // auto const paths = core::mpi::collect(keyPath, core::mpi::size()); - for (auto const& path : paths) - PHARE_LOG_LINE_STR(std::to_string(core::mpi::size()) << " " << path) - if (!core::all(paths, [&](auto const& path) { return path == paths[0]; })) - throw std::runtime_error("Function does not support different paths per mpi core"); + if (!core::all(paths, [&](auto const& path) { return path == paths[0]; })) throw std:: + runtime_error("Function does not support different paths per mpi core"); // ) - // clang-format on constexpr bool data_is_vector = core::is_std_vector_v; diff --git a/src/phare/phare.hpp b/src/phare/phare.hpp index d6519253f..e27c2a772 100644 --- a/src/phare/phare.hpp +++ b/src/phare/phare.hpp @@ -4,6 +4,8 @@ #include "core/def/phlop.hpp" // scope timing +#include "core/env.hpp" // scope timing + #include "simulator/simulator.hpp" #include "core/utilities/algorithm.hpp" #include "core/utilities/mpi_utils.hpp" @@ -41,7 +43,7 @@ class SamraiLifeCycle = std::make_shared(StreamAppender{&std::cout}); SAMRAI::tbox::Logger::getInstance()->setWarningAppender(appender); PHARE_WITH_PHLOP( // - if (auto e = core::get_env("PHARE_SCOPE_TIMING", "false"); e == "1" || e == "true") + if (auto e = Env::INSTANCE().PHARE_SCOPE_TIMING(); e == "1" || e == "true") phlop::ScopeTimerMan::INSTANCE() .file_name(".phare_times." + std::to_string(core::mpi::rank()) + ".txt") .init(); // diff --git a/src/python3/cpp_etc.cpp b/src/python3/cpp_etc.cpp index 0550c5a0a..1f41e21eb 100644 --- a/src/python3/cpp_etc.cpp +++ b/src/python3/cpp_etc.cpp @@ -1,16 +1,18 @@ +#include "pybind11/stl.h" +#include "pybind11/stl_bind.h" #include "python3/pybind_def.hpp" #include "simulator/simulator.hpp" +#include "core/env.hpp" #include "core/def/phare_config.hpp" - #include "amr/wrappers/hierarchy.hpp" // for HierarchyRestarter::getRestartFileFullPath - - namespace py = pybind11; +PYBIND11_MAKE_OPAQUE(std::unordered_map); + namespace PHARE::pydata { auto pybind_version() @@ -55,5 +57,25 @@ PYBIND11_MODULE(cpp_etc, m) }); m.def("phare_build_config", []() { return PHARE::build_config(); }); + + m.def("phare_env_exists", + [](std::string const& s) { return Env::INSTANCE().vars.count(s) > 0; }); + m.def("phare_env_val", [](std::string const& s) { return Env::INSTANCE()(s)(); }); + m.def("phare_env_result", [](std::string const& s) { + assert(Env::INSTANCE().vars.count(s) > 0); + return Env::INSTANCE().vars.at(s)->result(); + }); + py::class_(m, "phare_env_var") + .def_readonly("id", &env::Var::id) + .def_readonly("desc", &env::Var::desc) + .def_readonly("options", &env::Var::options) + .def_readonly("default", &env::Var::_default) + .def_readonly("result", &env::Var::r); + + py::bind_map>(m, "EnvVarMap"); + + m.def( + "phare_env_vars", []() -> auto& { return Env::INSTANCE().vars; }, + py::return_value_policy::reference); } } // namespace PHARE::pydata diff --git a/src/simulator/simulator.hpp b/src/simulator/simulator.hpp index 395d02831..bf0f5e823 100644 --- a/src/simulator/simulator.hpp +++ b/src/simulator/simulator.hpp @@ -7,6 +7,7 @@ #include "phare_types.hpp" #include "core/def.hpp" +#include "core/env.hpp" #include "core/logger.hpp" #include "core/utilities/types.hpp" #include "core/utilities/mpi_utils.hpp" @@ -110,32 +111,7 @@ class Simulator : public ISimulator private: auto find_model(std::string name); - auto static log_file_name() - { - // ".log" directory is not created here, but in python if PHARE_LOG != "NONE" - if (auto log = core::get_env("PHARE_LOG")) - { - if (log == "RANK_FILES") - return ".log/" + std::to_string(core::mpi::rank()) + ".out"; - - - if (log == "DATETIME_FILES") - { - auto date_time = core::mpi::date_time(); - auto rank = std::to_string(core::mpi::rank()); - auto size = std::to_string(core::mpi::size()); - return ".log/" + date_time + "_" + rank + "_of_" + size + ".out"; - } - - if (log != "NONE") - throw std::runtime_error( - "PHARE_LOG invalid type, valid keys are RANK_FILES/DATETIME_FILES/NONE"); - } - - return std::string{""}; // unused - } - - std::ofstream log_out{log_file_name()}; + std::ofstream log_out{Env::INSTANCE().PHARE_LOG("PHARE_LOG_FILE")}; std::streambuf* coutbuf = nullptr; std::shared_ptr hierarchy_; std::unique_ptr integrator_; @@ -186,7 +162,7 @@ namespace inline auto logging(std::ofstream& log_out) { std::streambuf* buf = nullptr; - if (auto log = core::get_env("PHARE_LOG"); log != "NONE") + if (auto log = Env::INSTANCE().PHARE_LOG(); log and log != "NONE") { buf = std::cout.rdbuf(); std::cout.rdbuf(log_out.rdbuf()); diff --git a/tests/core/utilities/env/CMakeLists.txt b/tests/core/utilities/env/CMakeLists.txt new file mode 100644 index 000000000..ef9ac396f --- /dev/null +++ b/tests/core/utilities/env/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required (VERSION 3.20.1) + +project(test-phare-env) + +set(SOURCES test_env.cpp) + +add_executable(${PROJECT_NAME} ${SOURCES}) + +target_include_directories(${PROJECT_NAME} PRIVATE + ${GTEST_INCLUDE_DIRS} + ) + +target_link_directories(${PROJECT_NAME} PUBLIC ${MPI_LIBRARY_PATH}) + +target_link_libraries(${PROJECT_NAME} PRIVATE + phare_core + MPI::MPI_C + ${GTEST_LIBS} +) + +add_phare_test(${PROJECT_NAME} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/core/utilities/env/test_env.cpp b/tests/core/utilities/env/test_env.cpp new file mode 100644 index 000000000..cac4e1d1f --- /dev/null +++ b/tests/core/utilities/env/test_env.cpp @@ -0,0 +1,41 @@ + + +#include "core/env.hpp" +#include "gtest/gtest.h" +#include + +namespace PHARE +{ + +TEST(Env, env_setting_works) +{ + { + auto& env = Env::reinit(); // init + + EXPECT_EQ(env.PHARE_LOG(), std::nullopt); + } + + setenv("PHARE_LOG", "NONE", true); + auto& env = Env::reinit(); // init + + EXPECT_EQ(env.PHARE_LOG(), "NONE"); + EXPECT_EQ(env.PHARE_SCOPE_TIMING(), "0"); // default +} + + +TEST(Env, logging_works_rank_files) +{ + setenv("PHARE_LOG", "RANK_FILES", true); + auto& env = Env::reinit(); // init + EXPECT_EQ(env.PHARE_LOG("PHARE_LOG_FILE"), + ".log/" + std::to_string(core::mpi::rank()) + ".out"); +} + +} // namespace PHARE + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + PHARE::core::mpi::Lifecycle mpi_init(argc, argv); + return RUN_ALL_TESTS(); +}