From 6d5248659b40a0ae9723c032cd4aec9181168318 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 3 Jan 2024 11:22:02 -0600 Subject: [PATCH 1/3] GH-2021 Pause net_plugin during snapshot write --- libraries/chain/controller.cpp | 14 ++++++++- .../chain/include/eosio/chain/controller.hpp | 2 ++ plugins/net_plugin/net_plugin.cpp | 30 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 3d92818fe2..decbce140c 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -251,6 +251,7 @@ struct controller_impl { named_thread_pool thread_pool; deep_mind_handler* deep_mind_logger = nullptr; bool okay_to_print_integrity_hash_on_stop = false; + std::atomic writing_snapshot = false; thread_local static platform_timer timer; // a copy for main thread and each read-only thread #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) @@ -3246,7 +3247,18 @@ fc::sha256 controller::calculate_integrity_hash() { try { void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) { EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" ); - return my->add_to_snapshot(snapshot); + my->writing_snapshot = true; + try { + my->add_to_snapshot(snapshot); + my->writing_snapshot = false; + } catch(...) { + my->writing_snapshot = false; + throw; + } +} + +bool controller::is_writing_snapshot() const { + return my->writing_snapshot; } int64_t controller::set_proposed_producers( vector producers ) { diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index e25a201050..b601b2f6c4 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -256,6 +256,8 @@ namespace eosio { namespace chain { fc::sha256 calculate_integrity_hash(); void write_snapshot( const snapshot_writer_ptr& snapshot ); + // thread-safe + bool is_writing_snapshot()const; bool sender_avoids_whitelist_blacklist_enforcement( account_name sender )const; void check_actor_list( const flat_set& actors )const; diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 4784ff63c1..02d6ec0bf4 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -522,6 +522,11 @@ namespace eosio { mutable fc::mutex chain_info_mtx; // protects chain_info_t chain_info_t chain_info GUARDED_BY(chain_info_mtx); + alignas(hardware_destructive_interference_size) + std::mutex pause_mtx; + std::condition_variable pause_cv; + bool paused = false; + public: void update_chain_info(); chain_info_t get_chain_info() const; @@ -537,6 +542,28 @@ namespace eosio { void start_expire_timer(); void start_monitors(); + void wait_if_paused() { + controller& cc = chain_plug->chain(); + if (!cc.is_writing_snapshot()) + return; + + std::unique_lock l(pause_mtx); + paused = true; + while (!pause_cv.wait_for(l, std::chrono::milliseconds(10), [&]{ return !paused; } )) { + if (!cc.is_writing_snapshot()) { + paused = false; + } + } + } + + void unpause() { + std::lock_guard g(pause_mtx); + if (paused) { + paused = false; + pause_cv.notify_all(); + } + } + void expire(); /** \name Peer Timestamps * Time message handling @@ -2897,6 +2924,8 @@ namespace eosio { return; } + my_impl->wait_if_paused(); + boost::asio::async_read( *socket, pending_message_buffer.get_buffer_sequence_for_boost_async_read(), completion_handler, boost::asio::bind_executor( strand, @@ -3890,6 +3919,7 @@ namespace eosio { // called from application thread void net_plugin_impl::on_accepted_block_header(const block_state_ptr& bs) { + unpause(); update_chain_info(); dispatcher->strand.post([bs]() { From c4ed79a85927054836ee6cb6f8cdb62649c6a2b8 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 3 Jan 2024 13:37:55 -0600 Subject: [PATCH 2/3] GH-2021 Simplify pause implementation. --- libraries/chain/controller.cpp | 15 ++++++--------- plugins/net_plugin/net_plugin.cpp | 26 +++----------------------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index decbce140c..d61c23737e 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -3247,18 +3247,15 @@ fc::sha256 controller::calculate_integrity_hash() { try { void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) { EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" ); - my->writing_snapshot = true; - try { - my->add_to_snapshot(snapshot); - my->writing_snapshot = false; - } catch(...) { - my->writing_snapshot = false; - throw; - } + my->writing_snapshot.store(true, std::memory_order_release); + fc::scoped_exit> e = [&] { + my->writing_snapshot.store(false, std::memory_order_release); + }; + my->add_to_snapshot(snapshot); } bool controller::is_writing_snapshot() const { - return my->writing_snapshot; + return my->writing_snapshot.load(std::memory_order_acquire); } int64_t controller::set_proposed_producers( vector producers ) { diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 02d6ec0bf4..ce34dceafd 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -522,11 +522,6 @@ namespace eosio { mutable fc::mutex chain_info_mtx; // protects chain_info_t chain_info_t chain_info GUARDED_BY(chain_info_mtx); - alignas(hardware_destructive_interference_size) - std::mutex pause_mtx; - std::condition_variable pause_cv; - bool paused = false; - public: void update_chain_info(); chain_info_t get_chain_info() const; @@ -542,25 +537,11 @@ namespace eosio { void start_expire_timer(); void start_monitors(); + // we currently pause on snapshot generation void wait_if_paused() { controller& cc = chain_plug->chain(); - if (!cc.is_writing_snapshot()) - return; - - std::unique_lock l(pause_mtx); - paused = true; - while (!pause_cv.wait_for(l, std::chrono::milliseconds(10), [&]{ return !paused; } )) { - if (!cc.is_writing_snapshot()) { - paused = false; - } - } - } - - void unpause() { - std::lock_guard g(pause_mtx); - if (paused) { - paused = false; - pause_cv.notify_all(); + while (cc.is_writing_snapshot()) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } @@ -3919,7 +3900,6 @@ namespace eosio { // called from application thread void net_plugin_impl::on_accepted_block_header(const block_state_ptr& bs) { - unpause(); update_chain_info(); dispatcher->strand.post([bs]() { From 8fe4f3c6c550361562c0f83042e0e336c9960d74 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 3 Jan 2024 13:48:28 -0600 Subject: [PATCH 3/3] GH-2021 Add const --- plugins/net_plugin/net_plugin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index ce34dceafd..5b6e618178 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -538,7 +538,7 @@ namespace eosio { void start_monitors(); // we currently pause on snapshot generation - void wait_if_paused() { + void wait_if_paused() const { controller& cc = chain_plug->chain(); while (cc.is_writing_snapshot()) { std::this_thread::sleep_for(std::chrono::milliseconds(10));