From 73120f5bdf822b7dc1205a47b4be59592ac3db39 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 30 Oct 2024 13:52:22 -0500 Subject: [PATCH] GH-985 Add ability to interrupt transaction in apply_block --- libraries/chain/controller.cpp | 37 +++++++++++++++++-- .../chain/include/eosio/chain/controller.hpp | 3 ++ .../chain/include/eosio/chain/exceptions.hpp | 8 ++-- .../include/eosio/chain/platform_timer.hpp | 1 + .../chain/platform_timer_asio_fallback.cpp | 5 +++ libraries/chain/platform_timer_kqueue.cpp | 5 +++ libraries/chain/platform_timer_posix.cpp | 5 +++ libraries/chain/transaction_context.cpp | 5 ++- programs/nodeos/main.cpp | 5 +++ 9 files changed, 66 insertions(+), 8 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index fd953c585b..8fa55cadf0 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1006,6 +1006,8 @@ struct controller_impl { async_t async_aggregation = async_t::yes; // by default we process incoming votes asynchronously my_finalizers_t my_finalizers; std::atomic writing_snapshot = false; + std::atomic applying_block = false; + platform_timer& main_thread_timer; thread_local static platform_timer timer; // a copy for main thread and each read-only thread #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED) @@ -1285,6 +1287,7 @@ struct controller_impl { read_mode( cfg.read_mode ), thread_pool(), my_finalizers(cfg.finalizers_dir / config::safety_filename), + main_thread_timer(timer), // assumes constructor is called from main thread wasmif( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty() ) { assert(cfg.chain_thread_pool_size > 0); @@ -3780,6 +3783,9 @@ struct controller_impl { } } + applying_block = true; + auto apply = fc::make_scoped_exit([&](){ applying_block = false; }); + transaction_trace_ptr trace; size_t packed_idx = 0; @@ -3806,7 +3812,11 @@ struct controller_impl { std::holds_alternative(receipt.trx); if( transaction_failed && !transaction_can_fail) { - edump((*trace)); + if (trace->except->code() == interrupt_exception::code_value) { + ilog("Interrupt of trx: ${t}", ("t", *trace)); + } else { + edump((*trace)); + } throw *trace->except; } @@ -3875,7 +3885,8 @@ struct controller_impl { } catch ( const boost::interprocess::bad_alloc& ) { throw; } catch ( const fc::exception& e ) { - edump((e.to_detail_string())); + if (e.code() != interrupt_exception::code_value) + edump((e.to_detail_string())); abort_block(); throw; } catch ( const std::exception& e ) { @@ -4431,8 +4442,12 @@ struct controller_impl { } catch ( const boost::interprocess::bad_alloc& ) { throw; } catch (const fc::exception& e) { - elog("exception thrown while applying block ${bn} : ${id}, previous ${p}, error: ${e}", - ("bn", bsp->block_num())("id", bsp->id())("p", bsp->previous())("e", e.to_detail_string())); + if (e.code() == interrupt_exception::code_value) { + ilog("interrupt while applying block ${bn} : ${id}", ("bn", bsp->block_num())("id", bsp->id())); + } else { + elog("exception thrown while applying block ${bn} : ${id}, previous ${p}, error: ${e}", + ("bn", bsp->block_num())("id", bsp->id())("p", bsp->previous())("e", e.to_detail_string())); + } except = std::current_exception(); } catch (const std::exception& e) { elog("exception thrown while applying block ${bn} : ${id}, previous ${p}, error: ${e}", @@ -4495,6 +4510,16 @@ struct controller_impl { return applied_trxs; } + void interrupt_transaction() { + // Only interrupt transaction if applying a block. Speculative trxs already have a deadline set so they + // have limited run time already. This is to allow killing a long-running transaction in a block being + // validated. + if (applying_block) { + ilog("Interrupting apply block"); + main_thread_timer.expire_now(); + } + } + // @param if_active true if instant finality is active static checksum256_type calc_merkle( deque&& digests, bool if_active ) { if (if_active) { @@ -5255,6 +5280,10 @@ deque controller::abort_block() { return my->abort_block(); } +void controller::interrupt_transaction() { + my->interrupt_transaction(); +} + boost::asio::io_context& controller::get_thread_pool() { return my->thread_pool.get_executor(); } diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index cf8f12e7e4..fe58f39f95 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -206,6 +206,9 @@ namespace eosio::chain { */ deque abort_block(); + /// Expected to be called from signal handler + void interrupt_transaction(); + /** * */ diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 1539b7c3be..fc6ef200b3 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -381,6 +381,9 @@ namespace eosio { namespace chain { 3080005, "Transaction CPU usage is too much for the remaining allowable usage of the current block" ) FC_DECLARE_DERIVED_EXCEPTION( deadline_exception, resource_exhausted_exception, 3080006, "Transaction took too long" ) + FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception, + 3081001, "Transaction reached the deadline set due to leeway on account CPU limits" ) + FC_DECLARE_DERIVED_EXCEPTION( greylist_net_usage_exceeded, resource_exhausted_exception, 3080007, "Transaction exceeded the current greylisted account network usage limit" ) FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception, @@ -389,9 +392,8 @@ namespace eosio { namespace chain { 3080009, "Read-only transaction eos-vm-oc compile temporary failure" ) FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception, 3080010, "Read-only transaction eos-vm-oc compile permanent failure" ) - - FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception, - 3081001, "Transaction reached the deadline set due to leeway on account CPU limits" ) + FC_DECLARE_DERIVED_EXCEPTION( interrupt_exception, resource_exhausted_exception, + 3080011, "Transaction interrupted by signal" ) FC_DECLARE_DERIVED_EXCEPTION( authorization_exception, chain_exception, 3090000, "Authorization exception" ) diff --git a/libraries/chain/include/eosio/chain/platform_timer.hpp b/libraries/chain/include/eosio/chain/platform_timer.hpp index 29a8d62d46..72fb8d1fef 100644 --- a/libraries/chain/include/eosio/chain/platform_timer.hpp +++ b/libraries/chain/include/eosio/chain/platform_timer.hpp @@ -17,6 +17,7 @@ struct platform_timer { void start(fc::time_point tp); void stop(); + void expire_now(); /* Sets a callback for when timer expires. Be aware this could might fire from a signal handling context and/or on any particular thread. Only a single callback can be registered at once; trying to register more will diff --git a/libraries/chain/platform_timer_asio_fallback.cpp b/libraries/chain/platform_timer_asio_fallback.cpp index 28525c7968..7ec808e128 100644 --- a/libraries/chain/platform_timer_asio_fallback.cpp +++ b/libraries/chain/platform_timer_asio_fallback.cpp @@ -84,6 +84,11 @@ void platform_timer::start(fc::time_point tp) { } } +void platform_timer::expire_now() { + expired = 1; + call_expiration_callback(); +} + void platform_timer::stop() { if(expired) return; diff --git a/libraries/chain/platform_timer_kqueue.cpp b/libraries/chain/platform_timer_kqueue.cpp index 3cb341a031..3e9dc68104 100644 --- a/libraries/chain/platform_timer_kqueue.cpp +++ b/libraries/chain/platform_timer_kqueue.cpp @@ -106,6 +106,11 @@ void platform_timer::start(fc::time_point tp) { } } +void platform_timer::expire_now() { + expired = 1; + call_expiration_callback(); +} + void platform_timer::stop() { if(expired) return; diff --git a/libraries/chain/platform_timer_posix.cpp b/libraries/chain/platform_timer_posix.cpp index bb000de5c3..0a50ebcade 100644 --- a/libraries/chain/platform_timer_posix.cpp +++ b/libraries/chain/platform_timer_posix.cpp @@ -71,6 +71,11 @@ void platform_timer::start(fc::time_point tp) { } } +void platform_timer::expire_now() { + expired = 1; + call_expiration_callback(); +} + void platform_timer::stop() { if(expired) return; diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index c59a9f2f8b..279b64a982 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -459,7 +459,10 @@ namespace eosio::chain { return; auto now = fc::time_point::now(); - if( explicit_billed_cpu_time || deadline_exception_code == deadline_exception::code_value ) { + if (explicit_billed_cpu_time) { + EOS_THROW( interrupt_exception, "interrupt signaled, ran ${bt}us, start ${s}", + ("bt", now - pseudo_start)("s", start) ); + } else if( deadline_exception_code == deadline_exception::code_value ) { EOS_THROW( deadline_exception, "deadline exceeded ${billing_timer}us", ("billing_timer", now - pseudo_start)("now", now)("deadline", _deadline)("start", start) ); } else if( deadline_exception_code == block_cpu_usage_exceeded::code_value ) { diff --git a/programs/nodeos/main.cpp b/programs/nodeos/main.cpp index a73e7cc582..a3533da6ec 100644 --- a/programs/nodeos/main.cpp +++ b/programs/nodeos/main.cpp @@ -171,6 +171,8 @@ int main(int argc, char** argv) app->set_stop_executor_cb([&app]() { ilog("appbase quit called"); app->get_io_context().stop(); + auto& chain = app->get_plugin().chain(); + chain.interrupt_transaction(); }); app->set_version(htonl(short_hash)); app->set_version_string(eosio::version::version_client()); @@ -220,6 +222,9 @@ int main(int argc, char** argv) elog( "database dirty flag set (likely due to unclean shutdown): replay required" ); return DATABASE_DIRTY; } + } else if (e.code() == interrupt_exception::code_value) { + ilog("Interrupted, successfully exiting"); + return SUCCESS; } elog( "${e}", ("e", e.to_detail_string())); return OTHER_FAIL;