GH-985 Add ability to interrupt transaction in apply_block

libraries/chain/controller.cpp

@@ -1006,6 +1006,8 @@ struct controller_impl {
    async_t                         async_aggregation = async_t::yes; // by default we process incoming votes asynchronously
    my_finalizers_t                 my_finalizers;
    std::atomic<bool>               writing_snapshot = false;
+   std::atomic<bool>               applying_block = false;
+   platform_timer&                 main_thread_timer;
 
    thread_local static platform_timer timer; // a copy for main thread and each read-only thread
 #if defined(EOSIO_EOS_VM_RUNTIME_ENABLED) || defined(EOSIO_EOS_VM_JIT_RUNTIME_ENABLED)
@@ -1285,6 +1287,7 @@ struct controller_impl {
     read_mode( cfg.read_mode ),
     thread_pool(),
     my_finalizers(cfg.finalizers_dir / config::safety_filename),
+    main_thread_timer(timer), // assumes constructor is called from main thread
     wasmif( conf.wasm_runtime, conf.eosvmoc_tierup, db, conf.state_dir, conf.eosvmoc_config, !conf.profile_accounts.empty() )
    {
       assert(cfg.chain_thread_pool_size > 0);
@@ -3780,6 +3783,9 @@ struct controller_impl {
                }
             }
 
+            applying_block = true;
+            auto apply = fc::make_scoped_exit([&](){ applying_block = false; });
+
             transaction_trace_ptr trace;
 
             size_t packed_idx = 0;
@@ -3806,7 +3812,11 @@ struct controller_impl {
                                            std::holds_alternative<transaction_id_type>(receipt.trx);
 
                if( transaction_failed && !transaction_can_fail) {
-                  edump((*trace));
+                  if (trace->except->code() == interrupt_exception::code_value) {
+                     ilog("Interrupt of trx: ${t}", ("t", *trace));
+                  } else {
+                     edump((*trace));
+                  }
                   throw *trace->except;
                }
 
@@ -3875,7 +3885,8 @@ struct controller_impl {
          } catch ( const boost::interprocess::bad_alloc& ) {
             throw;
          } catch ( const fc::exception& e ) {
-            edump((e.to_detail_string()));
+            if (e.code() != interrupt_exception::code_value)
+               edump((e.to_detail_string()));
             abort_block();
             throw;
          } catch ( const std::exception& e ) {
@@ -4431,8 +4442,12 @@ struct controller_impl {
             } catch ( const boost::interprocess::bad_alloc& ) {
                throw;
             } catch (const fc::exception& e) {
-               elog("exception thrown while applying block ${bn} : ${id}, previous ${p}, error: ${e}",
-                    ("bn", bsp->block_num())("id", bsp->id())("p", bsp->previous())("e", e.to_detail_string()));
+               if (e.code() == interrupt_exception::code_value) {
+                  ilog("interrupt while applying block ${bn} : ${id}", ("bn", bsp->block_num())("id", bsp->id()));
+               } else {
+                  elog("exception thrown while applying block ${bn} : ${id}, previous ${p}, error: ${e}",
+                       ("bn", bsp->block_num())("id", bsp->id())("p", bsp->previous())("e", e.to_detail_string()));
+               }
                except = std::current_exception();
             } catch (const std::exception& e) {
                elog("exception thrown while applying block ${bn} : ${id}, previous ${p}, error: ${e}",
@@ -4495,6 +4510,16 @@ struct controller_impl {
       return applied_trxs;
    }
 
+   void interrupt_transaction() {
+      // Only interrupt transaction if applying a block. Speculative trxs already have a deadline set so they
+      // have limited run time already. This is to allow killing a long-running transaction in a block being
+      // validated.
+      if (applying_block) {
+         ilog("Interrupting apply block");
+         main_thread_timer.expire_now();
+      }
+   }
+
    // @param if_active true if instant finality is active
    static checksum256_type calc_merkle( deque<digest_type>&& digests, bool if_active ) {
       if (if_active) {
@@ -5255,6 +5280,10 @@ deque<transaction_metadata_ptr> controller::abort_block() {
    return my->abort_block();
 }
 
+void controller::interrupt_transaction() {
+   my->interrupt_transaction();
+}
+
 boost::asio::io_context& controller::get_thread_pool() {
    return my->thread_pool.get_executor();
 }

libraries/chain/include/eosio/chain/controller.hpp

@@ -206,6 +206,9 @@ namespace eosio::chain {
           */
          deque<transaction_metadata_ptr> abort_block();
 
+         /// Expected to be called from signal handler
+         void interrupt_transaction();
+
        /**
         *
         */

libraries/chain/include/eosio/chain/exceptions.hpp

@@ -381,6 +381,9 @@ namespace eosio { namespace chain {
                                     3080005, "Transaction CPU usage is too much for the remaining allowable usage of the current block" )
       FC_DECLARE_DERIVED_EXCEPTION( deadline_exception, resource_exhausted_exception,
                                     3080006, "Transaction took too long" )
+         FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
+                                       3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )
+
       FC_DECLARE_DERIVED_EXCEPTION( greylist_net_usage_exceeded, resource_exhausted_exception,
                                     3080007, "Transaction exceeded the current greylisted account network usage limit" )
       FC_DECLARE_DERIVED_EXCEPTION( greylist_cpu_usage_exceeded, resource_exhausted_exception,
@@ -389,9 +392,8 @@ namespace eosio { namespace chain {
                                     3080009, "Read-only transaction eos-vm-oc compile temporary failure" )
       FC_DECLARE_DERIVED_EXCEPTION( ro_trx_vm_oc_compile_permanent_failure, resource_exhausted_exception,
                                     3080010, "Read-only transaction eos-vm-oc compile permanent failure" )
-
-      FC_DECLARE_DERIVED_EXCEPTION( leeway_deadline_exception, deadline_exception,
-                                    3081001, "Transaction reached the deadline set due to leeway on account CPU limits" )
+      FC_DECLARE_DERIVED_EXCEPTION( interrupt_exception, resource_exhausted_exception,
+                                    3080011, "Transaction interrupted by signal" )
 
    FC_DECLARE_DERIVED_EXCEPTION( authorization_exception, chain_exception,
                                  3090000, "Authorization exception" )

libraries/chain/include/eosio/chain/platform_timer.hpp

@@ -17,6 +17,7 @@ struct platform_timer {
 
    void start(fc::time_point tp);
    void stop();
+   void expire_now();
 
    /* Sets a callback for when timer expires. Be aware this could might fire from a signal handling context and/or
       on any particular thread. Only a single callback can be registered at once; trying to register more will

libraries/chain/platform_timer_asio_fallback.cpp

@@ -84,6 +84,11 @@ void platform_timer::start(fc::time_point tp) {
    }
 }
 
+void platform_timer::expire_now() {
+   expired = 1;
+   call_expiration_callback();
+}
+
 void platform_timer::stop() {
    if(expired)
       return;

libraries/chain/platform_timer_kqueue.cpp

@@ -106,6 +106,11 @@ void platform_timer::start(fc::time_point tp) {
    }
 }
 
+void platform_timer::expire_now() {
+   expired = 1;
+   call_expiration_callback();
+}
+
 void platform_timer::stop() {
    if(expired)
       return;

libraries/chain/platform_timer_posix.cpp

@@ -71,6 +71,11 @@ void platform_timer::start(fc::time_point tp) {
    }
 }
 
+void platform_timer::expire_now() {
+   expired = 1;
+   call_expiration_callback();
+}
+
 void platform_timer::stop() {
    if(expired)
       return;

libraries/chain/transaction_context.cpp

@@ -459,7 +459,10 @@ namespace eosio::chain {
          return;
 
       auto now = fc::time_point::now();
-      if( explicit_billed_cpu_time || deadline_exception_code == deadline_exception::code_value ) {
+      if (explicit_billed_cpu_time) {
+         EOS_THROW( interrupt_exception, "interrupt signaled, ran ${bt}us, start ${s}",
+                    ("bt", now - pseudo_start)("s", start) );
+      } else if( deadline_exception_code == deadline_exception::code_value ) {
          EOS_THROW( deadline_exception, "deadline exceeded ${billing_timer}us",
                      ("billing_timer", now - pseudo_start)("now", now)("deadline", _deadline)("start", start) );
       } else if( deadline_exception_code == block_cpu_usage_exceeded::code_value ) {

programs/nodeos/main.cpp

@@ -171,6 +171,8 @@ int main(int argc, char** argv)
       app->set_stop_executor_cb([&app]() {
          ilog("appbase quit called");
          app->get_io_context().stop();
+         auto& chain = app->get_plugin<chain_plugin>().chain();
+         chain.interrupt_transaction();
       });
       app->set_version(htonl(short_hash));
       app->set_version_string(eosio::version::version_client());
@@ -220,6 +222,9 @@ int main(int argc, char** argv)
             elog( "database dirty flag set (likely due to unclean shutdown): replay required" );
             return DATABASE_DIRTY;
          }
+      } else if (e.code() == interrupt_exception::code_value) {
+         ilog("Interrupted, successfully exiting");
+         return SUCCESS;
       }
       elog( "${e}", ("e", e.to_detail_string()));
       return OTHER_FAIL;