SHiP hang on exit

[heifner]

See #813

• Create a repeatable test based on ship_streamer_test.py
• Investigate and find fix.

[heifner]

Stack trace of two hung threads on exit.

[Inlined] __futex_abstimed_wait_common 0x00007f218762efd9
boost::asio::detail::posix_mutex::lock() posix_mutex.hpp:52
boost::asio::detail::scoped_lock<boost::asio::detail::posix_mutex>::scoped_lock(boost::asio::detail::posix_mutex &) scoped_lock.hpp:46
boost::asio::detail::strand_executor_service::strand_impl::~strand_impl() strand_executor_service.ipp:85
boost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 4ul> >::~strand() strand.hpp:151
boost::asio::execution::detail::any_executor_base::destroy_object<boost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 4ul> > >(boost::asio::execution::detail::any_executor_base &) any_executor.hpp:921
boost::asio::execution::detail::any_executor_base::~any_executor_base() any_executor.hpp:681
boost::asio::execution::any_executor<boost::asio::execution::context_as_t<boost::asio::execution_context &>, boost::asio::execution::detail::blocking::never_t<0>, boost::asio::execution::prefer_only<boost::asio::execution::detail::blocking::possibly_t<0> >, boost::asio::execution::prefer_only<boost::asio::execution::detail::outstanding_work::tracked_t<0> >, boost::asio::execution::prefer_only<boost::asio::execution::detail::outstanding_work::untracked_t<0> >, boost::asio::execution::prefer_only<boost::asio::execution::detail::relationship::fork_t<0> >, boost::asio::execution::prefer_only<boost::asio::execution::detail::relationship::continuation_t<0> > >::~any_executor() any_executor.hpp:1566
boost::asio::any_io_executor::~any_io_executor() any_io_executor.ipp:87
boost::asio::detail::handler_work_base<boost::asio::any_io_executor, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor>, boost::asio::io_context, boost::asio::executor, void>::~handler_work_base() handler_work.hpp:68
boost::asio::detail::handler_work<boost::beast::websocket::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor> >, true>::read_some_op<boost::beast::websocket::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor> >, true>::read_op<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, boost::system::error_code, unsigned long>, boost::beast::basic_flat_buffer<std::allocator<char> > >, boost::asio::mutable_buffer>, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor>, void>::~handler_work() handler_work.hpp:463
boost::asio::detail::reactive_socket_recv_op<boost::beast::detail::buffers_pair<true>, boost::beast::websocket::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor> >, true>::read_some_op<boost::beast::websocket::stream<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor> >, true>::read_op<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, boost::system::error_code, unsigned long>, boost::beast::basic_flat_buffer<std::allocator<char> > >, boost::asio::mutable_buffer>, boost::asio::use_awaitable_t<boost::asio::any_io_executor>::executor_with_default<boost::asio::any_io_executor> >::do_complete(void *, boost::asio::detail::scheduler_operation *, const boost::system::error_code &, unsigned long) reactive_socket_recv_op.hpp:155
boost::asio::detail::scheduler_operation::destroy() scheduler_operation.hpp:45
boost::asio::detail::op_queue_access::destroy<boost::asio::detail::scheduler_operation>(boost::asio::detail::scheduler_operation *) op_queue.hpp:47
boost::asio::detail::op_queue<boost::asio::detail::scheduler_operation>::~op_queue() op_queue.hpp:81
boost::asio::detail::scheduler::abandon_operations(boost::asio::detail::op_queue<boost::asio::detail::scheduler_operation> &) scheduler.ipp:447
boost::asio::detail::epoll_reactor::shutdown() epoll_reactor.ipp:92
boost::asio::detail::service_registry::shutdown_services() service_registry.ipp:44
boost::asio::execution_context::shutdown() execution_context.ipp:41
boost::asio::io_context::~io_context() io_context.ipp:58
appbase::priority_queue_executor::~priority_queue_executor() application.hpp:25
appbase::application_base::~application_base() application_base.cpp:108
appbase::application_t<appbase::priority_queue_executor>::~application_t() application_base.hpp:328
appbase::application_t<appbase::priority_queue_executor>::reset_app_singleton() application_base.hpp:338
appbase::scoped_app::~scoped_app() application_instance.hpp:83
main main.cpp:210
__wrap_main(int, char **) compile_monitor.cpp:309


[Inlined] __futex_abstimed_wait_common64 0x00007f218762f117
boost::asio::detail::posix_event::wait<boost::asio::detail::conditionally_enabled_mutex::scoped_lock>(boost::asio::detail::conditionally_enabled_mutex::scoped_lock &) posix_event.hpp:119
boost::asio::detail::conditionally_enabled_event::wait(boost::asio::detail::conditionally_enabled_mutex::scoped_lock &) conditionally_enabled_event.hpp:97
boost::asio::detail::scheduler::do_run_one(boost::asio::detail::conditionally_enabled_mutex::scoped_lock &, boost::asio::detail::scheduler_thread_info &, const boost::system::error_code &) scheduler.ipp:502
boost::asio::detail::scheduler::run(boost::system::error_code &) scheduler.ipp:210
boost::asio::detail::resolver_service_base::work_scheduler_runner::operator()() resolver_service_base.ipp:38
boost::asio::detail::posix_thread::func<boost::asio::detail::resolver_service_base::work_scheduler_runner>::run() posix_thread.hpp:86
boost_asio_detail_posix_thread_function(void *) posix_thread.ipp:74