diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 786a0972ff..d9d9acf431 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -90,7 +90,6 @@ target_include_directories(eosio_rapidjson INTERFACE ../rapidjson/include) ## SORT .cpp by most likely to change / break compile add_library( eosio_chain - merkle.cpp name.cpp transaction.cpp block.cpp diff --git a/libraries/chain/block_state.cpp b/libraries/chain/block_state.cpp index 1528284875..bae2ea5d43 100644 --- a/libraries/chain/block_state.cpp +++ b/libraries/chain/block_state.cpp @@ -88,8 +88,8 @@ block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& b result.valid_qc = {}; // best qc received from the network inside block extension, empty until first savanna proper IF block // Calculate Merkle tree root in Savanna way so that it is stored in Leaf Node when building block_state. - auto digests = *bsp.action_receipt_digests_savanna; - auto action_mroot_svnn = calculate_merkle(std::move(digests)); + const auto& digests = *bsp.action_receipt_digests_savanna; + auto action_mroot_svnn = calculate_merkle(digests); // build leaf_node and validation_tree valid_t::finality_leaf_node_t leaf_node { diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 76a6a6f282..599d040dae 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -674,13 +674,13 @@ struct building_block { auto [transaction_mroot, action_mroot] = std::visit( overloaded{[&](digests_t& trx_receipts) { // calculate the two merkle roots in separate threads auto trx_merkle_fut = - post_async_task(ioc, [&]() { return legacy_merkle(std::move(trx_receipts)); }); + post_async_task(ioc, [&]() { return calculate_merkle_legacy(std::move(trx_receipts)); }); auto action_merkle_fut = - post_async_task(ioc, [&]() { return legacy_merkle(std::move(*action_receipts.digests_l)); }); + post_async_task(ioc, [&]() { return calculate_merkle_legacy(std::move(*action_receipts.digests_l)); }); return std::make_pair(trx_merkle_fut.get(), action_merkle_fut.get()); }, [&](const checksum256_type& trx_checksum) { - return std::make_pair(trx_checksum, legacy_merkle(std::move(*action_receipts.digests_l))); + return std::make_pair(trx_checksum, calculate_merkle_legacy(std::move(*action_receipts.digests_l))); }}, trx_mroot_or_receipt_digests()); @@ -706,15 +706,14 @@ struct building_block { [&](building_block_if& bb) -> assembled_block { // compute the action_mroot and transaction_mroot auto [transaction_mroot, action_mroot] = std::visit( - overloaded{[&](digests_t& trx_receipts) { // calculate the two merkle roots in separate threads - auto trx_merkle_fut = - post_async_task(ioc, [&]() { return calculate_merkle(std::move(trx_receipts)); }); - auto action_merkle_fut = - post_async_task(ioc, [&]() { return calculate_merkle(std::move(*action_receipts.digests_s)); }); - return std::make_pair(trx_merkle_fut.get(), action_merkle_fut.get()); + overloaded{[&](digests_t& trx_receipts) { + // calculate_merkle takes 3.2ms for 50,000 digests (legacy version took 11.1ms) + return std::make_pair(calculate_merkle(trx_receipts), + calculate_merkle(*action_receipts.digests_s)); }, [&](const checksum256_type& trx_checksum) { - return std::make_pair(trx_checksum, calculate_merkle(std::move(*action_receipts.digests_s))); + return std::make_pair(trx_checksum, + calculate_merkle(*action_receipts.digests_s)); }}, trx_mroot_or_receipt_digests()); @@ -1308,8 +1307,8 @@ struct controller_impl { // IRREVERSIBLE applies (validates) blocks when irreversible, new_valid will be done after apply in log_irreversible assert(read_mode == db_read_mode::IRREVERSIBLE || legacy->action_receipt_digests_savanna); if (legacy->action_receipt_digests_savanna) { - auto digests = *legacy->action_receipt_digests_savanna; - auto action_mroot = calculate_merkle(std::move(digests)); + const auto& digests = *legacy->action_receipt_digests_savanna; + auto action_mroot = calculate_merkle(digests); // Create the valid structure for producing new_bsp->valid = prev->new_valid(*new_bsp, action_mroot, new_bsp->strong_digest); } @@ -1522,8 +1521,8 @@ struct controller_impl { validator_t{}, skip_validate_signee); // legacy_branch is from head, all should be validated assert(bspl->action_receipt_digests_savanna); - auto digests = *bspl->action_receipt_digests_savanna; - auto action_mroot = calculate_merkle(std::move(digests)); + const auto& digests = *bspl->action_receipt_digests_savanna; + auto action_mroot = calculate_merkle(digests); // Create the valid structure for producing new_bsp->valid = prev->new_valid(*new_bsp, action_mroot, new_bsp->strong_digest); prev = new_bsp; @@ -4066,9 +4065,9 @@ struct controller_impl { // @param if_active true if instant finality is active static checksum256_type calc_merkle( deque&& digests, bool if_active ) { if (if_active) { - return calculate_merkle( std::move(digests) ); + return calculate_merkle( digests ); } else { - return legacy_merkle( std::move(digests) ); + return calculate_merkle_legacy( std::move(digests) ); } } diff --git a/libraries/chain/include/eosio/chain/block_header_state.hpp b/libraries/chain/include/eosio/chain/block_header_state.hpp index 29d5b47ee6..02291c3547 100644 --- a/libraries/chain/include/eosio/chain/block_header_state.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state.hpp @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include #include #include diff --git a/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp b/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp index 7e9e8abb6a..a96b73d414 100644 --- a/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp +++ b/libraries/chain/include/eosio/chain/block_header_state_legacy.hpp @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include #include #include @@ -34,7 +34,7 @@ namespace detail { uint32_t dpos_proposed_irreversible_blocknum = 0; uint32_t dpos_irreversible_blocknum = 0; producer_authority_schedule active_schedule; - incremental_legacy_merkle_tree blockroot_merkle; + incremental_merkle_tree_legacy blockroot_merkle; flat_map producer_to_last_produced; flat_map producer_to_last_implied_irb; block_signing_authority valid_block_signing_authority; diff --git a/libraries/chain/include/eosio/chain/block_state.hpp b/libraries/chain/include/eosio/chain/block_state.hpp index 61429a7bc4..9c1fc5a7d9 100644 --- a/libraries/chain/include/eosio/chain/block_state.hpp +++ b/libraries/chain/include/eosio/chain/block_state.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace eosio::chain { diff --git a/libraries/chain/include/eosio/chain/global_property_object.hpp b/libraries/chain/include/eosio/chain/global_property_object.hpp index 51130d228a..8709cf9667 100644 --- a/libraries/chain/include/eosio/chain/global_property_object.hpp +++ b/libraries/chain/include/eosio/chain/global_property_object.hpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include "multi_index_includes.hpp" diff --git a/libraries/chain/include/eosio/chain/incremental_merkle.hpp b/libraries/chain/include/eosio/chain/incremental_merkle.hpp index ff85e5b5d6..3e9f9dbb01 100644 --- a/libraries/chain/include/eosio/chain/incremental_merkle.hpp +++ b/libraries/chain/include/eosio/chain/incremental_merkle.hpp @@ -6,212 +6,92 @@ namespace eosio::chain { -namespace detail { - -/** - * Given a number of nodes return the depth required to store them - * in a fully balanced binary tree. - * - * @param node_count - the number of nodes in the implied tree - * @return the max depth of the minimal tree that stores them - */ -constexpr uint64_t calculate_max_depth(uint64_t node_count) { - if (node_count == 0) - return 0; - // following is non-floating point equivalent to `std::ceil(std::log2(node_count)) + 1)` (and about 9x faster) - return std::bit_width(std::bit_ceil(node_count)); -} - -template -inline void move_nodes(ContainerA& to, const ContainerB& from) { - to.clear(); - to.insert(to.begin(), from.begin(), from.end()); -} - -template -inline void move_nodes(Container& to, Container&& from) { - to = std::forward(from); -} - - -} /// detail - /** * A balanced merkle tree built in such that the set of leaf nodes can be - * appended to without triggering the reconstruction of inner nodes that - * represent a complete subset of previous nodes. + * appended to without triggering the reconstruction of previously + * constructed nodes. * - * to achieve this new nodes can either imply an set of future nodes - * that achieve a balanced tree OR realize one of these future nodes. + * this is achieved by keeping all possible power of two size trees, so + * for example: + * - after appending 3 digests, we have one `tree of two` digests, and a + * single digest. the mask os 0b11. + * - when appending another digest, a new `tree of two` is constructed with + * the single digest, and these two `trees of two` are conbined in a `tree + * of four`. The original tree of two is unchanged. + * Only the tree of four is stored, the mask 0b100 indicating its rank (4) + * - when appending another digest, the `tree of four` is unchanged, we store + * the new single digest. The mask 0b101 indicates that the tow digest stored + * are the roots of one `tree of four` and one `tree of one` (single digest) * - * Once a sub-tree contains only realized nodes its sub-root will never - * change. This allows proofs based on this merkle to be very stable - * after some time has past only needing to update or add a single + * Once a sub-tree is constructed, its sub-root will never change. + * This allows proofs based on this merkle to be very stable + * after some time has passed, only needing to update or add a single * value to maintain validity. - * - * @param canonical if true use the merkle make_canonical_pair which sets the left/right bits of the hash */ -template class Container = vector, typename ...Args> -class incremental_merkle_impl { - public: - incremental_merkle_impl() = default; - incremental_merkle_impl( const incremental_merkle_impl& ) = default; - incremental_merkle_impl( incremental_merkle_impl&& ) = default; - incremental_merkle_impl& operator= (const incremental_merkle_impl& ) = default; - incremental_merkle_impl& operator= ( incremental_merkle_impl&& ) = default; - - template, incremental_merkle_impl>::value, int> = 0> - incremental_merkle_impl( Allocator&& alloc ):_active_nodes(forward(alloc)){} - - /* - template class OtherContainer, typename ...OtherArgs> - incremental_merkle_impl( incremental_merkle_impl&& other ) - :_node_count(other._node_count) - ,_active_nodes(other._active_nodes.begin(), other.active_nodes.end()) - {} - - incremental_merkle_impl( incremental_merkle_impl&& other ) - :_node_count(other._node_count) - ,_active_nodes(std::forward(other._active_nodes)) - {} - */ - - /** - * Add a node to the incremental tree and recalculate the _active_nodes so they - * are prepared for the next append. - * - * The algorithm for this is to start at the new node and retreat through the tree - * for any node that is the concatenation of a fully-realized node and a partially - * realized node we must record the value of the fully-realized node in the new - * _active_nodes so that the next append can fetch it. Fully realized nodes and - * Fully implied nodes do not have an effect on the _active_nodes. - * - * For convention _AND_ to allow appends when the _node_count is a power-of-2, the - * current root of the incremental tree is always appended to the end of the new - * _active_nodes. - * - * In practice, this can be done iteratively by recording any "left" value that - * is to be combined with an implied node. - * - * If the appended node is a "left" node in its pair, it will immediately push itself - * into the new active nodes list. - * - * If the new node is a "right" node it will begin collapsing upward in the tree, - * reading and discarding the "left" node data from the old active nodes list, until - * it becomes a "left" node. It must then push the "top" of its current collapsed - * sub-tree into the new active nodes list. - * - * Once any value has been added to the new active nodes, all remaining "left" nodes - * should be present in the order they are needed in the previous active nodes as an - * artifact of the previous append. As they are read from the old active nodes, they - * will need to be copied in to the new active nodes list as they are still needed - * for future appends. - * - * As a result, if an append collapses the entire tree while always being the "right" - * node, the new list of active nodes will be empty and by definition the tree contains - * a power-of-2 number of nodes. - * - * Regardless of the contents of the new active nodes list, the top "collapsed" value - * is appended. If this tree is _not_ a power-of-2 number of nodes, this node will - * not be used in the next append but still serves as a conventional place to access - * the root of the current tree. If this _is_ a power-of-2 number of nodes, this node - * will be needed during then collapse phase of the next append so, it serves double - * duty as a legitimate active node and the conventional storage location of the root. - * - * - * @param digest - the node to add - * @return - the new root - */ - const DigestType& append(const DigestType& digest) { - bool partial = false; - auto max_depth = detail::calculate_max_depth(_node_count + 1); - auto current_depth = max_depth - 1; - auto index = _node_count; - auto top = digest; - auto active_iter = _active_nodes.begin(); - auto updated_active_nodes = vector(); - updated_active_nodes.reserve(max_depth); - - while (current_depth > 0) { - if (!(index & 0x1)) { - // we are collapsing from a "left" value and an implied "right" creating a partial node - - // we only need to append this node if it is fully-realized and by definition - // if we have encountered a partial node during collapse this cannot be - // fully-realized - if (!partial) { - updated_active_nodes.emplace_back(top); - } - - // calculate the partially realized node value by implying the "right" value is identical - // to the "left" value - if constexpr (canonical) { - top = DigestType::hash(make_canonical_pair(top, top)); - } else { - top = DigestType::hash(std::make_pair(std::cref(top), std::cref(top))); - } - partial = true; - } else { - // we are collapsing from a "right" value and an fully-realized "left" - - // pull a "left" value from the previous active nodes - const auto& left_value = *active_iter; - ++active_iter; - - // if the "right" value is a partial node we will need to copy the "left" as future appends still need it - // otherwise, it can be dropped from the set of active nodes as we are collapsing a fully-realized node - if (partial) { - updated_active_nodes.emplace_back(left_value); - } - - // calculate the node - if constexpr (canonical) { - top = DigestType::hash(make_canonical_pair(left_value, top)); - } else { - top = DigestType::hash(std::make_pair(std::cref(left_value), std::cref(top))); - } - } - - // move up a level in the tree - --current_depth; - index = index >> 1; - } - - // append the top of the collapsed tree (aka the root of the merkle) - updated_active_nodes.emplace_back(top); - - // store the new active_nodes - detail::move_nodes(_active_nodes, std::move(updated_active_nodes)); - - // update the node count - ++_node_count; - - return _active_nodes.back(); - - } - - /** - * return the current root of the incremental merkle - */ - DigestType get_root() const { - if (_node_count > 0) { - return _active_nodes.back(); +class incremental_merkle_tree { +public: + void append(const digest_type& digest) { + assert(trees.size() == static_cast(detail::popcount(mask))); + _append(digest, trees.end(), 0); + assert(trees.size() == static_cast(detail::popcount(mask))); + } + + digest_type get_root() const { + if (!mask) + return {}; + assert(!trees.empty()); + return _get_root(0); + } + + uint64_t num_digests_appended() const { + return mask; + } + +private: + friend struct fc::reflector; + using vec_it = std::vector::iterator; + + bool is_bit_set(size_t idx) const { return !!(mask & (1ull << idx)); } + void set_bit(size_t idx) { mask |= (1ull << idx); } + void clear_bit(size_t idx) { mask &= ~(1ull << idx); } + + digest_type _get_root(size_t idx) const { + if (idx + 1 == trees.size()) + return trees[idx]; + return detail::hash_combine(trees[idx], _get_root(idx + 1)); // log2 recursion OK + } + + // slot points to the current insertion point. *(slot-1) is the digest for the first bit set >= idx + void _append(const digest_type& digest, vec_it slot, size_t idx) { + if (is_bit_set(idx)) { + assert(!trees.empty()); + if (!is_bit_set(idx+1)) { + // next location is empty, replace its tree with new combination, same number of slots and one bits + *(slot-1) = detail::hash_combine(*(slot-1), digest); + clear_bit(idx); + set_bit(idx+1); } else { - return DigestType(); + assert(trees.size() >= 2); + clear_bit(idx); + clear_bit(idx+1); + digest_type d = detail::hash_combine(*(slot-2), detail::hash_combine(*(slot-1), digest)); + trees.erase(slot-2, slot); + _append(d, slot-2, idx+2); // log2 recursion OK, uses less than 5KB stack space for 4 billion digests + // appended (or 0.25% of default 2MB thread stack size on Ubuntu) } + } else { + trees.insert(slot, digest); + set_bit(idx); } + } - private: - friend struct fc::reflector; - uint64_t _node_count = 0; - Container _active_nodes; + uint64_t mask = 0; // bits set signify tree present in trees vector. + // least signif. bit set maps to smallest tree present. + std::vector trees; // digests representing power of 2 trees, smallest tree last + // to minimize digest copying when appending. + // invariant: `trees.size() == detail::popcount(mask)` }; -typedef incremental_merkle_impl incremental_legacy_merkle_tree; -typedef incremental_merkle_impl shared_incremental_legacy_merkle_tree; -typedef incremental_merkle_impl incremental_merkle_tree; - } /// eosio::chain -FC_REFLECT( eosio::chain::incremental_legacy_merkle_tree, (_active_nodes)(_node_count) ); -FC_REFLECT( eosio::chain::incremental_merkle_tree, (_active_nodes)(_node_count) ); +FC_REFLECT( eosio::chain::incremental_merkle_tree, (mask)(trees) ); diff --git a/libraries/chain/include/eosio/chain/incremental_merkle_legacy.hpp b/libraries/chain/include/eosio/chain/incremental_merkle_legacy.hpp new file mode 100644 index 0000000000..0607ee9e3b --- /dev/null +++ b/libraries/chain/include/eosio/chain/incremental_merkle_legacy.hpp @@ -0,0 +1,203 @@ +#pragma once +#include +#include +#include + +namespace eosio::chain { + +namespace detail { + +/** + * Given a number of nodes return the depth required to store them + * in a fully balanced binary tree. + * + * @param node_count - the number of nodes in the implied tree + * @return the max depth of the minimal tree that stores them + */ +constexpr uint64_t calculate_max_depth(uint64_t node_count) { + if (node_count == 0) + return 0; + // following is non-floating point equivalent to `std::ceil(std::log2(node_count)) + 1)` (and about 9x faster) + return std::bit_width(std::bit_ceil(node_count)); +} + +template +inline void move_nodes(ContainerA& to, const ContainerB& from) { + to.clear(); + to.insert(to.begin(), from.begin(), from.end()); +} + +template +inline void move_nodes(Container& to, Container&& from) { + to = std::forward(from); +} + + +} /// detail + +/** + * A balanced merkle tree built in such that the set of leaf nodes can be + * appended to without triggering the reconstruction of inner nodes that + * represent a complete subset of previous nodes. + * + * to achieve this new nodes can either imply an set of future nodes + * that achieve a balanced tree OR realize one of these future nodes. + * + * Once a sub-tree contains only realized nodes its sub-root will never + * change. This allows proofs based on this merkle to be very stable + * after some time has passed, only needing to update or add a single + * value to maintain validity. + */ +template class Container = vector, typename ...Args> +class incremental_merkle_impl { + public: + incremental_merkle_impl() = default; + incremental_merkle_impl( const incremental_merkle_impl& ) = default; + incremental_merkle_impl( incremental_merkle_impl&& ) = default; + incremental_merkle_impl& operator= (const incremental_merkle_impl& ) = default; + incremental_merkle_impl& operator= ( incremental_merkle_impl&& ) = default; + + template, incremental_merkle_impl>::value, int> = 0> + incremental_merkle_impl( Allocator&& alloc ):_active_nodes(forward(alloc)){} + + /* + template class OtherContainer, typename ...OtherArgs> + incremental_merkle_impl( incremental_merkle_impl&& other ) + :_node_count(other._node_count) + ,_active_nodes(other._active_nodes.begin(), other.active_nodes.end()) + {} + + incremental_merkle_impl( incremental_merkle_impl&& other ) + :_node_count(other._node_count) + ,_active_nodes(std::forward(other._active_nodes)) + {} + */ + + /** + * Add a node to the incremental tree and recalculate the _active_nodes so they + * are prepared for the next append. + * + * The algorithm for this is to start at the new node and retreat through the tree + * for any node that is the concatenation of a fully-realized node and a partially + * realized node we must record the value of the fully-realized node in the new + * _active_nodes so that the next append can fetch it. Fully realized nodes and + * Fully implied nodes do not have an effect on the _active_nodes. + * + * For convention _AND_ to allow appends when the _node_count is a power-of-2, the + * current root of the incremental tree is always appended to the end of the new + * _active_nodes. + * + * In practice, this can be done iteratively by recording any "left" value that + * is to be combined with an implied node. + * + * If the appended node is a "left" node in its pair, it will immediately push itself + * into the new active nodes list. + * + * If the new node is a "right" node it will begin collapsing upward in the tree, + * reading and discarding the "left" node data from the old active nodes list, until + * it becomes a "left" node. It must then push the "top" of its current collapsed + * sub-tree into the new active nodes list. + * + * Once any value has been added to the new active nodes, all remaining "left" nodes + * should be present in the order they are needed in the previous active nodes as an + * artifact of the previous append. As they are read from the old active nodes, they + * will need to be copied in to the new active nodes list as they are still needed + * for future appends. + * + * As a result, if an append collapses the entire tree while always being the "right" + * node, the new list of active nodes will be empty and by definition the tree contains + * a power-of-2 number of nodes. + * + * Regardless of the contents of the new active nodes list, the top "collapsed" value + * is appended. If this tree is _not_ a power-of-2 number of nodes, this node will + * not be used in the next append but still serves as a conventional place to access + * the root of the current tree. If this _is_ a power-of-2 number of nodes, this node + * will be needed during then collapse phase of the next append so, it serves double + * duty as a legitimate active node and the conventional storage location of the root. + * + * + * @param digest - the node to add + * @return - the new root + */ + const DigestType& append(const DigestType& digest) { + bool partial = false; + auto max_depth = detail::calculate_max_depth(_node_count + 1); + auto current_depth = max_depth - 1; + auto index = _node_count; + auto top = digest; + auto active_iter = _active_nodes.begin(); + auto updated_active_nodes = vector(); + updated_active_nodes.reserve(max_depth); + + while (current_depth > 0) { + if (!(index & 0x1)) { + // we are collapsing from a "left" value and an implied "right" creating a partial node + + // we only need to append this node if it is fully-realized and by definition + // if we have encountered a partial node during collapse this cannot be + // fully-realized + if (!partial) { + updated_active_nodes.emplace_back(top); + } + + // calculate the partially realized node value by implying the "right" value is identical + // to the "left" value + top = DigestType::hash(detail::make_legacy_digest_pair(top, top)); + partial = true; + } else { + // we are collapsing from a "right" value and an fully-realized "left" + + // pull a "left" value from the previous active nodes + const auto& left_value = *active_iter; + ++active_iter; + + // if the "right" value is a partial node we will need to copy the "left" as future appends still need it + // otherwise, it can be dropped from the set of active nodes as we are collapsing a fully-realized node + if (partial) { + updated_active_nodes.emplace_back(left_value); + } + + // calculate the node + top = DigestType::hash(detail::make_legacy_digest_pair(left_value, top)); + } + + // move up a level in the tree + --current_depth; + index = index >> 1; + } + + // append the top of the collapsed tree (aka the root of the merkle) + updated_active_nodes.emplace_back(top); + + // store the new active_nodes + detail::move_nodes(_active_nodes, std::move(updated_active_nodes)); + + // update the node count + ++_node_count; + + return _active_nodes.back(); + + } + + /** + * return the current root of the incremental merkle + */ + DigestType get_root() const { + if (_node_count > 0) { + return _active_nodes.back(); + } else { + return DigestType(); + } + } + + private: + friend struct fc::reflector; + uint64_t _node_count = 0; + Container _active_nodes; +}; + +typedef incremental_merkle_impl incremental_merkle_tree_legacy; + +} /// eosio::chain + +FC_REFLECT( eosio::chain::incremental_merkle_tree_legacy, (_active_nodes)(_node_count) ); diff --git a/libraries/chain/include/eosio/chain/merkle.hpp b/libraries/chain/include/eosio/chain/merkle.hpp index 70932926da..8bdaa2a308 100644 --- a/libraries/chain/include/eosio/chain/merkle.hpp +++ b/libraries/chain/include/eosio/chain/merkle.hpp @@ -1,29 +1,114 @@ #pragma once #include +#include +#include +#include +#include -namespace eosio { namespace chain { +namespace eosio::chain { - digest_type make_canonical_left(const digest_type& val); - digest_type make_canonical_right(const digest_type& val); +namespace detail { - bool is_canonical_left(const digest_type& val); - bool is_canonical_right(const digest_type& val); +#if __cplusplus >= 202002L + inline int popcount(uint64_t x) noexcept { return std::popcount(x); } + inline uint64_t bit_floor(uint64_t x) noexcept { return std::bit_floor(x); } +#else + inline int popcount(uint64_t x) noexcept { return __builtin_popcountll(x); } + inline uint64_t bit_floor(uint64_t x) noexcept { return x == 0 ? 0ull : 1ull << (64 - 1 - __builtin_clzll(x)); } +#endif +inline digest_type hash_combine(const digest_type& a, const digest_type& b) { + return digest_type::hash(std::make_pair(std::cref(a), std::cref(b))); +} - inline auto make_canonical_pair(const digest_type& l, const digest_type& r) { - return make_pair(make_canonical_left(l), make_canonical_right(r)); - }; +template +requires std::is_same_v::value_type>, digest_type> +inline digest_type calculate_merkle_pow2(const It& start, const It& end) { + assert(end >= start + 2); + auto size = static_cast(end - start); + assert(detail::bit_floor(size) == size); - /** - * Calculates the merkle root of a set of digests, if ids is odd it will duplicate the last id. - * Uses make_canonical_pair which before hashing sets the first bit of the previous hashes - * to 0 or 1 to indicate the side it is on. - */ - digest_type legacy_merkle( deque&& ids ); + if (size == 2) + return hash_combine(start[0], start[1]); + else { + if (async && size >= 256) { + auto async_calculate_merkle_pow2 = [&start, &size](auto fut) { + size_t slice_size = size / fut.size(); - /** - * Calculates the merkle root of a set of digests. Does not manipulate the digests. - */ - digest_type calculate_merkle( deque&& ids ); + for (size_t i=0; i, + start + slice_size * i, start + slice_size * (i+1)); -} } /// eosio::chain + std::array res; + + for (size_t i=0; i= 2048) { + // use 4 threads. Future array size dictates the number of threads (must be power of two) + return async_calculate_merkle_pow2(std::array, 4>()); + } + // use 2 threads. Future array size dictates the number of threads (must be power of two) + return async_calculate_merkle_pow2(std::array, 2>()); + } else { + auto mid = start + size / 2; + return hash_combine(calculate_merkle_pow2(start, mid), calculate_merkle_pow2(mid, end)); + } + } +} + +} // namespace detail + +// ************* public interface starts here ************************************************ + +// ------------------------------------------------------------------------ +// calculate_merkle: +// ----------------- +// takes two random access iterators delimiting a sequence of `digest_type`, +// returns the root digest for the provided sequence. +// +// does not overwrite passed sequence +// +// log2 recursion OK, uses less than 5KB stack space for 4 billion digests +// appended (or 0.25% of default 2MB thread stack size on Ubuntu). +// ------------------------------------------------------------------------ +template +#if __cplusplus >= 202002L +requires std::random_access_iterator && + std::is_same_v::value_type>, digest_type> +#endif +inline digest_type calculate_merkle(const It& start, const It& end) { + assert(end >= start); + auto size = static_cast(end - start); + if (size <= 1) + return (size == 0) ? digest_type{} : *start; + + auto midpoint = detail::bit_floor(size); + if (size == midpoint) + return detail::calculate_merkle_pow2(start, end); + + auto mid = start + midpoint; + return detail::hash_combine(detail::calculate_merkle_pow2(start, mid), + calculate_merkle(mid, end)); +} + +// -------------------------------------------------------------------------- +// calculate_merkle: +// ----------------- +// takes a container or `std::span` of `digest_type`, returns the root digest +// for the sequence of digests in the container. +// -------------------------------------------------------------------------- +template +#if __cplusplus >= 202002L +requires std::random_access_iterator && + std::is_same_v, digest_type> +#endif +inline digest_type calculate_merkle(const Cont& ids) { + return calculate_merkle(ids.begin(), ids.end()); // cbegin not supported for std::span until C++23. +} + + +} /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/merkle_legacy.hpp b/libraries/chain/include/eosio/chain/merkle_legacy.hpp new file mode 100644 index 0000000000..5746f2e07f --- /dev/null +++ b/libraries/chain/include/eosio/chain/merkle_legacy.hpp @@ -0,0 +1,57 @@ +#pragma once +#include +#include + +namespace eosio::chain { + +namespace detail { + + inline digest_type make_legacy_left_digest(const digest_type& val) { + digest_type left = val; + left._hash[0] &= 0xFFFFFFFFFFFFFF7FULL; + return left; + } + + inline digest_type make_legacy_right_digest(const digest_type& val) { + digest_type right = val; + right._hash[0] |= 0x0000000000000080ULL; + return right; + } + + inline bool is_legacy_left_digest(const digest_type& val) { + return (val._hash[0] & 0x0000000000000080ULL) == 0; + } + + inline bool is_legacy_right_digest(const digest_type& val) { + return (val._hash[0] & 0x0000000000000080ULL) != 0; + } + + inline auto make_legacy_digest_pair(const digest_type& l, const digest_type& r) { + return make_pair(make_legacy_left_digest(l), make_legacy_right_digest(r)); + }; + +} // namespace detail + +/** + * Calculates the merkle root of a set of digests, if ids is odd it will duplicate the last id. + * Uses make_legacy_digest_pair which before hashing sets the first bit of the previous hashes + * to 0 or 1 to indicate the side it is on. + */ +inline digest_type calculate_merkle_legacy( deque ids ) { + if( 0 == ids.size() ) { return digest_type(); } + + while( ids.size() > 1 ) { + if( ids.size() % 2 ) + ids.push_back(ids.back()); + + for (size_t i = 0; i < ids.size() / 2; i++) { + ids[i] = digest_type::hash(detail::make_legacy_digest_pair(ids[2 * i], ids[(2 * i) + 1])); + } + + ids.resize(ids.size() / 2); + } + + return ids.front(); +} + +} /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/snapshot_detail.hpp b/libraries/chain/include/eosio/chain/snapshot_detail.hpp index 23c37fc149..6bf0a8b7a9 100644 --- a/libraries/chain/include/eosio/chain/snapshot_detail.hpp +++ b/libraries/chain/include/eosio/chain/snapshot_detail.hpp @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include #include #include @@ -29,7 +29,7 @@ namespace eosio::chain::snapshot_detail { uint32_t dpos_proposed_irreversible_blocknum = 0; uint32_t dpos_irreversible_blocknum = 0; legacy::producer_schedule_type active_schedule; - incremental_legacy_merkle_tree blockroot_merkle; + incremental_merkle_tree_legacy blockroot_merkle; flat_map producer_to_last_produced; flat_map producer_to_last_implied_irb; public_key_type block_signing_key; @@ -57,7 +57,7 @@ namespace eosio::chain::snapshot_detail { uint32_t dpos_proposed_irreversible_blocknum = 0; uint32_t dpos_irreversible_blocknum = 0; producer_authority_schedule active_schedule; - incremental_legacy_merkle_tree blockroot_merkle; + incremental_merkle_tree_legacy blockroot_merkle; flat_map producer_to_last_produced; flat_map producer_to_last_implied_irb; block_signing_authority valid_block_signing_authority; diff --git a/libraries/chain/merkle.cpp b/libraries/chain/merkle.cpp deleted file mode 100644 index fc3ed100f5..0000000000 --- a/libraries/chain/merkle.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include - -namespace eosio { namespace chain { - -/** - * in order to keep proofs concise, before hashing we set the first bit - * of the previous hashes to 0 or 1 to indicate the side it is on - * - * this relieves our proofs from having to indicate left vs right contactenation - * as the node values will imply it - */ - -digest_type make_canonical_left(const digest_type& val) { - digest_type canonical_l = val; - canonical_l._hash[0] &= 0xFFFFFFFFFFFFFF7FULL; - return canonical_l; -} - -digest_type make_canonical_right(const digest_type& val) { - digest_type canonical_r = val; - canonical_r._hash[0] |= 0x0000000000000080ULL; - return canonical_r; -} - -bool is_canonical_left(const digest_type& val) { - return (val._hash[0] & 0x0000000000000080ULL) == 0; -} - -bool is_canonical_right(const digest_type& val) { - return (val._hash[0] & 0x0000000000000080ULL) != 0; -} - - -digest_type legacy_merkle(deque&& ids) { - if( 0 == ids.size() ) { return digest_type(); } - - while( ids.size() > 1 ) { - if( ids.size() % 2 ) - ids.push_back(ids.back()); - - for (size_t i = 0; i < ids.size() / 2; i++) { - ids[i] = digest_type::hash(make_canonical_pair(ids[2 * i], ids[(2 * i) + 1])); - } - - ids.resize(ids.size() / 2); - } - - return ids.front(); -} - -digest_type calculate_merkle( deque&& ids ) { - if( 0 == ids.size() ) { return digest_type(); } - - while( ids.size() > 1 ) { - if( ids.size() % 2 ) - ids.push_back(ids.back()); - - for (size_t i = 0; i < ids.size() / 2; ++i) { - ids[i] = digest_type::hash(std::make_pair(std::cref(ids[2 * i]), std::cref(ids[(2 * i) + 1]))); - } - - ids.resize(ids.size() / 2); - } - - return ids.front(); -} - -} } // eosio::chain diff --git a/unittests/block_tests.cpp b/unittests/block_tests.cpp index 8b1a23eb37..aa9733a15c 100644 --- a/unittests/block_tests.cpp +++ b/unittests/block_tests.cpp @@ -35,7 +35,7 @@ BOOST_AUTO_TEST_CASE(block_with_invalid_tx_test) const auto& trxs = copy_b->transactions; for( const auto& a : trxs ) trx_digests.emplace_back( a.digest() ); - copy_b->transaction_mroot = legacy_merkle( std::move(trx_digests) ); + copy_b->transaction_mroot = calculate_merkle_legacy( std::move(trx_digests) ); // Re-sign the block auto header_bmroot = digest_type::hash( std::make_pair( copy_b->digest(), main.control->head_block_state_legacy()->blockroot_merkle.get_root() ) ); @@ -115,7 +115,7 @@ std::pair corrupt_trx_in_block(validating_te const auto& trxs = copy_b->transactions; for( const auto& a : trxs ) trx_digests.emplace_back( a.digest() ); - copy_b->transaction_mroot = legacy_merkle( std::move(trx_digests) ); + copy_b->transaction_mroot = calculate_merkle_legacy( std::move(trx_digests) ); // Re-sign the block auto header_bmroot = digest_type::hash( std::make_pair( copy_b->digest(), main.control->head_block_state_legacy()->blockroot_merkle.get_root() ) ); diff --git a/unittests/forked_tests.cpp b/unittests/forked_tests.cpp index 28e4a46143..487677a36c 100644 --- a/unittests/forked_tests.cpp +++ b/unittests/forked_tests.cpp @@ -30,7 +30,7 @@ BOOST_AUTO_TEST_CASE( irrblock ) try { struct fork_tracker { vector blocks; - incremental_legacy_merkle_tree block_merkle; + incremental_merkle_tree_legacy block_merkle; }; BOOST_AUTO_TEST_CASE( fork_with_bad_block ) try { diff --git a/unittests/merkle_tree_tests.cpp b/unittests/merkle_tree_tests.cpp index db3fef8fa4..bc49a48578 100644 --- a/unittests/merkle_tree_tests.cpp +++ b/unittests/merkle_tree_tests.cpp @@ -1,24 +1,40 @@ #include -#include +#include #include #include +#include using namespace eosio::chain; +using eosio::chain::detail::make_legacy_digest_pair; + +std::vector create_test_digests(size_t n) { + std::vector v; + v.reserve(n); + for (size_t i=0; i digests { node1, node2, node3, node4, node5, node6, node7, node8, node9 }; + auto first = digests.cbegin(); + tree.append(node1); - BOOST_CHECK_EQUAL(tree.get_root().str(), node1.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1}).str(), node1.str()); + BOOST_CHECK_EQUAL(tree.get_root(), node1); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 1)), node1); tree.append(node2); - BOOST_CHECK_EQUAL(tree.get_root().str(), fc::sha256::hash(std::make_pair(node1, node2)).str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2}).str(), fc::sha256::hash(std::make_pair(node1, node2)).str()); + BOOST_CHECK_EQUAL(tree.get_root(), hash(node1, node2)); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 2)), hash(node1, node2)); tree.append(node3); - BOOST_CHECK_EQUAL(tree.get_root().str(), fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node3)))).str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3}).str(), fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node3)))).str()); + auto calculated_root = hash(hash(node1, node2), node3); + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 3)), calculated_root); tree.append(node4); - auto calculated_root = fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node4)))); - BOOST_CHECK_EQUAL(tree.get_root().str(), calculated_root.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3, node4}).str(), calculated_root.str()); + auto first_four_tree = hash(hash(node1, node2), hash(node3, node4)); + calculated_root = first_four_tree; + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 4)), calculated_root); tree.append(node5); - calculated_root = fc::sha256::hash( - std::make_pair( - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node4)) - )), - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node5, node5)), - fc::sha256::hash(std::make_pair(node5, node5)) - )) - ) - ); - BOOST_CHECK_EQUAL(tree.get_root().str(), calculated_root.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3, node4, node5}).str(), calculated_root.str()); + calculated_root = hash(first_four_tree, node5); + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 5)), calculated_root); tree.append(node6); - calculated_root = fc::sha256::hash( - std::make_pair( - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node4)) - )), - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node5, node6)), - fc::sha256::hash(std::make_pair(node5, node6)) - )) - ) - ); - BOOST_CHECK_EQUAL(tree.get_root().str(), calculated_root.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3, node4, node5, node6}).str(), calculated_root.str()); + calculated_root = hash(first_four_tree, hash(node5, node6)); + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 6)), calculated_root); tree.append(node7); - calculated_root = fc::sha256::hash( - std::make_pair( - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node4)) - )), - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node5, node6)), - fc::sha256::hash(std::make_pair(node7, node7)) - )) - ) - ); - BOOST_CHECK_EQUAL(tree.get_root().str(), calculated_root.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3, node4, node5, node6, node7}).str(), calculated_root.str()); + calculated_root = hash(first_four_tree, hash(hash(node5, node6), node7)); + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 7)), calculated_root); tree.append(node8); - calculated_root = fc::sha256::hash( - std::make_pair( - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node4)) - )), - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node5, node6)), - fc::sha256::hash(std::make_pair(node7, node8)) - )) - ) - ); - BOOST_CHECK_EQUAL(tree.get_root().str(), calculated_root.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3, node4, node5, node6, node7, node8}).str(), calculated_root.str()); + auto next_four_tree = hash(hash(node5, node6), hash(node7, node8)); + calculated_root = hash(first_four_tree, next_four_tree); + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 8)), calculated_root); tree.append(node9); - calculated_root = fc::sha256::hash(std::make_pair( - fc::sha256::hash( - std::make_pair( - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node1, node2)), - fc::sha256::hash(std::make_pair(node3, node4)) - )), - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node5, node6)), - fc::sha256::hash(std::make_pair(node7, node8)) - )) - ) - ), - fc::sha256::hash( - std::make_pair( - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node9, node9)), - fc::sha256::hash(std::make_pair(node9, node9)) - )), - fc::sha256::hash(std::make_pair( - fc::sha256::hash(std::make_pair(node9, node9)), - fc::sha256::hash(std::make_pair(node9, node9)) - )) - ) - ) )); - BOOST_CHECK_EQUAL(tree.get_root().str(), calculated_root.str()); - BOOST_CHECK_EQUAL(calculate_merkle({node1, node2, node3, node4, node5, node6, node7, node8, node9}).str(), calculated_root.str()); + calculated_root = hash(hash(first_four_tree, next_four_tree), node9); + BOOST_CHECK_EQUAL(tree.get_root(), calculated_root); + BOOST_CHECK_EQUAL(calculate_merkle(std::span(first, 9)), calculated_root); +} + +BOOST_AUTO_TEST_CASE(consistency_over_large_range) { + constexpr size_t num_digests = 1001ull; + + const std::vector digests = create_test_digests(num_digests); + for (size_t i=1; i; + return std::chrono::duration_cast(clock::now() - _start).count(); + } + + using clock = std::chrono::high_resolution_clock; + using point = std::chrono::time_point; + + std::string _msg; + point _start; +}; + +BOOST_AUTO_TEST_CASE(perf_test_one_large) { + auto perf_test = [](const std::string& type, auto&& incr_tree, auto&& calc_fn) { + using namespace std::string_literals; + constexpr size_t num_digests = 1000ull * 1000ull; // don't use exact powers of 2 as it is a special case + + const std::vector digests = create_test_digests(num_digests); + const deque deq { digests.begin(), digests.end() }; + + auto msg_header = "1 sequence of "s + std::to_string(num_digests) + " digests: time for "s; + + auto incr_root = [&]() { + stopwatch s(msg_header + type + " incremental_merkle: "); + for (const auto& d : digests) + incr_tree.append(d); + return incr_tree.get_root(); + }(); + + auto calc_root = [&]() { + stopwatch s(msg_header + type + " calculate_merkle: "); + return calc_fn(deq); + }(); + + return std::make_pair(incr_root, calc_root); + }; + + { + auto [incr_root, calc_root] = perf_test("savanna", incremental_merkle_tree(), + [](const deque& d) { return calculate_merkle(d); }); // gcc10 needs a lambda here + BOOST_CHECK_EQUAL(incr_root, calc_root); + } + + { + auto [incr_root, calc_root] = perf_test("legacy ", incremental_merkle_tree_legacy(), calculate_merkle_legacy); + BOOST_CHECK_EQUAL(incr_root, calc_root); + } +} + + +BOOST_AUTO_TEST_CASE(perf_test_many_small) { + + auto perf_test = [](const std::string& type, const auto& incr_tree, auto&& calc_fn) { + using namespace std::string_literals; + constexpr size_t num_digests = 10000; // don't use exact powers of 2 as it is a special case + constexpr size_t num_runs = 100; + + const std::vector digests = create_test_digests(num_digests); + const deque deq { digests.begin(), digests.end() }; + + deque results(num_runs); + + auto incr = [&]() { + auto work_tree = incr_tree; + for (const auto& d : digests) + work_tree.append(d); + return work_tree.get_root(); + }; + + auto calc = [&]() { return calc_fn(deq); }; + + auto msg_header = std::to_string(num_runs) + " runs for a sequence of "s + std::to_string(num_digests) + " digests: time for "s; + + auto incr_root = [&]() { + stopwatch s(msg_header + type + " incremental_merkle: "); + for (auto& r : results) + r = incr(); + return calc_fn(results); + }(); + + auto calc_root = [&]() { + stopwatch s(msg_header + type + " calculate_merkle: "); + for (auto& r : results) + r = calc(); + return calc_fn(results); + }(); + + return std::make_pair(incr_root, calc_root); + }; + + { + auto [incr_root, calc_root] = perf_test("savanna", incremental_merkle_tree(), + [](const deque& d) { return calculate_merkle(d); }); // gcc10 needs a lambda here + BOOST_CHECK_EQUAL(incr_root, calc_root); + } + + { + auto [incr_root, calc_root] = perf_test("legacy ", incremental_merkle_tree_legacy(), calculate_merkle_legacy); + BOOST_CHECK_EQUAL(incr_root, calc_root); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/protocol_feature_tests.cpp b/unittests/protocol_feature_tests.cpp index a0b2e96c40..f589745ec3 100644 --- a/unittests/protocol_feature_tests.cpp +++ b/unittests/protocol_feature_tests.cpp @@ -2330,7 +2330,7 @@ BOOST_AUTO_TEST_CASE( block_validation_after_stage_1_test ) { try { const auto& trxs = copy_b->transactions; for( const auto& a : trxs ) trx_digests.emplace_back( a.digest() ); - copy_b->transaction_mroot = legacy_merkle( std::move(trx_digests) ); + copy_b->transaction_mroot = calculate_merkle_legacy( std::move(trx_digests) ); // Re-sign the block auto header_bmroot = digest_type::hash( std::make_pair( copy_b->digest(), tester1.control->head_block_state_legacy()->blockroot_merkle.get_root() ) );