Skip to content

Commit

Permalink
Merge branch 'GH-2048-persist-if-fork-db' of github.com:AntelopeIO/le…
Browse files Browse the repository at this point in the history 
…ap into GH-2048_whitespace
  • Loading branch information
@greg7mdp
greg7mdp committed Jan 20, 2024
2 parents 5cae9d9 + 23bee43 commit 7a4b5d5
Show file tree
Hide file tree
Showing 9 changed files with 116 additions and 90 deletions.
6 changes: 2 additions & 4 deletions libraries/chain/block_header_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,7 @@ block_header_state block_header_state::next(block_header_state_input& input) con
instant_finality_extension new_if_ext {if_ext.qc_info,
std::move(input.new_finalizer_policy),
std::move(input.new_proposer_policy)};
if (input.validating)
new_if_ext.qc_info = input.qc_info;
else if (input.qc_info)
if (input.qc_info)
new_if_ext.qc_info = *input.qc_info;

emplace_extension(result.header.header_extensions, if_ext_id, fc::raw::pack(new_if_ext));
Expand Down Expand Up @@ -203,7 +201,7 @@ block_header_state block_header_state::next(const signed_block_header& h, const

block_header_state_input bhs_input{
bb_input, h.transaction_mroot, h.action_mroot, if_ext.new_proposer_policy, if_ext.new_finalizer_policy,
if_ext.qc_info, true};
if_ext.qc_info };

return next(bhs_input);
}
Expand Down
6 changes: 3 additions & 3 deletions libraries/chain/block_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ block_state::block_state(const block_header_state& bhs, deque<transaction_metada
{
block->transactions = std::move(trx_receipts);

if( qc && bhs.is_needed(*qc) ) {
if( qc ) {
emplace_extension(block->block_extensions, quorum_certificate_extension::extension_id(), fc::raw::pack( *qc ));
}
}
Expand All @@ -35,7 +35,7 @@ block_state::block_state(const block_header_state& bhs, deque<transaction_metada
block_state::block_state(const block_state_legacy& bsp) {
block_header_state::id = bsp.id();
header = bsp.header;
core.last_final_block_num = bsp.block_num();
core.last_final_block_num = bsp.block_num(); // [if todo] instant transition is not acceptable
activated_protocol_features = bsp.activated_protocol_features;
std::optional<block_header_extension> ext = bsp.block->extract_header_extension(instant_finality_extension::extension_id());
assert(ext); // required by current transition mechanism
Expand Down Expand Up @@ -83,7 +83,7 @@ std::pair<bool, std::optional<uint32_t>> block_state::aggregate_vote(const hs_vo
return {valid, strong ? core.final_on_strong_qc_block_num : std::optional<uint32_t>{}};
} else {
wlog( "finalizer_key (${k}) in vote is not in finalizer policy", ("k", vote.finalizer_key) );
return {false, {}};
return {};
}
}

Expand Down
98 changes: 55 additions & 43 deletions libraries/chain/controller.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,8 +323,8 @@ struct assembled_block {
v);
}

completed_block make_completed_block(const protocol_feature_set& pfs, validator_t validator,
const signer_callback_type& signer) {
completed_block complete_block(const protocol_feature_set& pfs, validator_t validator,
const signer_callback_type& signer) {
return std::visit(overloaded{[&](assembled_block_dpos& ab) {
auto bsp = std::make_shared<block_state_legacy>(
std::move(ab.pending_block_header_state), std::move(ab.unsigned_block),
Expand Down Expand Up @@ -621,7 +621,7 @@ struct building_block {
const protocol_feature_set& pfs,
fork_database& fork_db,
bool validating,
std::optional<qc_info_t> validating_qc_info) {
std::optional<qc_data_t> validating_qc_data) {
digests_t& action_receipts = action_receipt_digests();
return std::visit(
overloaded{
Expand Down Expand Up @@ -672,8 +672,10 @@ struct building_block {
// find most recent ancestor block that has a QC by traversing fork db
// branch from parent
std::optional<qc_data_t> qc_data;
if (!validating) {
// get fork_database so that we can search for the best qc to include in this block.
if (validating) {
// we are simulating a block received from the network. Use the embedded qc from the block
qc_data = std::move(validating_qc_data);
} else {
fork_db.apply_if<void>([&](const auto& forkdb) {
auto branch = forkdb.fetch_branch(parent_id());
for( auto it = branch.begin(); it != branch.end(); ++it ) {
Expand All @@ -696,16 +698,10 @@ struct building_block {
.new_protocol_feature_activations = new_protocol_feature_activations()
};

std::optional<qc_info_t> qc_info;
if (validating) {
qc_info = validating_qc_info;
} else if (qc_data) {
qc_info = qc_data->qc_info;
}
block_header_state_input bhs_input{
bb_input, transaction_mroot, action_mroot, std::move(bb.new_proposer_policy),
std::move(bb.new_finalizer_policy),
qc_info, validating
qc_data ? qc_data->qc_info : std::optional<qc_info_t>{}
};

assembled_block::assembled_block_if ab{std::move(bb.active_producer_authority), bb.parent.next(bhs_input),
Expand Down Expand Up @@ -1270,7 +1266,7 @@ struct controller_impl {
forkdb.chain_head->block = std::make_shared<signed_block>(genheader.header);
};

fork_db.apply_dpos<void>(init_blockchain); // assuming here that genesis_state is always dpos
fork_db.apply_legacy<void>(init_blockchain); // assuming here that genesis_state is always dpos

db.set_revision( head_block_num() );
initialize_database(genesis);
Expand Down Expand Up @@ -1652,7 +1648,7 @@ struct controller_impl {
section.template add_row<block_header_state_legacy>(*forkdb.chain_head, db);
});
};
fork_db.apply_dpos<void>(write_block_state_section);
fork_db.apply_legacy<void>(write_block_state_section);

controller_index_set::walk_indices([this, &snapshot]( auto utils ){
using value_t = typename decltype(utils)::index_t::value_type;
Expand Down Expand Up @@ -1729,7 +1725,7 @@ struct controller_impl {
forkdb.chain_head = std::make_shared<block_state_legacy>();
static_cast<block_header_state_legacy&>(*forkdb.chain_head) = head_header_state;
};
fork_db.apply_dpos<void>(read_block_state_section);
fork_db.apply_legacy<void>(read_block_state_section);

controller_index_set::walk_indices([this, &snapshot, &header]( auto utils ){
using value_t = typename decltype(utils)::index_t::value_type;
Expand Down Expand Up @@ -2497,16 +2493,17 @@ struct controller_impl {
"db revision is not on par with head block",
("db.revision()", db.revision())("controller_head_block", head_block_num())("fork_db_head_block", fork_db_head_block_num()) );

fork_db.apply_dpos<void>([&](auto& forkdb) {
maybe_session session = self.skip_db_sessions(s) ? maybe_session() : maybe_session(db);
pending.emplace(std::move(session), *forkdb.chain_head, when, confirm_block_count, new_protocol_feature_activations);
});
fork_db.apply_if<void>([&](auto& forkdb) {
maybe_session session = self.skip_db_sessions(s) ? maybe_session() : maybe_session(db);
building_block_input bbi{ forkdb.chain_head->id(), when, forkdb.chain_head->get_scheduled_producer(when).producer_name,
new_protocol_feature_activations };
pending.emplace(std::move(session), *forkdb.chain_head, bbi);
});
fork_db.apply<void>(
[&](auto& forkdb) { // legacy
maybe_session session = self.skip_db_sessions(s) ? maybe_session() : maybe_session(db);
pending.emplace(std::move(session), *forkdb.chain_head, when, confirm_block_count, new_protocol_feature_activations);
},
[&](auto& forkdb) { // instant-finality
maybe_session session = self.skip_db_sessions(s) ? maybe_session() : maybe_session(db);
building_block_input bbi{forkdb.chain_head->id(), when, forkdb.chain_head->get_scheduled_producer(when).producer_name,
new_protocol_feature_activations};
pending.emplace(std::move(session), *forkdb.chain_head, bbi);
});

pending->_block_status = s;
pending->_producer_block_id = producer_block_id;
Expand Down Expand Up @@ -2647,7 +2644,7 @@ struct controller_impl {
guard_pending.cancel();
} /// start_block

void finish_block(bool validating = false, std::optional<qc_info_t> validating_qc_info = {})
void assemble_block(bool validating = false, std::optional<qc_data_t> validating_qc_data = {})
{
EOS_ASSERT( pending, block_validate_exception, "it is not valid to finalize when there is no pending block");
EOS_ASSERT( std::holds_alternative<building_block>(pending->_block_stage), block_validate_exception, "already called finish_block");
Expand All @@ -2670,8 +2667,9 @@ struct controller_impl {
);
resource_limits.process_block_usage(bb.block_num());

auto assembled_block = bb.assemble_block(thread_pool.get_executor(),
protocol_features.get_protocol_feature_set(), fork_db, validating, validating_qc_info);
auto assembled_block =
bb.assemble_block(thread_pool.get_executor(), protocol_features.get_protocol_feature_set(), fork_db,
validating, std::move(validating_qc_data));

// Update TaPoS table:
create_block_summary( assembled_block.id() );
Expand Down Expand Up @@ -2713,7 +2711,7 @@ struct controller_impl {

fork_db.apply<void>(add_completed_block);

fork_db.apply_dpos<void>([this](auto& forkdb) {
fork_db.apply_legacy<void>([this](auto& forkdb) {
#warning todo: support deep_mind_logger even when in IF mode (use apply instead of apply_dpos)
// at block level, no transaction specific logging is possible
if (auto* dm_logger = get_deep_mind_logger(false)) {
Expand Down Expand Up @@ -2741,7 +2739,7 @@ struct controller_impl {
}
return false;
};
if (fork_db.apply_dpos<bool>(transition)) {
if (fork_db.apply_legacy<bool>(transition)) {
fork_db.switch_from_legacy();
}

Expand Down Expand Up @@ -2841,25 +2839,39 @@ struct controller_impl {
EOS_REPORT( "header_extensions", b.header_extensions, ab.header_extensions )

if (b.header_extensions != ab.header_extensions) {
{
flat_multimap<uint16_t, block_header_extension> bheader_exts = b.validate_and_extract_header_extensions();
if (bheader_exts.count(instant_finality_extension::extension_id())) {
const auto& if_extension =
std::get<instant_finality_extension>(bheader_exts.lower_bound(instant_finality_extension::extension_id())->second);
elog("b if: ${i}", ("i", if_extension));
}
flat_multimap<uint16_t, block_header_extension> bheader_exts = b.validate_and_extract_header_extensions();
if (bheader_exts.count(instant_finality_extension::extension_id())) {
const auto& if_extension =
std::get<instant_finality_extension>(bheader_exts.lower_bound(instant_finality_extension::extension_id())->second);
elog("b if: ${i}", ("i", if_extension));
}
flat_multimap<uint16_t, block_header_extension> abheader_exts = ab.validate_and_extract_header_extensions();
if (abheader_exts.count(instant_finality_extension::extension_id())) {
const auto& if_extension =
std::get<instant_finality_extension>(abheader_exts.lower_bound(instant_finality_extension::extension_id())->second);
std::get<instant_finality_extension>(abheader_exts.lower_bound(instant_finality_extension::extension_id())->second);
elog("ab if: ${i}", ("i", if_extension));
}
}

#undef EOS_REPORT
}

static std::optional<qc_data_t> extract_qc_data(const signed_block_ptr& b) {
std::optional<qc_data_t> qc_data;
auto exts = b->validate_and_extract_extensions();
if (auto entry = exts.lower_bound(quorum_certificate_extension::extension_id()); entry != exts.end()) {
auto& qc_ext = std::get<quorum_certificate_extension>(entry->second);

// get the matching header extension... should always be present
auto hexts = b->validate_and_extract_header_extensions();
auto if_entry = hexts.lower_bound(instant_finality_extension::extension_id());
assert(if_entry != hexts.end());
auto& if_ext = std::get<instant_finality_extension>(if_entry->second);
return qc_data_t{ std::move(qc_ext.qc), *if_ext.qc_info };
}
return {};
}

template<class BSP>
void apply_block( controller::block_report& br, const BSP& bsp, controller::block_status s,
const trx_meta_cache_lookup& trx_lookup ) {
Expand Down Expand Up @@ -2955,8 +2967,8 @@ struct controller_impl {
auto& if_ext = std::get<instant_finality_extension>(if_entry->second);
qc_info = if_ext.qc_info;
}
finish_block(true, qc_info);

assemble_block(true, extract_qc_data(b));
auto& ab = std::get<assembled_block>(pending->_block_stage);

if( producer_block_id != ab.id() ) {
Expand Down Expand Up @@ -3876,13 +3888,13 @@ void controller::start_block( block_timestamp_type when,
bs, std::optional<block_id_type>(), deadline );
}

void controller::finish_block( block_report& br, const signer_callback_type& signer_callback ) {
void controller::assemble_and_complete_block( block_report& br, const signer_callback_type& signer_callback ) {
validate_db_available_size();

my->finish_block();
my->assemble_block();

auto& ab = std::get<assembled_block>(my->pending->_block_stage);
my->pending->_block_stage = ab.make_completed_block(
my->pending->_block_stage = ab.complete_block(
my->protocol_features.get_protocol_feature_set(),
[](block_timestamp_type timestamp, const flat_set<digest_type>& cur_features, const vector<digest_type>& new_features) {},
signer_callback);
Expand Down Expand Up @@ -4008,7 +4020,7 @@ const block_header& controller::head_block_header()const {

block_state_legacy_ptr controller::head_block_state_legacy()const {
// returns null after instant finality activated
return my->fork_db.apply_dpos<block_state_legacy_ptr>(
return my->fork_db.apply_legacy<block_state_legacy_ptr>(
[](auto& forkdb) -> block_state_legacy_ptr { return forkdb.chain_head; });
}

Expand Down
17 changes: 12 additions & 5 deletions libraries/chain/fork_database.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,10 @@ namespace eosio::chain {
}

fork_database::~fork_database() {
close();
}

void fork_database::close() {
apply<void>([&](auto& forkdb) { forkdb.close(data_dir / config::forkdb_filename); });
}

Expand Down Expand Up @@ -599,26 +603,29 @@ namespace eosio::chain {
);

if (totem == fork_database_legacy_t::legacy_magic_number) {
apply_dpos<void>([&](auto& forkdb) {
apply_legacy<void>([&](auto& forkdb) {
forkdb.open(fork_db_file, validator);
});
} else {
// file is instant-finality data, so switch to fork_database_if_t
v.emplace<fork_database_if_t>(fork_database_if_t::magic_number);
vforkdb.emplace<fork_database_if_t>(fork_database_if_t::magic_number);
apply_if<void>([&](auto& forkdb) {
forkdb.open(fork_db_file, validator);
});
}
apply<void>([&](auto& forkdb) {
forkdb.open(fork_db_file, validator);
});
} FC_CAPTURE_AND_RETHROW( (fork_db_file) )
}
}

void fork_database::switch_from_legacy() {
std::lock_guard g(m);
// no need to close fork_db because we don't want to write anything out, file is removed on open
block_state_legacy_ptr head = std::get<fork_database_legacy_t>(v).chain_head; // will throw if called after transistion
block_state_legacy_ptr head = std::get<fork_database_legacy_t>(vforkdb).chain_head; // will throw if called after transistion
auto new_head = std::make_shared<block_state>(*head);
std::lock_guard g(m);
v.emplace<fork_database_if_t>(fork_database_if_t::magic_number);
vforkdb.emplace<fork_database_if_t>(fork_database_if_t::magic_number);
apply_if<void>([&](auto& forkdb) {
forkdb.chain_head = new_head;
forkdb.reset(*new_head);
Expand Down
7 changes: 1 addition & 6 deletions libraries/chain/include/eosio/chain/block_header_state.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ struct block_header_state_input : public building_block_input {
std::optional<finalizer_policy> new_finalizer_policy; // Comes from building_block::new_finalizer_policy
std::optional<qc_info_t> qc_info; // Comes from traversing branch from parent and calling get_best_qc()
// assert(qc->block_num <= num_from_id(previous));
bool validating = false;
};

struct block_header_state_core {
Expand Down Expand Up @@ -68,6 +67,7 @@ struct block_header_state {


// ------ functions -----------------------------------------------------------------
// [if todo] https://github.com/AntelopeIO/leap/issues/2080
digest_type compute_finalizer_digest() const { return id; };
block_timestamp_type timestamp() const { return header.timestamp; }
account_name producer() const { return header.producer; }
Expand All @@ -79,11 +79,6 @@ struct block_header_state {

block_header_state next(const signed_block_header& h, const protocol_feature_set& pfs, validator_t& validator) const;

// block descending from this need the provided qc in the block extension
bool is_needed(const quorum_certificate& qc) const {
return !core.last_qc_block_num || qc.block_height > *core.last_qc_block_num;
}

flat_set<digest_type> get_activated_protocol_features() const { return activated_protocol_features->protocol_features; }
const vector<digest_type>& get_new_protocol_feature_activations() const;
producer_authority get_scheduled_producer(block_timestamp_type t) const;
Expand Down
2 changes: 1 addition & 1 deletion libraries/chain/include/eosio/chain/controller.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ namespace eosio::chain {
fc::microseconds total_time{};
};

void finish_block( block_report& br, const signer_callback_type& signer_callback );
void assemble_and_complete_block( block_report& br, const signer_callback_type& signer_callback );
void sign_block( const signer_callback_type& signer_callback );
void commit_block();

Expand Down
Loading

0 comments on commit 7a4b5d5

Please sign in to comment.