diff --git a/config.toml.sample b/config.toml.sample index ea3fada5..bf42c488 100644 --- a/config.toml.sample +++ b/config.toml.sample @@ -7,4 +7,4 @@ rpc_password = "SomePassword" rpc_host = "https://127.0.0.1:38334" [misc] -external_sync = "" +batch_sync = "" diff --git a/src/blockchain/chain_state.rs b/src/blockchain/chain_state.rs index 16508572..4a81547b 100644 --- a/src/blockchain/chain_state.rs +++ b/src/blockchain/chain_state.rs @@ -91,6 +91,165 @@ impl ChainState { } Ok(true) } + #[inline] + /// Whether a node is the genesis block for this net + fn is_genesis(&self, header: &BlockHeader) -> bool { + header.block_hash() == self.chain_params().genesis.block_hash() + } + #[inline] + /// Returns the ancestor of a given block + fn get_ancestor(&self, header: &BlockHeader) -> Result { + self.get_disk_block_header(&header.prev_blockhash) + } + /// Returns the cumulative work in this branch + fn get_branch_work(&self, header: &BlockHeader) -> Result { + let mut header = header.clone(); + let mut work = Uint256::from_u64(0).unwrap(); + while !self.is_genesis(&header) { + work = work + header.work(); + header = *self.get_ancestor(&header)?; + } + + Ok(work) + } + fn check_branch(&self, branch_tip: &BlockHeader) -> Result<(), BlockchainError> { + let mut header = self.get_disk_block_header(&branch_tip.block_hash())?; + + while !self.is_genesis(&header) { + header = self.get_ancestor(&header)?; + match header { + DiskBlockHeader::Orphan(block) => { + return Err(BlockchainError::InvalidTip(format!( + "Block {} doesn't have a known ancestor (i.e an orphan block)", + block.block_hash() + ))) + } + _ => { /* do nothing */ } + } + } + + Ok(()) + } + fn get_chain_depth(&self, branch_tip: &BlockHeader) -> Result { + let mut header = self.get_disk_block_header(&branch_tip.block_hash())?; + + let mut counter = 0; + while !self.is_genesis(&header) { + header = self.get_ancestor(&header)?; + counter += 1; + } + + Ok(counter) + } + fn mark_chain_as_active(&self, new_tip: &BlockHeader) -> Result<(), BlockchainError> { + let mut header = self.get_disk_block_header(&new_tip.block_hash())?; + let height = self.get_chain_depth(new_tip)?; + let inner = read_lock!(self); + while !self.is_genesis(&header) { + header = self.get_ancestor(&header)?; + inner + .chainstore + .update_block_index(height, header.block_hash())?; + let new_header = DiskBlockHeader::HeadersOnly(*header, height); + inner.chainstore.save_header(&new_header)?; + } + + Ok(()) + } + /// Mark the current index as inactive, either because we found an invalid ancestor, + /// or we are in the middle of reorg + fn mark_chain_as_inactive(&self, new_tip: &BlockHeader) -> Result<(), BlockchainError> { + let mut header = self.get_disk_block_header(&new_tip.block_hash())?; + let inner = read_lock!(self); + while !self.is_genesis(&header) { + header = self.get_ancestor(&header)?; + let new_header = DiskBlockHeader::InFork(*header); + inner.chainstore.save_header(&new_header)?; + } + + Ok(()) + } + // This method should only be called after we validate the new branch + fn reorg(&self, new_tip: BlockHeader) -> Result<(), BlockchainError> { + let current_best_block = self.get_best_block().unwrap().1; + let current_best_block = self.get_block_header(¤t_best_block)?; + self.mark_chain_as_active(&new_tip)?; + self.mark_chain_as_inactive(¤t_best_block)?; + + let mut inner = self.inner.write().unwrap(); + inner.best_block.best_block = new_tip.block_hash(); + inner.best_block.validation_index = self.get_last_valid_block(&new_tip)?; + Ok(()) + } + + /// Grabs the last block we validated in this branch. We don't validate a fork, unless it + /// becomes the best chain. This function technically finds out what is the last common block + /// between two branches. + fn get_last_valid_block(&self, header: &BlockHeader) -> Result { + let mut header = self.get_disk_block_header(&header.block_hash())?; + + while !self.is_genesis(&header) { + match header { + DiskBlockHeader::FullyValid(_, _) => return Ok(header.block_hash()), + DiskBlockHeader::Orphan(_) => { + return Err(BlockchainError::InvalidTip(format!( + "Block {} doesn't have a known ancestor (i.e an orphan block)", + header.block_hash() + ))) + } + DiskBlockHeader::HeadersOnly(_, _) | DiskBlockHeader::InFork(_) => {} + } + header = self.get_ancestor(&header)?; + } + + unreachable!() + } + /// If we get a header that doesn't build on top of our best chain, it may cause a reorganization. + /// We check this here. + pub fn maybe_reorg(&self, branch_tip: BlockHeader) -> Result<(), BlockchainError> { + let current_tip = self.get_block_header(&self.get_best_block().unwrap().1)?; + self.check_branch(&branch_tip)?; + + let current_work = self.get_branch_work(¤t_tip)?; + let new_work = self.get_branch_work(&branch_tip)?; + + if new_work > current_work { + self.reorg(branch_tip)?; + return Ok(()); + } + self.push_alt_tip(&branch_tip)?; + + read_lock!(self) + .chainstore + .save_header(&super::chainstore::DiskBlockHeader::InFork(branch_tip))?; + Ok(()) + } + /// Stores a new tip for a branch that is not the best one + fn push_alt_tip(&self, branch_tip: &BlockHeader) -> Result<(), BlockchainError> { + let ancestor = self.get_ancestor(branch_tip); + let ancestor = match ancestor { + Ok(ancestor) => Some(ancestor), + Err(BlockchainError::BlockNotPresent) => None, + Err(e) => return Err(e), + }; + let mut inner = write_lock!(self); + if ancestor.is_some() { + let ancestor_hash = ancestor.unwrap().block_hash(); + if let Some(idx) = inner + .best_block + .alternative_tips + .iter() + .position(|hash| ancestor_hash == *hash) + { + inner.best_block.alternative_tips.remove(idx); + } + } + inner + .best_block + .alternative_tips + .push(branch_tip.block_hash()); + Ok(()) + } fn calc_next_work_required( last_block: &BlockHeader, first_block: &BlockHeader, @@ -239,11 +398,15 @@ impl ChainState { ) -> ChainState { let genesis = genesis_block(network); chainstore - .save_header( - &super::chainstore::DiskBlockHeader::FullyValid(genesis.header, 0), + .save_header(&super::chainstore::DiskBlockHeader::FullyValid( + genesis.header, 0, - ) + )) .expect("Error while saving genesis"); + chainstore + .update_block_index(0, genesis.block_hash()) + .expect("Error updating index"); + let assume_valid_hash = Self::get_assume_valid_value(network, assume_valid); ChainState { inner: RwLock::new(ChainStateInner { @@ -462,6 +625,7 @@ impl BlockchainProviderInterface for ChainState return Ok(()), + DiskBlockHeader::InFork(_) => return Ok(()), DiskBlockHeader::HeadersOnly(_, height) => height, }; let inner = self.inner.read().unwrap(); @@ -501,10 +665,15 @@ impl BlockchainProviderInterface for ChainState BlockchainProviderInterface for ChainState super::Result<()> { - todo!() - } - fn handle_transaction(&self) -> super::Result<()> { unimplemented!("This chain_state has no mempool") } @@ -553,21 +718,23 @@ impl BlockchainProviderInterface for ChainState BlockchainProviderInterface for ChainState Ok(height), DiskBlockHeader::FullyValid(_, height) => Ok(height), - DiskBlockHeader::Orphan(_) => unreachable!(), + _ => unreachable!(), } } } @@ -597,6 +764,7 @@ macro_rules! write_lock { $obj.inner.write().expect("get_block_hash: Poisoned lock") }; } + #[derive(Clone)] /// Internal representation of the chain we are in pub struct BestChain { diff --git a/src/blockchain/chainparams.rs b/src/blockchain/chainparams.rs index ce3739ae..dfa9a272 100644 --- a/src/blockchain/chainparams.rs +++ b/src/blockchain/chainparams.rs @@ -23,11 +23,30 @@ pub struct ChainParams { /// it's more, we decrease difficulty, if it's less we increase difficulty pub pow_target_timespan: u64, } -impl ChainParams {} +impl ChainParams { + fn max_target(net: Network) -> Uint256 { + match net { + Network::Bitcoin => max_target(net), + Network::Testnet => max_target(net), + Network::Signet => Uint256([ + 0x00000377ae000000, + 0x0000000000000000, + 0x0000000000000000, + 0x0000000000000000, + ]), + Network::Regtest => Uint256([ + 0x7fffffffffffffff, + 0xffffffffffffffff, + 0xffffffffffffffff, + 0xffffffffffffffff, + ]), + } + } +} impl From for ChainParams { fn from(net: Network) -> Self { let genesis = genesis_block(net); - let max_target = max_target(net); + let max_target = ChainParams::max_target(net); match net { Network::Bitcoin => ChainParams { genesis, @@ -59,7 +78,7 @@ impl From for ChainParams { Network::Regtest => ChainParams { genesis, max_target, - pow_allow_min_diff: true, + pow_allow_min_diff: false, pow_allow_no_retarget: true, pow_target_spacing: 10 * 60, // One block every 600 seconds (10 minutes) pow_target_timespan: 14 * 24 * 60 * 60, // two weeks diff --git a/src/blockchain/chainstore.rs b/src/blockchain/chainstore.rs index 6b212540..a4ba641f 100644 --- a/src/blockchain/chainstore.rs +++ b/src/blockchain/chainstore.rs @@ -16,14 +16,11 @@ pub enum DiskBlockHeader { FullyValid(BlockHeader, u32), Orphan(BlockHeader), HeadersOnly(BlockHeader, u32), + InFork(BlockHeader), } impl DiskBlockHeader { pub fn block_hash(&self) -> BlockHash { - match self { - DiskBlockHeader::FullyValid(header, _) => header.block_hash(), - DiskBlockHeader::Orphan(header) => header.block_hash(), - DiskBlockHeader::HeadersOnly(header, _) => header.block_hash(), - } + self.deref().block_hash() } } impl Deref for DiskBlockHeader { @@ -33,6 +30,7 @@ impl Deref for DiskBlockHeader { DiskBlockHeader::FullyValid(header, _) => header, DiskBlockHeader::Orphan(header) => header, DiskBlockHeader::HeadersOnly(header, _) => header, + DiskBlockHeader::InFork(header) => header, } } } @@ -53,6 +51,7 @@ impl Decodable for DiskBlockHeader { let height = u32::consensus_decode(reader)?; Ok(Self::HeadersOnly(header, height)) } + 0x03 => Ok(Self::InFork(header)), _ => unreachable!(), } } @@ -62,7 +61,7 @@ impl Encodable for DiskBlockHeader { &self, writer: &mut W, ) -> std::result::Result { - let len = 80 + 1 + 4; // Header + tag + hight + let len = 80 + 1 + 4; // Header + tag + height match self { DiskBlockHeader::FullyValid(header, height) => { 0x00_u8.consensus_encode(writer)?; @@ -78,6 +77,10 @@ impl Encodable for DiskBlockHeader { header.consensus_encode(writer)?; height.consensus_encode(writer)?; } + DiskBlockHeader::InFork(header) => { + 0x03_u8.consensus_encode(writer)?; + header.consensus_encode(writer)?; + } }; Ok(len) } @@ -94,9 +97,10 @@ pub trait ChainStore { fn load_height(&self) -> Result>; fn save_height(&self, height: &BestChain) -> Result<()>; fn get_header(&self, block_hash: &BlockHash) -> Result>; - fn save_header(&self, header: &DiskBlockHeader, height: u32) -> Result<()>; + fn save_header(&self, header: &DiskBlockHeader) -> Result<()>; fn get_block_hash(&self, height: u32) -> Result>; fn flush(&self) -> Result<()>; + fn update_block_index(&self, height: u32, hash: BlockHash) -> Result<()>; } pub struct KvChainStore(Store); @@ -162,14 +166,11 @@ impl ChainStore for KvChainStore { Ok(()) } - fn save_header(&self, header: &DiskBlockHeader, height: u32) -> Result<()> { + fn save_header(&self, header: &DiskBlockHeader) -> Result<()> { let ser_header = serialize(header); let block_hash = serialize(&header.block_hash()); let bucket = self.0.bucket::<&[u8], Vec>(Some("header"))?; bucket.set(&&*block_hash, &ser_header)?; - - let bucket = self.0.bucket::>(Some("index"))?; - bucket.set(&Integer::from(height), &block_hash)?; Ok(()) } @@ -181,4 +182,12 @@ impl ChainStore for KvChainStore { } Ok(None) } + + fn update_block_index(&self, height: u32, hash: BlockHash) -> Result<()> { + let bucket = self.0.bucket::>(Some("index"))?; + let block_hash = serialize(&hash); + + bucket.set(&Integer::from(height), &block_hash)?; + Ok(()) + } } diff --git a/src/blockchain/cli_blockchain/mod.rs b/src/blockchain/cli_blockchain/mod.rs index cdf008d0..b95e3320 100644 --- a/src/blockchain/cli_blockchain/mod.rs +++ b/src/blockchain/cli_blockchain/mod.rs @@ -12,7 +12,7 @@ use bitcoin::{ hex::{FromHex, ToHex}, sha256, Hash, }, - Block, BlockHash, OutPoint, + Block, BlockHash, BlockHeader, OutPoint, }; use btcd_rpc::{ client::{BTCDClient, BtcdRpc}, @@ -29,8 +29,8 @@ use super::{ use crate::try_and_log; pub struct UtreexodBackend { - pub use_external_sync: bool, - pub external_sync_hostname: String, + pub use_batch_sync: bool, + pub batch_sync_hostname: String, pub rpc: Arc, pub chainstate: Arc>, pub term: Arc, @@ -118,13 +118,12 @@ impl UtreexodBackend { } Ok(()) } - pub fn handle_tip_update(&self) -> Result<()> { + pub async fn handle_tip_update(&self) -> Result<()> { let height = self.get_height()?; let local_best = self.chainstate.get_best_block().unwrap().0; if height > local_best { - for block_height in (local_best + 1)..=height { - self.process_block(block_height)?; - } + self.get_headers()?; + self.download_blocks().await?; } Ok(()) } @@ -147,24 +146,24 @@ impl UtreexodBackend { for leaf in leaf_data { inputs.insert(leaf.prevout, leaf.utxo); } - if !self.chainstate.is_in_idb() { - self.chainstate.accept_header(block.header)?; - } self.chainstate .connect_block(&block, proof, inputs, del_hashes)?; Ok(()) } - async fn start_ibd(&self) -> Result<()> { + async fn download_blocks(&self) -> Result<()> { let height = self.get_height()?; let current = self.chainstate.get_validation_index()?; - info!("Start Initial Block Download at height {current} of {height}"); + // We don't download genesis, because utreexod will error out if we try to fetch + // proof for it. + let current = if current == 0 { 1 } else { current }; + if self.chainstate.is_in_idb() { + info!("Start Initial Block Download at height {current} of {height}"); + } for block_height in current..=height { if self.is_shutting_down() { return Ok(()); } - if block_height == 0 { - continue; - } + if block_height % 10_000 == 0 { info!("Sync at block {block_height}"); if block_height % 100_000 == 0 { @@ -173,15 +172,18 @@ impl UtreexodBackend { } self.process_block(block_height)?; } - info!("Leaving Initial Block Download at height {height}"); + if self.chainstate.is_in_idb() { + info!("Leaving Initial Block Download at height {height}"); + } else { + info!("New tip: {height}"); + } self.chainstate.toggle_ibd(false); self.chainstate.flush()?; Ok(()) } - #[allow(unused)] async fn process_batch_block(&self) -> Result<()> { - let socket = TcpStream::connect(self.external_sync_hostname.to_owned().as_str())?; + let socket = TcpStream::connect(self.batch_sync_hostname.to_owned().as_str())?; let height = self.get_height()?; let current = self.chainstate.get_validation_index()?; @@ -247,7 +249,7 @@ impl UtreexodBackend { .iter() .map(|header| { let header = Vec::from_hex(header).unwrap(); - deserialize(&header).unwrap() + deserialize::(&header).unwrap() }) .collect::>(); @@ -270,10 +272,10 @@ impl UtreexodBackend { try_and_log!(self.chainstate.flush()); return; } - if self.use_external_sync { + if self.use_batch_sync { try_and_log!(self.process_batch_block().await); } else { - try_and_log!(self.start_ibd().await); + try_and_log!(self.download_blocks().await); } self.chainstate.toggle_ibd(false); loop { @@ -284,7 +286,7 @@ impl UtreexodBackend { return; } try_and_log!(self.handle_broadcast()); - try_and_log!(self.handle_tip_update()); + try_and_log!(self.handle_tip_update().await); try_and_log!(self.chainstate.flush()); } } diff --git a/src/blockchain/error.rs b/src/blockchain/error.rs index a3e1df73..bc89ddc4 100644 --- a/src/blockchain/error.rs +++ b/src/blockchain/error.rs @@ -11,6 +11,7 @@ pub enum BlockchainError { DatabaseError(kv::Error), ConsensusDecodeError(bitcoin::consensus::encode::Error), ChainNotInitialized, + InvalidTip(String), IoError(std::io::Error), } #[derive(Debug)] diff --git a/src/blockchain/mod.rs b/src/blockchain/mod.rs index 6ef38a0c..366360b1 100644 --- a/src/blockchain/mod.rs +++ b/src/blockchain/mod.rs @@ -55,9 +55,6 @@ pub trait BlockchainProviderInterface { /// makes some basic checks on a header and saves it on disk. We only accept a block as /// valid after calling connect_block. fn accept_header(&self, header: BlockHeader) -> Result<()>; - /// If we detect a reorganization of blocks, this function should reconciliate our view of - /// the network. - fn handle_reorg(&self) -> Result<()>; /// Not used for now, but in a future blockchain with mempool, we can process transactions /// that are not in a block yet. fn handle_transaction(&self) -> Result<()>; diff --git a/src/cli.rs b/src/cli.rs index d633501b..cbbc954f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -64,10 +64,10 @@ pub enum Commands { /// Whether or not we want to sync with a external provider #[arg(long)] #[arg(default_value_t = false)] - use_external_sync: bool, - /// If use_external_sync is set, this option provides which server we use + use_batch_sync: bool, + /// If use_batch_sync is set, this option provides which server we use #[arg(long)] - external_sync: Option, + batch_sync: Option, /// Assume blocks before this one as having valid signatures, same with bitcoin core #[arg(long)] assume_valid: Option, diff --git a/src/config_file.rs b/src/config_file.rs index 2f808794..f7f56f8a 100644 --- a/src/config_file.rs +++ b/src/config_file.rs @@ -16,7 +16,7 @@ pub struct Rpc { } #[derive(Default, Debug, Deserialize)] pub struct Misc { - pub external_sync: Option, + pub batch_sync: Option, } #[derive(Default, Debug, Deserialize)] pub struct ConfigFile { diff --git a/src/main.rs b/src/main.rs index 9a89efca..0bb9b191 100644 --- a/src/main.rs +++ b/src/main.rs @@ -74,8 +74,8 @@ fn main() { rpc_user, rpc_password, rpc_host, - external_sync, - use_external_sync, + batch_sync, + use_batch_sync, rpc_port, wallet_xpub, assume_valid, @@ -133,12 +133,12 @@ fn main() { let chain_provider = UtreexodBackend { chainstate: blockchain_state.clone(), rpc, - external_sync_hostname: get_one_or_another( - external_sync, - data.misc.external_sync, + batch_sync_hostname: get_one_or_another( + batch_sync, + data.misc.batch_sync, "".into(), ), - use_external_sync, + use_batch_sync, term: shutdown, }; info!("Starting server");