diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index be265467..aea5cb81 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -15,7 +15,7 @@ jobs: - name: Check out source uses: actions/checkout@v2 - name: Install Linters - run: "curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1" + run: "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.2" - name: Build env: GO111MODULE: "on" diff --git a/blockchain/accept.go b/blockchain/accept.go index f2f89575..4c3f18a2 100644 --- a/blockchain/accept.go +++ b/blockchain/accept.go @@ -1,4 +1,5 @@ // Copyright (c) 2013-2017 The btcsuite developers +// Copyright (c) 2018-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -9,6 +10,7 @@ import ( "github.com/utreexo/utreexod/btcutil" "github.com/utreexo/utreexod/database" + "github.com/utreexo/utreexod/wire" ) // maybeAcceptBlock potentially accepts a block into the block chain and, if @@ -60,23 +62,29 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) return false, err } - // Create a new block node for the block and add it to the node index. Even - // if the block ultimately gets connected to the main chain, it starts out - // on a side chain. + // Lookup for the block node that was created at the time of header + // validation if the block node is not existent then create a new + // block node for the block and add it to the node index. Even if + // the block ultimately gets connected to the main chain, it starts + // out on a side chain. blockHeader := &block.MsgBlock().Header - newNode := newBlockNode(blockHeader, prevNode) - newNode.status = statusDataStored - - b.index.AddNode(newNode) - err = b.index.flushToDB() - if err != nil { - return false, err + blockHash := blockHeader.BlockHash() + node := b.index.LookupNode(&blockHash) + if node == nil { + newNode := newBlockNode(blockHeader, prevNode) + b.index.AddNode(newNode) + err = b.index.flushToDB() + if err != nil { + return false, err + } + node = newNode } + node.status = statusDataStored // Connect the passed block to the chain while respecting proper chain // selection according to the chain with the most proof of work. This // also handles validation of the transaction scripts. - isMainChain, err := b.connectBestChain(newNode, block, flags) + isMainChain, err := b.connectBestChain(node, block, flags) if err != nil { return false, err } @@ -90,3 +98,76 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags) return isMainChain, nil } + +// maybeAcceptBlockHeader potentially accepts the header to the block index and, +// if accepted, returns the block node associated with the header. It performs +// several context independent checks as well as those which depend on its +// position within the chain. +// +// The flag for check header sanity allows the additional header sanity checks +// to be skipped which is useful for the full block processing path which checks +// the sanity of the entire block, including the header, before attempting to +// accept its header in order to quickly eliminate blocks that are obviously +// incorrect. +// +// In the case the block header is already known, the associated block node is +// examined to determine if the block is already known to be invalid, in which +// case an appropriate error will be returned. Otherwise, the block node is +// returned. +// +// This function MUST be called with the chain lock held (for writes). +func (b *BlockChain) maybeAcceptBlockHeader(header *wire.BlockHeader, checkHeaderSanity bool) (*blockNode, error) { + // Avoid validating the header again if its validation status is already + // known. Invalid headers are never added to the block index, so if there + // is an entry for the block hash, the header itself is known to be valid. + // However, it might have since been marked invalid either due to the + // associated block, or an ancestor, later failing validation. + hash := header.BlockHash() + if node := b.index.LookupNode(&hash); node != nil { + if err := b.checkKnownInvalidBlock(node); err != nil { + return nil, err + } + + return node, nil + } + + // Perform context-free sanity checks on the block header. + if checkHeaderSanity { + err := checkBlockHeaderSanity(header, b.chainParams.PowLimit, b.timeSource, BFNone) + if err != nil { + return nil, err + } + } + // Orphan headers are not allowed and this function should never be called + // with the genesis block. + prevHash := &header.PrevBlock + prevNode := b.index.LookupNode(prevHash) + if prevNode == nil { + str := fmt.Sprintf("previous block %s is not known", prevHash) + return nil, ruleError(ErrMissingParent, str) + } + + // There is no need to validate the header if an ancestor is already known + // to be invalid. + prevNodeStatus := b.index.NodeStatus(prevNode) + if prevNodeStatus.KnownInvalid() { + str := fmt.Sprintf("previous block %s is known to be invalid", prevHash) + return nil, ruleError(ErrInvalidAncestorBlock, str) + } + + // The block must pass all of the validation rules which depend on the + // position of the block within the block chain. + err := b.checkBlockHeaderContext(header, prevNode, BFNone) + if err != nil { + return nil, err + } + + // Create a new block node for the block and add it to the block index. + // + // Note that the additional information for the actual transactions and + // witnesses in the block can't be populated until the full block data is + // known since that information is not available in the header. + newNode := newBlockNode(header, prevNode) + b.index.AddNode(newNode) + return newNode, nil +} diff --git a/blockchain/blockindex.go b/blockchain/blockindex.go index 97256488..c7b4d701 100644 --- a/blockchain/blockindex.go +++ b/blockchain/blockindex.go @@ -1,4 +1,5 @@ // Copyright (c) 2015-2017 The btcsuite developers +// Copyright (c) 2018-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -60,6 +61,22 @@ func (status blockStatus) KnownInvalid() bool { return status&(statusValidateFailed|statusInvalidAncestor) != 0 } +// KnownInvalidAncestor returns whether the block is known to have an invalid +// ancestor. A return value of false in no way implies the block only has valid +// ancestors. Thus, this will return false for blocks with invalid ancestors +// that have not been proven invalid yet. +func (status blockStatus) KnownInvalidAncestor() bool { + return status&(statusInvalidAncestor) != 0 +} + +// KnownValidateFailed returns whether the block is known to have failed +// validation. A return value of false in no way implies the block is valid. +// Thus, this will return false for blocks that have not been proven to fail +// validation yet. +func (status blockStatus) KnownValidateFailed() bool { + return status&(statusValidateFailed) != 0 +} + // blockNode represents a block within the block chain and is primarily used to // aid in selecting the best chain to be the main chain. The main chain is // stored into the block database. @@ -114,6 +131,7 @@ func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, parent *block nonce: blockHeader.Nonce, timestamp: blockHeader.Timestamp.Unix(), merkleRoot: blockHeader.MerkleRoot, + status: statusNone, } if parent != nil { node.parent = parent @@ -217,6 +235,68 @@ func (node *blockNode) CalcPastMedianTime() time.Time { return time.Unix(medianTimestamp, 0) } +// compareHashesAsUint256LE compares two raw hashes treated as if they were +// little-endian uint256s in a way that is more efficient than converting them +// to big integers first. It returns 1 when a > b, -1 when a < b, and 0 when a +// == b. +func compareHashesAsUint256LE(a, b *chainhash.Hash) int { + // Find the index of the first byte that differs. + index := len(a) - 1 + for ; index >= 0 && a[index] == b[index]; index-- { + // Nothing to do. + } + if index < 0 { + return 0 + } + if a[index] > b[index] { + return 1 + } + return -1 +} + +// betterCandidate returns whether node 'a' is a better candidate than 'b' for +// the purposes of best chain selection. +// +// The criteria for determining what constitutes a better candidate, in order of +// priority, is as follows: +// +// 1. More total cumulative work +// 2. Having block data available +// 3. Receiving data earlier +// 4. Hash that represents more work (smaller value as a little-endian uint256) +// +// This function MUST be called with the block index lock held (for reads). +func betterCandidate(a, b *blockNode) bool { + // First, sort by the total cumulative work. + // + // Blocks with more cumulative work are better candidates for best chain + // selection. + if workCmp := a.workSum.Cmp(b.workSum); workCmp != 0 { + return workCmp > 0 + } + + // Then sort according to block data availability. + // + // Blocks that already have all of their data available are better + // candidates than those that do not. They have the same priority if either + // both have their data available or neither do. + if aHasData := a.status.HaveData(); aHasData != b.status.HaveData() { + return aHasData + } + + // Finally, fall back to sorting based on the hash in the case the work, + // block data availability, and received order are all the same. In + // practice, the order will typically only be the same for blocks loaded + // from disk since the received order is only stored in memory, however it + // can be the same when the block data for a given header is not yet known + // as well. + // + // Note that it is more difficult to find hashes with more leading zeros + // when treated as a little-endian uint256, so smaller values represent more + // work and are therefore better candidates. + return compareHashesAsUint256LE(&a.hash, &b.hash) < 0 +} + // blockIndex provides facilities for keeping track of an in-memory index of the // block chain. Although the name block chain suggests a single chain of // blocks, it is actually a tree-shaped structure where any node can have @@ -229,6 +309,17 @@ type blockIndex struct { db database.DB chainParams *chaincfg.Params + // bestHeader tracks the highest work block node in the index that is not + // known to be invalid. This is not necessarily the same as the active best + // chain, especially when block data is not yet known. However, since block + // nodes are only added to the index for block headers that pass all sanity + // and positional checks, which include checking proof of work, it does + // represent the tip of the header chain with the highest known work that + // has a reasonably high chance of becoming the best chain tip and is useful + // for things such as reporting progress and discovering the most suitable + // blocks to download. + bestHeader *blockNode + sync.RWMutex index map[chainhash.Hash]*blockNode dirty map[*blockNode]struct{} @@ -284,6 +375,11 @@ func (bi *blockIndex) AddNode(node *blockNode) { // This function is NOT safe for concurrent access. func (bi *blockIndex) addNode(node *blockNode) { bi.index[node.hash] = node + if bi.bestHeader == nil { + bi.bestHeader = node + } else if !node.status.KnownInvalid() && betterCandidate(node, bi.bestHeader) { + bi.bestHeader = node + } } // NodeStatus provides concurrent-safe access to the status field of a node. diff --git a/blockchain/chain.go b/blockchain/chain.go index a4b19f5c..15052230 100644 --- a/blockchain/chain.go +++ b/blockchain/chain.go @@ -228,6 +228,39 @@ func (b *BlockChain) HaveBlock(hash *chainhash.Hash) (bool, error) { return exists || b.IsKnownOrphan(hash), nil } +// HaveBlockWithData returns whether or not the chain instance has the block +// represented by the passed hash This includes checking the various places +// a block can be like part of the main chain, on a side chain, or in the orphan +// pool. In addition to this the function also checks for presence of block +// data along with presence of node. +// +// This function is safe for concurrent access. +func (b *BlockChain) HaveBlockWithData(hash *chainhash.Hash) (bool, error) { + // Checking if node exists in blockIndex or database. + exists, err := b.blockExists(hash) + if err != nil { + return false, err + } + if exists || b.IsKnownOrphan(hash) { + // Retrieving node from the blockIndex so that we can check the node + // status. + node := b.IndexLookupNode(hash) + // If the returned node is nil then the block is stored in the + // database and not present in block index, we return true here + // as presence of data in database means that we already have + // block data. + if node == nil { + return true, nil + } + // If block data is present then we return true, otherwise we would + // return false. + return node.status.HaveData(), nil + } + // If the block is not present in blockIndex or in database or is not known + // orphan, then we will return false. + return false, nil +} + // IsKnownOrphan returns whether the passed hash is currently a known orphan. // Keep in mind that only a limited number of orphans are held onto for a // limited amount of time, so this function must not be used as an absolute @@ -1364,6 +1397,27 @@ func (b *BlockChain) BlockLocatorFromHash(hash *chainhash.Hash) BlockLocator { return locator } +// IndexLookupNode returns the block node identified by the provided hash. It will +// return nil if there is no entry for the hash. +// +// This function is safe for concurrent access. +func (b *BlockChain) IndexLookupNode(hash *chainhash.Hash) *blockNode { + b.chainLock.RLock() + node := b.index.LookupNode(hash) + b.chainLock.RUnlock() + return node +} + +// IndexNodeStatus provides concurrent-safe access to the status field of a node. +// +// This function is safe for concurrent access. +func (b *BlockChain) IndexNodeStatus(node *blockNode) blockStatus { + b.chainLock.RLock() + status := b.index.NodeStatus(node) + b.chainLock.RUnlock() + return status +} + // LatestBlockLocator returns a block locator for the latest known tip of the // main (best) chain. // @@ -1389,6 +1443,21 @@ func (b *BlockChain) BlockHeightByHash(hash *chainhash.Hash) (int32, error) { return node.height, nil } +// NodeHeightByHash returns the height of the block with the given hash in the +// main chain. Does not check if the node is on the main chain, so returns height +// of the node which is created by header validation +// +// This function is safe for concurrent access. +func (b *BlockChain) NodeHeightByHash(hash *chainhash.Hash) (int32, error) { + node := b.index.LookupNode(hash) + if node == nil { + str := fmt.Sprintf("block %s is not in the main chain", hash) + return 0, errNotInMainChain(str) + } + + return node.height, nil +} + // BlockHashByHeight returns the hash of the block at the given height in the // main chain. // diff --git a/blockchain/chain_test.go b/blockchain/chain_test.go index fb1dc2e4..ae29a809 100644 --- a/blockchain/chain_test.go +++ b/blockchain/chain_test.go @@ -36,7 +36,7 @@ func TestHaveBlock(t *testing.T) { } // Create a new database and chain instance to run tests against. - chain, teardownFunc, err := chainSetup("haveblock", + chain, teardownFunc, err := ChainSetup("haveblock", &chaincfg.MainNetParams) if err != nil { t.Errorf("Failed to setup chain instance: %v", err) diff --git a/blockchain/chainquery.go b/blockchain/chainquery.go new file mode 100644 index 00000000..5929ebcb --- /dev/null +++ b/blockchain/chainquery.go @@ -0,0 +1,96 @@ +// Copyright (c) 2013-2016 The btcsuite developers +// Copyright (c) 2018-2021 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package blockchain + +import "github.com/utreexo/utreexod/chaincfg/chainhash" + +// PutNextNeededBlocks populates the provided slice with hashes for the next +// blocks after the current best chain tip that are needed to make progress +// towards the current best known header skipping any blocks that already have +// their data available. +// +// The provided slice will be populated with either as many hashes as it will +// fit per its length or as many hashes it takes to reach best header, whichever +// is smaller. +// +// It returns a sub slice of the provided one with its bounds adjusted to the +// number of entries populated. +// +// This function is safe for concurrent access. +func (b *BlockChain) PutNextNeededBlocks(out []chainhash.Hash) []chainhash.Hash { + // Nothing to do when no results are requested. + maxResults := len(out) + if maxResults == 0 { + return out[:0] + } + + b.index.RLock() + defer b.index.RUnlock() + + // Populate the provided slice by making use of a sliding window. Note that + // the needed block hashes are populated in forwards order while it is + // necessary to walk the block index backwards to determine them. Further, + // an unknown number of blocks may already have their data and need to be + // skipped, so it's not possible to determine the precise height after the + // fork point to start iterating from. Using a sliding window efficiently + // handles these conditions without needing additional allocations. + // + // The strategy is to initially determine the common ancestor between the + // current best chain tip and the current best known header as the starting + // fork point and move the fork point forward by the window size after + // populating the output slice with all relevant nodes in the window until + // either there are no more results or the desired number of results have + // been populated. + const windowSize = 32 + var outputIdx int + var window [windowSize]chainhash.Hash + bestHeader := b.index.bestHeader + fork := b.bestChain.FindFork(bestHeader) + for outputIdx < maxResults && fork != nil && fork != bestHeader { + // Determine the final descendant block on the branch that leads to the + // best known header in this window by clamping the number of + // descendants to consider to the window size. + endNode := bestHeader + numBlocksToConsider := endNode.height - fork.height + if numBlocksToConsider > windowSize { + endNode = endNode.Ancestor(fork.height + windowSize) + } + + // Populate the blocks in this window from back to front by walking + // backwards from the final block to consider in the window to the first + // one excluding any blocks that already have their data available. + windowIdx := windowSize + for node := endNode; node != nil && node != fork; node = node.parent { + if node.status.HaveData() { + continue + } + + windowIdx-- + window[windowIdx] = node.hash + } + + // Populate the outputs with as many from the back of the window as + // possible (since the window might not have been fully populated due to + // skipped blocks) and move the output index forward to match. + outputIdx += copy(out[outputIdx:], window[windowIdx:]) + + // Move the fork point forward to the final block of the window. + fork = endNode + } + + return out[:outputIdx] +} + +// BestHeader returns the header with the most cumulative work that is NOT +// known to be invalid. +func (b *BlockChain) BestHeader() (chainhash.Hash, int32) { + b.index.RLock() + header := b.index.bestHeader + blockHash := b.index.bestHeader.hash + height, _ := b.NodeHeightByHash(&blockHash) + b.index.RUnlock() + return header.hash, height +} diff --git a/blockchain/common.go b/blockchain/common.go index 8998330c..183a441e 100644 --- a/blockchain/common.go +++ b/blockchain/common.go @@ -323,10 +323,10 @@ func loadBlocks(filename string) (blocks []*btcutil.Block, err error) { return } -// chainSetup is used to create a new db and chain instance with the genesis +// ChainSetup is used to create a new db and chain instance with the genesis // block already inserted. In addition to the new chain instance, it returns // a teardown function the caller should invoke when done testing to clean up. -func chainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), error) { +func ChainSetup(dbName string, params *chaincfg.Params) (*BlockChain, func(), error) { if !IsSupportedDbType(testDbType) { return nil, nil, fmt.Errorf("unsupported db type %v", testDbType) } diff --git a/blockchain/error.go b/blockchain/error.go index 1e7c879b..8ca34201 100644 --- a/blockchain/error.go +++ b/blockchain/error.go @@ -220,6 +220,13 @@ const ( // current chain tip. This is not a block validation rule, but is required // for block proposals submitted via getblocktemplate RPC. ErrPrevBlockNotBest + + // ErrKnownInvalidBlock indicates that this block has previously failed + // validation. + ErrKnownInvalidBlock + + // ErrMissingParent indicates that the block was an orphan. + ErrMissingParent ) // Map of ErrorCode values back to their constant names for pretty printing. @@ -267,6 +274,8 @@ var errorCodeStrings = map[ErrorCode]string{ ErrPreviousBlockUnknown: "ErrPreviousBlockUnknown", ErrInvalidAncestorBlock: "ErrInvalidAncestorBlock", ErrPrevBlockNotBest: "ErrPrevBlockNotBest", + ErrKnownInvalidBlock: "ErrKnownInvalidBlock", + ErrMissingParent: "ErrMissingParent", } // String returns the ErrorCode as a human-readable name. diff --git a/blockchain/fullblocks_test.go b/blockchain/fullblocks_test.go index d8465079..9b25b600 100644 --- a/blockchain/fullblocks_test.go +++ b/blockchain/fullblocks_test.go @@ -57,10 +57,10 @@ func isSupportedDbType(dbType string) bool { return false } -// chainSetup is used to create a new db and chain instance with the genesis +// ChainSetup is used to create a new db and chain instance with the genesis // block already inserted. In addition to the new chain instance, it returns // a teardown function the caller should invoke when done testing to clean up. -func chainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, func(), error) { +func ChainSetup(dbName string, params *chaincfg.Params) (*blockchain.BlockChain, func(), error) { if !isSupportedDbType(testDbType) { return nil, nil, fmt.Errorf("unsupported db type %v", testDbType) } @@ -138,7 +138,7 @@ func TestFullBlocks(t *testing.T) { } // Create a new database and chain instance to run tests against. - chain, teardownFunc, err := chainSetup("fullblocktest", + chain, teardownFunc, err := ChainSetup("fullblocktest", &chaincfg.RegressionNetParams) if err != nil { t.Errorf("Failed to setup chain instance: %v", err) diff --git a/blockchain/fullblocktests/common_test.go b/blockchain/fullblocktests/common_test.go new file mode 100644 index 00000000..35e28f43 --- /dev/null +++ b/blockchain/fullblocktests/common_test.go @@ -0,0 +1,205 @@ +// Copyright (c) 2022 The utreexod developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package fullblocktests + +import ( + "testing" + + "github.com/utreexo/utreexod/blockchain" + "github.com/utreexo/utreexod/btcutil" +) + +type chaingenHarness struct { + *testGenerator + + t *testing.T + chain *blockchain.BlockChain +} + +// AcceptHeader processes the block header associated with the given name in the +// harness generator and expects it to be accepted, but not necessarily to the +// main chain. It also ensures the underlying block index is consistent with +// the result. +func (g *chaingenHarness) AcceptHeader(blockName string) { + g.t.Helper() + + header := &g.blockByName(blockName).Header + blockHash := header.BlockHash() + g.t.Logf("Testing accept block header %q (hash %s)", blockName, + blockHash) + + // Determine if the header is already known before attempting to process it. + alreadyHaveHeader := g.chain.IndexLookupNode(&blockHash) != nil + err := g.chain.ProcessBlockHeader(header) + if err != nil { + g.t.Fatalf("block header %q (hash %s) should have been "+ + "accepted: %v", blockName, blockHash, err) + } + + // Ensure the accepted header now exists in the block index. + node := g.chain.IndexLookupNode(&blockHash) + if node == nil { + g.t.Fatalf("accepted block header %q (hash %s) should have "+ + "been added to the block index", blockName, blockHash) + } + + // Ensure the accepted header is not marked as known valid when it was not + // previously known since that implies the block data is not yet available + // and therefore it can't possibly be known to be valid. + // + // Also, ensure the accepted header is not marked as known invalid, as + // having known invalid ancestors, or as known to have failed validation. + status := g.chain.IndexNodeStatus(node) + if !alreadyHaveHeader && status.KnownValid() { + g.t.Fatalf("accepted block header %q (hash %s) was not "+ + "already known, but is marked as known valid", blockName, blockHash, + ) + } + if status.KnownInvalid() { + g.t.Fatalf("accepted block header %q (hash %s) is marked "+ + "as known invalid", blockName, blockHash) + } + if status.KnownInvalidAncestor() { + g.t.Fatalf("accepted block header %q (hash %s) is marked "+ + "as having a known invalid ancestor", blockName, blockHash, + ) + } + if status.KnownValidateFailed() { + g.t.Fatalf("accepted block header %q (hash %s) is marked "+ + "as having known to fail validation", blockName, blockHash, + ) + } +} + +// RejectHeader expects the block header associated with the given name in the +// harness generator to be rejected with the provided error code and also +// ensures the underlying block index is consistent with the result. +func (g *chaingenHarness) RejectHeader(blockName string, code blockchain.ErrorCode) { + g.t.Helper() + + header := &g.blockByName(blockName).Header + blockHash := header.BlockHash() + g.t.Logf("Testing reject block header %q (hash %s, reason %v)", + blockName, blockHash, code) + + // Determine if the header is already known before attempting to process it. + alreadyHaveHeader := g.chain.IndexLookupNode(&blockHash) != nil + + err := g.chain.ProcessBlockHeader(header) + if err == nil { + g.t.Fatalf("block header %q (hash %s) should not have been "+ + "accepted", blockName, blockHash) + } + + // Ensure the error matches the value specified in the test instance. + rerr, ok := err.(blockchain.RuleError) + if (!ok) || rerr.ErrorCode != code { + g.t.Fatalf("block header %q (hash %s) does not have "+ + "expected reject code -- got %v, want %v", blockName, blockHash, + err, code) + } + + // Ensure the rejected header was not added to the block index when it was + // not already previously successfully added and that it was not removed if + // it was already previously added. + node := g.chain.IndexLookupNode(&blockHash) + switch { + case !alreadyHaveHeader && node == nil: + // Header was not added as expected. + return + + case !alreadyHaveHeader && node != nil: + g.t.Fatalf("rejected block header %q (hash %s) was added "+ + "to the block index", blockName, blockHash) + + case alreadyHaveHeader && node == nil: + g.t.Fatalf("rejected block header %q (hash %s) was removed "+ + "from the block index", blockName, blockHash) + } + + // The header was previously added, so ensure it is not reported as having + // been validated and that it is now known invalid. + status := g.chain.IndexNodeStatus(node) + if status.KnownValid() { + g.t.Fatalf("rejected block header %q (hash %s) is marked "+ + "as known valid", blockName, blockHash) + } + if !status.KnownInvalid() { + g.t.Fatalf("rejected block header %q (hash %s) is NOT "+ + "marked as known invalid", blockName, blockHash) + } +} + +// testAcceptedBlock attempts to process the block in the provided test +// instance and ensures that it was accepted according to the flags +// specified in the test. +func (g *chaingenHarness) AcceptBlock(item AcceptedBlock) { + g.t.Helper() + + blockHeight := item.Height + block := btcutil.NewBlock(item.Block) + block.SetHeight(blockHeight) + g.t.Logf("Testing block %s (hash %s, height %d)", + item.Name, block.Hash(), blockHeight) + + isMainChain, isOrphan, err := g.chain.ProcessBlock(block, + blockchain.BFNone) + if err != nil { + g.t.Fatalf("block %q (hash %s, height %d) should "+ + "have been accepted: %v", item.Name, + block.Hash(), blockHeight, err) + } + + // Ensure the main chain and orphan flags match the values + // specified in the test. + if isMainChain != item.IsMainChain { + g.t.Fatalf("block %q (hash %s, height %d) unexpected main "+ + "chain flag -- got %v, want %v", item.Name, + block.Hash(), blockHeight, isMainChain, + item.IsMainChain) + } + if isOrphan != item.IsOrphan { + g.t.Fatalf("block %q (hash %s, height %d) unexpected "+ + "orphan flag -- got %v, want %v", item.Name, + block.Hash(), blockHeight, isOrphan, + item.IsOrphan) + } +} + +// testRejectedBlock attempts to process the block in the provided test +// instance and ensures that it was rejected with the reject code +// specified in the test. +func (g *chaingenHarness) RejectBlock(item RejectedBlock) { + g.t.Helper() + + blockHeight := item.Height + block := btcutil.NewBlock(item.Block) + block.SetHeight(blockHeight) + g.t.Logf("Testing block %s (hash %s, height %d)", + item.Name, block.Hash(), blockHeight) + + _, _, err := g.chain.ProcessBlock(block, blockchain.BFNone) + if err == nil { + g.t.Fatalf("block %q (hash %s, height %d) should not "+ + "have been accepted", item.Name, block.Hash(), + blockHeight) + } + + // Ensure the error code is of the expected type and the reject + // code matches the value specified in the test instance. + rerr, ok := err.(blockchain.RuleError) + if !ok { + g.t.Fatalf("block %q (hash %s, height %d) returned "+ + "unexpected error type -- got %T, want "+ + "blockchain.RuleError", item.Name, block.Hash(), + blockHeight, err) + } + if rerr.ErrorCode != item.RejectCode { + g.t.Fatalf("block %q (hash %s, height %d) does not have "+ + "expected reject code -- got %v, want %v", + item.Name, block.Hash(), blockHeight, + rerr.ErrorCode, item.RejectCode) + } +} diff --git a/blockchain/fullblocktests/generate.go b/blockchain/fullblocktests/generate.go index c2416397..cd79bfc3 100644 --- a/blockchain/fullblocktests/generate.go +++ b/blockchain/fullblocktests/generate.go @@ -98,6 +98,42 @@ var _ TestInstance = RejectedBlock{} // This implements the TestInstance interface. func (b RejectedBlock) FullBlockTestInstance() {} +// AcceptedHeader defines a test instance that expects a header to be accepted to +// the blockchain either by extending the main chain, on a side chain, but can't be +// an orphan +type AcceptedHeader struct { + Name string + Block *wire.MsgBlock + Height int32 +} + +// Ensure AcceptedHeader implements the TestInstance interface. +var _ TestInstance = AcceptedHeader{} + +// FullBlockTestInstance only exists to allow AcceptedHeader to be treated as a +// TestInstance. +// +// This implements the TestInstance interface. +func (b AcceptedHeader) FullBlockTestInstance() {} + +// RejectedHeader defines a test instance that expects a header to be rejected by +// the blockchain consensus rules. +type RejectedHeader struct { + Name string + Block *wire.MsgBlock + Height int32 + RejectCode blockchain.ErrorCode +} + +// Ensure RejectedHeader implements the TestInstance interface. +var _ TestInstance = RejectedHeader{} + +// FullBlockTestInstance only exists to allow RejectedHeader to be treated as a +// TestInstance. +// +// This implements the TestInstance interface. +func (b RejectedHeader) FullBlockTestInstance() {} + // OrphanOrRejectedBlock defines a test instance that expects a block to either // be accepted as an orphan or rejected. This is useful since some // implementations might optimize the immediate rejection of orphan blocks when @@ -756,6 +792,16 @@ func (g *testGenerator) assertTipBlockMerkleRoot(expected chainhash.Hash) { } } +// blockByName returns the block associated with the provided block name. It +// will panic if the specified block name does not exist. +func (g *testGenerator) blockByName(blockName string) *wire.MsgBlock { + block, ok := g.blocksByName[blockName] + if !ok { + panic(fmt.Sprintf("block name %s does not exist", blockName)) + } + return block +} + // assertTipBlockTxOutOpReturn panics if the current tip block associated with // the generator does not have an OP_RETURN script for the transaction output at // the provided tx index and output index. @@ -887,7 +933,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) { acceptedToSideChainWithExpectedTip := func(tipName string) { tests = append(tests, []TestInstance{ acceptBlock(g.tipName, g.tip, false, false), - expectTipBlock(tipName, g.blocksByName[tipName]), + expectTipBlock(tipName, g.blockByName(tipName)), }) } rejected := func(code blockchain.ErrorCode) { @@ -906,6 +952,8 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) { }) } + // Checking that duplicate blocks are not allowed + tests = append(tests, []TestInstance{rejectBlock("genesis", g.tip, blockchain.ErrDuplicateBlock)}) // --------------------------------------------------------------------- // Generate enough blocks to have mature coinbase outputs to work with. // @@ -2107,7 +2155,7 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) { g.tip, false, false)) } testInstances = append(testInstances, expectTipBlock(chain1TipName, - g.blocksByName[chain1TipName])) + g.blockByName(chain1TipName))) tests = append(tests, testInstances) // Extend the side chain by one to force the large reorg. @@ -2133,3 +2181,240 @@ func Generate(includeLargeReorg bool) (tests [][]TestInstance, err error) { return tests, nil } + +// GenerateHeaders returns a slice of tests can be used to exercise the +// concensus and validation rules on the headers. These tests consists +// of blocks which only have header information that illustrates the header +// vaidation +func GenerateHeaders() (generator *testGenerator, tests [][]TestInstance) { + // Create a new database and chain instance needed to create the generator + // populated with the desired blocks. + params := chaincfg.RegressionNetParams + // Create a test generator instance initialized with the genesis block + // as the tip. + g, err := makeTestGenerator(regressionNetParams) + if err != nil { + return nil, nil + } + coinbaseMaturity := params.CoinbaseMaturity + + // Define some convenience helper functions to return an individual test + // instance that has the described characteristics. + // + // acceptHeader creates a test instance that expects the provided header + // to be accepted by the consensus rules. + // + // rejectHeader creates a test instance that expects the provided header + // to be rejected by the consensus rules. + acceptHeader := func(blockName string, blockHeader *wire.MsgBlock) TestInstance { + blockHeight := g.blockHeights[blockName] + return AcceptedHeader{blockName, blockHeader, blockHeight} + } + rejectHeader := func(blockName string, blockHeader *wire.MsgBlock, code blockchain.ErrorCode) TestInstance { + blockHeight := g.blockHeights[blockName] + return RejectedHeader{blockName, blockHeader, blockHeight, code} + } + acceptBlock := func(blockName string, block *wire.MsgBlock, isMainChain, isOrphan bool) TestInstance { + blockHeight := g.blockHeights[blockName] + return AcceptedBlock{blockName, block, blockHeight, isMainChain, + isOrphan} + } + rejectBlock := func(blockName string, block *wire.MsgBlock, code blockchain.ErrorCode) TestInstance { + blockHeight := g.blockHeights[blockName] + return RejectedBlock{blockName, block, blockHeight, code} + } + // Define some convenience helper functions to populate the tests slice + // with test instances that have the described characteristics. + // + // accepted creates and appends a single acceptHeader test instance for + // the current tip which expects the header to be accepted to the main + // chain. + // + // rejected creates and appends a single rejectHeader test instance for + // the current tip. + // + // acceptedBlock creates and appends a single acceptBlock test instance for + // the current tip which expects the block to be accepted to the main + // chain. + // + // rejectedBlock creates and appends a single rejectBlock test instance for + // the current tip. + accepted := func() { + tests = append(tests, []TestInstance{ + acceptHeader(g.tipName, g.tip), + }) + } + rejected := func(code blockchain.ErrorCode) { + tests = append(tests, []TestInstance{ + rejectHeader(g.tipName, g.tip, code), + }) + } + acceptedBlock := func() { + tests = append(tests, []TestInstance{ + acceptBlock(g.tipName, g.tip, true, false), + }) + } + rejectedBlock := func(code blockchain.ErrorCode) { + tests = append(tests, []TestInstance{ + rejectBlock(g.tipName, g.tip, code), + }) + } + // --------------------------------------------------------------------- + // Generate enough blocks to have mature coinbase outputs to work with. + // + // genesis -> bm0 -> bm1 -> ... -> bm99 + // --------------------------------------------------------------------- + var testInstances []TestInstance + for i := uint16(0); i < coinbaseMaturity; i++ { + blockName := fmt.Sprintf("bm%d", i) + g.nextBlock(blockName, nil) + g.saveTipCoinbaseOut() + testInstances = append(testInstances, acceptHeader(g.tipName, + g.tip)) + } + tests = append(tests, testInstances) + + // Collect spendable outputs. This simplifies the code below. + var outs []*spendableOut + for i := uint16(0); i < coinbaseMaturity; i++ { + op := g.oldestCoinbaseOut() + outs = append(outs, &op) + } + // Adding a block to the existing chain and accepting it + // + // ... -> b0 + g.nextBlock("b0", nil) + accepted() + // Create a block with invalid proof of work and reject that header + // + // ... -> b0() + // \-> b1(1) + g.setTip("b0") + b1 := g.nextBlock("b1", outs[1]) + + // Create a block with the parent as a header that will be rejected. + // This block's header will trigger the the error "missing parent". + g.nextBlock("b1a", outs[1]) + + // This can't be done inside a munge function passed to nextBlock + // because the block is solved after the function returns and this test + // requires an unsolved block. + { + origHash := b1.BlockHash() + for { + // Keep incrementing the nonce until the hash treated as + // a uint256 is higher than the limit. + b1.Header.Nonce++ + blockHash := b1.BlockHash() + hashNum := blockchain.HashToBig(&blockHash) + if hashNum.Cmp(g.params.PowLimit) >= 0 { + break + } + } + g.updateBlockState("b46", origHash, "b46", b1) + } + g.setTip(("b1")) + rejected(blockchain.ErrHighHash) + + g.setTip(("b1a")) + rejected(blockchain.ErrMissingParent) + // Create block with an invalid proof-of-work limit. + // + // ... -> b0() + // \-> b2(1) + g.setTip("b0") + g.nextBlock("b2", outs[1], func(b *wire.MsgBlock) { + b.Header.Bits-- + }) + rejected(blockchain.ErrUnexpectedDifficulty) + // Create block with an invalid negative proof-of-work limit. + // + // ... -> b0() + // \-> b3(1) + g.setTip("b0") + b3 := g.nextBlock("b3", outs[1]) + // This can't be done inside a munge function passed to nextBlock + // because the block is solved after the function returns and this test + // involves an unsolvable block. + { + origHash := b3.BlockHash() + b3.Header.Bits = 0x01810000 // -1 in compact form. + g.updateBlockState("b3", origHash, "b3", b3) + } + rejected(blockchain.ErrUnexpectedDifficulty) + // Adding a block with valid header but invalid spend + // + // ... -> b0() -> b4(2) + g.setTip("b0") + g.nextBlock("b4", outs[2]) + accepted() + // Adding a block with valid header and valid spend, but invalid parent + // + // ... -> b0() -> b5(1) + g.nextBlock("b5", outs[1]) + accepted() + // Adding a block with valid header and valid spend and valid parent + // + // ... -> b0() -> b6(1) + g.setTip("b0") + g.nextBlock("b6", outs[1]) + // Accepting/Rejecting the blocks for the headers that were + // accepted/rejected + testInstances = make([]TestInstance, 0) + for i := uint16(0); i < coinbaseMaturity; i++ { + blockName := fmt.Sprintf("bm%d", i) + g.setTip(blockName) + testInstances = append(testInstances, acceptBlock(g.tipName, + g.tip, true, false)) + } + tests = append(tests, testInstances) + // Accepting the block b0 + // + // ... -> b0 + g.setTip("b0") + acceptedBlock() + // Rejecting the block data for b1 because of high hash + // ... -> b0() + // \-> b1(1) + g.setTip("b1") + rejectedBlock(blockchain.ErrHighHash) + // Acccept the block as orphan + // + // -> b1a(1) + g.setTip("b1a") + tests = append(tests, []TestInstance{ + acceptBlock(g.tipName, g.tip, false, true), + }) + // Reject the block with invalid proof of work + // + // ... -> b0() + // \-> b2(1) + g.setTip("b2") + rejectedBlock(blockchain.ErrUnexpectedDifficulty) + // Reject the block with invalid negative proof of work + // + // ... -> b0() + // \-> b3(1) + g.setTip("b3") + rejectedBlock(blockchain.ErrUnexpectedDifficulty) + // Rejecting the block with valid header but invalid spend + // + // ... -> b0() + // \-> b4(2) + g.setTip("b4") + rejectedBlock(blockchain.ErrImmatureSpend) + // Since the block is rejected, so rejecting the header also + // which was earlier accepted. + rejected(blockchain.ErrKnownInvalidBlock) + // Rejecting a block with valid header and valid spend, but invalid parent + // + // -> b4(2) -> b5(1) + g.setTip("b5") + rejectedBlock(blockchain.ErrInvalidAncestorBlock) + // Accepting the block + // + // ... -> b0() -> b6(1) + g.setTip("b6") + acceptedBlock() + return &g, tests +} diff --git a/blockchain/fullblocktests/process_test.go b/blockchain/fullblocktests/process_test.go new file mode 100644 index 00000000..49319314 --- /dev/null +++ b/blockchain/fullblocktests/process_test.go @@ -0,0 +1,54 @@ +// Copyright (c) 2022 The utreexod developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package fullblocktests + +import ( + "testing" + + "github.com/utreexo/utreexod/blockchain" + "github.com/utreexo/utreexod/chaincfg" +) + +// TestProcessLogic ensures processing a mix of headers and blocks under a wide +// variety of fairly complex scenarios selects the expected best chain and +// properly tracks the header with the most cumulative work that is not known to +// be invalid as well as the one that is known to be invalid (when it exists). +func TestProcessLogic(t *testing.T) { + // Generate or reuse a shared chain generator with a set of blocks that form + // a fairly complex overall block tree including multiple forks such that + // some branches are valid and others contain invalid headers and/or blocks + // with multiple valid descendants as well as further forks at various + // heights from those invalid branches. + g, tests := GenerateHeaders() + // Create a new database and chain instance to run tests against. + chain, teardownFunc, err := blockchain.ChainSetup("fullblocktest", + &chaincfg.RegressionNetParams) + if err != nil { + t.Errorf("Failed to setup chain instance: %v", err) + return + } + defer teardownFunc() + harness := chaingenHarness{ + g, t, chain, + } + for testNum, test := range tests { + for itemNum, item := range test { + switch item := item.(type) { + case AcceptedHeader: + harness.AcceptHeader(item.Name) + case RejectedHeader: + harness.RejectHeader(item.Name, item.RejectCode) + case AcceptedBlock: + harness.AcceptBlock(item) + case RejectedBlock: + harness.RejectBlock(item) + default: + t.Fatalf("test #%d, item #%d is not one of "+ + "the supported test instance types -- "+ + "got type: %T", testNum, itemNum, item) + } + } + } +} diff --git a/blockchain/indexers/indexers_test.go b/blockchain/indexers/indexers_test.go index be99de73..e60e6f68 100644 --- a/blockchain/indexers/indexers_test.go +++ b/blockchain/indexers/indexers_test.go @@ -565,8 +565,8 @@ func TestProveUtxos(t *testing.T) { for _, spendable := range spendables { utxo, err := chain.FetchUtxoEntry(spendable.PrevOut) if err != nil { - t.Fatal(fmt.Sprintf("TestProveUtxos fail. err: outpoint %s not found.", - spendable.PrevOut.String())) + t.Fatalf("TestProveUtxos fail. err: outpoint %s not found.", + spendable.PrevOut.String()) } utxos = append(utxos, utxo) @@ -581,15 +581,15 @@ func TestProveUtxos(t *testing.T) { var err error flatProof, err = idxType.ProveUtxos(utxos, &outpoints) if err != nil { - t.Fatal(fmt.Sprintf("TestProveUtxos fail."+ - "Failed to create proof. err: %v", err)) + t.Fatalf("TestProveUtxos fail."+ + "Failed to create proof. err: %v", err) } case *UtreexoProofIndex: var err error proof, err = idxType.ProveUtxos(utxos, &outpoints) if err != nil { - t.Fatal(fmt.Sprintf("TestProveUtxos fail."+ - "Failed to create proof. err: %v", err)) + t.Fatalf("TestProveUtxos fail."+ + "Failed to create proof. err: %v", err) } } } @@ -610,7 +610,7 @@ func TestProveUtxos(t *testing.T) { uView := csnChain.GetUtreexoView() err = uView.VerifyAccProof(proof.HashesProven, proof.AccProof) if err != nil { - t.Fatal(fmt.Sprintf("TestProveUtxos fail. Failed to verify proof err: %v", err)) + t.Fatalf("TestProveUtxos fail. Failed to verify proof err: %v", err) } } } @@ -656,7 +656,7 @@ func TestUtreexoProofIndex(t *testing.T) { // same indexes. err := testUtreexoProof(newBlock, chain, indexes) if err != nil { - t.Fatal(fmt.Sprintf("TestUtreexoProofIndex failed testUtreexoProof. err: %v", err)) + t.Fatalf("TestUtreexoProofIndex failed testUtreexoProof. err: %v", err) } if b%10 == 0 { diff --git a/blockchain/notifications_test.go b/blockchain/notifications_test.go index 66f408e1..d59db20b 100644 --- a/blockchain/notifications_test.go +++ b/blockchain/notifications_test.go @@ -18,7 +18,7 @@ func TestNotifications(t *testing.T) { } // Create a new database and chain instance to run tests against. - chain, teardownFunc, err := chainSetup("notifications", + chain, teardownFunc, err := ChainSetup("notifications", &chaincfg.MainNetParams) if err != nil { t.Fatalf("Failed to setup chain instance: %v", err) diff --git a/blockchain/process.go b/blockchain/process.go index 13851292..30b92f31 100644 --- a/blockchain/process.go +++ b/blockchain/process.go @@ -1,4 +1,5 @@ // Copyright (c) 2013-2017 The btcsuite developers +// Copyright (c) 2018-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -11,6 +12,7 @@ import ( "github.com/utreexo/utreexod/btcutil" "github.com/utreexo/utreexod/chaincfg/chainhash" "github.com/utreexo/utreexod/database" + "github.com/utreexo/utreexod/wire" ) // BehaviorFlags is a bitmask defining tweaks to the normal behavior when @@ -31,6 +33,11 @@ const ( // BFNone is a convenience value to specifically indicate no flags. BFNone BehaviorFlags = 0 + + // BFHeaderValidated sets the third bit of the flag and indicates that + // the header has already been validated and the header validation can be + // skipped when block validation is being performed. + BFHeaderValidated BehaviorFlags = 1 << 2 ) // blockExists determines whether a block with the given hash exists either in @@ -129,6 +136,57 @@ func (b *BlockChain) processOrphans(hash *chainhash.Hash, flags BehaviorFlags) e return nil } +// checkKnownInvalidBlock returns an appropriate error when the provided block +// is known to be invalid either due to failing validation itself or due to +// having a known invalid ancestor (aka being part of an invalid branch). +// +// This function is safe for concurrent access. +func (b *BlockChain) checkKnownInvalidBlock(node *blockNode) error { + status := b.index.NodeStatus(node) + if status.KnownValidateFailed() { + str := fmt.Sprintf("block %s is known to be invalid", node.hash) + return ruleError(ErrKnownInvalidBlock, str) + } + if status.KnownInvalidAncestor() { + str := fmt.Sprintf("block %s is known to be part of an invalid branch", + node.hash) + return ruleError(ErrInvalidAncestorBlock, str) + } + + return nil +} + +// ProcessBlockHeader is the main workhorse for handling insertion of new block +// headers into the block chain using headers-first semantics. It includes +// functionality such as rejecting headers that do not connect to an existing +// known header, ensuring headers follow all rules that do not depend on having +// all ancestor block data available, and insertion into the block index. +// +// Block headers that have already been inserted are ignored, unless they have +// subsequently been marked invalid, in which case an appropriate error is +// returned. +// +// It should be noted that this function intentionally does not accept block +// headers that do not connect to an existing known header or to headers which +// are already known to be a part of an invalid branch. This means headers must +// be processed in order. +// +// This function is safe for concurrent access. +func (b *BlockChain) ProcessBlockHeader(header *wire.BlockHeader) error { + b.chainLock.Lock() + defer b.chainLock.Unlock() + const checkHeaderSanity = true + _, err := b.maybeAcceptBlockHeader(header, checkHeaderSanity) + if err != nil { + return err + } + err = b.index.flushToDB() + if err != nil { + return err + } + return nil +} + // ProcessBlock is the main workhorse for handling insertion of new blocks into // the block chain. It includes functionality such as rejecting duplicate // blocks, ensuring blocks follow all rules, orphan handling, and insertion into @@ -147,13 +205,26 @@ func (b *BlockChain) ProcessBlock(block *btcutil.Block, flags BehaviorFlags) (bo blockHash := block.Hash() log.Tracef("Processing block %v", blockHash) - // The block must not already exist in the main chain or side chains. exists, err := b.blockExists(blockHash) if err != nil { return false, false, err } + // If the block node is present, it means that the header is already verified. + // So we add this to the behavioural flag and the flag is then passed to further + // functions, where we don't need to reavalidate the header. if exists { + flags = flags | BFHeaderValidated + } + // Looking up for node and checking if it have block data or not + node := b.index.LookupNode(blockHash) + haveData := false + if node != nil { + haveData = node.status.HaveData() + } + // Returning error of duplicate block only if block data is already present + // at the node and we recieved duplicated block data. + if exists && haveData { str := fmt.Sprintf("already have block %v", blockHash) return false, false, ruleError(ErrDuplicateBlock, str) } @@ -164,7 +235,14 @@ func (b *BlockChain) ProcessBlock(block *btcutil.Block, flags BehaviorFlags) (bo return false, false, ruleError(ErrDuplicateBlock, str) } - // Perform preliminary sanity checks on the block and its transactions. + // Reject blocks that are already known to be invalid immediately to avoid + // additional work when possible. + if node != nil { + if err := b.checkKnownInvalidBlock(node); err != nil { + return false, false, err + } + } + // Perform preliminary sanity checks on the block and its transactions.a err = checkBlockSanity(block, b.chainParams.PowLimit, b.timeSource, flags) if err != nil { return false, false, err diff --git a/blockchain/utxocache_test.go b/blockchain/utxocache_test.go index 8b0eea1e..3ec7da98 100644 --- a/blockchain/utxocache_test.go +++ b/blockchain/utxocache_test.go @@ -76,7 +76,7 @@ func assertNbEntriesOnDisk(t *testing.T, chain *BlockChain, expectedNumber int) // the cache size limit to 10 MiB. func utxoCacheTestChain(testName string) (*BlockChain, *chaincfg.Params, func()) { params := chaincfg.RegressionNetParams - chain, tearDown, err := chainSetup(testName, ¶ms) + chain, tearDown, err := ChainSetup(testName, ¶ms) if err != nil { panic(fmt.Sprintf("error loading blockchain with database: %v", err)) } @@ -96,7 +96,7 @@ func TestUtxoCache_SimpleFlush(t *testing.T) { cache := chain.utxoCache tip := btcutil.NewBlock(params.GenesisBlock) - // The chainSetup init triggered write of consistency status of genesis. + // The ChainSetup init triggered write of consistency status of genesis. assertConsistencyState(t, chain, ucsConsistent, params.GenesisHash) assertNbEntriesOnDisk(t, chain, 0) diff --git a/blockchain/validate.go b/blockchain/validate.go index f2606054..6784ce44 100644 --- a/blockchain/validate.go +++ b/blockchain/validate.go @@ -467,9 +467,15 @@ func checkBlockHeaderSanity(header *wire.BlockHeader, powLimit *big.Int, timeSou func checkBlockSanity(block *btcutil.Block, powLimit *big.Int, timeSource MedianTimeSource, flags BehaviorFlags) error { msgBlock := block.MsgBlock() header := &msgBlock.Header - err := checkBlockHeaderSanity(header, powLimit, timeSource, flags) - if err != nil { - return err + + // If the third bit of the flag is set, then it means that the header + // is already been validated, so we don't need to revalidate it. So we + // check block header sanity, only when the second bit is not set to 1 + if (flags >> 2 & 1) != 1 { + err := checkBlockHeaderSanity(header, powLimit, timeSource, flags) + if err != nil { + return err + } } // A block must have at least one transaction. @@ -726,9 +732,15 @@ func (b *BlockChain) checkBlockHeaderContext(header *wire.BlockHeader, prevNode func (b *BlockChain) checkBlockContext(block *btcutil.Block, prevNode *blockNode, flags BehaviorFlags) error { // Perform all block header related validation checks. header := &block.MsgBlock().Header - err := b.checkBlockHeaderContext(header, prevNode, flags) - if err != nil { - return err + + // If the third bit of the flag is set, then it means that the header + // is already been validated, so we don't need to revalidate it. So we + // check block header context, only when the second bit is not set to 1 + if (flags >> 2 & 1) != 1 { + err := b.checkBlockHeaderContext(header, prevNode, flags) + if err != nil { + return err + } } fastAdd := flags&BFFastAdd == BFFastAdd diff --git a/blockchain/validate_test.go b/blockchain/validate_test.go index 86469a17..370dc9c1 100644 --- a/blockchain/validate_test.go +++ b/blockchain/validate_test.go @@ -67,7 +67,7 @@ func TestSequenceLocksActive(t *testing.T) { // ensure it fails. func TestCheckConnectBlockTemplate(t *testing.T) { // Create a new database and chain instance to run tests against. - chain, teardownFunc, err := chainSetup("checkconnectblocktemplate", + chain, teardownFunc, err := ChainSetup("checkconnectblocktemplate", &chaincfg.MainNetParams) if err != nil { t.Errorf("Failed to setup chain instance: %v", err) diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 060ea278..7beefcd7 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -560,7 +560,7 @@ func TestOrphanReject(t *testing.T) { // Ensure no transactions were reported as accepted. if len(acceptedTxns) != 0 { - t.Fatal("ProcessTransaction: reported %d accepted "+ + t.Fatalf("ProcessTransaction: reported %d accepted "+ "transactions from failed orphan attempt", len(acceptedTxns)) } diff --git a/netsync/manager.go b/netsync/manager.go index 737cd7ec..2228da9b 100644 --- a/netsync/manager.go +++ b/netsync/manager.go @@ -1,4 +1,5 @@ // Copyright (c) 2013-2017 The btcsuite developers +// Copyright (c) 2018-2021 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -28,6 +29,10 @@ const ( // more. minInFlightBlocks = 10 + // maxInFlightBlocks is the maximum number of blocks to allow in the sync + // peer request queue. + maxInFlightBlocks = 16 + // maxRejectedTxns is the maximum number of rejected transactions // hashes to store in memory. maxRejectedTxns = 1000 @@ -148,6 +153,8 @@ type headerNode struct { // peerSyncState stores additional information that the SyncManager tracks // about a peer. type peerSyncState struct { + *peerpkg.Peer + syncCandidate bool requestQueue []*wire.InvVect requestedTxns map[chainhash.Hash]struct{} @@ -204,6 +211,28 @@ type SyncManager struct { startHeader *list.Element nextCheckpoint *chaincfg.Checkpoint + // The following fields are used to track the list of the next blocks to + // download in the branch leading up to the best known header. + // + // nextBlocksHeader is the hash of the best known header when the list was + // last updated. + // + // nextBlocksBuf houses an overall list of blocks needed (up to the size of + // the array) regardless of whether or not they have been requested and + // provides what is effectively a reusable lookahead buffer. Note that + // since it is a fixed size and acts as a backing array, not all entries + // will necessarily refer to valid data, especially once the chain is + // synced. nextNeededBlocks slices into the valid part of the array. + // + // nextNeededBlocks subslices into nextBlocksBuf such that it provides an + // upper bound on the entries of the backing array that are valid and also + // acts as a list of needed blocks that are not already known to be in + // flight. + // nextBlocksHeader chainhash.Hash + nextBlocksHeader chainhash.Hash + nextBlocksBuf [1024]chainhash.Hash + nextNeededBlocks []chainhash.Hash + // An optional fee estimator. feeEstimator *mempool.FeeEstimator } @@ -842,7 +871,7 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { if !isCheckpointBlock { if sm.startHeader != nil && len(state.requestedBlocks) < minInFlightBlocks { - sm.fetchHeaderBlocks() + sm.fetchNextBlocks(peer, sm.peerStates[peer]) } return } @@ -883,6 +912,71 @@ func (sm *SyncManager) handleBlockMsg(bmsg *blockMsg) { } } +func (sm *SyncManager) maybeUpdateNextNeededBlocks() { + + // Update the list if the best known header changed since the last time it + // was updated or it is not empty, is getting short, does not already + // end at the best known header. + chain := sm.chain + bestHeader, _ := chain.BestHeader() + numNeeded := len(sm.nextNeededBlocks) + needsUpdate := sm.nextBlocksHeader != bestHeader || (numNeeded > 0 && + numNeeded < minInFlightBlocks && + sm.nextNeededBlocks[numNeeded-1] != bestHeader) + if needsUpdate { + sm.nextNeededBlocks = chain.PutNextNeededBlocks(sm.nextBlocksBuf[:]) + sm.nextBlocksHeader = bestHeader + } +} + +// fetchNextBlocks creates and sends a request to the provided peer for the next +// blocks to be downloaded based on the current headers. +func (m *SyncManager) fetchNextBlocks(peer *peerpkg.Peer, peerSync *peerSyncState) { + // Nothing to do if the target maximum number of blocks to request from the + // peer at the same time are already in flight. + numInFlight := len(peerSync.requestedBlocks) + if numInFlight >= maxInFlightBlocks { + return + } + + // Potentially update the list of the next blocks to download in the branch + // leading up to the best known header. + m.maybeUpdateNextNeededBlocks() + + // Build and send a getdata request for the needed blocks. + numNeeded := len(m.nextNeededBlocks) + if numNeeded == 0 { + return + } + maxNeeded := maxInFlightBlocks - numInFlight + if numNeeded > maxNeeded { + numNeeded = maxNeeded + } + gdmsg := wire.NewMsgGetDataSizeHint(uint(numNeeded)) + for i := 0; i < numNeeded && len(gdmsg.InvList) < wire.MaxInvPerMsg; i++ { + // The block is either going to be skipped because it has already been + // requested or it will be requested, but in either case, the block is + // no longer needed for future iterations. + hash := &m.nextNeededBlocks[0] + m.nextNeededBlocks = m.nextNeededBlocks[1:] + + // Skip blocks that have already been requested. The needed blocks + // might have been updated above thereby potentially repopulating some + // blocks that are still in flight. + if _, ok := m.requestedBlocks[*hash]; ok { + continue + } + + iv := wire.NewInvVect(wire.InvTypeBlock, hash) + m.requestedBlocks[*hash] = struct{}{} + peerSync.requestedBlocks[*hash] = struct{}{} + gdmsg.AddInvVect(iv) + } + if len(gdmsg.InvList) > 0 { + peer.QueueMessage(gdmsg, nil) + } +} + // fetchHeaderBlocks creates and sends a request to the syncPeer for the next // list of blocks to be downloaded based on the current list of headers. func (sm *SyncManager) fetchHeaderBlocks() { @@ -905,13 +999,13 @@ func (sm *SyncManager) fetchHeaderBlocks() { } iv := wire.NewInvVect(wire.InvTypeBlock, node.hash) - haveInv, err := sm.haveInventory(iv) + haveBlock, err := sm.chain.HaveBlockWithData(node.hash) if err != nil { log.Warnf("Unexpected failure when checking for "+ "existing inventory during header block "+ "fetch: %v", err) } - if !haveInv { + if !haveBlock { syncPeerState := sm.peerStates[sm.syncPeer] sm.requestedBlocks[*node.hash] = struct{}{} @@ -1008,7 +1102,17 @@ func (sm *SyncManager) handleHeadersMsg(hmsg *headersMsg) { peer.Disconnect() return } - + // Processing the block headers that are downloaded and if it passes + // all the checks, creating a block node that only contains header. + err := sm.chain.ProcessBlockHeader(blockHeader) + if err != nil { + // Note that there is no need to check for an orphan header here + // because they were already verified to connect above. + log.Debugf("Failed to process block header %s from peer %s: %v -- "+ + "disconnecting", blockHeader.BlockHash(), peer, err) + peer.Disconnect() + return + } // Verify the header at the next checkpoint height matches. if node.height == sm.nextCheckpoint.Height { if node.hash.IsEqual(sm.nextCheckpoint.Hash) { @@ -1041,7 +1145,7 @@ func (sm *SyncManager) handleHeadersMsg(hmsg *headersMsg) { log.Infof("Received %v block headers: Fetching blocks", sm.headerList.Len()) sm.progressLogger.SetLastLogTime(time.Now()) - sm.fetchHeaderBlocks() + sm.fetchNextBlocks(peer, sm.peerStates[peer]) return }