Sliding blocks window

.github/workflows/go.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Check out source
         uses: actions/checkout@v2
       - name: Install Linters
-        run: "curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.1"
+        run: "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.2"
       - name: Build
         env:
           GO111MODULE: "on"

blockchain/accept.go

@@ -1,4 +1,5 @@
 // Copyright (c) 2013-2017 The btcsuite developers
+// Copyright (c) 2018-2021 The Decred developers
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
@@ -9,6 +10,7 @@ import (
 
 	"github.com/utreexo/utreexod/btcutil"
 	"github.com/utreexo/utreexod/database"
+	"github.com/utreexo/utreexod/wire"
 )
 
 // maybeAcceptBlock potentially accepts a block into the block chain and, if
@@ -60,23 +62,29 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags)
 		return false, err
 	}
 
-	// Create a new block node for the block and add it to the node index. Even
-	// if the block ultimately gets connected to the main chain, it starts out
-	// on a side chain.
+	// Lookup for the block node that was created at the time of header
+	// validation if the block node is not existent then create a new
+	// block node for the block and add it to the node index. Even if
+	// the block ultimately gets connected to the main chain, it starts
+	// out on a side chain.
 	blockHeader := &block.MsgBlock().Header
-	newNode := newBlockNode(blockHeader, prevNode)
-	newNode.status = statusDataStored
-
-	b.index.AddNode(newNode)
-	err = b.index.flushToDB()
-	if err != nil {
-		return false, err
+	blockHash := blockHeader.BlockHash()
+	node := b.index.LookupNode(&blockHash)
+	if node == nil {
+		newNode := newBlockNode(blockHeader, prevNode)
+		b.index.AddNode(newNode)
+		err = b.index.flushToDB()
+		if err != nil {
+			return false, err
+		}
+		node = newNode
 	}
+	node.status = statusDataStored
 
 	// Connect the passed block to the chain while respecting proper chain
 	// selection according to the chain with the most proof of work.  This
 	// also handles validation of the transaction scripts.
-	isMainChain, err := b.connectBestChain(newNode, block, flags)
+	isMainChain, err := b.connectBestChain(node, block, flags)
 	if err != nil {
 		return false, err
 	}
@@ -90,3 +98,76 @@ func (b *BlockChain) maybeAcceptBlock(block *btcutil.Block, flags BehaviorFlags)
 
 	return isMainChain, nil
 }
+
+// maybeAcceptBlockHeader potentially accepts the header to the block index and,
+// if accepted, returns the block node associated with the header.  It performs
+// several context independent checks as well as those which depend on its
+// position within the chain.
+//
+// The flag for check header sanity allows the additional header sanity checks
+// to be skipped which is useful for the full block processing path which checks
+// the sanity of the entire block, including the header, before attempting to
+// accept its header in order to quickly eliminate blocks that are obviously
+// incorrect.
+//
+// In the case the block header is already known, the associated block node is
+// examined to determine if the block is already known to be invalid, in which
+// case an appropriate error will be returned.  Otherwise, the block node is
+// returned.
+//
+// This function MUST be called with the chain lock held (for writes).
+func (b *BlockChain) maybeAcceptBlockHeader(header *wire.BlockHeader, checkHeaderSanity bool) (*blockNode, error) {
+	// Avoid validating the header again if its validation status is already
+	// known.  Invalid headers are never added to the block index, so if there
+	// is an entry for the block hash, the header itself is known to be valid.
+	// However, it might have since been marked invalid either due to the
+	// associated block, or an ancestor, later failing validation.
+	hash := header.BlockHash()
+	if node := b.index.LookupNode(&hash); node != nil {
+		if err := b.checkKnownInvalidBlock(node); err != nil {
+			return nil, err
+		}
+
+		return node, nil
+	}
+
+	// Perform context-free sanity checks on the block header.
+	if checkHeaderSanity {
+		err := checkBlockHeaderSanity(header, b.chainParams.PowLimit, b.timeSource, BFNone)
+		if err != nil {
+			return nil, err
+		}
+	}
+	// Orphan headers are not allowed and this function should never be called
+	// with the genesis block.
+	prevHash := &header.PrevBlock
+	prevNode := b.index.LookupNode(prevHash)
+	if prevNode == nil {
+		str := fmt.Sprintf("previous block %s is not known", prevHash)
+		return nil, ruleError(ErrMissingParent, str)
+	}
+
+	// There is no need to validate the header if an ancestor is already known
+	// to be invalid.
+	prevNodeStatus := b.index.NodeStatus(prevNode)
+	if prevNodeStatus.KnownInvalid() {
+		str := fmt.Sprintf("previous block %s is known to be invalid", prevHash)
+		return nil, ruleError(ErrInvalidAncestorBlock, str)
+	}
+
+	// The block must pass all of the validation rules which depend on the
+	// position of the block within the block chain.
+	err := b.checkBlockHeaderContext(header, prevNode, BFNone)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create a new block node for the block and add it to the block index.
+	//
+	// Note that the additional information for the actual transactions and
+	// witnesses in the block can't be populated until the full block data is
+	// known since that information is not available in the header.
+	newNode := newBlockNode(header, prevNode)
+	b.index.AddNode(newNode)
+	return newNode, nil
+}

blockchain/blockindex.go

@@ -1,4 +1,5 @@
 // Copyright (c) 2015-2017 The btcsuite developers
+// Copyright (c) 2018-2021 The Decred developers
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
@@ -60,6 +61,22 @@ func (status blockStatus) KnownInvalid() bool {
 	return status&(statusValidateFailed|statusInvalidAncestor) != 0
 }
 
+// KnownInvalidAncestor returns whether the block is known to have an invalid
+// ancestor.  A return value of false in no way implies the block only has valid
+// ancestors.  Thus, this will return false for blocks with invalid ancestors
+// that have not been proven invalid yet.
+func (status blockStatus) KnownInvalidAncestor() bool {
+	return status&(statusInvalidAncestor) != 0
+}
+
+// KnownValidateFailed returns whether the block is known to have failed
+// validation.  A return value of false in no way implies the block is valid.
+// Thus, this will return false for blocks that have not been proven to fail
+// validation yet.
+func (status blockStatus) KnownValidateFailed() bool {
+	return status&(statusValidateFailed) != 0
+}
+
 // blockNode represents a block within the block chain and is primarily used to
 // aid in selecting the best chain to be the main chain.  The main chain is
 // stored into the block database.
@@ -114,6 +131,7 @@ func initBlockNode(node *blockNode, blockHeader *wire.BlockHeader, parent *block
 		nonce:      blockHeader.Nonce,
 		timestamp:  blockHeader.Timestamp.Unix(),
 		merkleRoot: blockHeader.MerkleRoot,
+		status:     statusNone,
 	}
 	if parent != nil {
 		node.parent = parent
@@ -217,6 +235,68 @@ func (node *blockNode) CalcPastMedianTime() time.Time {
 	return time.Unix(medianTimestamp, 0)
 }
 
+// compareHashesAsUint256LE compares two raw hashes treated as if they were
+// little-endian uint256s in a way that is more efficient than converting them
+// to big integers first.  It returns 1 when a > b, -1 when a < b, and 0 when a
+// == b.
+func compareHashesAsUint256LE(a, b *chainhash.Hash) int {
+	// Find the index of the first byte that differs.
+	index := len(a) - 1
+	for ; index >= 0 && a[index] == b[index]; index-- {
+		// Nothing to do.
+	}
+	if index < 0 {
+		return 0
+	}
+	if a[index] > b[index] {
+		return 1
+	}
+	return -1
+}
+
+// betterCandidate returns whether node 'a' is a better candidate than 'b' for
+// the purposes of best chain selection.
+//
+// The criteria for determining what constitutes a better candidate, in order of
+// priority, is as follows:
+//
+// 1. More total cumulative work
+// 2. Having block data available
+// 3. Receiving data earlier
+// 4. Hash that represents more work (smaller value as a little-endian uint256)
+//
+// This function MUST be called with the block index lock held (for reads).
+func betterCandidate(a, b *blockNode) bool {
+	// First, sort by the total cumulative work.
+	//
+	// Blocks with more cumulative work are better candidates for best chain
+	// selection.
+	if workCmp := a.workSum.Cmp(b.workSum); workCmp != 0 {
+		return workCmp > 0
+	}
+
+	// Then sort according to block data availability.
+	//
+	// Blocks that already have all of their data available are better
+	// candidates than those that do not.  They have the same priority if either
+	// both have their data available or neither do.
+	if aHasData := a.status.HaveData(); aHasData != b.status.HaveData() {
+		return aHasData
+	}
+
+	// Finally, fall back to sorting based on the hash in the case the work,
+	// block data availability, and received order are all the same.  In
+	// practice, the order will typically only be the same for blocks loaded
+	// from disk since the received order is only stored in memory, however it
+	// can be the same when the block data for a given header is not yet known
+	// as well.
+	//
+	// Note that it is more difficult to find hashes with more leading zeros
+	// when treated as a little-endian uint256, so smaller values represent more
+	// work and are therefore better candidates.
+	return compareHashesAsUint256LE(&a.hash, &b.hash) < 0
+}
+
 // blockIndex provides facilities for keeping track of an in-memory index of the
 // block chain.  Although the name block chain suggests a single chain of
 // blocks, it is actually a tree-shaped structure where any node can have
@@ -229,6 +309,17 @@ type blockIndex struct {
 	db          database.DB
 	chainParams *chaincfg.Params
 
+	// bestHeader tracks the highest work block node in the index that is not
+	// known to be invalid.  This is not necessarily the same as the active best
+	// chain, especially when block data is not yet known.  However, since block
+	// nodes are only added to the index for block headers that pass all sanity
+	// and positional checks, which include checking proof of work, it does
+	// represent the tip of the header chain with the highest known work that
+	// has a reasonably high chance of becoming the best chain tip and is useful
+	// for things such as reporting progress and discovering the most suitable
+	// blocks to download.
+	bestHeader *blockNode
+
 	sync.RWMutex
 	index map[chainhash.Hash]*blockNode
 	dirty map[*blockNode]struct{}
@@ -284,6 +375,11 @@ func (bi *blockIndex) AddNode(node *blockNode) {
 // This function is NOT safe for concurrent access.
 func (bi *blockIndex) addNode(node *blockNode) {
 	bi.index[node.hash] = node
+	if bi.bestHeader == nil {
+		bi.bestHeader = node
+	} else if !node.status.KnownInvalid() && betterCandidate(node, bi.bestHeader) {
+		bi.bestHeader = node
+	}
 }
 
 // NodeStatus provides concurrent-safe access to the status field of a node.

blockchain/chain.go

@@ -228,6 +228,39 @@ func (b *BlockChain) HaveBlock(hash *chainhash.Hash) (bool, error) {
 	return exists || b.IsKnownOrphan(hash), nil
 }
 
+// HaveBlockWithData returns whether or not the chain instance has the block
+// represented by the passed hash This includes checking the various places
+// a block can be like part of the main chain, on a side chain, or in the orphan
+// pool. In addition to this the function also checks for presence of block
+// data along with presence of node.
+//
+// This function is safe for concurrent access.
+func (b *BlockChain) HaveBlockWithData(hash *chainhash.Hash) (bool, error) {
+	// Checking if node exists in blockIndex or database.
+	exists, err := b.blockExists(hash)
+	if err != nil {
+		return false, err
+	}
+	if exists || b.IsKnownOrphan(hash) {
+		// Retrieving node from the blockIndex so that we can check the node
+		// status.
+		node := b.IndexLookupNode(hash)
+		// If the returned node is nil then the block is stored in the
+		// database and not present in block index, we return true here
+		// as presence of data in database means that we already have
+		// block data.
+		if node == nil {
+			return true, nil
+		}
+		// If block data is present then we return true, otherwise we would
+		// return false.
+		return node.status.HaveData(), nil
+	}
+	// If the block is not present in blockIndex or in database or is not known
+	// orphan, then we will return false.
+	return false, nil
+}
+
 // IsKnownOrphan returns whether the passed hash is currently a known orphan.
 // Keep in mind that only a limited number of orphans are held onto for a
 // limited amount of time, so this function must not be used as an absolute
@@ -1364,6 +1397,27 @@ func (b *BlockChain) BlockLocatorFromHash(hash *chainhash.Hash) BlockLocator {
 	return locator
 }
 
+// IndexLookupNode returns the block node identified by the provided hash.  It will
+// return nil if there is no entry for the hash.
+//
+// This function is safe for concurrent access.
+func (b *BlockChain) IndexLookupNode(hash *chainhash.Hash) *blockNode {
+	b.chainLock.RLock()
+	node := b.index.LookupNode(hash)
+	b.chainLock.RUnlock()
+	return node
+}
+
+// IndexNodeStatus provides concurrent-safe access to the status field of a node.
+//
+// This function is safe for concurrent access.
+func (b *BlockChain) IndexNodeStatus(node *blockNode) blockStatus {
+	b.chainLock.RLock()
+	status := b.index.NodeStatus(node)
+	b.chainLock.RUnlock()
+	return status
+}
+
 // LatestBlockLocator returns a block locator for the latest known tip of the
 // main (best) chain.
 //
@@ -1389,6 +1443,21 @@ func (b *BlockChain) BlockHeightByHash(hash *chainhash.Hash) (int32, error) {
 	return node.height, nil
 }
 
+// NodeHeightByHash returns the height of the block with the given hash in the
+// main chain. Does not check if the node is on the main chain, so returns height
+// of the node which is created by header validation
+//
+// This function is safe for concurrent access.
+func (b *BlockChain) NodeHeightByHash(hash *chainhash.Hash) (int32, error) {
+	node := b.index.LookupNode(hash)
+	if node == nil {
+		str := fmt.Sprintf("block %s is not in the main chain", hash)
+		return 0, errNotInMainChain(str)
+	}
+
+	return node.height, nil
+}
+
 // BlockHashByHeight returns the hash of the block at the given height in the
 // main chain.
 //

blockchain/chain_test.go

@@ -36,7 +36,7 @@ func TestHaveBlock(t *testing.T) {
 	}
 
 	// Create a new database and chain instance to run tests against.
-	chain, teardownFunc, err := chainSetup("haveblock",
+	chain, teardownFunc, err := ChainSetup("haveblock",
 		&chaincfg.MainNetParams)
 	if err != nil {
 		t.Errorf("Failed to setup chain instance: %v", err)