Add new getcfilters message for utreexo nodes

blockchain/chainio.go

@@ -1080,6 +1080,32 @@ func SerializeUtreexoRoots(numLeaves uint64, roots []utreexo.Hash) ([]byte, erro
 	return w.Bytes(), nil
 }
 
+// SerializeUtreexoRootsHash serializes the numLeaves and the roots into a byte slice.
+// it takes in a slice of chainhash.Hash instead of utreexo.Hash. chainhash.Hash is the hashed
+// value of the utreexo.Hash.
+func SerializeUtreexoRootsHash(numLeaves uint64, roots []*chainhash.Hash) ([]byte, error) {
+	// 8 byte NumLeaves + (32 byte roots * len(roots))
+	w := bytes.NewBuffer(make([]byte, 0, 8+(len(roots)*chainhash.HashSize)))
+
+	// Write the NumLeaves first.
+	var buf [8]byte
+	byteOrder.PutUint64(buf[:], numLeaves)
+	_, err := w.Write(buf[:])
+	if err != nil {
+		return nil, err
+	}
+
+	// Then write the roots.
+	for _, root := range roots {
+		_, err = w.Write(root[:])
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return w.Bytes(), nil
+}
+
 // DeserializeUtreexoRoots deserializes the provided byte slice into numLeaves and roots.
 func DeserializeUtreexoRoots(serializedUView []byte) (uint64, []utreexo.Hash, error) {
 	totalLen := len(serializedUView)

blockchain/indexers/cfindex.go

@@ -33,7 +33,7 @@ var (
 	// cfIndexKeys is an array of db bucket names used to house indexes of
 	// block hashes to cfilters.
 	cfIndexKeys = [][]byte{
-		[]byte("cf0byhashidx"),
+		[]byte("cf0byhashidx"), // bucket for basic filter indexes
 	}
 
 	// cfHeaderKeys is an array of db bucket names used to house indexes of

blockchain/indexers/utreexocfindex.go

@@ -0,0 +1,301 @@
+package indexers
+
+import (
+	"errors"
+
+	"github.com/utreexo/utreexod/blockchain"
+	"github.com/utreexo/utreexod/btcutil"
+	"github.com/utreexo/utreexod/btcutil/gcs/builder"
+	"github.com/utreexo/utreexod/chaincfg"
+	"github.com/utreexo/utreexod/chaincfg/chainhash"
+	"github.com/utreexo/utreexod/database"
+	"github.com/utreexo/utreexod/wire"
+)
+
+// utreexoProofIndexName is the human-readable name for the index.
+const (
+	utreexoCFIndexName = "utreexo custom cfilter index"
+)
+
+// utreexocfilter is a custom commited filter which serves utreexo roots
+// these roots are already present, so they need not be created/stored, their
+// headers could be stored though
+var (
+	// utreexoCFIndexParentBucketKey is the name of the parent bucket used to
+	// house the index. The rest of the buckets live below this bucket.
+	utreexoCFIndexParentBucketKey = []byte("utreexocfindexparentbucket")
+
+	// utreexoCfHeaderKeys is an array of db bucket names used to house indexes of
+	// block hashes to cf headers.
+	utreexoCfHeaderKeys = [][]byte{
+		[]byte("utreexocfheaderbyhashidx"),
+	}
+)
+
+// dbFetchFilterIdxEntry retrieves a data blob from the filter index database.
+// An entry's absence is not considered an error.
+func dbFetchUtreexoCFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) ([]byte, error) {
+	idx := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey).Bucket(key)
+	return idx.Get(h[:]), nil
+}
+
+// dbStoreFilterIdxEntry stores a data blob in the filter index database.
+func dbStoreUtreexoCFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash, f []byte) error {
+	idx := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey).Bucket(key)
+	return idx.Put(h[:], f)
+}
+
+// dbDeleteFilterIdxEntry deletes a data blob from the filter index database.
+func dbDeleteUtreexoCFilterIdxEntry(dbTx database.Tx, key []byte, h *chainhash.Hash) error {
+	idx := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey).Bucket(key)
+	return idx.Delete(h[:])
+}
+
+var _ Indexer = (*UtreexoCFIndex)(nil)
+
+var _ NeedsInputser = (*UtreexoCFIndex)(nil)
+
+type UtreexoCFIndex struct {
+	db          database.DB
+	chainParams *chaincfg.Params
+
+	chain *blockchain.BlockChain
+
+	utreexoProofIndex *UtreexoProofIndex
+
+	flatUtreexoProofIndex *FlatUtreexoProofIndex
+}
+
+func (idx *UtreexoCFIndex) NeedsInputs() bool {
+	return true
+}
+
+// Init initializes the utreexo cf index. This is part of the Indexer
+// interface.
+func (idx *UtreexoCFIndex) Init(_ *blockchain.BlockChain) error {
+	return nil // Nothing to do.
+}
+
+// Key returns the database key to use for the index as a byte slice. This is
+// part of the Indexer interface.
+func (idx *UtreexoCFIndex) Key() []byte {
+	return utreexoCFIndexParentBucketKey
+}
+
+// Name returns the human-readable name of the index. This is part of the
+// Indexer interface.
+func (idx *UtreexoCFIndex) Name() string {
+	return utreexoCFIndexName
+}
+
+// Create is invoked when the index manager determines the index needs to
+// be created for the first time. It creates buckets for the custom utreexo
+// filter index.
+func (idx *UtreexoCFIndex) Create(dbTx database.Tx) error {
+	meta := dbTx.Metadata()
+
+	utreexoCfIndexParentBucket, err := meta.CreateBucket(utreexoCFIndexParentBucketKey)
+	if err != nil {
+		return err
+	}
+
+	for _, bucketName := range utreexoCfHeaderKeys {
+		_, err = utreexoCfIndexParentBucket.CreateBucket(bucketName)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// storeUtreexoCFilter stores a given utreexocfilter header
+func storeUtreexoCFHeader(dbTx database.Tx, block *btcutil.Block, filterData []byte,
+	filterType wire.FilterType) error {
+	if filterType != wire.UtreexoCFilter {
+		return errors.New("invalid filter type")
+	}
+
+	// Figure out which header bucket to use.
+	hkey := utreexoCfHeaderKeys[0]
+	h := block.Hash()
+
+	// fetch the previous block's filter header.
+	var prevHeader *chainhash.Hash
+	ph := &block.MsgBlock().Header.PrevBlock
+	if ph.IsEqual(&zeroHash) {
+		prevHeader = &zeroHash
+	} else {
+		pfh, err := dbFetchUtreexoCFilterIdxEntry(dbTx, hkey, ph)
+		if err != nil {
+			return err
+		}
+
+		// Construct the new block's filter header, and store it.
+		prevHeader, err = chainhash.NewHash(pfh)
+		if err != nil {
+			return err
+		}
+	}
+
+	fh, err := builder.MakeHeaderForUtreexoCFilter(filterData, *prevHeader)
+	if err != nil {
+		return err
+	}
+	return dbStoreUtreexoCFilterIdxEntry(dbTx, hkey, h, fh[:])
+}
+
+// ConnectBlock is invoked by the index manager when a new block has been
+// connected to the main chain.
+// This is part of the Indexer interface.
+func (idx *UtreexoCFIndex) ConnectBlock(dbTx database.Tx, block *btcutil.Block,
+	stxos []blockchain.SpentTxOut) error {
+
+	blockHash := block.Hash()
+	roots, leaves, err := idx.fetchUtreexoRoots(dbTx, blockHash)
+
+	if err != nil {
+		return err
+	}
+
+	// serialize the hashes of the utreexo roots hash
+	serializedUtreexo, err := blockchain.SerializeUtreexoRootsHash(leaves, roots)
+	if err != nil {
+		return err
+	}
+
+	return storeUtreexoCFHeader(dbTx, block, serializedUtreexo, wire.UtreexoCFilter)
+}
+
+// fetches the utreexo roots for a given block hash
+func (idx *UtreexoCFIndex) fetchUtreexoRoots(dbTx database.Tx,
+	blockHash *chainhash.Hash) ([]*chainhash.Hash, uint64, error) {
+
+	var leaves uint64
+	var roots []*chainhash.Hash
+
+	// For compact state nodes
+	if idx.chain.IsUtreexoViewActive() && idx.chain.IsAssumeUtreexo() {
+		viewPoint, err := idx.chain.FetchUtreexoViewpoint(blockHash)
+		if err != nil {
+			return nil, 0, err
+		}
+		roots = viewPoint.GetRoots()
+		leaves = viewPoint.NumLeaves()
+	}
+	// for bridge nodes
+	if idx.utreexoProofIndex != nil {
+		roots, leaves, err := idx.utreexoProofIndex.FetchUtreexoState(dbTx, blockHash)
+		if err != nil {
+			return nil, 0, err
+		}
+		return roots, leaves, nil
+	} else if idx.flatUtreexoProofIndex != nil {
+		height, err := idx.chain.BlockHeightByHash(blockHash)
+		if err != nil {
+			return nil, 0, err
+		}
+		roots, leaves, err := idx.flatUtreexoProofIndex.FetchUtreexoState(height)
+		if err != nil {
+			return nil, 0, err
+		}
+		return roots, leaves, nil
+	}
+
+	return roots, leaves, nil
+}
+
+// DisconnectBlock is invoked by the index manager when a block has been
+// disconnected from the main chain.  This indexer removes the hash-to-cf
+// mapping for every passed block. This is part of the Indexer interface.
+func (idx *UtreexoCFIndex) DisconnectBlock(dbTx database.Tx, block *btcutil.Block,
+	_ []blockchain.SpentTxOut) error {
+
+	for _, key := range utreexoCfHeaderKeys {
+		err := dbDeleteUtreexoCFilterIdxEntry(dbTx, key, block.Hash())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// PruneBlock is invoked when an older block is deleted after it's been
+// processed.
+// TODO (kcalvinalvin): Consider keeping the filters at a later date to help with
+// reindexing as a pruned node.
+//
+// This is part of the Indexer interface.
+func (idx *UtreexoCFIndex) PruneBlock(dbTx database.Tx, blockHash *chainhash.Hash) error {
+
+	for _, key := range utreexoCfHeaderKeys {
+		err := dbDeleteUtreexoCFilterIdxEntry(dbTx, key, blockHash)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// entryByBlockHash fetches a filter index entry of a particular type
+// (eg. filter, filter header, etc) for a filter type and block hash.
+func (idx *UtreexoCFIndex) entryByBlockHash(dbTx database.Tx,
+	filterType wire.FilterType, h *chainhash.Hash) ([]byte, error) {
+
+	if uint8(filterType) != uint8(wire.UtreexoCFilter) {
+		return nil, errors.New("unsupported filter type")
+	}
+
+	roots, leaves, err := idx.fetchUtreexoRoots(dbTx, h)
+
+	if err != nil {
+		return nil, err
+	}
+
+	// serialize the hashes of the utreexo roots hash
+	serializedUtreexo, err := blockchain.SerializeUtreexoRootsHash(leaves, roots)
+	if err != nil {
+		return nil, err
+	}
+
+	return serializedUtreexo, err
+}
+
+// FilterByBlockHash returns the serialized contents of a block's utreexo
+// cfilter.
+func (idx *UtreexoCFIndex) FilterByBlockHash(dbTx database.Tx, h *chainhash.Hash,
+	filterType wire.FilterType) ([]byte, error) {
+	return idx.entryByBlockHash(dbTx, filterType, h)
+}
+
+// NewCfIndex returns a new instance of an indexer that is used to create a
+// mapping of the hashes of all blocks in the blockchain to their respective
+// committed filters.
+//
+// It implements the Indexer interface which plugs into the IndexManager that
+// in turn is used by the blockchain package. This allows the index to be
+// seamlessly maintained along with the chain.
+func NewUtreexoCfIndex(db database.DB, chainParams *chaincfg.Params, utreexoProofIndex *UtreexoProofIndex,
+	flatUtreexoProofIndex *FlatUtreexoProofIndex) *UtreexoCFIndex {
+	return &UtreexoCFIndex{db: db, chainParams: chainParams, utreexoProofIndex: utreexoProofIndex,
+		flatUtreexoProofIndex: flatUtreexoProofIndex}
+}
+
+// DropCfIndex drops the CF index from the provided database if exists.
+func DropUtreexoCfIndex(db database.DB, interrupt <-chan struct{}) error {
+	return dropIndex(db, utreexoCFIndexParentBucketKey, utreexoCFIndexName, interrupt)
+}
+
+// CfIndexInitialized returns true if the cfindex has been created previously.
+func UtreexoCfIndexInitialized(db database.DB) bool {
+	var exists bool
+	db.View(func(dbTx database.Tx) error {
+		bucket := dbTx.Metadata().Bucket(utreexoCFIndexParentBucketKey)
+		exists = bucket != nil
+		return nil
+	})
+
+	return exists
+}

btcutil/gcs/builder/builder.go

@@ -369,3 +369,19 @@ func MakeHeaderForFilter(filter *gcs.Filter, prevHeader chainhash.Hash) (chainha
 	// above.
 	return chainhash.DoubleHashH(filterTip), nil
 }
+
+// MakeHeaderForUtreexoCFilter makes a filter chain header for a utreexoc filter, given the
+// filter data and the previous filter chain header.
+func MakeHeaderForUtreexoCFilter(filterData []byte, prevHeader chainhash.Hash) (chainhash.Hash, error) {
+	filterTip := make([]byte, 2*chainhash.HashSize)
+	filterHash := chainhash.DoubleHashH(filterData)
+
+	// In the buffer we created above we'll compute hash || prevHash as an
+	// intermediate value.
+	copy(filterTip, filterHash[:])
+	copy(filterTip[chainhash.HashSize:], prevHeader[:])
+
+	// The final filter hash is the double-sha256 of the hash computed
+	// above.
+	return chainhash.DoubleHashH(filterTip), nil
+}

config.go

@@ -204,9 +204,11 @@ type config struct {
 	FlatUtreexoProofIndex      bool  `long:"flatutreexoproofindex" description:"Maintain a utreexo proof for all blocks in flat files"`
 	UtreexoProofIndexMaxMemory int64 `long:"utreexoproofindexmaxmemory" description:"The maxmimum memory in mebibytes (MiB) that the utreexo proof indexes will use up. Passing in 0 will make the entire proof index stay on disk. Passing in a negative value will make the entire proof index stay in memory. Default of 250MiB."`
 	CFilters                   bool  `long:"cfilters" description:"Enable committed filtering (CF) support"`
+	UtreexoCFilters            bool  `long:"utreexocfilters" description:"Enable committed filtering (CF) support serving utreexo roots."`
 	NoPeerBloomFilters         bool  `long:"nopeerbloomfilters" description:"Disable bloom filtering support"`
 	DropAddrIndex              bool  `long:"dropaddrindex" description:"Deletes the address-based transaction index from the database on start up and then exits."`
 	DropCfIndex                bool  `long:"dropcfindex" description:"Deletes the index used for committed filtering (CF) support from the database on start up and then exits."`
+	DropUtreexoCfIndex         bool  `long:"droputreexocfindex" description:"Deletes the index used for custom utreexo commited filter indexing support serving utreexo roots from the database on start up and then exits."`
 	DropTxIndex                bool  `long:"droptxindex" description:"Deletes the hash-based transaction index from the database on start up and then exits."`
 	DropTTLIndex               bool  `long:"dropttlindex" description:"Deletes the time to live index from the database on start up and then exits."`
 	DropUtreexoProofIndex      bool  `long:"droputreexoproofindex" description:"Deletes the utreexo proof index from the database on start up and then exits."`