Skip to content

Commit

Permalink
move trie generation helper out of trie/ (#1053)
Browse files Browse the repository at this point in the history 
* remove ethdb

* refactor: move AllowUnfinalizedQueries out of vm.Config

* move trie generation helper out of trie/ (#359)
  • Loading branch information
@darioush
darioush authored Jan 23, 2024
1 parent 955377b commit d305e3f
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 59 deletions.
7 changes: 4 additions & 3 deletions sync/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
clientstats "github.com/ava-labs/subnet-evm/sync/client/stats"
"github.com/ava-labs/subnet-evm/sync/handlers"
handlerstats "github.com/ava-labs/subnet-evm/sync/handlers/stats"
"github.com/ava-labs/subnet-evm/sync/syncutils"
"github.com/ava-labs/subnet-evm/trie"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
Expand Down Expand Up @@ -410,8 +411,8 @@ func TestGetLeafs(t *testing.T) {
const leafsLimit = 1024

trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase())
largeTrieRoot, largeTrieKeys, _ := trie.GenerateTrie(t, trieDB, 100_000, common.HashLength)
smallTrieRoot, _, _ := trie.GenerateTrie(t, trieDB, leafsLimit, common.HashLength)
largeTrieRoot, largeTrieKeys, _ := syncutils.GenerateTrie(t, trieDB, 100_000, common.HashLength)
smallTrieRoot, _, _ := syncutils.GenerateTrie(t, trieDB, leafsLimit, common.HashLength)

handler := handlers.NewLeafsRequestHandler(trieDB, nil, message.Codec, handlerstats.NewNoopHandlerStats())
client := NewClient(&ClientConfig{
Expand Down Expand Up @@ -781,7 +782,7 @@ func TestGetLeafsRetries(t *testing.T) {
rand.Seed(1)

trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase())
root, _, _ := trie.GenerateTrie(t, trieDB, 100_000, common.HashLength)
root, _, _ := syncutils.GenerateTrie(t, trieDB, 100_000, common.HashLength)

handler := handlers.NewLeafsRequestHandler(trieDB, nil, message.Codec, handlerstats.NewNoopHandlerStats())
mockNetClient := &mockNetwork{}
Expand Down
15 changes: 10 additions & 5 deletions sync/handlers/leafs_request_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/ava-labs/subnet-evm/core/types"
"github.com/ava-labs/subnet-evm/plugin/evm/message"
"github.com/ava-labs/subnet-evm/sync/handlers/stats"
"github.com/ava-labs/subnet-evm/sync/syncutils"
"github.com/ava-labs/subnet-evm/trie"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
Expand All @@ -29,13 +30,17 @@ func TestLeafsRequestHandler_OnLeafsRequest(t *testing.T) {
memdb := rawdb.NewMemoryDatabase()
trieDB := trie.NewDatabase(memdb)

corruptedTrieRoot, _, _ := trie.GenerateTrie(t, trieDB, 100, common.HashLength)
corruptedTrieRoot, _, _ := syncutils.GenerateTrie(t, trieDB, 100, common.HashLength)
tr, err := trie.New(trie.TrieID(corruptedTrieRoot), trieDB)
if err != nil {
t.Fatal(err)
}
// Corrupt [corruptedTrieRoot]
trie.CorruptTrie(t, trieDB, corruptedTrieRoot, 5)
syncutils.CorruptTrie(t, memdb, tr, 5)

largeTrieRoot, largeTrieKeys, _ := trie.GenerateTrie(t, trieDB, 10_000, common.HashLength)
smallTrieRoot, _, _ := trie.GenerateTrie(t, trieDB, 500, common.HashLength)
accountTrieRoot, accounts := trie.FillAccounts(
largeTrieRoot, largeTrieKeys, _ := syncutils.GenerateTrie(t, trieDB, 10_000, common.HashLength)
smallTrieRoot, _, _ := syncutils.GenerateTrie(t, trieDB, 500, common.HashLength)
accountTrieRoot, accounts := syncutils.FillAccounts(
t,
trieDB,
common.Hash{},
Expand Down
43 changes: 26 additions & 17 deletions sync/statesync/sync_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
statesyncclient "github.com/ava-labs/subnet-evm/sync/client"
"github.com/ava-labs/subnet-evm/sync/handlers"
handlerstats "github.com/ava-labs/subnet-evm/sync/handlers/stats"
"github.com/ava-labs/subnet-evm/sync/syncutils"
"github.com/ava-labs/subnet-evm/trie"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
Expand Down Expand Up @@ -120,15 +121,15 @@ func TestSimpleSyncCases(t *testing.T) {
prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) {
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, nil)
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, nil)
return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root
},
},
"accounts with code": {
prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) {
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
if index%3 == 0 {
codeBytes := make([]byte, 256)
_, err := rand.Read(codeBytes)
Expand Down Expand Up @@ -157,9 +158,9 @@ func TestSimpleSyncCases(t *testing.T) {
prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) {
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, i int, account types.StateAccount) types.StateAccount {
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, i int, account types.StateAccount) types.StateAccount {
if i%5 == 0 {
account.Root, _, _ = trie.GenerateTrie(t, serverTrieDB, 16, common.HashLength)
account.Root, _, _ = syncutils.GenerateTrie(t, serverTrieDB, 16, common.HashLength)
}

return account
Expand All @@ -179,7 +180,7 @@ func TestSimpleSyncCases(t *testing.T) {
prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) {
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccountsSmall, nil)
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, numAccountsSmall, nil)
return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root
},
GetLeafsIntercept: func(_ message.LeafsRequest, _ message.LeafsResponse) (message.LeafsResponse, error) {
Expand Down Expand Up @@ -279,8 +280,8 @@ func TestResumeSyncLargeStorageTrieInterrupted(t *testing.T) {
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)

largeStorageRoot, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, 2000, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
largeStorageRoot, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 2000, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
// Set the root for a single account
if index == 10 {
account.Root = largeStorageRoot
Expand Down Expand Up @@ -311,16 +312,16 @@ func TestResumeSyncToNewRootAfterLargeStorageTrieInterrupted(t *testing.T) {
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)

largeStorageRoot1, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
largeStorageRoot2, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root1, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, 2000, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
largeStorageRoot1, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
largeStorageRoot2, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root1, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 2000, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
// Set the root for a single account
if index == 10 {
account.Root = largeStorageRoot1
}
return account
})
root2, _ := trie.FillAccounts(t, serverTrieDB, root1, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
root2, _ := syncutils.FillAccounts(t, serverTrieDB, root1, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
if index == 20 {
account.Root = largeStorageRoot2
}
Expand Down Expand Up @@ -352,8 +353,8 @@ func TestResumeSyncLargeStorageTrieWithConsecutiveDuplicatesInterrupted(t *testi
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)

largeStorageRoot, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
largeStorageRoot, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
// Set the root for 2 successive accounts
if index == 10 || index == 11 {
account.Root = largeStorageRoot
Expand Down Expand Up @@ -384,8 +385,8 @@ func TestResumeSyncLargeStorageTrieWithSpreadOutDuplicatesInterrupted(t *testing
serverDB := rawdb.NewMemoryDatabase()
serverTrieDB := trie.NewDatabase(serverDB)

largeStorageRoot, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
largeStorageRoot, _, _ := syncutils.GenerateTrie(t, serverTrieDB, 2000, common.HashLength)
root, _ := syncutils.FillAccounts(t, serverTrieDB, common.Hash{}, 100, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
if index == 10 || index == 90 {
account.Root = largeStorageRoot
}
Expand Down Expand Up @@ -464,7 +465,11 @@ func TestResyncNewRootAfterDeletes(t *testing.T) {
continue
}
corruptedStorageRoots[acc.Root] = struct{}{}
trie.CorruptTrie(t, clientTrieDB, acc.Root, 2)
tr, err := trie.New(trie.TrieID(acc.Root), clientTrieDB)
if err != nil {
t.Fatal(err)
}
syncutils.CorruptTrie(t, clientDB, tr, 2)
}
if err := it.Err; err != nil {
t.Fatal(err)
Expand All @@ -474,7 +479,11 @@ func TestResyncNewRootAfterDeletes(t *testing.T) {
"delete intermediate account trie nodes": {
deleteBetweenSyncs: func(t *testing.T, root common.Hash, clientDB ethdb.Database) {
clientTrieDB := trie.NewDatabase(clientDB)
trie.CorruptTrie(t, clientTrieDB, root, 5)
tr, err := trie.New(trie.TrieID(root), clientTrieDB)
if err != nil {
t.Fatal(err)
}
syncutils.CorruptTrie(t, clientDB, tr, 5)
},
},
} {
Expand Down
15 changes: 8 additions & 7 deletions sync/statesync/test_sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/ava-labs/subnet-evm/core/rawdb"
"github.com/ava-labs/subnet-evm/core/state/snapshot"
"github.com/ava-labs/subnet-evm/core/types"
"github.com/ava-labs/subnet-evm/sync/syncutils"
"github.com/ava-labs/subnet-evm/trie"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
Expand All @@ -38,7 +39,7 @@ func assertDBConsistency(t testing.TB, root common.Hash, clientDB ethdb.Database
}
trieAccountLeaves := 0

trie.AssertTrieConsistency(t, root, serverTrieDB, clientTrieDB, func(key, val []byte) error {
syncutils.AssertTrieConsistency(t, root, serverTrieDB, clientTrieDB, func(key, val []byte) error {
trieAccountLeaves++
accHash := common.BytesToHash(key)
var acc types.StateAccount
Expand Down Expand Up @@ -73,7 +74,7 @@ func assertDBConsistency(t testing.TB, root common.Hash, clientDB ethdb.Database
storageTrieLeavesCount := 0

// check storage trie and storage snapshot consistency
trie.AssertTrieConsistency(t, acc.Root, serverTrieDB, clientTrieDB, func(key, val []byte) error {
syncutils.AssertTrieConsistency(t, acc.Root, serverTrieDB, clientTrieDB, func(key, val []byte) error {
storageTrieLeavesCount++
snapshotVal := rawdb.ReadStorageSnapshot(clientDB, accHash, common.BytesToHash(key))
assert.Equal(t, val, snapshotVal)
Expand All @@ -89,7 +90,7 @@ func assertDBConsistency(t testing.TB, root common.Hash, clientDB ethdb.Database
}

func fillAccountsWithStorage(t *testing.T, serverDB ethdb.Database, serverTrieDB *trie.Database, root common.Hash, numAccounts int) common.Hash {
newRoot, _ := trie.FillAccounts(t, serverTrieDB, root, numAccounts, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
newRoot, _ := syncutils.FillAccounts(t, serverTrieDB, root, numAccounts, func(t *testing.T, index int, account types.StateAccount) types.StateAccount {
codeBytes := make([]byte, 256)
_, err := rand.Read(codeBytes)
if err != nil {
Expand All @@ -102,7 +103,7 @@ func fillAccountsWithStorage(t *testing.T, serverDB ethdb.Database, serverTrieDB

// now create state trie
numKeys := 16
account.Root, _, _ = trie.GenerateTrie(t, serverTrieDB, numKeys, common.HashLength)
account.Root, _, _ = syncutils.GenerateTrie(t, serverTrieDB, numKeys, common.HashLength)
return account
})
return newRoot
Expand All @@ -119,18 +120,18 @@ func FillAccountsWithOverlappingStorage(
) (common.Hash, map[*keystore.Key]*types.StateAccount) {
storageRoots := make([]common.Hash, 0, numOverlappingStorageRoots)
for i := 0; i < numOverlappingStorageRoots; i++ {
storageRoot, _, _ := trie.GenerateTrie(t, trieDB, 16, common.HashLength)
storageRoot, _, _ := syncutils.GenerateTrie(t, trieDB, 16, common.HashLength)
storageRoots = append(storageRoots, storageRoot)
}
storageRootIndex := 0
return trie.FillAccounts(t, trieDB, root, numAccounts, func(t *testing.T, i int, account types.StateAccount) types.StateAccount {
return syncutils.FillAccounts(t, trieDB, root, numAccounts, func(t *testing.T, i int, account types.StateAccount) types.StateAccount {
switch i % 3 {
case 0: // unmodified account
case 1: // account with overlapping storage root
account.Root = storageRoots[storageRootIndex%numOverlappingStorageRoots]
storageRootIndex++
case 2: // account with unique storage root
account.Root, _, _ = trie.GenerateTrie(t, trieDB, 16, common.HashLength)
account.Root, _, _ = syncutils.GenerateTrie(t, trieDB, 16, common.HashLength)
}

return account
Expand Down
39 changes: 18 additions & 21 deletions trie/test_trie.go → sync/syncutils/test_trie.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// (c) 2021-2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package trie
package syncutils

import (
cryptoRand "crypto/rand"
Expand All @@ -13,9 +13,11 @@ import (
"github.com/ava-labs/avalanchego/utils/wrappers"
"github.com/ava-labs/subnet-evm/accounts/keystore"
"github.com/ava-labs/subnet-evm/core/types"
"github.com/ava-labs/subnet-evm/trie"
"github.com/ava-labs/subnet-evm/trie/trienode"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/rlp"
"github.com/stretchr/testify/assert"
)
Expand All @@ -24,11 +26,11 @@ import (
// Returns the root of the generated trie, the slice of keys inserted into the trie in lexicographical
// order, and the slice of corresponding values.
// GenerateTrie reads from [rand] and the caller should call rand.Seed(n) for deterministic results
func GenerateTrie(t *testing.T, trieDB *Database, numKeys int, keySize int) (common.Hash, [][]byte, [][]byte) {
func GenerateTrie(t *testing.T, trieDB *trie.Database, numKeys int, keySize int) (common.Hash, [][]byte, [][]byte) {
if keySize < wrappers.LongLen+1 {
t.Fatal("key size must be at least 9 bytes (8 bytes for uint64 and 1 random byte)")
}
testTrie := NewEmpty(trieDB)
testTrie := trie.NewEmpty(trieDB)

keys, values := FillTrie(t, numKeys, keySize, testTrie)

Expand All @@ -45,7 +47,7 @@ func GenerateTrie(t *testing.T, trieDB *Database, numKeys int, keySize int) (com
// FillTrie fills a given trie with [numKeys] number of keys, each of size [keySize]
// returns inserted keys and values
// FillTrie reads from [rand] and the caller should call rand.Seed(n) for deterministic results
func FillTrie(t *testing.T, numKeys int, keySize int, testTrie *Trie) ([][]byte, [][]byte) {
func FillTrie(t *testing.T, numKeys int, keySize int, testTrie *trie.Trie) ([][]byte, [][]byte) {
keys := make([][]byte, 0, numKeys)
values := make([][]byte, 0, numKeys)

Expand All @@ -70,18 +72,18 @@ func FillTrie(t *testing.T, numKeys int, keySize int, testTrie *Trie) ([][]byte,

// AssertTrieConsistency ensures given trieDB [a] and [b] both have the same
// non-empty trie at [root]. (all key/value pairs must be equal)
func AssertTrieConsistency(t testing.TB, root common.Hash, a, b *Database, onLeaf func(key, val []byte) error) {
trieA, err := New(TrieID(root), a)
func AssertTrieConsistency(t testing.TB, root common.Hash, a, b *trie.Database, onLeaf func(key, val []byte) error) {
trieA, err := trie.New(trie.TrieID(root), a)
if err != nil {
t.Fatalf("error creating trieA, root=%s, err=%v", root, err)
}
trieB, err := New(TrieID(root), b)
trieB, err := trie.New(trie.TrieID(root), b)
if err != nil {
t.Fatalf("error creating trieB, root=%s, err=%v", root, err)
}

itA := NewIterator(trieA.NodeIterator(nil))
itB := NewIterator(trieB.NodeIterator(nil))
itA := trie.NewIterator(trieA.NodeIterator(nil))
itB := trie.NewIterator(trieB.NodeIterator(nil))
count := 0
for itA.Next() && itB.Next() {
count++
Expand All @@ -100,16 +102,11 @@ func AssertTrieConsistency(t testing.TB, root common.Hash, a, b *Database, onLea
assert.Greater(t, count, 0)
}

// CorruptTrie deletes every [n]th trie node from the trie given by [root] from the trieDB.
// Assumes that the trie given by root can be iterated without issue.
func CorruptTrie(t *testing.T, trieDB *Database, root common.Hash, n int) {
batch := trieDB.diskdb.NewBatch()
// next delete some trie nodes
tr, err := New(TrieID(root), trieDB)
if err != nil {
t.Fatal(err)
}

// CorruptTrie deletes every [n]th trie node from the trie given by [tr] from the underlying [db].
// Assumes [tr] can be iterated without issue.
func CorruptTrie(t *testing.T, diskdb ethdb.Batcher, tr *trie.Trie, n int) {
// Delete some trie nodes
batch := diskdb.NewBatch()
nodeIt := tr.NodeIterator(nil)
count := 0
for nodeIt.Next(true) {
Expand All @@ -133,7 +130,7 @@ func CorruptTrie(t *testing.T, trieDB *Database, root common.Hash, n int) {
// [onAccount] is called if non-nil (so the caller can modify the account before it is stored in the secure trie).
// returns the new trie root and a map of funded keys to StateAccount structs.
func FillAccounts(
t *testing.T, trieDB *Database, root common.Hash, numAccounts int,
t *testing.T, trieDB *trie.Database, root common.Hash, numAccounts int,
onAccount func(*testing.T, int, types.StateAccount) types.StateAccount,
) (common.Hash, map[*keystore.Key]*types.StateAccount) {
var (
Expand All @@ -143,7 +140,7 @@ func FillAccounts(
accounts = make(map[*keystore.Key]*types.StateAccount, numAccounts)
)

tr, err := NewStateTrie(TrieID(root), trieDB)
tr, err := trie.NewStateTrie(trie.TrieID(root), trieDB)
if err != nil {
t.Fatalf("error opening trie: %v", err)
}
Expand Down
Loading

0 comments on commit d305e3f

Please sign in to comment.