Feature/path based merkle tree

src/Nethermind/Nethermind.Blockchain.Test/BlockchainProcessorTests.cs

@@ -22,6 +22,7 @@
 using Nethermind.Db.Blooms;
 using Nethermind.State;
 using Nethermind.Trie;
+using Nethermind.Trie.Pruning;
 using NSubstitute;
 using NUnit.Framework;
 using Nethermind.Trie.Pruning;

src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs

@@ -156,8 +156,6 @@ public void Proper_transactions_selected(ProperTransactionsSelectedTestCase test
             MemDb codeDb = new();
             TrieStore trieStore = new(stateDb, LimboLogs.Instance);
             StateProvider stateProvider = new(trieStore, codeDb, LimboLogs.Instance);
-            StateReader stateReader =
-                new(new TrieStore(stateDb, LimboLogs.Instance), new TrieStore(stateDb, LimboLogs.Instance), codeDb, LimboLogs.Instance);
             ISpecProvider specProvider = Substitute.For<ISpecProvider>();
 
             void SetAccountStates(IEnumerable<Address> missingAddresses)

src/Nethermind/Nethermind.Blockchain.Test/Visitors/StartupTreeFixerTests.cs

@@ -74,7 +74,7 @@ public async Task Deletes_everything_after_the_missing_level()
 
             tree = new BlockTree(blocksDb, headersDb, blockInfosDb, new ChainLevelInfoRepository(blockInfosDb), MainnetSpecProvider.Instance, NullBloomStorage.Instance, LimboLogs.Instance);
 
-            StartupBlockTreeFixer fixer = new(new SyncConfig(), tree, new MemDb(), LimboNoErrorLogger.Instance);
+            StartupBlockTreeFixer fixer = new(new SyncConfig(), tree, NullTrieStore.Instance, LimboNoErrorLogger.Instance);
             await tree.Accept(fixer, CancellationToken.None);
 
             Assert.Null(blockInfosDb.Get(3), "level 3");
@@ -112,7 +112,7 @@ public async Task Suggesting_blocks_works_correctly_after_processor_restart(int
             testRpc.BlockchainProcessor = newBlockchainProcessor;
 
             // fixing after restart
-            StartupBlockTreeFixer fixer = new(new SyncConfig(), tree, testRpc.DbProvider.StateDb, LimboNoErrorLogger.Instance, 5);
+            StartupBlockTreeFixer fixer = new(new SyncConfig(), tree, testRpc.TrieStore, LimboNoErrorLogger.Instance, 5);
             await tree.Accept(fixer, CancellationToken.None);
 
             // waiting for N new heads
@@ -146,8 +146,7 @@ public async Task Fixer_should_not_suggest_block_without_state(int suggestedBloc
             testRpc.BlockchainProcessor = newBlockchainProcessor;
 
             // we create a new empty db for stateDb so we shouldn't suggest new blocks
-            MemDb stateDb = new();
-            IBlockTreeVisitor fixer = new StartupBlockTreeFixer(new SyncConfig(), tree, stateDb, LimboNoErrorLogger.Instance, 5);
+            IBlockTreeVisitor fixer = new StartupBlockTreeFixer(new SyncConfig(), tree, NullTrieStore.Instance, LimboNoErrorLogger.Instance, 5);
             BlockVisitOutcome result = await fixer.VisitBlock(tree.Head!, CancellationToken.None);
 
             Assert.AreEqual(BlockVisitOutcome.None, result);
@@ -168,7 +167,7 @@ public async Task Fixer_should_not_suggest_block_with_null_block()
             newBlockchainProcessor.Start();
             testRpc.BlockchainProcessor = newBlockchainProcessor;
 
-            IBlockTreeVisitor fixer = new StartupBlockTreeFixer(new SyncConfig(), tree, testRpc.DbProvider.StateDb, LimboNoErrorLogger.Instance, 5);
+            IBlockTreeVisitor fixer = new StartupBlockTreeFixer(new SyncConfig(), tree, testRpc.TrieStore, LimboNoErrorLogger.Instance, 5);
             BlockVisitOutcome result = await fixer.VisitBlock(null, CancellationToken.None);
 
             Assert.AreEqual(BlockVisitOutcome.None, result);
@@ -213,7 +212,7 @@ public async Task When_head_block_is_followed_by_a_block_bodies_gap_it_should_de
 
             tree.UpdateMainChain(block2);
 
-            StartupBlockTreeFixer fixer = new(new SyncConfig(), tree, new MemDb(), LimboNoErrorLogger.Instance);
+            StartupBlockTreeFixer fixer = new(new SyncConfig(), tree,  NullTrieStore.Instance, LimboNoErrorLogger.Instance);
             await tree.Accept(fixer, CancellationToken.None);
 
             Assert.Null(blockInfosDb.Get(3), "level 3");

src/Nethermind/Nethermind.Blockchain/Visitors/StartupBlockTreeFixer.cs

@@ -12,14 +12,15 @@
 using Nethermind.Core.Crypto;
 using Nethermind.Db;
 using Nethermind.Logging;
+using Nethermind.Trie.Pruning;
 
 namespace Nethermind.Blockchain.Visitors
 {
     public class StartupBlockTreeFixer : IBlockTreeVisitor
     {
         public const int DefaultBatchSize = 4000;
         private readonly IBlockTree _blockTree;
-        private readonly IDb _stateDb;
+        private readonly ITrieNodeResolver _stateNodeResolver;
         private readonly ILogger _logger;
         private long _startNumber;
         private long _blocksToLoad;
@@ -42,12 +43,12 @@ public class StartupBlockTreeFixer : IBlockTreeVisitor
         public StartupBlockTreeFixer(
             ISyncConfig syncConfig,
             IBlockTree blockTree,
-            IDb stateDb,
+            ITrieNodeResolver stateNodeResolver,
             ILogger logger,
             long batchSize = DefaultBatchSize)
         {
             _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree));
-            _stateDb = stateDb;
+            _stateNodeResolver = stateNodeResolver;
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
 
             _batchSize = batchSize;
@@ -242,7 +243,7 @@ private bool CanSuggestBlocks(Block block)
             {
                 BlockHeader? parentHeader = _blockTree.FindParentHeader(block.Header, BlockTreeLookupOptions.TotalDifficultyNotNeeded);
                 if (parentHeader is null || parentHeader.StateRoot is null ||
-                    _stateDb.Get(parentHeader.StateRoot) is null)
+                    !_stateNodeResolver.ExistsInDB(parentHeader.StateRoot, Array.Empty<byte>()))
                     return false;
             }
             else

src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs

@@ -132,8 +132,8 @@ protected virtual async Task<TestBlockchain> Build(ISpecProvider? specProvider =
         State.Commit(SpecProvider.GenesisSpec);
         State.CommitTree(0);
 
-            ReadOnlyTrieStore = TrieStore.AsReadOnly(StateDb);
-            StateReader = new StateReader(ReadOnlyTrieStore, ReadOnlyTrieStore, CodeDb, LogManager);
+        ReadOnlyTrieStore = TrieStore.AsReadOnly(StateDb);
+        StateReader = new StateReader(ReadOnlyTrieStore, ReadOnlyTrieStore, CodeDb, LogManager);
 
         IDb blockDb = new MemDb();
         IDb headerDb = new MemDb();

src/Nethermind/Nethermind.Core.Test/Builders/TestItem.Tree.cs

@@ -72,6 +72,17 @@ public static void FillStateTreeWithTestAccounts(StateTree stateTree)
                 stateTree.Commit(0);
             }
 
+            public static void FillStateTreeWithTestAccounts(StateTreeByPath stateTree)
+            {
+                stateTree.Set(AccountsWithPaths[0].Path, AccountsWithPaths[0].Account);
+                stateTree.Set(AccountsWithPaths[1].Path, AccountsWithPaths[1].Account);
+                stateTree.Set(AccountsWithPaths[2].Path, AccountsWithPaths[2].Account);
+                stateTree.Set(AccountsWithPaths[3].Path, AccountsWithPaths[3].Account);
+                stateTree.Set(AccountsWithPaths[4].Path, AccountsWithPaths[4].Account);
+                stateTree.Set(AccountsWithPaths[5].Path, AccountsWithPaths[5].Account);
+                stateTree.Commit(0);
+            }
+
             public static (StateTree stateTree, StorageTree storageTree) GetTrees(ITrieStore? store)
             {
                 store ??= new TrieStore(new MemDb(), LimboLogs.Instance);

src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs

@@ -38,6 +38,7 @@
 using Nethermind.State.Witnesses;
 using Nethermind.Synchronization.Witness;
 using Nethermind.Trie;
+using Nethermind.Trie.ByPath;
 using Nethermind.Trie.Pruning;
 using Nethermind.TxPool;
 using Nethermind.Wallet;
@@ -106,55 +107,14 @@ private Task InitBlockchain()
             setApi.MainStateDbWithCache = cachedStateDb;
             IKeyValueStore codeDb = getApi.DbProvider.CodeDb
                 .WitnessedBy(witnessCollector);
-
-            ITrieStore trieStore;
-            TrieStore storageStore;
             IKeyValueStoreWithBatching stateWitnessedBy = setApi.MainStateDbWithCache.WitnessedBy(witnessCollector);
-            if (pruningConfig.Mode.IsMemory())
-            {
-                IPersistenceStrategy persistenceStrategy = Persist.IfBlockOlderThan(pruningConfig.PersistenceInterval); // TODO: this should be based on time
-                if (pruningConfig.Mode.IsFull())
-                {
-                    PruningTriggerPersistenceStrategy triggerPersistenceStrategy = new((IFullPruningDb)getApi.DbProvider!.StateDb, getApi.BlockTree!, getApi.LogManager);
-                    getApi.DisposeStack.Push(triggerPersistenceStrategy);
-                    persistenceStrategy = persistenceStrategy.Or(triggerPersistenceStrategy);
-                }
 
-                setApi.TrieStore = trieStore = new TrieStoreByPath(
-                    stateWitnessedBy,
-                    Prune.WhenCacheReaches(pruningConfig.CacheMb.MB()), // TODO: memory hint should define this
-                    persistenceStrategy,
-                    getApi.LogManager);
-                //TODO - remove seprate store for this
-                storageStore = new TrieStore(
-                    stateWitnessedBy,
-                    Prune.WhenCacheReaches(pruningConfig.CacheMb.MB()), // TODO: memory hint should define this
-                    persistenceStrategy,
-                    getApi.LogManager);
-
-                if (pruningConfig.Mode.IsFull())
-                {
-                    IFullPruningDb fullPruningDb = (IFullPruningDb)getApi.DbProvider!.StateDb;
-                    fullPruningDb.PruningStarted += (_, args) =>
-                    {
-                        cachedStateDb.PersistCache(args.Context);
-                        //trieStore.PersistCache(args.Context, args.Context.CancellationTokenSource.Token);
-                    };
-                }
-            }
-            else
-            {
-                setApi.TrieStore = trieStore = new TrieStoreByPath(
-                    stateWitnessedBy,
-                    No.Pruning,
-                    Persist.EveryBlock,
-                    getApi.LogManager);
-                storageStore = new TrieStore(
-                    stateWitnessedBy,
-                    No.Pruning,
-                    Persist.EveryBlock,
-                    getApi.LogManager);
-            }
+            ITrieStore trieStore = setApi.TrieStore = new TrieStoreByPath(
+                stateWitnessedBy,
+                No.Pruning,
+                Persist.EveryBlock,
+                getApi.LogManager,
+                new FullLeafHistory(128)); //TDOD - change, so that object init is done in class based on strategy - should be same as persistance strategy?
 
             TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(trieStore, _api.BlockTree!, _api.LogManager);
             getApi.DisposeStack.Push(trieStoreBoundaryWatcher);
@@ -169,7 +129,11 @@ private Task InitBlockchain()
 
             ReadOnlyDbProvider readOnly = new(getApi.DbProvider, false);
 
-            IStateReader stateReader = setApi.StateReader = new StateReader(readOnlyTrieStore, storageStore.AsReadOnly(), readOnly.GetDb<IDb>(DbNames.Code), getApi.LogManager);
+            IStateReader stateReader = setApi.StateReader = new StateReader(
+                readOnlyTrieStore,
+                readOnlyTrieStore,
+                readOnly.GetDb<IDb>(DbNames.Code),
+                getApi.LogManager);
 
             setApi.TransactionComparerProvider = new TransactionComparerProvider(getApi.SpecProvider!, getApi.BlockTree.AsReadOnly());
             setApi.ChainHeadStateProvider = new ChainHeadReadOnlyStateProvider(getApi.BlockTree, stateReader);

src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs

@@ -39,6 +39,7 @@
 using Nethermind.Synchronization.Peers;
 using Nethermind.Synchronization.Reporting;
 using Nethermind.Synchronization.SnapSync;
+using Nethermind.Trie.Pruning;
 
 namespace Nethermind.Init.Steps;
 
@@ -107,7 +108,7 @@ private async Task Initialize(CancellationToken cancellationToken)
             _api.BlockTree!,
             _api.ReceiptStorage!,
             _api.DbProvider.StateDb,
-            _api.ReadOnlyTrieStore!,
+            new TrieStoreByPath(_api.DbProvider.StateDb, Trie.Pruning.No.Pruning, Persist.EveryBlock, _api.LogManager),
             progressTracker,
             _syncConfig,
             _api.LogManager);
@@ -271,7 +272,7 @@ await StartDiscovery().ContinueWith(initDiscoveryTask =>
     }
 
     protected virtual MultiSyncModeSelector CreateMultiSyncModeSelector(SyncProgressResolver syncProgressResolver)
-        => new(syncProgressResolver, _api.SyncPeerPool!, _syncConfig, No.BeaconSync, _api.BetterPeerStrategy!, _api.LogManager, _api.ChainSpec?.SealEngineType == SealEngineType.Clique);
+        => new(syncProgressResolver, _api.SyncPeerPool!, _syncConfig, Synchronization.No.BeaconSync, _api.BetterPeerStrategy!, _api.LogManager, _api.ChainSpec?.SealEngineType == SealEngineType.Clique);
 
     private Task StartDiscovery()
     {

src/Nethermind/Nethermind.Init/Steps/ReviewBlockTree.cs

@@ -63,7 +63,7 @@ await _api.BlockTree.Accept(loader, cancellationToken).ContinueWith(t =>
             }
             else
             {
-                StartupBlockTreeFixer fixer = new(syncConfig, _api.BlockTree, _api.DbProvider!.StateDb, _logger!);
+                StartupBlockTreeFixer fixer = new(syncConfig, _api.BlockTree, _api.ReadOnlyTrieStore!, _logger!);
                 await _api.BlockTree.Accept(fixer, cancellationToken).ContinueWith(t =>
                 {
                     if (t.IsFaulted)

src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs

@@ -64,7 +64,6 @@ public void Initialize()
             peerManager.MaxActivePeers.Returns(15);
 
             StateProvider stateProvider = new(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance);
-            StateReader stateReader = new(new TrieStore(new MemDb(), LimboLogs.Instance), new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance);
 
             IDb blockDb = new MemDb();
             IDb headerDb = new MemDb();

src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs

@@ -193,7 +193,7 @@ public ResultWrapper<byte[]> eth_getStorageAt(Address address, UInt256 positionI
             return ResultWrapper<byte[]>.Success(Array.Empty<byte>());
         }
 
-        byte[] storage = _stateReader.GetStorage(account.StorageRoot, positionIndex);
+        byte[] storage = _stateReader.GetStorage(header.StateRoot, address, positionIndex);
         return ResultWrapper<byte[]>.Success(storage.PadLeft(32));
     }
 

src/Nethermind/Nethermind.State.Test/PatriciaTreeTests.cs

@@ -1,13 +1,15 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System;
 using FluentAssertions;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Db;
 using Nethermind.Int256;
 using Nethermind.Logging;
+using Nethermind.Serialization.Rlp;
 using Nethermind.State;
 using Nethermind.Trie;
 using Nethermind.Trie.Pruning;
@@ -18,6 +20,65 @@ namespace Nethermind.Store.Test
     [TestFixture, Parallelizable(ParallelScope.All)]
     public class PatriciaTreeTests
     {
+
+        [Test]
+        public void Something()
+        {
+            AccountDecoder accountDecoder = new AccountDecoder();
+
+            PatriciaTree tree1 = new PatriciaTree(new MemDb());
+            PatriciaTree tree2 = new PatriciaTree(new MemDb());
+
+            Span<byte> key1 = stackalloc byte[52];
+            Span<byte> key2 = stackalloc byte[32];
+
+            Account[] accounts =
+            {
+                new Account(1),
+                new Account(2),
+                new Account(3),
+                new Account(4),
+                new Account(5),
+            };
+
+            TestItem.AddressA.Bytes.CopyTo(key1);
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressB.Bytes, key1.Slice(20));
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressB.Bytes, key2);
+            tree1.Set(key1, accountDecoder.Encode(accounts[0]));
+            tree2.Set(key2, accountDecoder.Encode(accounts[0]));
+
+            TestItem.AddressA.Bytes.CopyTo(key1);
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressC.Bytes, key1.Slice(20));
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressC.Bytes, key2);
+            tree1.Set(key1, accountDecoder.Encode(accounts[1]));
+            tree2.Set(key2, accountDecoder.Encode(accounts[1]));
+
+            TestItem.AddressA.Bytes.CopyTo(key1);
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressD.Bytes, key1.Slice(20));
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressB.Bytes, key2);
+            tree1.Set(key1, accountDecoder.Encode(accounts[2]));
+            tree2.Set(key2, accountDecoder.Encode(accounts[2]));
+
+            TestItem.AddressA.Bytes.CopyTo(key1);
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressE.Bytes, key1.Slice(20));
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressB.Bytes, key2);
+            tree1.Set(key1, accountDecoder.Encode(accounts[3]));
+            tree2.Set(key2, accountDecoder.Encode(accounts[3]));
+
+            TestItem.AddressA.Bytes.CopyTo(key1);
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressF.Bytes, key1.Slice(20));
+            KeccakHash.ComputeHashBytesToSpan(TestItem.AddressB.Bytes, key2);
+            tree1.Set(key1, accountDecoder.Encode(accounts[4]));
+            tree2.Set(key2, accountDecoder.Encode(accounts[4]));
+
+            tree1.Commit(0);
+            tree2.Commit(0);
+
+            Console.WriteLine(string.Join(", ", tree1.RootHash));
+            Console.WriteLine(string.Join(", ", tree2.RootHash));
+        }
+
+
         [Test]
         public void Create_commit_change_balance_get()
         {

src/Nethermind/Nethermind.State.Test/StateProviderTests.cs

@@ -41,7 +41,7 @@ public void Setup()
         public static (string name, ITrieStore trieStore)[] Variants
             => LazyInitializer.EnsureInitialized(ref _variants, InitVariants);
 
-        public static (string Name, ITrieStore TrieStore)[] InitVariants()
+        private static (string Name, ITrieStore TrieStore)[] InitVariants()
         {
             return new (string, ITrieStore)[]
             {