Head tracking batch

src/Paprika.Cli/VerifyWholeTreeSettings.cs

@@ -11,16 +11,16 @@ public class Command : Command<VerifyWholeTreeSettings>
     {
         public override int Execute(CommandContext context, VerifyWholeTreeSettings settings)
         {
-            using var db = settings.BuildDb();
-
-            using var read = db.BeginReadOnlyBatch();
-            using var latest = Blockchain.StartReadOnlyLatestFromDb(db);
-
-            AnsiConsole.WriteLine("Checking whole tree...");
-
-            var keccak = new ComputeMerkleBehavior().CalculateStateRootHash(latest);
-
-            AnsiConsole.WriteLine($"Keccak {keccak.ToString()}");
+            // using var db = settings.BuildDb();
+            //
+            // using var read = db.BeginReadOnlyBatch();
+            // using var latest = Blockchain.StartReadOnlyLatestFromDb(db);
+            //
+            // AnsiConsole.WriteLine("Checking whole tree...");
+            //
+            // var keccak = new ComputeMerkleBehavior().CalculateStateRootHash(read);
+            //
+            // AnsiConsole.WriteLine($"Keccak {keccak.ToString()}");
 
 
             return 0;

src/Paprika.Tests/Chain/BlockchainTests.cs

@@ -157,21 +157,21 @@ public async Task Account_destruction_same_block()
         using var db = PagedDb.NativeMemoryDb(1 * Mb, 2);
         await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior());
 
-        using var block = blockchain.StartNew(Keccak.EmptyTreeHash);
+        var before = Keccak.EmptyTreeHash;
 
-        var before = block.Hash;
+        using var worldState = blockchain.StartNew(before);
 
-        block.SetAccount(Key0, new Account(1, 1));
-        block.SetStorage(Key0, Key1, stackalloc byte[1] { 1 });
+        worldState.SetAccount(Key0, new Account(1, 1));
+        worldState.SetStorage(Key0, Key1, stackalloc byte[1] { 1 });
 
         // force hash calculation
-        var mid = block.Hash;
+        var mid = worldState.Hash;
 
-        block.DestroyAccount(Key0);
-        block.GetAccount(Key0).Should().Be(new Account(0, 0));
-        block.AssertNoStorageAt(Key0, Key1);
+        worldState.DestroyAccount(Key0);
+        worldState.GetAccount(Key0).Should().Be(new Account(0, 0));
+        worldState.AssertNoStorageAt(Key0, Key1);
 
-        var after = block.Hash;
+        var after = worldState.Hash;
 
         before.Should().Be(after);
         before.Should().NotBe(mid);
@@ -235,13 +235,13 @@ public async Task Account_destruction_spin()
 
             if (finality.Count > 64)
             {
-                blockchain.Finalize(finality.Dequeue());
+                await blockchain.Finalize(finality.Dequeue());
             }
         }
 
         while (finality.TryDequeue(out var finalized))
         {
-            blockchain.Finalize(finalized);
+            await blockchain.Finalize(finalized);
         }
     }
 
@@ -251,9 +251,8 @@ public async Task Account_destruction_multi_block()
         using var db = PagedDb.NativeMemoryDb(1 * Mb, 2);
         await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior());
 
-        using var block1 = blockchain.StartNew(Keccak.EmptyTreeHash);
-
-        var before = block1.Hash;
+        var before = Keccak.EmptyTreeHash;
+        using var block1 = blockchain.StartNew(before);
 
         block1.SetAccount(Key0, new Account(1, 1));
         block1.SetStorage(Key0, Key1, stackalloc byte[1] { 1 });
@@ -358,34 +357,28 @@ public async Task Account_destruction_database_flushed()
         using var db = PagedDb.NativeMemoryDb(1 * Mb, 2);
         await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior());
 
-        using var block1 = blockchain.StartNew(Keccak.EmptyTreeHash);
-
-        var before = block1.Hash;
-
-        block1.SetAccount(Key0, new Account(1, 1));
-        block1.SetStorage(Key0, Key1, stackalloc byte[1] { 1 });
+        using var worldState = blockchain.StartNew(Keccak.EmptyTreeHash);
 
-        var hash = block1.Commit(blockNo++);
+        worldState.SetAccount(Key1, new Account(2, 2));
 
-        blockchain.Finalize(hash);
+        var before = worldState.Commit(blockNo++);
+        await blockchain.Finalize(before);
 
-        // Poor man's await on finalization flushed
-        await blockchain.WaitTillFlush(hash);
+        worldState.SetAccount(Key0, new Account(1, 1));
+        worldState.SetStorage(Key0, Key1, [1]);
 
-        using var block2 = blockchain.StartNew(hash);
+        var hash = worldState.Commit(blockNo++);
 
-        block2.DestroyAccount(Key0);
-        var hash2 = block2.Commit(blockNo);
+        await blockchain.Finalize(hash);
 
-        var wait = blockchain.WaitTillFlush(blockNo);
+        worldState.DestroyAccount(Key0);
+        var hash2 = worldState.Commit(blockNo);
 
-        blockchain.Finalize(hash2);
-
-        await wait;
+        await blockchain.Finalize(hash2);
 
         using var read = db.BeginReadOnlyBatch();
 
-        read.Metadata.BlockNumber.Should().Be(2);
+        read.Metadata.BlockNumber.Should().Be(3);
 
         read.AssertNoAccount(Key0);
         read.AssertNoStorageAt(Key0, Key1);
@@ -406,7 +399,7 @@ public async Task BiggerTest()
 
         await using (var blockchain = new Blockchain(db, behavior))
         {
-            var hash = Keccak.Zero;
+            var hash = Keccak.EmptyTreeHash;
 
             for (uint i = 1; i < blockCount + 1; i++)
             {
@@ -474,11 +467,7 @@ public async Task Start_in_the_past()
                 const int block2 = 2;
 
                 var keccak2A = block2A.Commit(block2);
-                var task = blockchain.WaitTillFlush(block2);
-
-                blockchain.Finalize(keccak2A);
-
-                await task;
+                await blockchain.Finalize(keccak2A);
 
                 // start in the past
                 using (var block2B = blockchain.StartNew(keccak1A))
@@ -546,39 +535,6 @@ public async Task Respects_commit_cache_budget()
         }
     }
 
-    [Test]
-    public async Task Reports_ancestor_blocks()
-    {
-        using var db = PagedDb.NativeMemoryDb(1 * Mb);
-
-        await using var blockchain = new Blockchain(db, new PreCommit(), null, CacheBudget.Options.None,
-            CacheBudget.Options.None, 1);
-
-        // Arrange
-        const uint block1 = 1;
-        var (hash1, _) = BuildBlock(block1, Keccak.EmptyTreeHash);
-
-        const uint block2 = 2;
-        var (hash2, _) = BuildBlock(block2, hash1);
-
-        const uint block3 = 3;
-        var (hash3, last) = BuildBlock(block3, hash2);
-
-        // Assert stats in order
-        last.Stats.Ancestors
-            .Should()
-            .BeEquivalentTo(new[] { (block2, hash2), (block1, hash1) });
-
-        return;
-
-        (Keccak hash, IWorldState state) BuildBlock(uint number, in Keccak parent)
-        {
-            using var block = blockchain.StartNew(parent);
-            block.SetAccount(Keccak.OfAnEmptyString, new Account(number, number));
-            return (block.Commit(number), block);
-        }
-    }
-
     [Test]
     public async Task Read_accessor()
     {
@@ -605,9 +561,7 @@ public async Task Read_accessor()
 
         var h = hashes[historyDepth];
 
-        var task = blockchain.WaitTillFlush(h);
-        blockchain.Finalize(h);
-        await task;
+        await blockchain.Finalize(h);
 
         // omit 0th
         for (uint i = 10; i < count; i++)
@@ -630,29 +584,6 @@ static Keccak Key(uint i)
         static Account Value(uint i) => new(i, i);
     }
 
-    [Test]
-    public async Task StartNew_when_throws_should_not_lock_db_readonly_batch()
-    {
-        using var db = PagedDb.NativeMemoryDb(1 * Mb, 2);
-
-        await using var blockchain = new Blockchain(db, new PreCommit());
-
-        const int none = 0;
-
-        db.CountReadOnlyBatches().Should().Be(none);
-
-        var nonExistentParent = new Random(13).NextKeccak();
-        try
-        {
-            var exception = Assert.Throws<Exception>(() => blockchain.StartNew(nonExistentParent));
-            exception.Message.Should().Contain("dependencies");
-        }
-        finally
-        {
-            db.CountReadOnlyBatches().Should().Be(none);
-        }
-    }
-
     private static Account GetAccount(int i) => new((UInt256)i + 1, (UInt256)i + 1);
 
     private static Keccak BuildKey(int i)

src/Paprika.Tests/Chain/PagePoolTests.cs

@@ -51,7 +51,7 @@ public void Rented_is_clear()
     }
 
     [Test]
-    public void Big_pool()
+    public unsafe void Big_pool()
     {
         const int pageCount = 1024;
         using var pool = new BufferPool(pageCount, assertCountOnDispose: false);
@@ -64,5 +64,10 @@ public void Big_pool()
         }
 
         set.Count.Should().Be(pageCount);
+
+        foreach (var page in set)
+        {
+            pool.Return(new Page((byte*)page.ToPointer()));
+        }
     }
 }

src/Paprika.Tests/Chain/PreCommitBehaviorTests.cs

@@ -58,7 +58,19 @@ public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget)
 
             _keccaks.SetEquals(_found).Should().BeTrue();
 
-            return Keccak.Zero;
+            return _found.Aggregate((keccak1, keccak2) =>
+            {
+                var result = default(Keccak);
+
+                var span = result.BytesAsSpan;
+
+                for (int i = 0; i < Keccak.Size; i++)
+                {
+                    span[i] = (byte)(keccak1.Span[i] ^ keccak2.Span[i] ^ i);
+                }
+
+                return result;
+            });
         }
 
         private void OnKey(in Key key, ReadOnlySpan<byte> value)

src/Paprika.Tests/Chain/PrefetchingTests.cs

@@ -23,9 +23,6 @@ public async Task Prefetches_properly_on_not_changed_structure()
         var merkle = new ComputeMerkleBehavior(ComputeMerkleBehavior.ParallelismNone);
         await using var blockchain = new Blockchain(db, merkle);
 
-        var wait = new SemaphoreSlim(0, 1000);
-        blockchain.Flushed += (_, _) => wait.Release();
-
         // Nicely fill 2 levels of the tree, so that RlpMemo.Compress does not remove branches with two children
         const int blocks = 256;
         var bigNonce = new UInt256(100, 100, 100, 100);
@@ -43,9 +40,7 @@ public async Task Prefetches_properly_on_not_changed_structure()
         }
 
         parent = start.Commit(1);
-        blockchain.Finalize(parent);
-
-        await wait.WaitAsync();
+        await blockchain.Finalize(parent);
 
         for (uint i = 2; i < blocks; i++)
         {
@@ -66,8 +61,7 @@ public async Task Prefetches_properly_on_not_changed_structure()
             block.SetAccount(keccak, new Account(i, i));
             parent = block.Commit(i);
 
-            blockchain.Finalize(parent);
-            await wait.WaitAsync();
+            await blockchain.Finalize(parent);
         }
     }
 
@@ -82,8 +76,7 @@ public async Task Makes_all_decompression_on_prefetch()
         var hash = Keccak.EmptyTreeHash;
 
         hash = BuildBlock(blockchain, hash, 1);
-        blockchain.Finalize(hash);
-        await blockchain.WaitTillFlush(hash);
+        await blockchain.Finalize(hash);
 
         hash = BuildBlock(blockchain, hash, 2);
 
@@ -169,8 +162,7 @@ public async Task Spin(bool prefetch, bool storage)
         using var first = blockchain.StartNew(parent);
         SetAccounts(new ReadOnlyMemory<Keccak>(keccaks), first, startBlockNumber, storage);
         parent = first.Commit(startBlockNumber);
-        blockchain.Finalize(parent);
-        await blockchain.WaitTillFlush(startBlockNumber);
+        await blockchain.Finalize(parent);
 
         // Run commits now with a prefetching
         await RunBlocksWithPrefetching(blockchain, keccaks, parent, prefetch, storage);