Skip to content

Commit

Permalink
Merge branch 'main' into importer
Browse files Browse the repository at this point in the history
  • Loading branch information
Scooletz committed Oct 17, 2024
2 parents f862389 + cafd97e commit 80b8e22
Show file tree
Hide file tree
Showing 23 changed files with 955 additions and 869 deletions.
32 changes: 16 additions & 16 deletions src/Paprika.Runner/StatisticsForPagedDb.cs
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,11 @@ private static Layout BuildReport(StatisticsVisitor.Stats stats, string name, in

var t = new Table();
t.AddColumn(new TableColumn("Depth"));
t.AddColumn(new TableColumn("Page count"));
t.AddColumn(new TableColumn("Leaf page count"));
t.AddColumn(new TableColumn("Overflow page count"));
t.AddColumn(new TableColumn($"{nameof(DataPage)}-inner (P50) % usage"));
t.AddColumn(new TableColumn($"{nameof(DataPage)}-leaf (P50) % usage"));
t.AddColumn(new TableColumn($"{nameof(LeafOverflowPage)} (P50) % usage"));
t.AddColumn(new TableColumn("Total page count"));
t.AddColumn(new TableColumn("Data page count"));
t.AddColumn(new TableColumn("Bottom page count"));
t.AddColumn(new TableColumn($"{nameof(DataPage)} P50 % usage"));
t.AddColumn(new TableColumn($"{nameof(BottomPage)} P50 % usage"));

if (fanOutLevels != null)
{
Expand All @@ -98,28 +97,29 @@ private static Layout BuildReport(StatisticsVisitor.Stats stats, string name, in
new Text("-"),
new Text("-"),
new Text("-"),
new Text("-"),
new Text("-")
);
}
}

var maxDepth = stats.PageCountPerNibblePathDepth.AsSpan().LastIndexOfAnyExcept(0) + 1;
var maxDepthDataPage = stats.DataPagePageCountPerNibblePathDepth.AsSpan().LastIndexOfAnyExcept(0) + 1;
var maxDepthBottomPage = stats.BottomPageCountPerNibblePathDepth.AsSpan().LastIndexOfAnyExcept(0) + 1;

var maxDepth = Math.Max(maxDepthDataPage, maxDepthBottomPage);

for (var depth = 0; depth < maxDepth; depth++)
{
var count = stats.PageCountPerNibblePathDepth[depth];
var leafPageCount = stats.LeafPageCountPerNibblePathDepth[depth];
var overflowCount = stats.OverflowPageCountPerNibblePathDepth[depth];
var dataPageCount = stats.DataPagePageCountPerNibblePathDepth[depth];
var bottomPageCount = stats.BottomPageCountPerNibblePathDepth[depth];
var count = dataPageCount + bottomPageCount;

t.AddRow(
new Text(depth.ToString()),
new Text(count.ToString()),
new Text(leafPageCount.ToString()),
new Text(overflowCount.ToString()),
new Text(GetP50(stats.InnerDataPagePercentageUsed, depth)),
new Text(GetP50(stats.LeafDataPagePercentageUsed, depth)),
new Text(GetP50(stats.OverflowPagePercentageUsed, depth))
new Text(dataPageCount.ToString()),
new Text(bottomPageCount.ToString()),
new Text(GetP50(stats.DataPagePercentageUsed, depth)),
new Text(GetP50(stats.BottomPagePercentageUsed, depth))
);
}

Expand Down
106 changes: 106 additions & 0 deletions src/Paprika.Tests/Data/NibbleSelectorTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
using FluentAssertions;
using Paprika.Data;
using static Paprika.Data.NibbleSelector;

namespace Paprika.Tests.Data;

[Parallelizable(ParallelScope.None)]
public class NibbleSelectorTests
{
private static readonly Type[] Selectors = typeof(NibbleSelector).GetNestedTypes();
private readonly HashSet<Type> _asserted = new();

[SetUp]
public void Setup() => _asserted.Clear();

[TestCase((byte)0)]
[TestCase((byte)1)]
[TestCase((byte)2)]
[TestCase((byte)3)]
public void Range_Q0(byte nibble)
{
True<All>(nibble);
True<HalfLow>(nibble);
True<Q0>(nibble);

RestIsFalse(nibble);
}

[TestCase((byte)4)]
[TestCase((byte)5)]
[TestCase((byte)6)]
[TestCase((byte)7)]
public void Range_Q1(byte nibble)
{
True<All>(nibble);
True<HalfLow>(nibble);
True<Q1>(nibble);

RestIsFalse(nibble);
}

[TestCase((byte)8)]
[TestCase((byte)9)]
[TestCase((byte)10)]
[TestCase((byte)11)]
public void Range_Q2(byte nibble)
{
True<All>(nibble);
True<HalfHigh>(nibble);
True<Q2>(nibble);

RestIsFalse(nibble);
}

[TestCase((byte)12)]
[TestCase((byte)13)]
[TestCase((byte)14)]
[TestCase((byte)15)]
public void Range_Q3(byte nibble)
{
True<All>(nibble);
True<HalfHigh>(nibble);
True<Q3>(nibble);

RestIsFalse(nibble);
}

[Test(Description = "Should throw on non asserted one")]
public void SanityCheck()
{
var ex = Assert.Throws<Exception>(() => RestIsFalse(1));

ex.Message.Should().ContainAll(nameof(All), nameof(HalfLow), nameof(Q0));
}

private void RestIsFalse(byte nibble)
{
List<Type> failed = new();

foreach (var selector in Selectors)
{
if (_asserted.Contains(selector))
continue;

var result = (bool)((selector.GetMethod(nameof(INibbleSelector.Should)).Invoke(null, [nibble])));
if (result)
{
failed.Add(selector);
}
}

_asserted.Clear();

if (failed.Count > 0)
{
throw new Exception(
"The following selectors returned true while not asserted: " + string.Join(", ", failed));
}
}

private void True<TSelector>(byte nibble) where TSelector : INibbleSelector
{
TSelector.Should(nibble).Should().BeTrue();
_asserted.Add(typeof(TSelector));
}
}
50 changes: 45 additions & 5 deletions src/Paprika.Tests/Data/SlottedArrayTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,47 @@ public void Move_to_1()
original.SetAssert(key4, Data(4));
original.SetAssert(key5, Data(5));

original.MoveNonEmptyKeysTo(new(copy0));
original.MoveNonEmptyKeysTo(new MapSource(copy0));

// original should have only empty
original.Count.Should().Be(1);
original.GetAssert(NibblePath.Empty, Data(0));
original.GetShouldFail(key1);
original.GetShouldFail(key2);
original.GetShouldFail(key3);
original.GetShouldFail(key4);
original.GetShouldFail(key5);

// copy should have all but empty
copy0.Count.Should().Be(5);
copy0.GetShouldFail(NibblePath.Empty);
copy0.GetAssert(key1, Data(1));
copy0.GetAssert(key2, Data(2));
copy0.GetAssert(key3, Data(3));
copy0.GetAssert(key4, Data(4));
copy0.GetAssert(key5, Data(5));
}

[Test]
public void Move_to_SlottedArray()
{
var original = new SlottedArray(stackalloc byte[256]);
var copy0 = new SlottedArray(stackalloc byte[256]);

var key1 = NibblePath.Parse("1");
var key2 = NibblePath.Parse("23");
var key3 = NibblePath.Parse("345");
var key4 = NibblePath.Parse("4567");
var key5 = NibblePath.Parse("56789");

original.SetAssert(NibblePath.Empty, Data(0));
original.SetAssert(key1, Data(1));
original.SetAssert(key2, Data(2));
original.SetAssert(key3, Data(3));
original.SetAssert(key4, Data(4));
original.SetAssert(key5, Data(5));

original.MoveNonEmptyKeysTo<NibbleSelector.All>(copy0);

// original should have only empty
original.Count.Should().Be(1);
Expand Down Expand Up @@ -653,7 +693,7 @@ public void Move_to_respects_tombstones()
original.SetAssert(key4, tombstone);
original.SetAssert(key5, Data(5));

original.MoveNonEmptyKeysTo(new(copy0), true);
original.MoveNonEmptyKeysTo(new MapSource(copy0), true);

// original should have only empty
original.Count.Should().Be(0);
Expand Down Expand Up @@ -694,7 +734,7 @@ public void Move_to_2()
original.SetAssert(key4, Data(4));
original.SetAssert(key5, Data(5));

original.MoveNonEmptyKeysTo(new(copy0, copy1));
original.MoveNonEmptyKeysTo(new MapSource(copy0, copy1));

// original should have only empty
original.Count.Should().Be(1);
Expand Down Expand Up @@ -747,7 +787,7 @@ public void Move_to_4()
original.SetAssert(key4, Data(4));
original.SetAssert(key5, Data(5));

original.MoveNonEmptyKeysTo(new(copy0, copy1, copy2, copy3));
original.MoveNonEmptyKeysTo(new MapSource(copy0, copy1, copy2, copy3));

// original should have only empty
original.Count.Should().Be(1);
Expand Down Expand Up @@ -828,7 +868,7 @@ public void Move_to_8()
original.SetAssert(key7, Data(7));
original.SetAssert(key8, Data(8));

original.MoveNonEmptyKeysTo(new(copy0, copy1, copy2, copy3, copy4, copy5, copy6, copy7));
original.MoveNonEmptyKeysTo(new MapSource(copy0, copy1, copy2, copy3, copy4, copy5, copy6, copy7));

// original should have only empty
HasOnly(original, 0);
Expand Down
7 changes: 4 additions & 3 deletions src/Paprika.Tests/Store/AbandonedTests.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System.Runtime.InteropServices;
using FluentAssertions;
using NUnit.Framework;
using Paprika.Crypto;
using Paprika.Data;
using Paprika.Store;
Expand Down Expand Up @@ -83,10 +84,10 @@ public void Properly_handles_page_addresses_that_are_packed_2()

[TestCase(20, 1, 10_000, false, TestName = "Accounts - 1")]
[TestCase(428, 100, 10_000, false, TestName = "Accounts - 100")]
[TestCase(19278, 4000, 200, false,
[TestCase(21796, 4000, 200, false,
TestName = "Accounts - 4000 to get a bit reuse",
Category = Categories.LongRunning)]
[TestCase(48228, 10_000, 50, false,
[TestCase(55480, 10_000, 50, false,
TestName = "Accounts - 10000 to breach the AbandonedPage",
Category = Categories.LongRunning)]
[TestCase(118364, 20_000, 50, true,
Expand Down Expand Up @@ -261,7 +262,7 @@ public async Task Abandoned_chain_creation_with_overflow()
var accountValue = new byte[2900];
new Random(17).NextBytes(accountValue);

using var db = PagedDb.NativeMemoryDb(150000 * Page.PageSize, HistoryDepth);
using var db = PagedDb.NativeMemoryDb(165_000 * Page.PageSize, HistoryDepth);

// Start read only batch to ensure that new pages are allocated instead of reusing
// the abandoned pages.
Expand Down
5 changes: 3 additions & 2 deletions src/Paprika.Tests/Store/BasePageTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,10 @@ public override Page GetNewPage(out DbAddress addr, bool clear)

public override void RegisterForFutureReuse(Page page, bool possibleImmediateReuse = false)
{
_toReuse.Add(GetAddress(page))
var addr = GetAddress(page);
_toReuse.Add(addr)
.Should()
.BeTrue("Page should not be registered as reusable before");
.BeTrue($"Page at {addr} should not be registered as reusable before");
}

public override Dictionary<Keccak, uint> IdCache { get; } = new();
Expand Down
61 changes: 61 additions & 0 deletions src/Paprika.Tests/Store/BottomPageTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
using FluentAssertions;
using Paprika.Data;
using Paprika.Store;

namespace Paprika.Tests.Store;

public class BottomPageTests : BasePageTests
{
private const uint BatchId = 1;

[Test]
public void Sufficient_to_set()
{
var batch = NewBatch(BatchId);
var bottom = ((IBatchContext)batch).GetNewPage<BottomPage>(out _);

var key = NibblePath.Empty;

// construct keys so that they fall into child, grand-child left, grand-child rigth
// Left child
var key0 = NibblePath.FromKey([0x0A]); // 0 is 0th nibble
var key1 = NibblePath.FromKey([0x4A]); // 4 is 0th nibble
var key2 = NibblePath.FromKey([0x1A]); // 1 is 0th nibble

// Right child
var key8 = NibblePath.FromKey([0x8A]); // 8 is 0th nibble
var key9 = NibblePath.FromKey([0xFA]); // 9 is 0th nibble
var key10 = NibblePath.FromKey([0x9A]); // A is 0th nibble

var v0 = new byte[3002];
var v1 = new byte[2999];
var v2 = new byte[2998];
var v8 = new byte[3003];
var v9 = new byte[3006];
var v10 = new byte[2980];
var v = new byte[3001];

bottom.Set(key0, v0, batch);
bottom.Set(key1, v1, batch);
bottom.Set(key2, v2, batch);
bottom.Set(key8, v8, batch);
bottom.Set(key9, v9, batch);
bottom.Set(key10, v10, batch);
bottom.Set(key, v, batch);

Assert(key, v);
Assert(key0, v0);
Assert(key1, v1);
Assert(key2, v2);
Assert(key8, v8);
Assert(key9, v9);
Assert(key10, v10);
return;

void Assert(in NibblePath key, in ReadOnlySpan<byte> expected)
{
bottom.TryGet(batch, key, out var actual).Should().BeTrue();
actual.SequenceEqual(expected).Should().BeTrue();
}
}
}
2 changes: 1 addition & 1 deletion src/Paprika.Tests/Store/ClearableTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public unsafe class ClearableTests : IDisposable
public void AbandonedPage() => TestPage<AbandonedPage>();

[Test]
public void LeafOverflowPage() => TestPage<LeafOverflowPage>();
public void BottomPage() => TestPage<BottomPage>();

[Test]
public void StorageFanOut_Level1Page() => TestPage<StorageFanOut.Level1Page>();
Expand Down
Loading

0 comments on commit 80b8e22

Please sign in to comment.