add predicate function and test for atree's PersistentSlabStorage.Fix…

…LoadedBrokenReferences

migrations/broken_dictionary.go

@@ -0,0 +1,47 @@
+/*
+ * Cadence - The resource-oriented smart contract programming language
+ *
+ * Copyright Dapper Labs, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrations
+
+import (
+	"github.com/onflow/atree"
+
+	"github.com/onflow/cadence/runtime/interpreter"
+)
+
+// ShouldFixBrokenCompositeKeyedDictionary returns true if the given value is a dictionary with a composite key type.
+//
+// It is useful for use with atree's PersistentSlabStorage.FixLoadedBrokenReferences.
+//
+// NOTE: The intended use case is to enable migration programs in onflow/flow-go to fix broken references.
+// As of April 2024, only 10 registers in testnet (not mainnet) were found to have broken references,
+// and they seem to have resulted from a bug that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565.
+func ShouldFixBrokenCompositeKeyedDictionary(atreeValue atree.Value) bool {
+	orderedMap, ok := atreeValue.(*atree.OrderedMap)
+	if !ok {
+		return false
+	}
+
+	dictionaryStaticType, ok := orderedMap.Type().(*interpreter.DictionaryStaticType)
+	if !ok {
+		return false
+	}
+
+	_, ok = dictionaryStaticType.KeyType.(*interpreter.CompositeStaticType)
+	return ok
+}

migrations/migration_test.go

@@ -19,6 +19,10 @@
 package migrations
 
 import (
+	"bytes"
+	_ "embed"
+	"encoding/csv"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"testing"
@@ -2659,3 +2663,98 @@ func TestDictionaryKeyConflict(t *testing.T) {
 		test(t, false)
 	})
 }
+
+//go:embed testdata/missing-slabs-payloads.csv
+var missingSlabsPayloadsData []byte
+
+// '$' + 8 byte index
+const slabKeyLength = 9
+
+func isSlabStorageKey(key []byte) bool {
+	return len(key) == slabKeyLength && key[0] == '$'
+}
+
+func TestFixLoadedBrokenReferences(t *testing.T) {
+
+	t.Parallel()
+
+	// Read CSV file with test data
+
+	reader := csv.NewReader(bytes.NewReader(missingSlabsPayloadsData))
+
+	// account, key, value
+	reader.FieldsPerRecord = 3
+
+	records, err := reader.ReadAll()
+	require.NoError(t, err)
+
+	// Load data into ledger. Skip header
+
+	ledger := NewTestLedger(nil, nil)
+
+	for _, record := range records[1:] {
+		account, err := hex.DecodeString(record[0])
+		require.NoError(t, err)
+
+		key, err := hex.DecodeString(record[1])
+		require.NoError(t, err)
+
+		value, err := hex.DecodeString(record[2])
+		require.NoError(t, err)
+
+		err = ledger.SetValue(account, key, value)
+		require.NoError(t, err)
+	}
+
+	storage := runtime.NewStorage(ledger, nil)
+
+	// Check health.
+	// Retrieve all slabs before migration
+
+	err = ledger.ForEach(func(owner, key, value []byte) error {
+
+		if !isSlabStorageKey(key) {
+			return nil
+		}
+
+		// Convert the owner/key to a storage ID.
+
+		var storageIndex atree.StorageIndex
+		copy(storageIndex[:], key[1:])
+
+		storageID := atree.NewStorageID(atree.Address(owner), storageIndex)
+
+		// Retrieve the slab.
+		_, _, err = storage.Retrieve(storageID)
+		require.NoError(t, err)
+
+		return nil
+	})
+	require.NoError(t, err)
+
+	address, err := common.HexToAddress("0x5d63c34d7f05e5a4")
+	require.NoError(t, err)
+
+	for _, domain := range common.AllPathDomains {
+		_ = storage.GetStorageMap(address, domain.Identifier(), false)
+	}
+
+	err = storage.CheckHealth()
+	require.Error(t, err)
+
+	require.ErrorContains(t, err, "slab (0x0.49) not found: slab not found during slab iteration")
+
+	// Fix the broken slab references
+
+	fixedSlabs, skippedSlabIDs, err := storage.PersistentSlabStorage.
+		FixLoadedBrokenReferences(ShouldFixBrokenCompositeKeyedDictionary)
+	require.NoError(t, err)
+
+	require.NotEmpty(t, fixedSlabs)
+	require.Empty(t, skippedSlabIDs)
+
+	// Re-run health check. This time it should pass.
+
+	err = storage.CheckHealth()
+	require.NoError(t, err)
+}

migrations/testdata/missing-slabs-payloads.csv

@@ -0,0 +1,25 @@
+owner,key,value
+5d63c34d7f05e5a4,240000000000000009,008883d88483d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7402021b6f605c4ae015732a008883005b00000000000000100727f5bf434b68c18638e048b5b7cdb49b0000000000000002826475756964d8a41a00a4a65d826762616c616e6365d8bc1b0000001748783b3a
+5d63c34d7f05e5a4,612e73,0000000000000012b9000000000000002b0000000000000001
+5d63c34d7f05e5a4,24000000000000000f,00c883d8d982d8d41830d8d582d8c08248631e88ae7f1d7c20704e6f6e46756e6769626c65546f6b656e744e6f6e46756e6769626c65546f6b656e2e4e4654031bb9d0e9f36650574100c883005b00000000000000180a5d0065f54d3b70efeefcbe52b8e6bbfe71d66e8625c1019b000000000000000382d8a419bfedd8ff505d63c34d7f05e5a4000000000000001682d8a419bfecd8ff505d63c34d7f05e5a4000000000000001082d8a419c026d8ff505d63c34d7f05e5a40000000000000028
+5d63c34d7f05e5a4,240000000000000005,00c883d88483d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e744173736574781b4469676974616c436f6e74656e7441737365742e4e46544461746101041bdb477d045820e23f00c883005b000000000000002001e9dead718715827af7a9558c406fd09c308d168ef2bca4bbd2c23d2c1c98209b0000000000000004826c73657269616c4e756d626572d8a30182666974656d4964d8876e746573742d6974656d2d69642d3282686d65746164617461d8ff505d63c34d7f05e5a40000000000000006826b6974656d56657273696f6ed8a301
+5d63c34d7f05e5a4,240000000000000006,008883d8d982d8d408d8d408011bd3ad92ee82a688cf008883005b00000000000000087d4660d52656b0f19b000000000000000182d887666578496e666fd887781b4164646974696f6e616c20696e666f206561636820746f6b656e2e
+5d63c34d7f05e5a4,24000000000000002a,008883d8d982d8d408d8d408001b7d63d5c2a25d9570008883005b00000000000000009b0000000000000000
+5d63c34d7f05e5a4,24000000000000000d,008883f6051bbe10d8ef134f8081008883005b00000000000000283c74aa8be0b9f0e484cd59f126f4e5908c69f0b6aca96e5f9c1af60840e775e3c06c0966bef02b939b0000000000000005826d444341436f6c6c656374696f6ed8cb82d8c882016d444341436f6c6c656374696f6ed8db82f4d8dc82d8d40581d8d682d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e74417373657478244469676974616c436f6e74656e7441737365742e436f6c6c656374696f6e5075626c6963827046616e546f705065726d697373696f6ed8cb82d8c882017046616e546f705065726d697373696f6ed8db82f4d8dc82d8d40581d8d682d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e781946616e546f705065726d697373696f6e2e52656365697665728270666c6f77546f6b656e42616c616e6365d8cb82d8c882016e666c6f77546f6b656e5661756c74d8db82f4d8dc82d8d582d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7481d8d682d8c082489a0766d93b6608b76d46756e6769626c65546f6b656e7546756e6769626c65546f6b656e2e42616c616e63658271666c6f77546f6b656e5265636569766572d8cb82d8c882016e666c6f77546f6b656e5661756c74d8db82f4d8dc82d8d582d8c082487e60df042a9c086869466c6f77546f6b656e6f466c6f77546f6b656e2e5661756c7481d8d682d8c082489a0766d93b6608b76d46756e6769626c65546f6b656e7646756e6769626c65546f6b656e2e5265636569766572827546616e546f70546f6b656e436f6c6c656374696f6ed8cb82d8c882017546616e546f70546f6b656e436f6c6c656374696f6ed8db82f4d8dc82d8d40581d8d682d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e781c46616e546f70546f6b656e2e436f6c6c656374696f6e5075626c6963
+5d63c34d7f05e5a4,7075626c6963,000000000000000d
+5d63c34d7f05e5a4,240000000000000029,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7346616e546f70546f6b656e2e4e46544461746101041bee2eefad66dc79bb00c883005b000000000000002038fc822d312df7f38ef037256b43eed596a75665ca80dfe6aec0bc45428aa9e59b0000000000000004826b6974656d56657273696f6ed8a301826c73657269616c4e756d626572d8a30182686d65746164617461d8ff505d63c34d7f05e5a4000000000000002a82666974656d4964d887781a746573742d6974656d2d69642d31363534393133393635393138
+5d63c34d7f05e5a4,73746f72616765,000000000000000c
+5d63c34d7f05e5a4,24000000000000000c,00c883f6041bd6bd24f14bb87b1d00c883005b000000000000002011e0799c7eee7e48e100d7590eb37bbcfc5fbcac8f0e7c49fdc92797c5d69e9f9b0000000000000004826e666c6f77546f6b656e5661756c74d8ff505d63c34d7f05e5a40000000000000009827546616e546f70546f6b656e436f6c6c656374696f6ed8ff505d63c34d7f05e5a4000000000000000e827046616e546f705065726d697373696f6ed8ff505d63c34d7f05e5a4000000000000000a826d444341436f6c6c656374696f6ed8ff505d63c34d7f05e5a40000000000000008
+5d63c34d7f05e5a4,240000000000000028,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e6f46616e546f70546f6b656e2e4e465402041bb77700225b8d6b5200c883005b00000000000000202183e3a62217fe8a36d1395a6a775e199d29ed46c773a0acc6d7ec7f9e97bfc79b0000000000000004826464617461d8ff505d63c34d7f05e5a40000000000000029826475756964d8a41a05cb6bc982657265664964d8877819746573742d7265662d69642d3136353439313339363539313882626964d8a419c026
+5d63c34d7f05e5a4,240000000000000016,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e6f46616e546f70546f6b656e2e4e465402041bb77700225b8d6b5200c883005b00000000000000202183e3a62217fe8a36d1395a6a775e199d29ed46c773a0acc6d7ec7f9e97bfc79b0000000000000004826464617461d8ff505d63c34d7f05e5a40000000000000017826475756964d8a41a05c8448882657265664964d8877819746573742d7265662d69642d3136353436363835373330383182626964d8a419bfed
+5d63c34d7f05e5a4,24000000000000000b,00c883d8d982d8d582d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e7546616e546f705065726d697373696f6e2e526f6c65d8ddf6011b535c9de83a38cab000c883005b00000000000000089bd8ae8dd553d9479b000000000000000182d8ff5000000000000000000000000000000031d8c983d88348a47a2d3a3b7e9133d8c882026c46616e546f704d696e746572d8db82f4d8d582d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e7746616e546f705065726d697373696f6e2e4d696e746572
+5d63c34d7f05e5a4,240000000000000007,00c883d8d982d8d41830d8d582d8c08248631e88ae7f1d7c20704e6f6e46756e6769626c65546f6b656e744e6f6e46756e6769626c65546f6b656e2e4e4654011b095f1b185a2800d900c883005b00000000000000085521726dbf8b37549b000000000000000182d8a4182bd8ff505d63c34d7f05e5a40000000000000004
+5d63c34d7f05e5a4,240000000000000004,00c883d88483d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e744173736574774469676974616c436f6e74656e7441737365742e4e465402041bf62e63ac21a3216600c883005b00000000000000200507c593895292f5ab364b56d50468c7d0766f8c688ebb74f8c1a4433ffbccf29b000000000000000482626964d8a4182b826464617461d8ff505d63c34d7f05e5a40000000000000005826475756964d8a41a00a5641882657265664964d8877830746573742d7265662d69642d61353662363764392d306634352d346339622d613264352d636534306439376339333230
+5d63c34d7f05e5a4,240000000000000008,00c883d88483d8c08248a47a2d3a3b7e9133734469676974616c436f6e74656e744173736574781e4469676974616c436f6e74656e7441737365742e436f6c6c656374696f6e02021bf7f58ec9906ea65700c883005b0000000000000010e5df9d08d0a2430ce91971a214b62eaa9b0000000000000002826475756964d8a41a00a4a66682696f776e65644e465473d8ff505d63c34d7f05e5a40000000000000007
+5d63c34d7f05e5a4,240000000000000017,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7346616e546f70546f6b656e2e4e46544461746101041bee2eefad66dc79bb00c883005b000000000000002038fc822d312df7f38ef037256b43eed596a75665ca80dfe6aec0bc45428aa9e59b0000000000000004826b6974656d56657273696f6ed8a301826c73657269616c4e756d626572d8a30182686d65746164617461d8ff505d63c34d7f05e5a4000000000000001882666974656d4964d887781a746573742d6974656d2d69642d31363534363638353733303831
+5d63c34d7f05e5a4,240000000000000018,008883d8d982d8d408d8d408001b7d63d5c2a25d9570008883005b00000000000000009b0000000000000000
+5d63c34d7f05e5a4,240000000000000010,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e6f46616e546f70546f6b656e2e4e465402041bb77700225b8d6b5200c883005b00000000000000202183e3a62217fe8a36d1395a6a775e199d29ed46c773a0acc6d7ec7f9e97bfc79b0000000000000004826464617461d8ff505d63c34d7f05e5a40000000000000011826475756964d8a41a05c838bf82657265664964d8877819746573742d7265662d69642d3136353436363539323439343382626964d8a419bfec
+5d63c34d7f05e5a4,240000000000000011,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7346616e546f70546f6b656e2e4e46544461746101041bee2eefad66dc79bb00c883005b000000000000002038fc822d312df7f38ef037256b43eed596a75665ca80dfe6aec0bc45428aa9e59b0000000000000004826b6974656d56657273696f6ed8a301826c73657269616c4e756d626572d8a30182686d65746164617461d8ff505d63c34d7f05e5a4000000000000001282666974656d4964d887781a746573742d6974656d2d69642d31363534363635393234393433
+5d63c34d7f05e5a4,24000000000000000a,00c883d88483d8c08248a47a2d3a3b7e91337046616e546f705065726d697373696f6e7746616e546f705065726d697373696f6e2e486f6c64657202031bb9d0e9f36650574100c883005b000000000000001820d6c23f2e85e694b0070dbc21a9822de5725916c4a005e99b0000000000000003826475756964d8a41a00d858bc8265726f6c6573d8ff505d63c34d7f05e5a4000000000000000b8269726563697069656e74d883485d63c34d7f05e5a4
+5d63c34d7f05e5a4,240000000000000012,008883d8d982d8d408d8d408001b7d63d5c2a25d9570008883005b00000000000000009b0000000000000000
+5d63c34d7f05e5a4,24000000000000000e,00c883d88483d8c08248a47a2d3a3b7e91336b46616e546f70546f6b656e7646616e546f70546f6b656e2e436f6c6c656374696f6e02021b85d7c70d054429ef00c883005b000000000000001012dab75ddc75f021eec8d5b5338fb70a9b000000000000000282696f776e65644e465473d8ff505d63c34d7f05e5a4000000000000000f826475756964d8a41a05c83883

runtime/tests/runtime_utils/testledger.go

@@ -22,6 +22,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -54,6 +55,34 @@ func (s TestLedger) AllocateStorageIndex(owner []byte) (atree.StorageIndex, erro
 	return s.OnAllocateStorageIndex(owner)
 }
 
+const testLedgerKeySeparator = "|"
+
+func (s TestLedger) ForEach(f func(owner, key, value []byte) error) error {
+
+	var keys []string
+	for key := range s.StoredValues { //nolint:maprange
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys { //nolint:maprange
+		value := s.StoredValues[key]
+
+		keyParts := strings.Split(key, testLedgerKeySeparator)
+		owner := []byte(keyParts[0])
+		key := []byte(keyParts[1])
+
+		if err := f(owner, key, value); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func TestStorageKey(owner, key string) string {
+	return strings.Join([]string{owner, key}, testLedgerKeySeparator)
+}
+
 func (s TestLedger) Dump() {
 	// Only used for testing/debugging purposes
 	for key, data := range s.StoredValues { //nolint:maprange
@@ -68,29 +97,25 @@ func NewTestLedger(
 	onWrite func(owner, key, value []byte),
 ) TestLedger {
 
-	storageKey := func(owner, key string) string {
-		return strings.Join([]string{owner, key}, "|")
-	}
-
 	storedValues := map[string][]byte{}
 
 	storageIndices := map[string]uint64{}
 
 	return TestLedger{
 		StoredValues: storedValues,
 		OnValueExists: func(owner, key []byte) (bool, error) {
-			value := storedValues[storageKey(string(owner), string(key))]
+			value := storedValues[TestStorageKey(string(owner), string(key))]
 			return len(value) > 0, nil
 		},
 		OnGetValue: func(owner, key []byte) (value []byte, err error) {
-			value = storedValues[storageKey(string(owner), string(key))]
+			value = storedValues[TestStorageKey(string(owner), string(key))]
 			if onRead != nil {
 				onRead(owner, key, value)
 			}
 			return value, nil
 		},
 		OnSetValue: func(owner, key, value []byte) (err error) {
-			storedValues[storageKey(string(owner), string(key))] = value
+			storedValues[TestStorageKey(string(owner), string(key))] = value
 			if onWrite != nil {
 				onWrite(owner, key, value)
 			}