From 39b60befeb9782cb9370d24a5ccfce89e1937547 Mon Sep 17 00:00:00 2001 From: nikkolasg Date: Wed, 29 Nov 2023 18:09:00 +0100 Subject: [PATCH 1/6] blockdata fetcher --- .github/workflows/rust.yml | 2 + Cargo.lock | 800 ++++++++++++++++++++++++++++++++++++- Cargo.toml | 10 +- src/lib.rs | 4 +- src/mpt_tx.rs | 489 ----------------------- 5 files changed, 805 insertions(+), 500 deletions(-) delete mode 100644 src/mpt_tx.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 5ee948aa7..843f27c22 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -57,6 +57,8 @@ jobs: with: command: test args: --release + env: + CI_RPC_URL: ${{ secrets.CI_RPC_URL }} fmt: name: Rustfmt diff --git a/Cargo.lock b/Cargo.lock index 7756fc328..41a437112 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -87,6 +93,15 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + [[package]] name = "async-trait" version = "0.1.74" @@ -172,6 +187,21 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -242,6 +272,27 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "camino" version = "1.1.6" @@ -280,6 +331,7 @@ version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] @@ -403,6 +455,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation" version = "0.9.3" @@ -428,6 +486,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -533,6 +600,12 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "data-encoding" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" + [[package]] name = "der" version = "0.7.8" @@ -564,6 +637,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -576,6 +655,48 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dunce" version = "1.0.4" @@ -621,6 +742,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ena" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1" +dependencies = [ + "log", +] + [[package]] name = "encoding_rs" version = "0.8.33" @@ -686,6 +816,19 @@ dependencies = [ "uuid", ] +[[package]] +name = "eth_trie" +version = "0.4.0" +source = "git+https://github.com/nikkolasg/eth-trie.rs#27a9c35a2c27a442444d86e9f7a84d25e69d0e9c" +dependencies = [ + "ethereum-types", + "hashbrown 0.14.2", + "keccak-hash 0.10.0", + "log", + "parking_lot", + "rlp", +] + [[package]] name = "ethabi" version = "18.0.0" @@ -743,9 +886,11 @@ dependencies = [ "ethers-addressbook", "ethers-contract", "ethers-core", + "ethers-etherscan", "ethers-middleware", "ethers-providers", "ethers-signers", + "ethers-solc", ] [[package]] @@ -789,11 +934,13 @@ dependencies = [ "const-hex", "dunce", "ethers-core", + "ethers-etherscan", "eyre", "prettyplease", "proc-macro2", "quote", "regex", + "reqwest", "serde", "serde_json", "syn 2.0.39", @@ -847,6 +994,22 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "ethers-etherscan" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abbac2c890bdbe0f1b8e549a53b00e2c4c1de86bb077c1094d1f38cdf9381a56" +dependencies = [ + "chrono", + "ethers-core", + "reqwest", + "semver", + "serde", + "serde_json", + "thiserror", + "tracing", +] + [[package]] name = "ethers-middleware" version = "2.0.11" @@ -900,6 +1063,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "tokio-tungstenite", "tracing", "tracing-futures", "url", @@ -928,6 +1092,38 @@ dependencies = [ "tracing", ] +[[package]] +name = "ethers-solc" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64f710586d147864cff66540a6d64518b9ff37d73ef827fee430538265b595f" +dependencies = [ + "cfg-if", + "const-hex", + "dirs", + "dunce", + "ethers-core", + "glob", + "home", + "md-5", + "num_cpus", + "once_cell", + "path-slash", + "rayon", + "regex", + "semver", + "serde", + "serde_json", + "solang-parser", + "svm-rs", + "thiserror", + "tiny-keccak", + "tokio", + "tracing", + "walkdir", + "yansi", +] + [[package]] name = "eyre" version = "0.6.9" @@ -975,6 +1171,22 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -990,6 +1202,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "2.0.0" @@ -1144,6 +1366,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -1199,6 +1427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash", + "allocator-api2", "rayon", "serde", ] @@ -1218,6 +1447,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + [[package]] name = "hex" version = "0.4.3" @@ -1233,6 +1468,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "http" version = "0.2.11" @@ -1291,6 +1535,20 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "iana-time-zone" version = "0.1.58" @@ -1419,6 +1677,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1452,6 +1721,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jobserver" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.65" @@ -1469,7 +1747,7 @@ checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.5", "pem", - "ring", + "ring 0.16.20", "serde", "serde_json", "simple_asn1", @@ -1508,6 +1786,44 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "keccak-hash" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b286e6b663fb926e1eeb68528e69cb70ed46c6d65871a21b2215ae8154c6d3c" +dependencies = [ + "primitive-types 0.12.2", + "tiny-keccak", +] + +[[package]] +name = "lalrpop" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8" +dependencies = [ + "ascii-canvas", + "bit-set", + "diff", + "ena", + "is-terminal", + "itertools 0.10.5", + "lalrpop-util", + "petgraph", + "regex", + "regex-syntax 0.7.5", + "string_cache", + "term", + "tiny-keccak", + "unicode-xid", +] + +[[package]] +name = "lalrpop-util" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1526,12 +1842,33 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall", +] + [[package]] name = "linux-raw-sys" version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.20" @@ -1543,6 +1880,7 @@ name = "mapreduce-plonky2" version = "0.1.0" dependencies = [ "anyhow", + "eth_trie", "ethers", "hex", "itertools 0.12.0", @@ -1552,6 +1890,17 @@ dependencies = [ "rlp", "serde", "sha3", + "tokio", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", ] [[package]] @@ -1595,6 +1944,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + [[package]] name = "num" version = "0.4.1" @@ -1674,6 +2029,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "num_enum" version = "0.7.1" @@ -1735,6 +2100,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "parity-scale-codec" version = "3.6.5" @@ -1761,6 +2132,46 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "path-slash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91099d4268b0e11973f036e885d652fb0b21fedcf69738c627f94db6a44f42" + [[package]] name = "pbkdf2" version = "0.11.0" @@ -1768,6 +2179,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest", + "hmac", + "password-hash", + "sha2", ] [[package]] @@ -1795,6 +2209,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap 2.1.0", +] + [[package]] name = "pharos" version = "0.5.3" @@ -1805,6 +2229,57 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared 0.11.2", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator", + "phf_shared 0.11.2", + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.3" @@ -1847,6 +2322,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + [[package]] name = "plonky2" version = "0.1.4" @@ -1858,7 +2339,7 @@ dependencies = [ "getrandom", "hashbrown 0.14.2", "itertools 0.11.0", - "keccak-hash", + "keccak-hash 0.8.0", "log", "num", "plonky2_field", @@ -1932,6 +2413,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" version = "0.2.15" @@ -2030,7 +2517,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax", + "regex-syntax 0.8.2", "unarray", ] @@ -2117,6 +2604,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.10.2" @@ -2126,7 +2624,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax", + "regex-syntax 0.8.2", ] [[package]] @@ -2137,9 +2635,15 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.2", ] +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "regex-syntax" version = "0.8.2" @@ -2161,6 +2665,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "ipnet", "js-sys", "log", @@ -2168,16 +2673,20 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots", "winreg", ] @@ -2200,12 +2709,26 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "ripemd" version = "0.1.3" @@ -2271,6 +2794,37 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustls" +version = "0.21.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +dependencies = [ + "log", + "ring 0.17.6", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.5", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.6", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -2343,6 +2897,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.6", + "untrusted 0.9.0", +] + [[package]] name = "sec1" version = "0.7.3" @@ -2458,6 +3022,17 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.8" @@ -2501,6 +3076,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "slab" version = "0.4.9" @@ -2510,6 +3091,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" + [[package]] name = "socket2" version = "0.4.10" @@ -2530,17 +3117,37 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "solang-parser" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c425ce1c59f4b154717592f0bdf4715c3a1d55058883622d3157e1f0908a5b26" +dependencies = [ + "itertools 0.11.0", + "lalrpop", + "lalrpop-util", + "phf", + "thiserror", + "unicode-xid", +] + [[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "spki" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", "der", @@ -2552,6 +3159,19 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", +] + [[package]] name = "strsim" version = "0.10.0" @@ -2586,6 +3206,26 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "svm-rs" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20689c7d03b6461b502d0b95d6c24874c7d24dea2688af80486a130a06af3b07" +dependencies = [ + "dirs", + "fs2", + "hex", + "once_cell", + "reqwest", + "semver", + "serde", + "serde_json", + "sha2", + "thiserror", + "url", + "zip", +] + [[package]] name = "syn" version = "1.0.109" @@ -2648,6 +3288,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.50" @@ -2731,11 +3382,49 @@ dependencies = [ "bytes", "libc", "mio", + "num_cpus", "pin-project-lite", "socket2 0.5.5", + "tokio-macros", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +dependencies = [ + "futures-util", + "log", + "rustls", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots", +] + [[package]] name = "tokio-util" version = "0.7.10" @@ -2859,6 +3548,26 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "rustls", + "sha1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" @@ -2926,6 +3635,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.0" @@ -2937,6 +3652,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "uuid" version = "0.8.2" @@ -3054,6 +3775,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" + [[package]] name = "winapi" version = "0.3.9" @@ -3273,6 +4000,12 @@ dependencies = [ "tap", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zerocopy" version = "0.7.26" @@ -3298,3 +4031,52 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2 0.11.0", + "sha1", + "time", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index e60f532f7..6e588f1f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,16 +9,24 @@ edition = "2021" anyhow = "1.0.75" itertools = "0.12.0" plonky2 = "0.1.4" +ethers = { version ="2.0.11", default-features = false, features = ["rustls"]} # supporting latest plonky2 plonky2_crypto = { git = "https://github.com/nikkolasg/plonky2-crypto" } rand = "0.8.5" serde = "1.0.193" +# TODO: see if we can revert to upstream repo: originally used +# to fetch proof with "node" instead of already encoded struct +eth_trie = { git = "https://github.com/nikkolasg/eth-trie.rs" } +rlp = "0.5.2" [dev-dependencies] anyhow = "1.0.75" -ethers = { version ="2.0.11", default-features = false } hex = "0.4.3" itertools = "0.12.0" rlp = "0.5.2" sha3 = "0.10.8" +tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] } + +[features] +ci = [] diff --git a/src/lib.rs b/src/lib.rs index 415274303..101f49ad0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![warn(missing_docs)] #![feature(generic_const_exprs)] use plonky2::plonk::{ @@ -5,8 +6,9 @@ use plonky2::plonk::{ proof::ProofWithPublicInputs, }; -mod mpt_tx; +mod eth; mod rlp; +mod transaction; mod utils; /// Bundle containing the raw proof, the verification key, and some common data diff --git a/src/mpt_tx.rs b/src/mpt_tx.rs deleted file mode 100644 index 653608db4..000000000 --- a/src/mpt_tx.rs +++ /dev/null @@ -1,489 +0,0 @@ -use anyhow::Result; -use plonky2::{ - field::extension::Extendable, - hash::hash_types::RichField, - iop::{ - target::Target, - witness::{PartialWitness, WitnessWrite}, - }, - plonk::{circuit_builder::CircuitBuilder, circuit_data::CircuitConfig, config::GenericConfig}, - util::ceil_div_usize, -}; -use plonky2_crypto::{ - biguint::BigUintTarget, - hash::{ - keccak256::{CircuitBuilderHashKeccak, KECCAK256_R}, - HashInputTarget, - }, - u32::arithmetic_u32::U32Target, -}; - -use crate::{ - rlp::{decode_fixed_list, decode_tuple, extract_array}, - utils::{convert_u8_to_u32, less_than}, - ProofTuple, -}; - -/// The maximum length of a RLP encoded leaf node in a MPT tree holding a legacy tx. -const MAX_LEGACY_TX_NODE_LENGTH: usize = 532; -/// The maximum size a RLP encoded legacy tx can take. This is different from -/// `LEGACY_TX_NODE_LENGTH` because the latter contains the key in the path -/// as well. -const MAX_LEGACY_TX_LENGTH: usize = 532; -/// Maximum size the gas value can take in bytes. -const MAX_GAS_VALUE_LEN: usize = 32; - -/// There are different ways to extract values from a transaction. This enum -/// list some. -pub(crate) enum ExtractionMethod { - /// RLPBased decodes each header consecutively and extract the gas value - /// TODO: Currently hardcode that the gas value is 3rd item in the tx list - /// because we use const generics and can't pass the index as a parameter. - RLPBased, - /// Directly reads at the specified offset. - /// Offset of the item in the tx list - length is assumed to be constant - /// OffsetBased is NOT secure, it is only useful for testing & quick prototyping purposes. - OffsetBased(usize), -} - -/// Provides a proof for a leaf node in a MPT tree holding a legacy tx. It exposes -/// the hash of the node as public input, as well as the gas value of the tx. -pub fn legacy_tx_leaf_node_proof< - F: RichField + Extendable, - C: GenericConfig, - const D: usize, ->( - config: &CircuitConfig, - mut node: Vec, - extract: ExtractionMethod, -) -> Result> { - let mut b = CircuitBuilder::::new(config.clone()); - let mut pw = PartialWitness::new(); - - let node_length = node.len(); - node.resize(MAX_LEGACY_TX_NODE_LENGTH, 0); - let node_targets = b.add_virtual_targets(MAX_LEGACY_TX_NODE_LENGTH); - - // Witness assignement - for i in 0..MAX_LEGACY_TX_NODE_LENGTH { - pw.set_target(node_targets[i], F::from_canonical_u8(node[i])); - } - - // Hash computation and exposing as public input - let length_target = b.add_virtual_target(); - pw.set_target(length_target, F::from_canonical_usize(node_length)); - hash_node(&mut b, &mut pw, &node_targets, length_target, node_length); - - // Gas value extraction and exposing as public input - let gas_value_array = match extract { - // gas is at 3rd position - ExtractionMethod::RLPBased => { - extract_item_from_tx_list::(&mut b, &node_targets) - } - ExtractionMethod::OffsetBased(offset) => { - // NOTE: It does NOT guarantee the offset is _correct_. The prover CAN give - // any offset within the given slice that has been hashed, and claim it is - // the gas value. - let gas_offset_target = b.add_virtual_target(); - pw.set_target(gas_offset_target, F::from_canonical_usize(offset)); - extract_array::(&mut b, &node_targets, gas_offset_target) - } - }; - // maximum length that the RLP(gas) == RLP(U256) can take: - // * 32 bytes for the value (U256 = 32 bytes) - // TODO: pack the gas value into U32Target - more compact - b.register_public_inputs(&gas_value_array); - - // proving part - let data = b.build::(); - let proof = data.prove(pw)?; - - Ok((proof, data.verifier_only, data.common)) -} - -/// Reads the header of the RLP node, then reads the header of the TX item -/// then reads all headers of the items in the list until it reaches the given -/// header at position N. It reads that header and returns the offset from the array -/// where the data is starting -fn extract_item_from_tx_list< - F: RichField + Extendable, - const D: usize, - const N_FIELDS: usize, - const MAX_VALUE_SIZE: usize, ->( - b: &mut CircuitBuilder, - node: &[Target], - // TODO: make that const generic -) -> [Target; MAX_VALUE_SIZE] { - // First, decode headers of RLP ( RLP (key), RLP(tx) ) - let tuple_headers = decode_tuple(b, node); - let rlp_tx_index = 1; - // extract the RLP(tx) from the node encoding - let tx_offset = tuple_headers.offset[rlp_tx_index]; - let rlp_tx = extract_array::(b, node, tx_offset); - - // then extract the gas fees: it's the third item in the tx list (out of 9 for legacy tx) - // NOTE: we should only decode the things we need, so for example here - // the gas fee is at the 3rd position then we only need to decode up to the 3rd - // headers in the list and keep the rest untouched. However, later user query might - // want the whole thing. - let tx_list = decode_fixed_list::(b, &rlp_tx); - let item_index = N_FIELDS - 1; - let item_offset = tx_list.offset[item_index]; - extract_array::(b, &rlp_tx, item_offset) -} - -fn hash_node, const D: usize>( - b: &mut CircuitBuilder, - pw: &mut PartialWitness, - node: &[Target], // assume constant size : TODO make it const generic - length_target: Target, // the size of the data inside this fixed size array - length: usize, // could maybe be done with a generator but simpler this way -) { - let total_len = node.len(); - // the computation of the padding length can be done outside the circuit - // because the important thing is that we prove in crcuit (a) we did some padding - // starting from the end of the message and (b) that padded array is transformed - // into u32 array correctly. - // We don't care if the _padding length_ if done incorrectly, - // because the hash output will be incorrect because hash computation is constrained. - // If the prover gave a incorrect length_target, that means either the data buffer - // will be changed, OR the the padding "buffer" will be changed from what is expected - // -> in both cases, the resulting hash will be different. - // (a) is necessary to allow the circuit to take as witness this length_target such - // that we can _directly_ lookup the data that is interesting for us _without_ passing - // through the expensive RLP decoding steps. To do this, we need to make sure, the prover - // can NOT give a target_length value which points to an index > to where we actually - // start padding the data. Otherwise, target_length could point to _any_ byte after - // the end of the data slice up to the end of the fixed size array. - let input_len_bits = length * 8; // only pad the data that is inside the fixed buffer - let num_actual_blocks = 1 + input_len_bits / KECCAK256_R; - let padded_len_bits = num_actual_blocks * KECCAK256_R; - // reason why ^: this is annoying to do in circuit. - let num_bytes = ceil_div_usize(padded_len_bits, 8); - let diff = num_bytes - length; - - let diff_target = b.add_virtual_target(); - pw.set_target(diff_target, F::from_canonical_usize(diff)); - let end_padding = b.add(length_target, diff_target); - let one = b.one(); - let end_padding = b.sub(end_padding, one); // inclusive range - // little endian so we start padding from the end of the byte - let single_pad = b.constant(F::from_canonical_usize(0x81)); // 1000 0001 - let begin_pad = b.constant(F::from_canonical_usize(0x01)); // 0000 0001 - let end_pad = b.constant(F::from_canonical_usize(0x80)); // 1000 0000 - // TODO : make that const generic - let padded_node = node - .iter() - .enumerate() - .map(|(i, byte)| { - let i_target = b.constant(F::from_canonical_usize(i)); - // condition if we are within the data range ==> i < length - let is_data = less_than(b, i_target, length_target, 32); - // condition if we start the padding ==> i == length - let is_start_padding = b.is_equal(i_target, length_target); - // condition if we are done with the padding ==> i == length + diff - 1 - let is_end_padding = b.is_equal(i_target, end_padding); - // condition if we only need to add one byte 1000 0001 to pad - // because we work on u8 data, we know we're at least adding 1 byte and in - // this case it's 0x81 = 1000 0001 - // i == length == diff - 1 - let is_start_and_end = b.and(is_start_padding, is_end_padding); - - // nikko XXX: Is this sound ? I think so but not 100% sure. - // I think it's ok to not use `quin_selector` or `b.random_acess` because - // if the prover gives another byte target, then the resulting hash would be invalid, - let item_data = b.mul(is_data.target, *byte); - let item_start_padding = b.mul(is_start_padding.target, begin_pad); - let item_end_padding = b.mul(is_end_padding.target, end_pad); - let item_start_and_end = b.mul(is_start_and_end.target, single_pad); - // if all of these conditions are false, then item will be 0x00,i.e. the padding - let mut item = item_data; - item = b.add(item, item_start_padding); - item = b.add(item, item_end_padding); - item = b.add(item, item_start_and_end); - item - }) - .collect::>(); - - // NOTE we don't pad anymore because we enforce that the resulting length is already a multiple - // of 4 so it will fit the conversion to u32 and circuit vk would stay the same for different - // data length - assert!(total_len % 4 == 0); - - // convert padded node to u32 - let node_u32_target: Vec = convert_u8_to_u32(b, &padded_node); - - // fixed size block delimitation: this is where we tell the hash function gadget - // to only look at a certain portion of our data, each bool says if the hash function - // will update its state for this block or not. - let rate_bytes = b.constant(F::from_canonical_usize(KECCAK256_R / 8)); - let end_padding_offset = b.add(end_padding, one); - let nb_blocks = b.div(end_padding_offset, rate_bytes); - // - 1 because keccak always take first block so we don't count it - let nb_actual_blocks = b.sub(nb_blocks, one); - let total_num_blocks = total_len / (KECCAK256_R / 8) - 1; - let blocks = (0..total_num_blocks) - .map(|i| { - let i_target = b.constant(F::from_canonical_usize(i)); - less_than(b, i_target, nb_actual_blocks, 8) - }) - .collect::>(); - - let hash_target = HashInputTarget { - input: BigUintTarget { - limbs: node_u32_target, - }, - input_bits: padded_len_bits, - blocks, - }; - - let hash_output = b.hash_keccak256(&hash_target); - b.register_public_inputs( - &hash_output - .limbs - .iter() - .map(|limb| limb.0) - .collect::>(), - ); -} - -#[cfg(test)] -mod test { - use anyhow::Result; - use ethers::types::Transaction; - use plonky2::field::extension::Extendable; - use plonky2::hash::hash_types::RichField; - use plonky2::{ - iop::witness::PartialWitness, - plonk::{ - circuit_builder::CircuitBuilder, - circuit_data::{CircuitConfig, VerifierCircuitData}, - config::{GenericConfig, PoseidonGoldilocksConfig}, - }, - }; - use rlp::{Decodable, Encodable, Rlp}; - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - type F = >::F; - - const STRING: usize = 0; - const LIST: usize = 1; - - use crate::utils::test::data_to_constant_targets; - use crate::utils::test::{connect, keccak256}; - use crate::ProofTuple; - use crate::{ - mpt_tx::MAX_LEGACY_TX_NODE_LENGTH, - rlp::{decode_header, decode_tuple}, - }; - - use super::{legacy_tx_leaf_node_proof, ExtractionMethod}; - fn run_leaf_proof_test< - F: RichField + Extendable, - C: GenericConfig, - const D: usize, - Circuit, - >( - circuit: Circuit, - ) -> Result<()> - where - Circuit: FnOnce(&CircuitConfig, &[u8]) -> Result>, - { - // The following test data comes from: - // ``` - // let block_number = 10593417; - // let tx_index = U64::from(3); - // ``` - let leaf_node_hex= "f87420b871f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"; - let leaf_node_buff = hex::decode(leaf_node_hex).unwrap(); - let node_hash = keccak256(&leaf_node_buff); - let tx_hex = "f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"; - let tx_buff = hex::decode(tx_hex).unwrap(); - let tx = Transaction::decode(&Rlp::new(&tx_buff)).unwrap(); - - let config = CircuitConfig::standard_recursion_config(); - let leaf_proof = circuit(&config, &leaf_node_buff)?; - let vcd = VerifierCircuitData { - verifier_only: leaf_proof.1.clone(), - common: leaf_proof.2.clone(), - }; - - vcd.verify(leaf_proof.0.clone())?; - // verify hash of the node - let expected_hash = crate::utils::test::hash_to_fields::(&node_hash); - let proof_hash = &leaf_proof.0.public_inputs[0..8]; - assert!(expected_hash == proof_hash, "hashes not equal?"); - // verify gas value - let gas_buff = tx.gas.rlp_bytes(); - let gas_rlp = rlp::Rlp::new(&gas_buff); - let gas_header = gas_rlp.payload_info()?; - let gas_value = gas_rlp.data().unwrap().to_vec(); - assert_eq!( - &leaf_proof.0.public_inputs[8..8 + gas_header.value_len], - gas_value - .iter() - .take(gas_header.value_len) - .map(|byte| F::from_canonical_u8(*byte)) - .collect::>() - .as_slice() - ); - Ok(()) - } - #[test] - fn test_legacy_tx_leaf_proof_rlp_extract() -> Result<()> { - run_leaf_proof_test(|c, n| { - legacy_tx_leaf_node_proof::(c, n.to_vec(), ExtractionMethod::RLPBased) - }) - } - #[test] - fn test_legacy_tx_leaf_proof_offset() -> Result<()> { - run_leaf_proof_test(|c, n| { - let (gas_offset, _) = gas_offset_from_rlp_node(n); - legacy_tx_leaf_node_proof::( - c, - n.to_vec(), - ExtractionMethod::OffsetBased(gas_offset), - ) - }) - } - #[test] - fn test_rlp_mpt_node_list() -> Result<()> { - // come from last tx in block 10593417, leaf node for tx idx 03 in the MPT - let data_str = "f87420b871f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"; - let mut data = hex::decode(data_str).unwrap(); - assert!(data.len() > 55); - - let r = rlp::Rlp::new(&data); - let prototype = r.prototype().expect("error reading prototype"); - assert!( - matches!(prototype, rlp::Prototype::List(2)), - "prototype is {:?}", - prototype - ); - let header = r.payload_info().expect("can't get payload info"); - let key_rlp = r.at(0).expect("can't get key rlp"); - let value_rlp = r.at(1).expect("can't get value rlp"); - let key_header = key_rlp.payload_info().expect("can't get key payload info"); - let value_header = value_rlp - .payload_info() - .expect("can't get value payload info"); - assert!(key_header.header_len == 0); // this is short value so directly single byte! 0x20 - assert!(key_header.value_len > 0); // there is a value to be read - assert!(value_header.header_len > 0); // tx is more than 55 bytes long - assert!(key_header.value_len > 0); - - // check total value checks out for sub items length - let computed_len = header.header_len - + key_header.value_len - + value_header.value_len - + key_header.header_len - + value_header.header_len; - // add redundant header_len to mimick the circuit function - assert!(header.value_len + header.header_len == computed_len); - - let config = CircuitConfig::standard_recursion_config(); - - let mut pw = PartialWitness::new(); - let mut b = CircuitBuilder::::new(config); - - // before transforming to targets, we pad to constant size so circuit always work for different sizes - // Note we can't do it when reading rlp data offcircuit because rlp library continues to read until the - // end of the array so it's not gonna be a list(2) anymore but much longer list. - data.resize(MAX_LEGACY_TX_NODE_LENGTH, 0); - let node_targets = data_to_constant_targets(&mut b, &data); - - // check the header of the list is correctly decoded - let rlp_header = decode_header(&mut b, &node_targets); - connect(&mut b, &mut pw, rlp_header.offset, header.header_len as u32); - connect(&mut b, &mut pw, rlp_header.len, header.value_len as u32); - // it's a list so type = 1 - connect(&mut b, &mut pw, rlp_header.data_type, LIST as u32); - - // decode all the sub headers now, we know there are only two - let rlp_list = decode_tuple(&mut b, &node_targets); - // check the first sub header which is the key of the MPT leaf node - // value of the key header starts after first header and after header of the key item - let expected_key_value_offset = key_header.header_len + header.header_len; - - connect( - &mut b, - &mut pw, - rlp_list.offset[0], - expected_key_value_offset as u32, - ); - connect(&mut b, &mut pw, rlp_list.data_type[0], STRING as u32); - connect( - &mut b, - &mut pw, - rlp_list.len[0], - key_header.value_len as u32, - ); - // check the second sub header which is the key of the MPT leaf node - // value starts after first header, after key header, after key value and after value header - let expected_value_value_offset = value_header.header_len - + key_header.header_len - + key_header.value_len - + header.header_len; - connect( - &mut b, - &mut pw, - rlp_list.offset[1], - expected_value_value_offset as u32, - ); - connect(&mut b, &mut pw, rlp_list.data_type[1], STRING as u32); - connect( - &mut b, - &mut pw, - rlp_list.len[1], - value_header.value_len as u32, - ); - - let data = b.build::(); - let proof = data.prove(pw)?; - data.verify(proof) - } - use std::cmp::Ordering; - /// Function that returns the offset of the gas value in the RLP encoded - /// node containing a transaction. It also returns the gas length. - fn gas_offset_from_rlp_node(node: &[u8]) -> (usize, usize) { - let node_rlp = rlp::Rlp::new(node); - let tuple_info = node_rlp.payload_info().unwrap(); - let tuple_offset = tuple_info.header_len; - assert_eq!(node_rlp.item_count().unwrap(), 2); - let tx_index = 1; - let gas_index = 2; - let mut tx_offset = tuple_offset; - let mut gas_value_len = 0; - let mut gas_offset = 0; - node_rlp.iter().enumerate().for_each(|(i, r)| { - let h = r.payload_info().unwrap(); - tx_offset += h.header_len; - match i.cmp(&tx_index) { - Ordering::Less => tx_offset += h.value_len, - Ordering::Greater => panic!("node should not have more than 2 items"), - Ordering::Equal => { - let tx_rlp = rlp::Rlp::new(r.data().unwrap()); - gas_offset += tx_rlp.payload_info().unwrap().header_len; - tx_rlp.iter().enumerate().for_each(|(j, rr)| { - let hh = rr.payload_info().unwrap(); - match j.cmp(&gas_index) { - Ordering::Less => { - gas_offset += hh.header_len; - gas_offset += hh.value_len; - } - // do nothing as we don't care about the other items - Ordering::Greater => {} - Ordering::Equal => { - // we want the value directly - we skip the header - gas_offset += hh.header_len; - gas_value_len = hh.value_len; - } - } - }); - } - } - }); - (tx_offset + gas_offset, gas_value_len) - } -} From 000ccf3e27d7b7a0cc85a8a4ec172d9a05bda503 Mon Sep 17 00:00:00 2001 From: nikkolasg Date: Wed, 29 Nov 2023 18:15:16 +0100 Subject: [PATCH 2/6] blockdata fetcher - with file .... --- src/eth.rs | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 src/eth.rs diff --git a/src/eth.rs b/src/eth.rs new file mode 100644 index 000000000..772996bd2 --- /dev/null +++ b/src/eth.rs @@ -0,0 +1,149 @@ +//! Module containing several structure definitions for Ethereum related operations +//! such as fetching blocks, transactions, creating MPTs, getting proofs, etc. +use anyhow::{Ok, Result}; +use eth_trie::{EthTrie, MemoryDB, Trie}; +use ethers::{ + providers::{Http, Middleware, Provider}, + types::{BlockId, Bytes, Transaction, TransactionReceipt, U64}, +}; +use rlp::{Encodable, RlpStream}; +use std::{env, sync::Arc}; +/// A wrapper around a transaction and its receipt. The receipt is used to filter +/// bad transactions, so we only compute over valid transactions. +pub(crate) struct TxAndReceipt(Transaction, TransactionReceipt); + +impl TxAndReceipt { + pub fn tx(&self) -> &Transaction { + &self.0 + } + pub fn receipt(&self) -> &TransactionReceipt { + &self.1 + } + pub fn tx_rlp(&self) -> Bytes { + self.0.rlp() + } + // TODO: this should be upstreamed to ethers-rs + pub fn receipt_rlp(&self) -> Bytes { + let tx_type = self.tx().transaction_type; + let mut rlp = RlpStream::new(); + rlp.begin_unbounded_list(); + match &self.1.status { + Some(s) if s.as_u32() == 1 => rlp.append(s), + _ => rlp.append_empty_data(), + }; + rlp.append(&self.1.cumulative_gas_used) + .append(&self.1.logs_bloom) + .append_list(&self.1.logs); + + rlp.finalize_unbounded_list(); + let rlp_bytes: Bytes = rlp.out().freeze().into(); + let mut encoded = vec![]; + match tx_type { + // EIP-2930 (0x01) + Some(x) if x == U64::from(0x1) => { + encoded.extend_from_slice(&[0x1]); + encoded.extend_from_slice(rlp_bytes.as_ref()); + encoded.into() + } + // EIP-1559 (0x02) + Some(x) if x == U64::from(0x2) => { + encoded.extend_from_slice(&[0x2]); + encoded.extend_from_slice(rlp_bytes.as_ref()); + encoded.into() + } + _ => rlp_bytes, + } + } +} + +/// Structure containing the block header and its transactions / receipts. Amongst other things, +/// it is used to create a proof of inclusion for any transaction inside this block. +pub struct BlockData { + pub block: ethers::types::Block, + pub txs: Vec, + // TODO: add generics later - this may be re-used amongst different workers + pub tx_trie: EthTrie, + pub receipts_trie: EthTrie, +} + +impl BlockData { + pub async fn fetch + Send + Sync>(blockid: T) -> Result { + #[cfg(feature = "ci")] + let url = env::var("CI_RPC_URL").expect("RPC_URL env var not set"); + #[cfg(not(feature = "ci"))] + let url = "https://eth.llamarpc.com"; + //let provider = Provider::::try_from + let provider = + Provider::::try_from(url).expect("could not instantiate HTTP Provider"); + + let block = provider + .get_block_with_txs(blockid) + .await? + .expect("should have been a block"); + let receipts = provider.get_block_receipts(block.number.unwrap()).await?; + + let tx_with_receipt = block + .transactions + .clone() + .into_iter() + .map(|tx| { + let tx_hash = tx.hash(); + let r = receipts + .iter() + .find(|r| r.transaction_hash == tx_hash) + .expect("RPC sending invalid data"); + // TODO remove cloning + TxAndReceipt(tx, r.clone()) + }) + .collect::>(); + + // check transaction root + let memdb = Arc::new(MemoryDB::new(true)); + let mut tx_trie = EthTrie::new(Arc::clone(&memdb)); + for tr in tx_with_receipt.iter() { + tx_trie + .insert(&tr.receipt().transaction_index.rlp_bytes(), &tr.tx().rlp()) + .expect("can't insert tx"); + } + + // check receipt root + let memdb = Arc::new(MemoryDB::new(true)); + let mut receipts_trie = EthTrie::new(Arc::clone(&memdb)); + for tr in tx_with_receipt.iter() { + receipts_trie + .insert( + &tr.receipt().transaction_index.rlp_bytes(), + // TODO: make getter value for rlp encoding + &tr.receipt_rlp(), + ) + .expect("can't insert tx"); + } + let computed = tx_trie.root_hash().expect("root hash problem"); + let expected = block.transactions_root; + assert_eq!(expected, computed); + + let computed = receipts_trie.root_hash().expect("root hash problem"); + let expected = block.receipts_root; + assert_eq!(expected, computed); + + Ok(BlockData { + block, + tx_trie, + receipts_trie, + txs: tx_with_receipt, + }) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[tokio::test] + async fn fetch_block() -> Result<()> { + let block_number = 10593417; + let block = BlockData::fetch(block_number).await?; + assert_eq!(block.block.number.unwrap(), block_number.into()); + Ok(()) + } +} From e11e31ec9dd906b0d21d8ae57f58be42cc651050 Mon Sep 17 00:00:00 2001 From: nikkolasg Date: Thu, 30 Nov 2023 17:20:20 +0100 Subject: [PATCH 3/6] full proof verification --- src/rlp.rs | 29 ++ src/transaction/mod.rs | 2 + src/transaction/mpt.rs | 637 +++++++++++++++++++++++++++++++++++++++++ src/utils.rs | 11 + 4 files changed, 679 insertions(+) create mode 100644 src/transaction/mod.rs create mode 100644 src/transaction/mpt.rs diff --git a/src/rlp.rs b/src/rlp.rs index 1b6aa396d..4b38bfca5 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -279,7 +279,36 @@ mod tests { use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig}; use crate::rlp::{RlpHeader, MAX_LEN_BYTES}; + #[test] + fn test_vk() -> Result<()> { + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + type F = >::F; + let config = CircuitConfig::standard_recursion_config(); + use plonky2::field::types::Sample; + let n = 15; + let a = F::rand_vec(n); + let b = F::rand_vec(n); + let mut b1 = CircuitBuilder::::new(config.clone()); + let pw1 = PartialWitness::new(); + b1.constants(&a); + let d1 = b1.build::(); + let p1 = d1.prove(pw1)?; + d1.verify(p1)?; + let digest1 = d1.verifier_only.circuit_digest; + + let mut b2 = CircuitBuilder::::new(config); + b2.constants(&b); + let d2 = b2.build::(); + let pw2 = PartialWitness::new(); + let p2 = d2.prove(pw2)?; + d2.verify(p2)?; + let digest2 = d2.verifier_only.circuit_digest; + assert_eq!(digest1, digest2); + Ok(()) + } + // TODO: replace these tests by deterministic tests by cr #[test] fn test_data_len() -> Result<()> { const D: usize = 2; diff --git a/src/transaction/mod.rs b/src/transaction/mod.rs new file mode 100644 index 000000000..a4003003c --- /dev/null +++ b/src/transaction/mod.rs @@ -0,0 +1,2 @@ +mod mpt; +//mod prover; diff --git a/src/transaction/mpt.rs b/src/transaction/mpt.rs new file mode 100644 index 000000000..7f82b1ba0 --- /dev/null +++ b/src/transaction/mpt.rs @@ -0,0 +1,637 @@ +use anyhow::Result; +use plonky2::{ + field::extension::Extendable, + hash::hash_types::RichField, + iop::{ + target::Target, + witness::{PartialWitness, WitnessWrite}, + }, + plonk::{ + circuit_builder::CircuitBuilder, + circuit_data::{CircuitConfig, CommonCircuitData, ProverCircuitData}, + config::{AlgebraicHasher, GenericConfig}, + proof::ProofWithPublicInputs, + }, + util::ceil_div_usize, +}; +use plonky2_crypto::{ + biguint::BigUintTarget, + hash::{ + keccak256::{CircuitBuilderHashKeccak, KECCAK256_R}, + HashInputTarget, + }, + u32::arithmetic_u32::U32Target, +}; + +use crate::{ + rlp::{decode_fixed_list, decode_tuple, extract_array}, + utils::{convert_u8_to_u32, less_than}, + ProofTuple, +}; + +/// The maximum length of a RLP encoded leaf node in a MPT tree holding a legacy tx. +const MAX_LEGACY_TX_NODE_LENGTH: usize = 532; +/// The maximum size a RLP encoded legacy tx can take. This is different from +/// `LEGACY_TX_NODE_LENGTH` because the latter contains the key in the path +/// as well. +const MAX_LEGACY_TX_LENGTH: usize = 532; +/// Maximum size the gas value can take in bytes. +const MAX_GAS_VALUE_LEN: usize = 32; + +/// Size of an intermediate (branch or leaf) node in the MPT trie. +/// A branch node can take up to 17*32 = 544 bytes. +const MAX_INTERMEDIATE_NODE_LENGTH: usize = 544; + +/// Length of a "key" (a hash) in the MPT trie. +const HASH_LENGTH: usize = 32; + +/// There are different ways to extract values from a transaction. This enum +/// list some. +pub(crate) enum ExtractionMethod { + /// RLPBased decodes each header consecutively and extract the gas value + /// TODO: Currently hardcode that the gas value is 3rd item in the tx list + /// because we use const generics and can't pass the index as a parameter. + RLPBased, + /// Directly reads at the specified offset. + /// Offset of the item in the tx list - length is assumed to be constant + /// OffsetBased is NOT secure, it is only useful for testing & quick prototyping purposes. + OffsetBased(usize), +} + +/// Provides a proof for a leaf node in a MPT tree holding a legacy tx. It exposes +/// the hash of the node as public input, as well as the gas value of the tx. +pub fn legacy_tx_leaf_node_proof< + F: RichField + Extendable, + C: GenericConfig, + const D: usize, +>( + config: &CircuitConfig, + mut node: Vec, + extract: ExtractionMethod, +) -> Result> { + let mut b = CircuitBuilder::::new(config.clone()); + let mut pw = PartialWitness::new(); + + let node_length = node.len(); + node.resize(MAX_LEGACY_TX_NODE_LENGTH, 0); + let node_targets = b.add_virtual_targets(MAX_LEGACY_TX_NODE_LENGTH); + + // Witness assignement + for i in 0..MAX_LEGACY_TX_NODE_LENGTH { + pw.set_target(node_targets[i], F::from_canonical_u8(node[i])); + } + + // Hash computation and exposing as public input + let length_target = b.add_virtual_target(); + pw.set_target(length_target, F::from_canonical_usize(node_length)); + hash_node(&mut b, &mut pw, &node_targets, length_target, node_length); + + // Gas value extraction and exposing as public input + let gas_value_array = match extract { + // gas is at 3rd position + ExtractionMethod::RLPBased => { + extract_item_from_tx_list::(&mut b, &node_targets) + } + ExtractionMethod::OffsetBased(offset) => { + // NOTE: It does NOT guarantee the offset is _correct_. The prover CAN give + // any offset within the given slice that has been hashed, and claim it is + // the gas value. + let gas_offset_target = b.add_virtual_target(); + pw.set_target(gas_offset_target, F::from_canonical_usize(offset)); + extract_array::(&mut b, &node_targets, gas_offset_target) + } + }; + // maximum length that the RLP(gas) == RLP(U256) can take: + // * 32 bytes for the value (U256 = 32 bytes) + // TODO: pack the gas value into U32Target - more compact + b.register_public_inputs(&gas_value_array); + + // proving part + let data = b.build::(); + let proof = data.prove(pw)?; + + Ok((proof, data.verifier_only, data.common)) +} + +pub fn recursive_node_proof< + F: RichField + Extendable, + C: GenericConfig, + InnerC: GenericConfig, + const D: usize, +>( + config: &CircuitConfig, + mut node: Vec, + inner_proofs: &[ProofTuple], + hash_offsets: &[usize], + hash: &[u8], +) -> Result> +where + InnerC::Hasher: AlgebraicHasher, +{ + let node_length = node.len(); + node.resize(MAX_INTERMEDIATE_NODE_LENGTH, 0); + assert_ne!(inner_proofs.len(), 0); + assert_eq!(inner_proofs.len(), hash_offsets.len()); + assert_eq!(hash.len(), 32); + assert!(node_length <= MAX_INTERMEDIATE_NODE_LENGTH); + assert!(node_length > 0); + + let mut b = CircuitBuilder::::new(config.clone()); + let mut pw = PartialWitness::new(); + + let offset_targets = b.add_virtual_targets(hash_offsets.len()); + let node_targets = b.add_virtual_targets(MAX_INTERMEDIATE_NODE_LENGTH); + let length_target = b.add_virtual_target(); + + pw.set_target(length_target, F::from_canonical_usize(node_length)); + for (offset_tgt, offset) in offset_targets.iter().zip(hash_offsets) { + pw.set_target(*offset_tgt, F::from_canonical_u32(*offset as u32)); + } + for i in 0..MAX_INTERMEDIATE_NODE_LENGTH { + pw.set_target(node_targets[i], F::from_canonical_u8(node[i])); + } + + hash_node(&mut b, &mut pw, &node_targets, length_target, node_length); + + // verify each inner proof - can be up to 16 if a branch node is full + //let mut proofs_canonical_hash_targets = vec![]; + for (offset, prooft) in offset_targets.iter().zip(inner_proofs.iter()) { + let children_hash = extract_array::(&mut b, &node_targets, *offset); + // connect the lookup hash to the proof hash + // hash is exposed as 8 target elements so need to convert from u8 array to u32 + // 32 bytes hash output = 256 bits = 32 bits * 8 => 8 u32 targets exposed + let extracted_hash = convert_u8_to_u32(&mut b, &children_hash); + let (inner_proof, inner_vd, inner_cd) = prooft; + let pt = b.add_virtual_proof_with_pis(inner_cd); + pw.set_proof_with_pis_target(&pt, inner_proof); + + // nikko: historically been done like this - we could just connect the extract hash directly + extracted_hash + .iter() + .zip(pt.public_inputs[0..8].iter()) + .for_each(|(l, r)| { + b.connect(l.0, *r); + }); + + let inner_data = b.add_virtual_verifier_data(inner_cd.config.fri_config.cap_height); + // nikko: XXX WHy do we need these two lines ? + // In plonky2 benchmarks they dont use it but if we remove them from here + // it just fails + // See https://github.com/0xPolygonZero/plonky2/blob/main/plonky2/examples/bench_recursion.rs#L212-L217 + pw.set_cap_target( + &inner_data.constants_sigmas_cap, + &inner_vd.constants_sigmas_cap, + ); + pw.set_hash_target(inner_data.circuit_digest, inner_vd.circuit_digest); + b.verify_proof::(&pt, &inner_data, inner_cd); + } + + let data = b.build::(); + let proof = data.prove(pw)?; + + Ok((proof, data.verifier_only, data.common)) +} + +/// Reads the header of the RLP node, then reads the header of the TX item +/// then reads all headers of the items in the list until it reaches the given +/// header at position N. It reads that header and returns the offset from the array +/// where the data is starting +fn extract_item_from_tx_list< + F: RichField + Extendable, + const D: usize, + const N_FIELDS: usize, + const MAX_VALUE_SIZE: usize, +>( + b: &mut CircuitBuilder, + node: &[Target], + // TODO: make that const generic +) -> [Target; MAX_VALUE_SIZE] { + // First, decode headers of RLP ( RLP (key), RLP(tx) ) + let tuple_headers = decode_tuple(b, node); + let rlp_tx_index = 1; + // extract the RLP(tx) from the node encoding + let tx_offset = tuple_headers.offset[rlp_tx_index]; + let rlp_tx = extract_array::(b, node, tx_offset); + + // then extract the gas fees: it's the third item in the tx list (out of 9 for legacy tx) + // NOTE: we should only decode the things we need, so for example here + // the gas fee is at the 3rd position then we only need to decode up to the 3rd + // headers in the list and keep the rest untouched. However, later user query might + // want the whole thing. + let tx_list = decode_fixed_list::(b, &rlp_tx); + let item_index = N_FIELDS - 1; + let item_offset = tx_list.offset[item_index]; + extract_array::(b, &rlp_tx, item_offset) +} + +fn hash_node, const D: usize>( + b: &mut CircuitBuilder, + pw: &mut PartialWitness, + node: &[Target], // assume constant size : TODO make it const generic + length_target: Target, // the size of the data inside this fixed size array + length: usize, // could maybe be done with a generator but simpler this way +) { + let total_len = node.len(); + // the computation of the padding length can be done outside the circuit + // because the important thing is that we prove in crcuit (a) we did some padding + // starting from the end of the message and (b) that padded array is transformed + // into u32 array correctly. + // We don't care if the _padding length_ if done incorrectly, + // because the hash output will be incorrect because hash computation is constrained. + // If the prover gave a incorrect length_target, that means either the data buffer + // will be changed, OR the the padding "buffer" will be changed from what is expected + // -> in both cases, the resulting hash will be different. + // (a) is necessary to allow the circuit to take as witness this length_target such + // that we can _directly_ lookup the data that is interesting for us _without_ passing + // through the expensive RLP decoding steps. To do this, we need to make sure, the prover + // can NOT give a target_length value which points to an index > to where we actually + // start padding the data. Otherwise, target_length could point to _any_ byte after + // the end of the data slice up to the end of the fixed size array. + let input_len_bits = length * 8; // only pad the data that is inside the fixed buffer + let num_actual_blocks = 1 + input_len_bits / KECCAK256_R; + let padded_len_bits = num_actual_blocks * KECCAK256_R; + // reason why ^: this is annoying to do in circuit. + let num_bytes = ceil_div_usize(padded_len_bits, 8); + let diff = num_bytes - length; + + let diff_target = b.add_virtual_target(); + pw.set_target(diff_target, F::from_canonical_usize(diff)); + let end_padding = b.add(length_target, diff_target); + let one = b.one(); + let end_padding = b.sub(end_padding, one); // inclusive range + // little endian so we start padding from the end of the byte + let single_pad = b.constant(F::from_canonical_usize(0x81)); // 1000 0001 + let begin_pad = b.constant(F::from_canonical_usize(0x01)); // 0000 0001 + let end_pad = b.constant(F::from_canonical_usize(0x80)); // 1000 0000 + // TODO : make that const generic + let padded_node = node + .iter() + .enumerate() + .map(|(i, byte)| { + let i_target = b.constant(F::from_canonical_usize(i)); + // condition if we are within the data range ==> i < length + let is_data = less_than(b, i_target, length_target, 32); + // condition if we start the padding ==> i == length + let is_start_padding = b.is_equal(i_target, length_target); + // condition if we are done with the padding ==> i == length + diff - 1 + let is_end_padding = b.is_equal(i_target, end_padding); + // condition if we only need to add one byte 1000 0001 to pad + // because we work on u8 data, we know we're at least adding 1 byte and in + // this case it's 0x81 = 1000 0001 + // i == length == diff - 1 + let is_start_and_end = b.and(is_start_padding, is_end_padding); + + // nikko XXX: Is this sound ? I think so but not 100% sure. + // I think it's ok to not use `quin_selector` or `b.random_acess` because + // if the prover gives another byte target, then the resulting hash would be invalid, + let item_data = b.mul(is_data.target, *byte); + let item_start_padding = b.mul(is_start_padding.target, begin_pad); + let item_end_padding = b.mul(is_end_padding.target, end_pad); + let item_start_and_end = b.mul(is_start_and_end.target, single_pad); + // if all of these conditions are false, then item will be 0x00,i.e. the padding + let mut item = item_data; + item = b.add(item, item_start_padding); + item = b.add(item, item_end_padding); + item = b.add(item, item_start_and_end); + item + }) + .collect::>(); + + // NOTE we don't pad anymore because we enforce that the resulting length is already a multiple + // of 4 so it will fit the conversion to u32 and circuit vk would stay the same for different + // data length + assert!(total_len % 4 == 0); + + // convert padded node to u32 + let node_u32_target: Vec = convert_u8_to_u32(b, &padded_node); + + // fixed size block delimitation: this is where we tell the hash function gadget + // to only look at a certain portion of our data, each bool says if the hash function + // will update its state for this block or not. + let rate_bytes = b.constant(F::from_canonical_usize(KECCAK256_R / 8)); + let end_padding_offset = b.add(end_padding, one); + let nb_blocks = b.div(end_padding_offset, rate_bytes); + // - 1 because keccak always take first block so we don't count it + let nb_actual_blocks = b.sub(nb_blocks, one); + let total_num_blocks = total_len / (KECCAK256_R / 8) - 1; + let blocks = (0..total_num_blocks) + .map(|i| { + let i_target = b.constant(F::from_canonical_usize(i)); + less_than(b, i_target, nb_actual_blocks, 8) + }) + .collect::>(); + + let hash_target = HashInputTarget { + input: BigUintTarget { + limbs: node_u32_target, + }, + input_bits: padded_len_bits, + blocks, + }; + + let hash_output = b.hash_keccak256(&hash_target); + b.register_public_inputs( + &hash_output + .limbs + .iter() + .map(|limb| limb.0) + .collect::>(), + ); +} + +#[cfg(test)] +mod test { + use anyhow::Result; + use ethers::types::Transaction; + use plonky2::field::extension::Extendable; + use plonky2::hash::hash_types::RichField; + use plonky2::hash::keccak; + use plonky2::plonk::circuit_data::{CommonCircuitData, VerifierOnlyCircuitData}; + use plonky2::plonk::config::AlgebraicHasher; + use plonky2::plonk::proof::ProofWithPublicInputs; + use plonky2::{ + iop::witness::PartialWitness, + plonk::{ + circuit_builder::CircuitBuilder, + circuit_data::{CircuitConfig, VerifierCircuitData}, + config::{GenericConfig, PoseidonGoldilocksConfig}, + }, + }; + use rlp::{Decodable, Encodable, Rlp}; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + type F = >::F; + + const STRING: usize = 0; + const LIST: usize = 1; + + use super::{recursive_node_proof, MAX_LEGACY_TX_NODE_LENGTH}; + use crate::rlp::{decode_header, decode_tuple}; + use crate::utils::test::{connect, keccak256}; + use crate::utils::test::{data_to_constant_targets, hash_output_to_field}; + use crate::ProofTuple; + + use super::{legacy_tx_leaf_node_proof, ExtractionMethod}; + // Returns the index where the subvector starts in v, if any. + fn find_index_subvector(v: &[u8], sub: &[u8]) -> Option { + (0..(v.len() - sub.len())).find(|&i| &v[i..i + sub.len()] == sub) + } + + #[test] + fn test_legacy_full_proof() -> Result<()> { + run_legacy_mpt_proof::() + } + fn run_legacy_mpt_proof< + F: RichField + Extendable, + C: GenericConfig, + const D: usize, + >() -> Result<()> + where + C::Hasher: AlgebraicHasher, + { + let mpt_proof_hex = ["f851a098f3110a26a9ee7d32d5da055ebd725f09d6b5223aaa66a1f46382262830895680808080808080a00e6e346926890fbe0443125f7eed828ab63e0f4ebcd37722254eb097bac002528080808080808080","f87180a02675bad1a2403f7724522e6105b2279e66791dcd4a8a2165480b199d4cea6594a0fed7ac70c74e6148971bca70502ce65d28fd4060d8d4fa3acb29a5f84faff324a07f009b48d17653d95d8fd7974e26e03083d84f6a1262d03c076b272f545af3e580808080808080808080808080","f87420b871f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"]; + let mpt_proof = mpt_proof_hex + .iter() + .map(|hex| hex::decode(hex).unwrap()) + .collect::>(); + let root_hash_hex = "ab41f886be23cd786d8a69a72b0f988ea72e0b2e03970d0798f5e03763a442cc"; + let root_hash = hex::decode(root_hash_hex).unwrap(); + let config = CircuitConfig::standard_recursion_config(); + + let mut last_proof = None; + let mut last_hash = vec![]; + for (i, node) in mpt_proof.into_iter().rev().enumerate() { + let node_hash = keccak256(&node); + let proof = if i == 0 { + legacy_tx_leaf_node_proof::(&config, node, ExtractionMethod::RLPBased)? + } else { + let hash_offset = find_index_subvector(&node, &last_hash).unwrap(); + let p = last_proof.unwrap(); + recursive_node_proof(&config, node, &[p], &[hash_offset], &last_hash)? + }; + + let vcd = VerifierCircuitData { + verifier_only: proof.1.clone(), + common: proof.2.clone(), + }; + println!("[+] Proof index {} computed", i); + vcd.verify(proof.0.clone())?; + println!("[+] Proof index {} verified", i); + let expected_hash = hash_output_to_field::(&node_hash); + let proof_hash = &proof.0.public_inputs[0..8]; + assert!(expected_hash == proof_hash, "hashes not equal?"); + + last_proof = Some(proof); + last_hash = node_hash; + } + assert_eq!(last_hash, root_hash); + Ok(()) + } + fn run_leaf_proof_test< + F: RichField + Extendable, + C: GenericConfig, + const D: usize, + Circuit, + >( + circuit: Circuit, + ) -> Result<()> + where + Circuit: FnOnce(&CircuitConfig, &[u8]) -> Result>, + { + // The following test data comes from: + // ``` + // let block_number = 10593417; + // let tx_index = U64::from(3); + // ``` + let leaf_node_hex= "f87420b871f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"; + let leaf_node_buff = hex::decode(leaf_node_hex).unwrap(); + let node_hash = keccak256(&leaf_node_buff); + let tx_hex = "f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"; + let tx_buff = hex::decode(tx_hex).unwrap(); + let tx = Transaction::decode(&Rlp::new(&tx_buff)).unwrap(); + + let config = CircuitConfig::standard_recursion_config(); + let leaf_proof = circuit(&config, &leaf_node_buff)?; + let vcd = VerifierCircuitData { + verifier_only: leaf_proof.1.clone(), + common: leaf_proof.2.clone(), + }; + + vcd.verify(leaf_proof.0.clone())?; + // verify hash of the node + let expected_hash = crate::utils::test::hash_to_fields::(&node_hash); + let proof_hash = &leaf_proof.0.public_inputs[0..8]; + assert!(expected_hash == proof_hash, "hashes not equal?"); + // verify gas value + let gas_buff = tx.gas.rlp_bytes(); + let gas_rlp = rlp::Rlp::new(&gas_buff); + let gas_header = gas_rlp.payload_info()?; + let gas_value = gas_rlp.data().unwrap().to_vec(); + assert_eq!( + &leaf_proof.0.public_inputs[8..8 + gas_header.value_len], + gas_value + .iter() + .take(gas_header.value_len) + .map(|byte| F::from_canonical_u8(*byte)) + .collect::>() + .as_slice() + ); + Ok(()) + } + #[test] + fn test_legacy_tx_leaf_proof_rlp_extract() -> Result<()> { + run_leaf_proof_test(|c, n| { + legacy_tx_leaf_node_proof::(c, n.to_vec(), ExtractionMethod::RLPBased) + }) + } + #[test] + fn test_legacy_tx_leaf_proof_offset() -> Result<()> { + run_leaf_proof_test(|c, n| { + let (gas_offset, _) = gas_offset_from_rlp_node(n); + legacy_tx_leaf_node_proof::( + c, + n.to_vec(), + ExtractionMethod::OffsetBased(gas_offset), + ) + }) + } + #[test] + fn test_rlp_mpt_node_list() -> Result<()> { + // come from last tx in block 10593417, leaf node for tx idx 03 in the MPT + let data_str = "f87420b871f86f826b2585199c82cc0083015f9094e955ede0a3dbf651e2891356ecd0509c1edb8d9c8801051fdc4efdc0008025a02190f26e70a82d7f66354a13cda79b6af1aa808db768a787aeb348d425d7d0b3a06a82bd0518bc9b69dc551e20d772a1b06222edfc5d39b6973e4f4dc46ed8b196"; + let mut data = hex::decode(data_str).unwrap(); + assert!(data.len() > 55); + + let r = rlp::Rlp::new(&data); + let prototype = r.prototype().expect("error reading prototype"); + assert!( + matches!(prototype, rlp::Prototype::List(2)), + "prototype is {:?}", + prototype + ); + let header = r.payload_info().expect("can't get payload info"); + let key_rlp = r.at(0).expect("can't get key rlp"); + let value_rlp = r.at(1).expect("can't get value rlp"); + let key_header = key_rlp.payload_info().expect("can't get key payload info"); + let value_header = value_rlp + .payload_info() + .expect("can't get value payload info"); + assert!(key_header.header_len == 0); // this is short value so directly single byte! 0x20 + assert!(key_header.value_len > 0); // there is a value to be read + assert!(value_header.header_len > 0); // tx is more than 55 bytes long + assert!(key_header.value_len > 0); + + // check total value checks out for sub items length + let computed_len = header.header_len + + key_header.value_len + + value_header.value_len + + key_header.header_len + + value_header.header_len; + // add redundant header_len to mimick the circuit function + assert!(header.value_len + header.header_len == computed_len); + + let config = CircuitConfig::standard_recursion_config(); + + let mut pw = PartialWitness::new(); + let mut b = CircuitBuilder::::new(config); + + // before transforming to targets, we pad to constant size so circuit always work for different sizes + // Note we can't do it when reading rlp data offcircuit because rlp library continues to read until the + // end of the array so it's not gonna be a list(2) anymore but much longer list. + data.resize(MAX_LEGACY_TX_NODE_LENGTH, 0); + let node_targets = data_to_constant_targets(&mut b, &data); + + // check the header of the list is correctly decoded + let rlp_header = decode_header(&mut b, &node_targets); + connect(&mut b, &mut pw, rlp_header.offset, header.header_len as u32); + connect(&mut b, &mut pw, rlp_header.len, header.value_len as u32); + // it's a list so type = 1 + connect(&mut b, &mut pw, rlp_header.data_type, LIST as u32); + + // decode all the sub headers now, we know there are only two + let rlp_list = decode_tuple(&mut b, &node_targets); + // check the first sub header which is the key of the MPT leaf node + // value of the key header starts after first header and after header of the key item + let expected_key_value_offset = key_header.header_len + header.header_len; + + connect( + &mut b, + &mut pw, + rlp_list.offset[0], + expected_key_value_offset as u32, + ); + connect(&mut b, &mut pw, rlp_list.data_type[0], STRING as u32); + connect( + &mut b, + &mut pw, + rlp_list.len[0], + key_header.value_len as u32, + ); + // check the second sub header which is the key of the MPT leaf node + // value starts after first header, after key header, after key value and after value header + let expected_value_value_offset = value_header.header_len + + key_header.header_len + + key_header.value_len + + header.header_len; + connect( + &mut b, + &mut pw, + rlp_list.offset[1], + expected_value_value_offset as u32, + ); + connect(&mut b, &mut pw, rlp_list.data_type[1], STRING as u32); + connect( + &mut b, + &mut pw, + rlp_list.len[1], + value_header.value_len as u32, + ); + + let data = b.build::(); + let proof = data.prove(pw)?; + data.verify(proof) + } + use std::cmp::Ordering; + /// Function that returns the offset of the gas value in the RLP encoded + /// node containing a transaction. It also returns the gas length. + fn gas_offset_from_rlp_node(node: &[u8]) -> (usize, usize) { + let node_rlp = rlp::Rlp::new(node); + let tuple_info = node_rlp.payload_info().unwrap(); + let tuple_offset = tuple_info.header_len; + assert_eq!(node_rlp.item_count().unwrap(), 2); + let tx_index = 1; + let gas_index = 2; + let mut tx_offset = tuple_offset; + let mut gas_value_len = 0; + let mut gas_offset = 0; + node_rlp.iter().enumerate().for_each(|(i, r)| { + let h = r.payload_info().unwrap(); + tx_offset += h.header_len; + match i.cmp(&tx_index) { + Ordering::Less => tx_offset += h.value_len, + Ordering::Greater => panic!("node should not have more than 2 items"), + Ordering::Equal => { + let tx_rlp = rlp::Rlp::new(r.data().unwrap()); + gas_offset += tx_rlp.payload_info().unwrap().header_len; + tx_rlp.iter().enumerate().for_each(|(j, rr)| { + let hh = rr.payload_info().unwrap(); + match j.cmp(&gas_index) { + Ordering::Less => { + gas_offset += hh.header_len; + gas_offset += hh.value_len; + } + // do nothing as we don't care about the other items + Ordering::Greater => {} + Ordering::Equal => { + // we want the value directly - we skip the header + gas_offset += hh.header_len; + gas_value_len = hh.value_len; + } + } + }); + } + } + }); + (tx_offset + gas_offset, gas_value_len) + } +} diff --git a/src/utils.rs b/src/utils.rs index 53ec8f0da..efc57a1c2 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -141,6 +141,17 @@ pub(crate) mod test { use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig}; use sha3::{Digest, Keccak256}; + pub(crate) fn hash_output_to_field(expected: &[u8]) -> Vec { + let iter_u32 = expected.iter().chunks(4); + iter_u32 + .into_iter() + .map(|chunk| { + let mut chunk_buff = chunk.map(|b| *b).collect::>(); + let u32_num = read_le_u32(&mut chunk_buff.as_slice()); + F::from_canonical_u32(u32_num) + }) + .collect::>() + } pub(crate) fn keccak256(data: &[u8]) -> Vec { let mut hasher = Keccak256::new(); hasher.update(data); From c4b2e7e5a917a7ad684b2f9f2aa5a62f46f1a9fa Mon Sep 17 00:00:00 2001 From: nikkolasg Date: Thu, 30 Nov 2023 18:54:36 +0100 Subject: [PATCH 4/6] full block proving --- Cargo.toml | 1 + src/eth.rs | 43 +++++- src/rlp.rs | 28 ---- src/transaction/mod.rs | 2 +- src/transaction/mpt.rs | 14 +- src/transaction/prover.rs | 300 ++++++++++++++++++++++++++++++++++++++ src/utils.rs | 57 +++++--- 7 files changed, 381 insertions(+), 64 deletions(-) create mode 100644 src/transaction/prover.rs diff --git a/Cargo.toml b/Cargo.toml index 6e588f1f9..3f670fa0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ serde = "1.0.193" # to fetch proof with "node" instead of already encoded struct eth_trie = { git = "https://github.com/nikkolasg/eth-trie.rs" } rlp = "0.5.2" +sha3 = "0.10.8" [dev-dependencies] anyhow = "1.0.75" diff --git a/src/eth.rs b/src/eth.rs index 772996bd2..fbb1c78c8 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -1,12 +1,12 @@ //! Module containing several structure definitions for Ethereum related operations //! such as fetching blocks, transactions, creating MPTs, getting proofs, etc. use anyhow::{Ok, Result}; -use eth_trie::{EthTrie, MemoryDB, Trie}; +use eth_trie::{EthTrie, MemoryDB, Trie, Node}; use ethers::{ providers::{Http, Middleware, Provider}, types::{BlockId, Bytes, Transaction, TransactionReceipt, U64}, }; -use rlp::{Encodable, RlpStream}; +use rlp::{Encodable, Rlp, RlpStream}; use std::{env, sync::Arc}; /// A wrapper around a transaction and its receipt. The receipt is used to filter /// bad transactions, so we only compute over valid transactions. @@ -135,6 +135,45 @@ impl BlockData { } } +/// Extract the hash in case of Extension node, or all the hashes in case of a Branch node. +pub(crate) fn extract_child_hashes(rlp_data: &[u8]) -> Vec> { + let rlp = Rlp::new(rlp_data); + let mut hashes = Vec::new(); + + // Check for branch node (length of 17 items) + if rlp.item_count().unwrap_or(0) == 17 { + for i in 0..16 { + let item = rlp.at(i).unwrap(); + if item.is_data() && item.data().unwrap().len() == 32 { + hashes.push(item.data().unwrap().to_vec()); + } + } + } else if rlp.item_count().unwrap_or(0) == 2 { + // Check for extension or leaf node + let possible_hash = rlp.at(1).unwrap(); + if possible_hash.is_data() && possible_hash.data().unwrap().len() == 32 { + hashes.push(possible_hash.data().unwrap().to_vec()); + } + } + hashes +} +/// Computes the length of the radix, of the "key" to lookup in the MPT trie, from +/// the path of nodes given. +/// TODO: transform that to only use the raw encoded bytes, instead of the nodes. Would +/// allow us to remove the need to give the proofs as nodes. +pub(crate) fn compute_key_length(path: &[Node]) -> usize { + let mut key_len = 0; + for node in path { + match node { + Node::Branch(b) => key_len += 1, + Node::Extension(e) => key_len += e.read().unwrap().prefix.len(), + Node::Leaf(l) => key_len += l.key.len(), + Node::Hash(h) => panic!("what is a hash node!?"), + Node::Empty => panic!("should not be an empty node in the path"), + } + } + key_len + } #[cfg(test)] mod test { use super::*; diff --git a/src/rlp.rs b/src/rlp.rs index 4b38bfca5..2d61eb0ea 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -279,34 +279,6 @@ mod tests { use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig}; use crate::rlp::{RlpHeader, MAX_LEN_BYTES}; - #[test] - fn test_vk() -> Result<()> { - const D: usize = 2; - type C = PoseidonGoldilocksConfig; - type F = >::F; - let config = CircuitConfig::standard_recursion_config(); - use plonky2::field::types::Sample; - let n = 15; - let a = F::rand_vec(n); - let b = F::rand_vec(n); - let mut b1 = CircuitBuilder::::new(config.clone()); - let pw1 = PartialWitness::new(); - b1.constants(&a); - let d1 = b1.build::(); - let p1 = d1.prove(pw1)?; - d1.verify(p1)?; - let digest1 = d1.verifier_only.circuit_digest; - - let mut b2 = CircuitBuilder::::new(config); - b2.constants(&b); - let d2 = b2.build::(); - let pw2 = PartialWitness::new(); - let p2 = d2.prove(pw2)?; - d2.verify(p2)?; - let digest2 = d2.verifier_only.circuit_digest; - assert_eq!(digest1, digest2); - Ok(()) - } // TODO: replace these tests by deterministic tests by cr #[test] diff --git a/src/transaction/mod.rs b/src/transaction/mod.rs index a4003003c..e918a38ea 100644 --- a/src/transaction/mod.rs +++ b/src/transaction/mod.rs @@ -1,2 +1,2 @@ mod mpt; -//mod prover; +mod prover; diff --git a/src/transaction/mpt.rs b/src/transaction/mpt.rs index 7f82b1ba0..01c7a61f4 100644 --- a/src/transaction/mpt.rs +++ b/src/transaction/mpt.rs @@ -123,7 +123,6 @@ pub fn recursive_node_proof< mut node: Vec, inner_proofs: &[ProofTuple], hash_offsets: &[usize], - hash: &[u8], ) -> Result> where InnerC::Hasher: AlgebraicHasher, @@ -132,7 +131,6 @@ where node.resize(MAX_INTERMEDIATE_NODE_LENGTH, 0); assert_ne!(inner_proofs.len(), 0); assert_eq!(inner_proofs.len(), hash_offsets.len()); - assert_eq!(hash.len(), 32); assert!(node_length <= MAX_INTERMEDIATE_NODE_LENGTH); assert!(node_length > 0); @@ -338,7 +336,6 @@ fn hash_node, const D: usize>( .collect::>(), ); } - #[cfg(test)] mod test { use anyhow::Result; @@ -367,15 +364,12 @@ mod test { use super::{recursive_node_proof, MAX_LEGACY_TX_NODE_LENGTH}; use crate::rlp::{decode_header, decode_tuple}; - use crate::utils::test::{connect, keccak256}; + use crate::utils::find_index_subvector; use crate::utils::test::{data_to_constant_targets, hash_output_to_field}; + use crate::utils::{keccak256, test::connect}; use crate::ProofTuple; use super::{legacy_tx_leaf_node_proof, ExtractionMethod}; - // Returns the index where the subvector starts in v, if any. - fn find_index_subvector(v: &[u8], sub: &[u8]) -> Option { - (0..(v.len() - sub.len())).find(|&i| &v[i..i + sub.len()] == sub) - } #[test] fn test_legacy_full_proof() -> Result<()> { @@ -407,7 +401,7 @@ mod test { } else { let hash_offset = find_index_subvector(&node, &last_hash).unwrap(); let p = last_proof.unwrap(); - recursive_node_proof(&config, node, &[p], &[hash_offset], &last_hash)? + recursive_node_proof(&config, node, &[p], &[hash_offset])? }; let vcd = VerifierCircuitData { @@ -459,7 +453,7 @@ mod test { vcd.verify(leaf_proof.0.clone())?; // verify hash of the node - let expected_hash = crate::utils::test::hash_to_fields::(&node_hash); + let expected_hash = crate::utils::hash_to_fields::(&node_hash); let proof_hash = &leaf_proof.0.public_inputs[0..8]; assert!(expected_hash == proof_hash, "hashes not equal?"); // verify gas value diff --git a/src/transaction/prover.rs b/src/transaction/prover.rs new file mode 100644 index 000000000..86bb0f262 --- /dev/null +++ b/src/transaction/prover.rs @@ -0,0 +1,300 @@ +use crate::utils::find_index_subvector; +use anyhow::Result; +use eth_trie::{Node, Trie}; +use ethers::{types::BlockId, utils::hex}; +use plonky2::{ + field::extension::Extendable, + hash::hash_types::RichField, + plonk::{ + circuit_data::{CircuitConfig, VerifierCircuitData}, + config::{AlgebraicHasher, GenericConfig}, + proof::ProofWithPublicInputs, + }, +}; +use rlp::Encodable; +use std::{collections::HashMap, marker::PhantomData}; +use anyhow::anyhow; +use crate::{ + eth::{compute_key_length, extract_child_hashes, BlockData}, + transaction::mpt::{legacy_tx_leaf_node_proof, recursive_node_proof, ExtractionMethod}, + utils::keccak256, + ProofTuple, +}; + + +struct TxBlockProver, C: GenericConfig, const D: usize> { + data: BlockData, + config: CircuitConfig, + _pf: PhantomData, + _pc: PhantomData, +} + +struct RecursiveProofData, C: GenericConfig, const D: usize> +{ + // The nodes left to traverse in the path to go to the root + mpt_proof: Vec, + mpt_bytes: Vec>, + key: Vec, + key_ptr: usize, + // the current proof proving the whole subtree of the node with the + // given "hash" + proof: ProofWithPublicInputs, + hash: Vec, +} + +struct MPTNode, C: GenericConfig, const D: usize> { + node_bytes: Vec, // RLP byte representation of the node + hash: Vec, // its hash + key_ptr: usize, // ptr to any key in the subtree - act like a "height" in some sort + key: Vec, // any key that leads to this node - key[0..key_ptr] is the same for the whole subtree of this node + // child i : (key, proof) - key needed locate where is the hash of the child in the node + children_proofs: Vec>, // will be filled in + children_hashes: Vec>, // potential hashes of the children if any (zero if leaf for example) + parent_hash: Option>, // indicator to go up one level when this node has been "proven" +} + +struct ProverOutput, C: GenericConfig, const D: usize> { + parent_hash: Option>, + // hash of this node + hash: Vec, + // plonky2 proof for this node + proof: ProofTuple, +} + +type HashTrie = HashMap, MPTNode>; +impl TxBlockProver +where + F: RichField + Extendable, + C: GenericConfig + 'static, + C::Hasher: AlgebraicHasher, +{ + pub async fn init + Send + Sync>(id: T) -> Result { + let data = BlockData::fetch(id).await?; + Ok(Self { + data, + config: CircuitConfig::standard_recursion_config(), + _pf: PhantomData, + _pc: PhantomData, + }) + } + + pub fn prove(&mut self) -> Result> { + let (mut trie, leaves_hashes) = self.init_proofs_trie(); + println!("[+] Built internal trie with {} nodes in total", trie.len()); + let mut current_proofs = leaves_hashes + .iter() + .map(|h| self.run_leaf_proof(&trie, h.clone())) + .collect::>>()?; + while let Some(output) = current_proofs.pop() { + let node_hash = hex::encode(&output.hash); + if output.parent_hash.is_none() { + // we have reached the root node ! + println!("[+] Reached root node {}", node_hash); + return Ok(output.proof); + } + let parent_hash = output.parent_hash.as_ref().unwrap(); + let proof_node = trie + .get_mut(parent_hash) + .expect("every node should be in the trie"); + // a proof for one of the children of this node in the MPT has been computed! + proof_node.children_proofs.push(output); + // look if we have all the individual children proofs to start proving this node now + let exp_children_hashes = &proof_node.children_hashes; + let rcvd_children = proof_node.children_proofs.len(); + let children_proofs_done = if exp_children_hashes.len() != rcvd_children { + println!( + "[+] Node proof {} : only {}/{} children proofs", + &node_hash, + rcvd_children, + exp_children_hashes.len() + ); + false // not the same number of proofs than children in branch node + } else { + // make sure we have all the same hashes + let all = exp_children_hashes + .iter() + .all(|h| proof_node.children_proofs.iter().any(|p| *p.hash == *h)); + println!( + "[+] Node proof {} : ALL {}/{} children proofs!", + &node_hash, + rcvd_children, + exp_children_hashes.len() + ); + assert!(all, "same number of proofs but different hashes !?"); + true + }; + if children_proofs_done { + let parent_proof = self.run_recursive_proof(proof_node)?; + current_proofs.push(parent_proof); + } + } + Err(anyhow!("no root node found")) + } + + fn run_recursive_proof(&self, node: &MPTNode) -> Result> { + let inner_proofs = node + .children_proofs + .iter() + .map(|p| p.proof.clone()) + .collect::>(); + let node_bytes = node.node_bytes.clone(); + let node_hash = keccak256(&node_bytes); + let parent_hash = node.parent_hash.clone(); + let config = self.config.clone(); + let node_key = node.key.clone(); + // where to find the hashes for each children of the node + let children_hash_offsets = node + .children_proofs + .iter() + .map(|p| p.hash.clone()) + .map(|hash| find_index_subvector(&node_bytes, &hash).expect("invalid hash")) + .collect::>(); + println!( + "[+] GO recursive proof for node {} with {} children", + hex::encode(&node_hash), + node.children_hashes.len() + ); + // F, C, C, D because we use same recursive config at each step + let plonk_proof = recursive_node_proof::( + &config, + node_bytes, + inner_proofs.as_slice(), + &children_hash_offsets, + )?; + Self::verify_proof_tuple(&plonk_proof)?; + println!( + "[+] OK Valid recursive proof for node hash {}", + hex::encode(&node_hash) + ); + Ok(ProverOutput { + parent_hash, + proof: plonk_proof, + hash: node_hash, + }) + } + + fn run_leaf_proof( + &self, + trie: &HashTrie, + leaf_hash: Vec, + ) -> Result> { + let mpt_node = trie.get(&leaf_hash).expect("leaf should be inside trie"); + let key = mpt_node.key.clone(); + let node_bytes = mpt_node.node_bytes.clone(); + let parent_hash = mpt_node.parent_hash.clone(); + let config = self.config.clone(); + println!( + "[+] GO leaf proof idx {} - hash {}", + hex::encode(&key), + hex::encode(&leaf_hash) + ); + let plonk_proof = + legacy_tx_leaf_node_proof(&config, node_bytes, ExtractionMethod::RLPBased)?; + Self::verify_proof_tuple(&plonk_proof)?; + + println!( + "[+] OK Valid proof for leaf idx {} - hash {}", + hex::encode(&key), + hex::encode(&leaf_hash) + ); + Ok(ProverOutput { + parent_hash, + proof: plonk_proof, + hash: leaf_hash, + }) + } + + // Returns the hashmap filled with the trie info + // and returns the initial list of nodes's hash, which happen to be leaves, to prove + #[allow(clippy::type_complexity)] + fn init_proofs_trie(&mut self) -> (HashMap, MPTNode>, Vec>) { + // H(node) => { MPTNode() } + let mut tree = HashMap::new(); + let mut leaves = Vec::new(); + for txr in self.data.txs.iter() { + let idx = txr.receipt().transaction_index; + let key = idx.rlp_bytes().to_vec(); + // nikko TODO: only kept for computing key length but can be done only with + // the raw bytes - should change. + let proof_nodes = self.data.tx_trie.get_proof_nodes(&key).unwrap(); + let proof_bytes = self.data.tx_trie.get_proof(&key).unwrap(); + for (i, node_bytes) in proof_bytes.iter().rev().enumerate() { + let idx_in_path = proof_nodes.len() - 1 - i; + let hash = keccak256(node_bytes); + let key_ptr = compute_key_length(&proof_nodes[..idx_in_path]); + if i == 0 { + leaves.push(hash.clone()); + } + let node_bytes = node_bytes.to_vec(); + let children_proofs = vec![]; + let parent_hash = if idx_in_path > 0 { + Some(keccak256(&proof_bytes[idx_in_path - 1])) + } else { + None // root node ! + }; + // nikko TODO: This assumes there is no value in the branch node. + // Will need to make sure this assumption is true in practice for tx at least + let children_hashes = extract_child_hashes(&node_bytes); + let trie_node = MPTNode { + node_bytes, + hash: hash.clone(), + key_ptr, + children_proofs, + children_hashes, + parent_hash, + // nikko: note this will be different for different for diff. leaves under the same + // subtree. However, currently the code expects a key when proving recursively + // so we only need to put one, regardless of all the others in the same subtree. + // in this example, only the key of the last tx in the subtree will be stored/used. + // TODO: remove the key from the recursive API and only pass the new nibble + key: key.clone(), + }; + #[cfg(test)] + { + // if entry is already in the tree, we don't need to add it, but we + // check it's correct still - i.e. all ptr in the subtree starting + // at this node should be the same since all leafs in this subtree + // have the same key until at least this node. + if tree.contains_key(&hash) { + let present_trie_node: &MPTNode = tree.get(&hash).unwrap(); + assert!(present_trie_node.key_ptr == trie_node.key_ptr); + } + } + tree.entry(hash).or_insert(trie_node); + } + } + (tree, leaves) + } + + fn verify_proof_tuple(proof: &ProofTuple) -> Result<()> { + let vcd = VerifierCircuitData { + verifier_only: proof.1.clone(), + common: proof.2.clone(), + }; + vcd.verify(proof.0.clone()) + } +} + +#[cfg(test)] +mod test { + use eth_trie::Trie; + use ethers::types::BlockNumber; + use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig}; + + use crate::{transaction::prover::TxBlockProver, utils::hash_to_fields}; + use anyhow::Result; + + #[tokio::test] + pub async fn prove_all_tx() -> Result<()> { + let block_number = 10593417; + const D: usize = 2; + type C = PoseidonGoldilocksConfig; + type F = >::F; + let mut prover = TxBlockProver::::init(BlockNumber::from(block_number)).await?; + let root_proof = prover.prove()?; + let root_hash = prover.data.tx_trie.root_hash()?.as_bytes().to_vec(); + let expected_pub_inputs = hash_to_fields::(&root_hash); + assert_eq!(expected_pub_inputs, root_proof.0.public_inputs[0..8]); + Ok(()) + } +} diff --git a/src/utils.rs b/src/utils.rs index efc57a1c2..b57da1d92 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,7 +4,21 @@ use plonky2::hash::hash_types::RichField; use plonky2::iop::target::{BoolTarget, Target}; use plonky2::plonk::circuit_builder::CircuitBuilder; use plonky2_crypto::u32::arithmetic_u32::U32Target; +use sha3::Digest; +use sha3::Keccak256; +// Returns the index where the subvector starts in v, if any. +pub(crate) fn find_index_subvector(v: &[u8], sub: &[u8]) -> Option { + (0..(v.len() - sub.len())).find(|&i| &v[i..i + sub.len()] == sub) +} + +/// Compute the keccak256 hash of the given data. +/// NOTE: probably should have two modules for circuit related stuff and non-circuit related stuff +pub(crate) fn keccak256(data: &[u8]) -> Vec { + let mut hasher = Keccak256::new(); + hasher.update(data); + hasher.finalize().to_vec() +} pub(crate) fn convert_u8_to_u32, const D: usize>( b: &mut CircuitBuilder, data: &[Target], @@ -122,6 +136,24 @@ pub fn greater_than_or_equal_to, const D: usize>( let a_plus_1 = builder.add(a, one); less_than(builder, b, a_plus_1, n) } +pub(crate) fn hash_to_fields(expected: &[u8]) -> Vec { + let iter_u32 = expected.iter().chunks(4); + iter_u32 + .into_iter() + .map(|chunk| { + let chunk_buff = chunk.copied().collect::>(); + let u32_num = read_le_u32(&mut chunk_buff.as_slice()); + F::from_canonical_u32(u32_num) + }) + .collect::>() +} + +// taken from rust doc https://doc.rust-lang.org/std/primitive.u32.html#method.from_be_bytes +fn read_le_u32(input: &mut &[u8]) -> u32 { + let (int_bytes, rest) = input.split_at(std::mem::size_of::()); + *input = rest; + u32::from_le_bytes(int_bytes.try_into().unwrap()) +} #[cfg(test)] pub(crate) mod test { @@ -139,7 +171,8 @@ pub(crate) mod test { use plonky2::plonk::circuit_builder::CircuitBuilder; use plonky2::plonk::circuit_data::CircuitConfig; use plonky2::plonk::config::{GenericConfig, PoseidonGoldilocksConfig}; - use sha3::{Digest, Keccak256}; + + use super::read_le_u32; pub(crate) fn hash_output_to_field(expected: &[u8]) -> Vec { let iter_u32 = expected.iter().chunks(4); @@ -152,29 +185,7 @@ pub(crate) mod test { }) .collect::>() } - pub(crate) fn keccak256(data: &[u8]) -> Vec { - let mut hasher = Keccak256::new(); - hasher.update(data); - hasher.finalize().to_vec() - } - pub(crate) fn hash_to_fields(expected: &[u8]) -> Vec { - let iter_u32 = expected.iter().chunks(4); - iter_u32 - .into_iter() - .map(|chunk| { - let chunk_buff = chunk.copied().collect::>(); - let u32_num = read_le_u32(&mut chunk_buff.as_slice()); - F::from_canonical_u32(u32_num) - }) - .collect::>() - } - // taken from rust doc https://doc.rust-lang.org/std/primitive.u32.html#method.from_be_bytes - fn read_le_u32(input: &mut &[u8]) -> u32 { - let (int_bytes, rest) = input.split_at(std::mem::size_of::()); - *input = rest; - u32::from_le_bytes(int_bytes.try_into().unwrap()) - } pub(crate) fn connect, const D: usize, I: Into>( b: &mut CircuitBuilder, pw: &mut PartialWitness, From 279ade5acf1cb3cf6c5c0f99a540106f33429916 Mon Sep 17 00:00:00 2001 From: nikkolasg Date: Thu, 30 Nov 2023 18:58:10 +0100 Subject: [PATCH 5/6] fmt --- src/eth.rs | 22 +++++++++++----------- src/transaction/prover.rs | 15 +++++++-------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/eth.rs b/src/eth.rs index fbb1c78c8..4a2823829 100644 --- a/src/eth.rs +++ b/src/eth.rs @@ -1,7 +1,7 @@ //! Module containing several structure definitions for Ethereum related operations //! such as fetching blocks, transactions, creating MPTs, getting proofs, etc. use anyhow::{Ok, Result}; -use eth_trie::{EthTrie, MemoryDB, Trie, Node}; +use eth_trie::{EthTrie, MemoryDB, Node, Trie}; use ethers::{ providers::{Http, Middleware, Provider}, types::{BlockId, Bytes, Transaction, TransactionReceipt, U64}, @@ -162,18 +162,18 @@ pub(crate) fn extract_child_hashes(rlp_data: &[u8]) -> Vec> { /// TODO: transform that to only use the raw encoded bytes, instead of the nodes. Would /// allow us to remove the need to give the proofs as nodes. pub(crate) fn compute_key_length(path: &[Node]) -> usize { - let mut key_len = 0; - for node in path { - match node { - Node::Branch(b) => key_len += 1, - Node::Extension(e) => key_len += e.read().unwrap().prefix.len(), - Node::Leaf(l) => key_len += l.key.len(), - Node::Hash(h) => panic!("what is a hash node!?"), - Node::Empty => panic!("should not be an empty node in the path"), - } + let mut key_len = 0; + for node in path { + match node { + Node::Branch(b) => key_len += 1, + Node::Extension(e) => key_len += e.read().unwrap().prefix.len(), + Node::Leaf(l) => key_len += l.key.len(), + Node::Hash(h) => panic!("what is a hash node!?"), + Node::Empty => panic!("should not be an empty node in the path"), } - key_len } + key_len +} #[cfg(test)] mod test { use super::*; diff --git a/src/transaction/prover.rs b/src/transaction/prover.rs index 86bb0f262..7f880397c 100644 --- a/src/transaction/prover.rs +++ b/src/transaction/prover.rs @@ -1,4 +1,11 @@ use crate::utils::find_index_subvector; +use crate::{ + eth::{compute_key_length, extract_child_hashes, BlockData}, + transaction::mpt::{legacy_tx_leaf_node_proof, recursive_node_proof, ExtractionMethod}, + utils::keccak256, + ProofTuple, +}; +use anyhow::anyhow; use anyhow::Result; use eth_trie::{Node, Trie}; use ethers::{types::BlockId, utils::hex}; @@ -13,14 +20,6 @@ use plonky2::{ }; use rlp::Encodable; use std::{collections::HashMap, marker::PhantomData}; -use anyhow::anyhow; -use crate::{ - eth::{compute_key_length, extract_child_hashes, BlockData}, - transaction::mpt::{legacy_tx_leaf_node_proof, recursive_node_proof, ExtractionMethod}, - utils::keccak256, - ProofTuple, -}; - struct TxBlockProver, C: GenericConfig, const D: usize> { data: BlockData, From 90925821a208f23252100c5022b5526f326f3695 Mon Sep 17 00:00:00 2001 From: nikkolasg Date: Mon, 4 Dec 2023 11:09:48 +0100 Subject: [PATCH 6/6] move prover to test --- src/transaction/mod.rs | 1 + src/transaction/mpt.rs | 6 +----- src/transaction/prover.rs | 1 - src/utils.rs | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/transaction/mod.rs b/src/transaction/mod.rs index e918a38ea..b16ccfee1 100644 --- a/src/transaction/mod.rs +++ b/src/transaction/mod.rs @@ -1,2 +1,3 @@ mod mpt; +#[cfg(test)] mod prover; diff --git a/src/transaction/mpt.rs b/src/transaction/mpt.rs index 01c7a61f4..434e33611 100644 --- a/src/transaction/mpt.rs +++ b/src/transaction/mpt.rs @@ -8,9 +8,8 @@ use plonky2::{ }, plonk::{ circuit_builder::CircuitBuilder, - circuit_data::{CircuitConfig, CommonCircuitData, ProverCircuitData}, + circuit_data::CircuitConfig, config::{AlgebraicHasher, GenericConfig}, - proof::ProofWithPublicInputs, }, util::ceil_div_usize, }; @@ -342,10 +341,7 @@ mod test { use ethers::types::Transaction; use plonky2::field::extension::Extendable; use plonky2::hash::hash_types::RichField; - use plonky2::hash::keccak; - use plonky2::plonk::circuit_data::{CommonCircuitData, VerifierOnlyCircuitData}; use plonky2::plonk::config::AlgebraicHasher; - use plonky2::plonk::proof::ProofWithPublicInputs; use plonky2::{ iop::witness::PartialWitness, plonk::{ diff --git a/src/transaction/prover.rs b/src/transaction/prover.rs index 7f880397c..5b262f172 100644 --- a/src/transaction/prover.rs +++ b/src/transaction/prover.rs @@ -274,7 +274,6 @@ where } } -#[cfg(test)] mod test { use eth_trie::Trie; use ethers::types::BlockNumber; diff --git a/src/utils.rs b/src/utils.rs index b57da1d92..5e1cf7a01 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -179,7 +179,7 @@ pub(crate) mod test { iter_u32 .into_iter() .map(|chunk| { - let mut chunk_buff = chunk.map(|b| *b).collect::>(); + let mut chunk_buff = chunk.copied().collect::>(); let u32_num = read_le_u32(&mut chunk_buff.as_slice()); F::from_canonical_u32(u32_num) })