feat(sync): MerkleStage (#994)

Co-authored-by: lambdaclass-user <github@lambdaclass.com>
Co-authored-by: Francisco Krause Arnim <fkrausear@gmail.com>
This commit is contained in:
Tomás
2023-02-03 23:50:33 -03:00
committed by GitHub
parent 6dce98cfa8
commit fd7dc11960
20 changed files with 1374 additions and 118 deletions

237
Cargo.lock generated
View File

@ -62,9 +62,9 @@ dependencies = [
[[package]] [[package]]
name = "ahash" name = "ahash"
version = "0.8.2" version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf6ccdb167abbf410dcb915cabd428929d7f6a04980b54a11f26a39f1c7f7107" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"once_cell", "once_cell",
@ -116,9 +116,9 @@ dependencies = [
[[package]] [[package]]
name = "arbitrary" name = "arbitrary"
version = "1.2.2" version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0224938f92e7aef515fac2ff2d18bd1115c1394ddf4a092e0c87e8be9499ee5" checksum = "3e90af4de65aa7b293ef2d09daff88501eb254f58edde2e1ac02c82d873eadad"
dependencies = [ dependencies = [
"derive_arbitrary", "derive_arbitrary",
] ]
@ -168,9 +168,9 @@ dependencies = [
[[package]] [[package]]
name = "async-trait" name = "async-trait"
version = "0.1.61" version = "0.1.64"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "705339e0e4a9690e2908d2b3d049d85682cf19fbd5782494498fbf7003a6a282" checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
dependencies = [ dependencies = [
"proc-macro2 1.0.50", "proc-macro2 1.0.50",
"quote 1.0.23", "quote 1.0.23",
@ -308,6 +308,15 @@ version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf"
[[package]]
name = "basic-toml"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e819b667739967cd44d308b8c7b71305d8bb0729ac44a248aa08f33d01950b4"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "bech32" name = "bech32"
version = "0.7.3" version = "0.7.3"
@ -547,9 +556,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.0.78" version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
[[package]] [[package]]
name = "cexpr" name = "cexpr"
@ -625,6 +634,17 @@ dependencies = [
"inout", "inout",
] ]
[[package]]
name = "cita_trie"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afe7baab47f510f52ca8dc9c0eb9082020c627c7f22285bea30edc3511f7ee29"
dependencies = [
"hasher",
"parking_lot 0.12.1",
"rlp",
]
[[package]] [[package]]
name = "clang-sys" name = "clang-sys"
version = "1.4.0" version = "1.4.0"
@ -650,9 +670,9 @@ dependencies = [
[[package]] [[package]]
name = "clap" name = "clap"
version = "4.1.1" version = "4.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ec7a4128863c188deefe750ac1d1dfe66c236909f845af04beed823638dc1b2" checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"clap_derive", "clap_derive",
@ -850,9 +870,9 @@ dependencies = [
[[package]] [[package]]
name = "crc" name = "crc"
version = "3.0.0" version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe"
dependencies = [ dependencies = [
"crc-catalog", "crc-catalog",
] ]
@ -1034,9 +1054,9 @@ dependencies = [
[[package]] [[package]]
name = "cxx" name = "cxx"
version = "1.0.86" version = "1.0.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d1075c37807dcf850c379432f0df05ba52cc30f279c5cfc43cc221ce7f8579" checksum = "322296e2f2e5af4270b54df9e85a02ff037e271af20ba3e7fe1575515dc840b8"
dependencies = [ dependencies = [
"cc", "cc",
"cxxbridge-flags", "cxxbridge-flags",
@ -1046,9 +1066,9 @@ dependencies = [
[[package]] [[package]]
name = "cxx-build" name = "cxx-build"
version = "1.0.86" version = "1.0.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5044281f61b27bc598f2f6647d480aed48d2bf52d6eb0b627d84c0361b17aa70" checksum = "017a1385b05d631e7875b1f151c9f012d37b53491e2a87f65bff5c262b2111d8"
dependencies = [ dependencies = [
"cc", "cc",
"codespan-reporting", "codespan-reporting",
@ -1061,15 +1081,15 @@ dependencies = [
[[package]] [[package]]
name = "cxxbridge-flags" name = "cxxbridge-flags"
version = "1.0.86" version = "1.0.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61b50bc93ba22c27b0d31128d2d130a0a6b3d267ae27ef7e4fae2167dfe8781c" checksum = "c26bbb078acf09bc1ecda02d4223f03bdd28bd4874edcb0379138efc499ce971"
[[package]] [[package]]
name = "cxxbridge-macro" name = "cxxbridge-macro"
version = "1.0.86" version = "1.0.88"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39e61fda7e62115119469c7b3591fd913ecca96fb766cfd3f2e2502ab7bc87a5" checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e"
dependencies = [ dependencies = [
"proc-macro2 1.0.50", "proc-macro2 1.0.50",
"quote 1.0.23", "quote 1.0.23",
@ -1187,9 +1207,9 @@ dependencies = [
[[package]] [[package]]
name = "derive_arbitrary" name = "derive_arbitrary"
version = "1.2.2" version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf460bbff5f571bfc762da5102729f59f338be7db17a21fade44c5c4f5005350" checksum = "8beee4701e2e229e8098bbdecdca12449bc3e322f137d269182fa1291e20bd00"
dependencies = [ dependencies = [
"proc-macro2 1.0.50", "proc-macro2 1.0.50",
"quote 1.0.23", "quote 1.0.23",
@ -1406,9 +1426,9 @@ dependencies = [
[[package]] [[package]]
name = "either" name = "either"
version = "1.8.0" version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]] [[package]]
name = "elliptic-curve" name = "elliptic-curve"
@ -1628,7 +1648,7 @@ dependencies = [
[[package]] [[package]]
name = "ethers-contract" name = "ethers-contract"
version = "1.0.2" version = "1.0.2"
source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1"
dependencies = [ dependencies = [
"ethers-core", "ethers-core",
"ethers-providers", "ethers-providers",
@ -1644,7 +1664,7 @@ dependencies = [
[[package]] [[package]]
name = "ethers-core" name = "ethers-core"
version = "1.0.2" version = "1.0.2"
source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1"
dependencies = [ dependencies = [
"arrayvec", "arrayvec",
"bytes", "bytes",
@ -1675,7 +1695,7 @@ dependencies = [
[[package]] [[package]]
name = "ethers-etherscan" name = "ethers-etherscan"
version = "1.0.2" version = "1.0.2"
source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1"
dependencies = [ dependencies = [
"ethers-core", "ethers-core",
"getrandom 0.2.8", "getrandom 0.2.8",
@ -1691,7 +1711,7 @@ dependencies = [
[[package]] [[package]]
name = "ethers-middleware" name = "ethers-middleware"
version = "1.0.2" version = "1.0.2"
source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"auto_impl 0.5.0", "auto_impl 0.5.0",
@ -1716,7 +1736,7 @@ dependencies = [
[[package]] [[package]]
name = "ethers-providers" name = "ethers-providers"
version = "1.0.2" version = "1.0.2"
source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"auto_impl 1.0.1", "auto_impl 1.0.1",
@ -1753,7 +1773,7 @@ dependencies = [
[[package]] [[package]]
name = "ethers-signers" name = "ethers-signers"
version = "1.0.2" version = "1.0.2"
source = "git+https://github.com/gakonst/ethers-rs#9ebc5b378c672e03cf2cc35cc7a1aa52d326dfae" source = "git+https://github.com/gakonst/ethers-rs#91d88288a652119c40dc4698a04feef8943ea3f1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"coins-bip32", "coins-bip32",
@ -1858,9 +1878,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -1873,9 +1893,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -1883,15 +1903,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@ -1900,9 +1920,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531"
[[package]] [[package]]
name = "futures-lite" name = "futures-lite"
@ -1931,9 +1951,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70"
dependencies = [ dependencies = [
"proc-macro2 1.0.50", "proc-macro2 1.0.50",
"quote 1.0.23", "quote 1.0.23",
@ -1942,15 +1962,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366"
[[package]] [[package]]
name = "futures-timer" name = "futures-timer"
@ -1964,9 +1984,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.25" version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -2063,9 +2083,9 @@ dependencies = [
[[package]] [[package]]
name = "gloo-net" name = "gloo-net"
version = "0.2.5" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9050ff8617e950288d7bf7f300707639fdeda5ca0d0ecf380cff448cfd52f4a6" checksum = "9902a044653b26b99f7e3693a42f171312d9be8b26b5697bd1e43ad1f8a35e10"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -2083,9 +2103,9 @@ dependencies = [
[[package]] [[package]]
name = "gloo-timers" name = "gloo-timers"
version = "0.2.5" version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98c4a8d6391675c6b2ee1a6c8d06e8e2d03605c44cec1270675985a4c2a5500b" checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -2181,10 +2201,19 @@ version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
dependencies = [ dependencies = [
"ahash 0.8.2", "ahash 0.8.3",
"serde", "serde",
] ]
[[package]]
name = "hasher"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbba678b6567f27ce22870d951f4208e1dc2fef64993bd4521b1d497ef8a3aa"
dependencies = [
"tiny-keccak",
]
[[package]] [[package]]
name = "hashers" name = "hashers"
version = "1.0.1" version = "1.0.1"
@ -2954,9 +2983,9 @@ dependencies = [
[[package]] [[package]]
name = "matches" name = "matches"
version = "0.1.9" version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]] [[package]]
name = "memchr" name = "memchr"
@ -3105,6 +3134,15 @@ dependencies = [
"minimal-lexical", "minimal-lexical",
] ]
[[package]]
name = "nom8"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "nu-ansi-term" name = "nu-ansi-term"
version = "0.46.0" version = "0.46.0"
@ -3142,9 +3180,9 @@ dependencies = [
[[package]] [[package]]
name = "num-complex" name = "num-complex"
version = "0.4.2" version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d"
dependencies = [ dependencies = [
"num-traits", "num-traits",
] ]
@ -3310,9 +3348,9 @@ dependencies = [
[[package]] [[package]]
name = "parity-scale-codec" name = "parity-scale-codec"
version = "3.2.1" version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "366e44391a8af4cfd6002ef6ba072bae071a96aafca98d7d448a34c5dca38b6a" checksum = "c3840933452adf7b3b9145e27086a5a3376c619dca1a21b1e5a5af0d54979bed"
dependencies = [ dependencies = [
"arrayvec", "arrayvec",
"bitvec 1.0.1", "bitvec 1.0.1",
@ -3325,9 +3363,9 @@ dependencies = [
[[package]] [[package]]
name = "parity-scale-codec-derive" name = "parity-scale-codec-derive"
version = "3.1.3" version = "3.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9299338969a3d2f491d65f140b00ddec470858402f888af98e8642fb5e8965cd" checksum = "86b26a931f824dd4eca30b3e43bb4f31cd5f0d3a403c5f5ff27106b805bfde7b"
dependencies = [ dependencies = [
"proc-macro-crate", "proc-macro-crate",
"proc-macro2 1.0.50", "proc-macro2 1.0.50",
@ -3446,9 +3484,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
[[package]] [[package]]
name = "pest" name = "pest"
version = "2.5.3" version = "2.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a" checksum = "4ab62d2fa33726dbe6321cc97ef96d8cde531e3eeaf858a058de53a8a6d40d8f"
dependencies = [ dependencies = [
"thiserror", "thiserror",
"ucd-trie", "ucd-trie",
@ -3594,13 +3632,12 @@ dependencies = [
[[package]] [[package]]
name = "proc-macro-crate" name = "proc-macro-crate"
version = "1.2.1" version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eda0fc3b0fb7c975631757e14d9049da17374063edb6ebbcbc54d880d4fe94e9" checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"thiserror", "toml_edit",
"toml",
] ]
[[package]] [[package]]
@ -3866,9 +3903,9 @@ dependencies = [
[[package]] [[package]]
name = "rayon-core" name = "rayon-core"
version = "1.10.1" version = "1.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
dependencies = [ dependencies = [
"crossbeam-channel", "crossbeam-channel",
"crossbeam-deque", "crossbeam-deque",
@ -3985,7 +4022,7 @@ name = "reth"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"backon", "backon",
"clap 4.1.1", "clap 4.1.4",
"comfy-table", "comfy-table",
"confy", "confy",
"crossterm", "crossterm",
@ -4650,12 +4687,16 @@ name = "reth-stages"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"aquamarine", "aquamarine",
"arbitrary",
"assert_matches", "assert_matches",
"async-trait", "async-trait",
"cita_trie",
"futures-util", "futures-util",
"hasher",
"itertools 0.10.5", "itertools 0.10.5",
"metrics", "metrics",
"paste", "paste",
"proptest",
"rand 0.8.5", "rand 0.8.5",
"rayon", "rayon",
"reth-db", "reth-db",
@ -4666,12 +4707,14 @@ dependencies = [
"reth-primitives", "reth-primitives",
"reth-provider", "reth-provider",
"reth-rlp", "reth-rlp",
"reth-staged-sync",
"serde", "serde",
"tempfile", "tempfile",
"thiserror", "thiserror",
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"tracing", "tracing",
"triehash",
] ]
[[package]] [[package]]
@ -5078,9 +5121,9 @@ dependencies = [
[[package]] [[package]]
name = "security-framework" name = "security-framework"
version = "2.7.0" version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"core-foundation", "core-foundation",
@ -5091,9 +5134,9 @@ dependencies = [
[[package]] [[package]]
name = "security-framework-sys" name = "security-framework-sys"
version = "2.6.1" version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4"
dependencies = [ dependencies = [
"core-foundation-sys", "core-foundation-sys",
"libc", "libc",
@ -5134,9 +5177,9 @@ checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0"
[[package]] [[package]]
name = "send_wrapper" name = "send_wrapper"
version = "0.5.0" version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "930c0acf610d3fdb5e2ab6213019aaa04e227ebe9547b0649ba599b16d788bd7" checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73"
[[package]] [[package]]
name = "serde" name = "serde"
@ -5778,9 +5821,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.24.2" version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"bytes", "bytes",
@ -5887,13 +5930,30 @@ dependencies = [
[[package]] [[package]]
name = "toml" name = "toml"
version = "0.5.10" version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
dependencies = [ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "toml_datetime"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5"
[[package]]
name = "toml_edit"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b"
dependencies = [
"indexmap",
"nom8",
"toml_datetime",
]
[[package]] [[package]]
name = "toolchain_find" name = "toolchain_find"
version = "0.2.0" version = "0.2.0"
@ -6158,17 +6218,17 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
[[package]] [[package]]
name = "trybuild" name = "trybuild"
version = "1.0.76" version = "1.0.77"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed2c57956f91546d4d33614265a85d55c8e1ab91484853a10335894786d7db6" checksum = "a44da5a6f2164c8e14d3bbc0657d69c5966af9f5f6930d4f600b1f5c4a673413"
dependencies = [ dependencies = [
"basic-toml",
"glob", "glob",
"once_cell", "once_cell",
"serde", "serde",
"serde_derive", "serde_derive",
"serde_json", "serde_json",
"termcolor", "termcolor",
"toml",
] ]
[[package]] [[package]]
@ -6229,9 +6289,9 @@ dependencies = [
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.8" version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58"
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
@ -6613,16 +6673,17 @@ dependencies = [
[[package]] [[package]]
name = "ws_stream_wasm" name = "ws_stream_wasm"
version = "0.7.3" version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47ca1ab42f5afed7fc332b22b6e932ca5414b209465412c8cdf0ad23bc0de645" checksum = "7999f5f4217fe3818726b66257a4475f71e74ffd190776ad053fa159e50737f5"
dependencies = [ dependencies = [
"async_io_stream", "async_io_stream",
"futures", "futures",
"js-sys", "js-sys",
"log",
"pharos", "pharos",
"rustc_version", "rustc_version",
"send_wrapper 0.5.0", "send_wrapper 0.6.0",
"thiserror", "thiserror",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",

View File

@ -43,6 +43,8 @@ pub enum Error {
HeaderGasUsedExceedsGasLimit { gas_used: u64, gas_limit: u64 }, HeaderGasUsedExceedsGasLimit { gas_used: u64, gas_limit: u64 },
#[error("Block ommer hash ({got:?}) is different then expected: ({expected:?})")] #[error("Block ommer hash ({got:?}) is different then expected: ({expected:?})")]
BodyOmmersHashDiff { got: H256, expected: H256 }, BodyOmmersHashDiff { got: H256, expected: H256 },
#[error("Block state root ({got:?}) is different then expected: ({expected:?})")]
BodyStateRootDiff { got: H256, expected: H256 },
#[error("Block transaction root ({got:?}) is different then expected: ({expected:?})")] #[error("Block transaction root ({got:?}) is different then expected: ({expected:?})")]
BodyTransactionRootDiff { got: H256, expected: H256 }, BodyTransactionRootDiff { got: H256, expected: H256 },
#[error("Block receipts root ({got:?}) is different then expected: ({expected:?}).")] #[error("Block receipts root ({got:?}) is different then expected: ({expected:?}).")]

View File

@ -173,7 +173,7 @@ pub fn random_eoa_account() -> (Address, Account) {
(addr, Account { nonce, balance, bytecode_hash: None }) (addr, Account { nonce, balance, bytecode_hash: None })
} }
/// Docs /// Generate random Externaly Owned Accounts
pub fn random_eoa_account_range(acc_range: &mut std::ops::Range<u64>) -> Vec<(Address, Account)> { pub fn random_eoa_account_range(acc_range: &mut std::ops::Range<u64>) -> Vec<(Address, Account)> {
let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize); let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize);
for _ in acc_range { for _ in acc_range {
@ -182,6 +182,19 @@ pub fn random_eoa_account_range(acc_range: &mut std::ops::Range<u64>) -> Vec<(Ad
accounts accounts
} }
/// Generate random Contract Accounts
pub fn random_contract_account_range(
acc_range: &mut std::ops::Range<u64>,
) -> Vec<(Address, Account)> {
let mut accounts = Vec::with_capacity(acc_range.end.saturating_sub(acc_range.start) as usize);
for _ in acc_range {
let (address, eoa_account) = random_eoa_account();
let account = Account { bytecode_hash: Some(H256::random()), ..eoa_account };
accounts.push((address, account))
}
accounts
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use std::str::FromStr; use std::str::FromStr;

View File

@ -54,7 +54,7 @@ pub use log::Log;
pub use net::NodeRecord; pub use net::NodeRecord;
pub use peer::{PeerId, WithPeerId}; pub use peer::{PeerId, WithPeerId};
pub use receipt::Receipt; pub use receipt::Receipt;
pub use storage::StorageEntry; pub use storage::{StorageEntry, StorageTrieEntry};
pub use transaction::{ pub use transaction::{
AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature, AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature,
Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559, Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559,

View File

@ -20,7 +20,7 @@ pub const EMPTY_ROOT: H256 =
/// A [Hasher] that calculates a keccak256 hash of the given data. /// A [Hasher] that calculates a keccak256 hash of the given data.
#[derive(Default, Debug, Clone, PartialEq, Eq)] #[derive(Default, Debug, Clone, PartialEq, Eq)]
pub(crate) struct KeccakHasher; pub struct KeccakHasher;
impl Hasher for KeccakHasher { impl Hasher for KeccakHasher {
type Out = H256; type Out = H256;

View File

@ -31,3 +31,34 @@ impl Compact for StorageEntry {
(Self { key, value }, out) (Self { key, value }, out)
} }
} }
/// Account storage trie node.
#[derive_arbitrary(compact)]
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)]
pub struct StorageTrieEntry {
/// Hashed storage key.
pub hash: H256,
/// Encoded node.
pub node: Vec<u8>,
}
// NOTE: Removing main_codec and manually encode subkey
// and compress second part of the value. If we have compression
// over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey
impl Compact for StorageTrieEntry {
fn to_compact(self, buf: &mut impl bytes::BufMut) -> usize {
// for now put full bytes and later compress it.
buf.put_slice(&self.hash.to_fixed_bytes()[..]);
buf.put_slice(&self.node[..]);
self.node.len() + 32
}
fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8])
where
Self: Sized,
{
let key = H256::from_slice(&buf[..32]);
let node = Vec::from(&buf[32..len]);
(Self { hash: key, node }, &buf[len..])
}
}

View File

@ -313,6 +313,7 @@ mod ethereum_types_support {
fixed_revm_uint_impl!(RU128, 16); fixed_revm_uint_impl!(RU128, 16);
fixed_revm_uint_impl!(RU256, 32); fixed_revm_uint_impl!(RU256, 32);
impl_max_encoded_len!(RU256, { length_of_length(32) + 32 });
} }
macro_rules! slice_impl { macro_rules! slice_impl {
@ -403,9 +404,9 @@ where
} }
} }
pub fn encode_iter<'a, K>(i: impl Iterator<Item = &'a K> + Clone, out: &mut dyn BufMut) pub fn encode_iter<K>(i: impl Iterator<Item = K> + Clone, out: &mut dyn BufMut)
where where
K: Encodable + 'a, K: Encodable,
{ {
let mut h = Header { list: true, payload_length: 0 }; let mut h = Header { list: true, payload_length: 0 };
for x in i.clone() { for x in i.clone() {

View File

@ -38,6 +38,15 @@ thiserror = "1.0.37"
aquamarine = "0.2.1" aquamarine = "0.2.1"
itertools = "0.10.5" itertools = "0.10.5"
rayon = "1.6.0" rayon = "1.6.0"
# trie
cita_trie = "4.0.0"
hasher = "0.1.4"
# arbitrary utils
arbitrary = { version = "1.1.7", features = ["derive"], optional = true }
proptest = { version = "1.0", optional = true }
[dev-dependencies] [dev-dependencies]
# reth # reth
reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] } reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] }
@ -50,6 +59,14 @@ assert_matches = "1.5.0"
rand = "0.8.5" rand = "0.8.5"
paste = "1.0" paste = "1.0"
# arbitrary utils
arbitrary = { version = "1.1.7", features = ["derive"] }
proptest = { version = "1.0" }
# trie
reth-staged-sync = { path = "../staged-sync" }
triehash = "0.8"
[features] [features]
default = ["serde"] default = ["serde"]
serde = ["dep:serde"] serde = ["dep:serde"]

View File

@ -13,7 +13,7 @@ use reth_db::{
transaction::{DbTx, DbTxMut}, transaction::{DbTx, DbTxMut},
Error, Error,
}; };
use reth_primitives::{BlockHash, BlockNumber, TransitionId, TxNumber}; use reth_primitives::{BlockHash, BlockNumber, Header, TransitionId, TxNumber};
use crate::{DatabaseIntegrityError, StageError}; use crate::{DatabaseIntegrityError, StageError};
@ -164,6 +164,15 @@ where
Ok((prev_body.start_tx_id + prev_body.tx_count, last_transition)) Ok((prev_body.start_tx_id + prev_body.tx_count, last_transition))
} }
/// Query the block header by number
pub(crate) fn get_header_by_num(&self, block: BlockNumber) -> Result<Header, StageError> {
let key = self.get_block_numhash(block)?;
let header = self
.get::<tables::Headers>(key)?
.ok_or(DatabaseIntegrityError::Header { number: block, hash: key.hash() })?;
Ok(header)
}
/// Unwind table by some number key /// Unwind table by some number key
#[inline] #[inline]
pub(crate) fn unwind_table_by_num<T>(&self, num: u64) -> Result<(), Error> pub(crate) fn unwind_table_by_num<T>(&self, num: u64) -> Result<(), Error>

View File

@ -54,6 +54,7 @@ mod error;
mod id; mod id;
mod pipeline; mod pipeline;
mod stage; mod stage;
mod trie;
mod util; mod util;
#[cfg(test)] #[cfg(test)]

View File

@ -164,10 +164,10 @@ pub struct HashingStages;
impl<DB: Database> StageSet<DB> for HashingStages { impl<DB: Database> StageSet<DB> for HashingStages {
fn builder(self) -> StageSetBuilder<DB> { fn builder(self) -> StageSetBuilder<DB> {
StageSetBuilder::default() StageSetBuilder::default()
.add_stage(MerkleStage::Unwind) .add_stage(MerkleStage::default_unwind())
.add_stage(AccountHashingStage::default()) .add_stage(AccountHashingStage::default())
.add_stage(StorageHashingStage::default()) .add_stage(StorageHashingStage::default())
.add_stage(MerkleStage::Execution) .add_stage(MerkleStage::default_execution())
} }
} }

View File

@ -56,8 +56,9 @@ impl<DB: Database> Stage<DB> for AccountHashingStage {
// if there are more blocks then threshold it is faster to go over Plain state and hash all // if there are more blocks then threshold it is faster to go over Plain state and hash all
// account otherwise take changesets aggregate the sets and apply hashing to // account otherwise take changesets aggregate the sets and apply hashing to
// AccountHashing table // AccountHashing table. Also, if we start from genesis, we need to hash from scratch, as
if to_transition - from_transition > self.clean_threshold { // genesis accounts are not in changeset.
if to_transition - from_transition > self.clean_threshold || stage_progress == 0 {
// clear table, load all accounts and hash it // clear table, load all accounts and hash it
tx.clear::<tables::HashedAccount>()?; tx.clear::<tables::HashedAccount>()?;
tx.commit()?; tx.commit()?;

View File

@ -54,8 +54,9 @@ impl<DB: Database> Stage<DB> for StorageHashingStage {
// if there are more blocks then threshold it is faster to go over Plain state and hash all // if there are more blocks then threshold it is faster to go over Plain state and hash all
// account otherwise take changesets aggregate the sets and apply hashing to // account otherwise take changesets aggregate the sets and apply hashing to
// AccountHashing table // AccountHashing table. Also, if we start from genesis, we need to hash from scratch, as
if to_transition - from_transition > self.clean_threshold { // genesis accounts are not in changeset, along with their storages.
if to_transition - from_transition > self.clean_threshold || stage_progress == 0 {
tx.clear::<tables::HashedStorage>()?; tx.clear::<tables::HashedStorage>()?;
tx.commit()?; tx.commit()?;

View File

@ -1,7 +1,9 @@
use crate::{ use crate::{
db::Transaction, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput, db::Transaction, trie::DBTrieLoader, ExecInput, ExecOutput, Stage, StageError, StageId,
UnwindInput, UnwindOutput,
}; };
use reth_db::database::Database; use reth_db::{database::Database, tables, transaction::DbTx};
use reth_interfaces::consensus;
use std::fmt::Debug; use std::fmt::Debug;
use tracing::*; use tracing::*;
@ -34,10 +36,29 @@ pub const MERKLE_UNWIND: StageId = StageId("MerkleUnwind");
/// - [`MerkleStage::Execution`] /// - [`MerkleStage::Execution`]
#[derive(Debug)] #[derive(Debug)]
pub enum MerkleStage { pub enum MerkleStage {
/// The execution portion of the hashing stage. /// The execution portion of the merkle stage.
Execution, Execution {
/// The unwind portion of the hasing stage. /// The threshold for switching from incremental trie building
/// of changes to whole rebuild. Num of transitions.
clean_threshold: u64,
},
/// The unwind portion of the merkle stage.
Unwind, Unwind,
#[cfg(test)]
Both { clean_threshold: u64 },
}
impl MerkleStage {
/// Stage default for the Execution variant.
pub fn default_execution() -> Self {
Self::Execution { clean_threshold: 5_000 }
}
/// Stage default for the Unwind variant.
pub fn default_unwind() -> Self {
Self::Unwind
}
} }
#[async_trait::async_trait] #[async_trait::async_trait]
@ -45,23 +66,64 @@ impl<DB: Database> Stage<DB> for MerkleStage {
/// Return the id of the stage /// Return the id of the stage
fn id(&self) -> StageId { fn id(&self) -> StageId {
match self { match self {
MerkleStage::Execution => MERKLE_EXECUTION, MerkleStage::Execution { .. } => MERKLE_EXECUTION,
MerkleStage::Unwind => MERKLE_UNWIND, MerkleStage::Unwind => MERKLE_UNWIND,
#[cfg(test)]
MerkleStage::Both { .. } => unreachable!(),
} }
} }
/// Execute the stage. /// Execute the stage.
async fn execute( async fn execute(
&mut self, &mut self,
_tx: &mut Transaction<'_, DB>, tx: &mut Transaction<'_, DB>,
input: ExecInput, input: ExecInput,
) -> Result<ExecOutput, StageError> { ) -> Result<ExecOutput, StageError> {
if matches!(self, MerkleStage::Unwind) { let threshold = match self {
info!(target: "sync::stages::merkle::unwind", "Stage is always skipped"); MerkleStage::Unwind => {
return Ok(ExecOutput { stage_progress: input.previous_stage_progress(), done: true }) info!(target: "sync::stages::merkle::unwind", "Stage is always skipped");
} return Ok(ExecOutput {
stage_progress: input.previous_stage_progress(),
done: true,
})
}
MerkleStage::Execution { clean_threshold } => *clean_threshold,
#[cfg(test)]
MerkleStage::Both { clean_threshold } => *clean_threshold,
};
// Iterate over changeset (similar to Hashing stages) and take new values let stage_progress = input.stage_progress.unwrap_or_default();
let previous_stage_progress = input.previous_stage_progress();
let from_transition = tx.get_block_transition(stage_progress)?;
let to_transition = tx.get_block_transition(previous_stage_progress)?;
let block_root = tx.get_header_by_num(previous_stage_progress)?.state_root;
let trie_root = if from_transition == to_transition {
block_root
} else if to_transition - from_transition > threshold || stage_progress == 0 {
debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Rebuilding trie");
// if there are more blocks than threshold it is faster to rebuild the trie
let loader = DBTrieLoader::default();
loader.calculate_root(tx).map_err(|e| StageError::Fatal(Box::new(e)))?
} else {
debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Updating trie");
// Iterate over changeset (similar to Hashing stages) and take new values
let current_root = tx.get_header_by_num(stage_progress)?.state_root;
let loader = DBTrieLoader::default();
loader
.update_root(tx, current_root, from_transition..to_transition)
.map_err(|e| StageError::Fatal(Box::new(e)))?
};
if block_root != trie_root {
warn!(target: "sync::stages::merkle::exec", ?previous_stage_progress, got = ?block_root, expected = ?trie_root, "Block's root state failed verification");
return Err(StageError::Validation {
block: previous_stage_progress,
error: consensus::Error::BodyStateRootDiff { got: trie_root, expected: block_root },
})
}
info!(target: "sync::stages::merkle::exec", "Stage finished"); info!(target: "sync::stages::merkle::exec", "Stage finished");
Ok(ExecOutput { stage_progress: input.previous_stage_progress(), done: true }) Ok(ExecOutput { stage_progress: input.previous_stage_progress(), done: true })
@ -70,15 +132,419 @@ impl<DB: Database> Stage<DB> for MerkleStage {
/// Unwind the stage. /// Unwind the stage.
async fn unwind( async fn unwind(
&mut self, &mut self,
_tx: &mut Transaction<'_, DB>, tx: &mut Transaction<'_, DB>,
input: UnwindInput, input: UnwindInput,
) -> Result<UnwindOutput, StageError> { ) -> Result<UnwindOutput, StageError> {
if matches!(self, MerkleStage::Execution) { if matches!(self, MerkleStage::Execution { .. }) {
info!(target: "sync::stages::merkle::exec", "Stage is always skipped"); info!(target: "sync::stages::merkle::exec", "Stage is always skipped");
return Ok(UnwindOutput { stage_progress: input.unwind_to }) return Ok(UnwindOutput { stage_progress: input.unwind_to })
} }
let target_root = tx.get_header_by_num(input.unwind_to)?.state_root;
// If the merkle stage fails to execute, the trie changes weren't commited
// and the root stayed the same
if tx.get::<tables::AccountsTrie>(target_root)?.is_some() {
info!(target: "sync::stages::merkle::unwind", "Stage skipped");
return Ok(UnwindOutput { stage_progress: input.unwind_to })
}
let loader = DBTrieLoader::default();
let current_root = tx.get_header_by_num(input.stage_progress)?.state_root;
let from_transition = tx.get_block_transition(input.unwind_to)?;
let to_transition = tx.get_block_transition(input.stage_progress)?;
loader
.update_root(tx, current_root, from_transition..to_transition)
.map_err(|e| StageError::Fatal(Box::new(e)))?;
info!(target: "sync::stages::merkle::unwind", "Stage finished"); info!(target: "sync::stages::merkle::unwind", "Stage finished");
Ok(UnwindOutput { stage_progress: input.unwind_to }) Ok(UnwindOutput { stage_progress: input.unwind_to })
} }
} }
#[cfg(test)]
mod tests {
use super::*;
use crate::test_utils::{
stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError,
TestTransaction, UnwindStageTestRunner, PREV_STAGE_ID,
};
use assert_matches::assert_matches;
use reth_db::{
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
models::{AccountBeforeTx, BlockNumHash, StoredBlockBody},
tables,
transaction::{DbTx, DbTxMut},
};
use reth_interfaces::test_utils::generators::{
random_block, random_block_range, random_contract_account_range,
};
use reth_primitives::{keccak256, Account, Address, SealedBlock, StorageEntry, H256, U256};
use std::collections::BTreeMap;
stage_test_suite_ext!(MerkleTestRunner, merkle);
/// Execute from genesis so as to merkelize whole state
#[tokio::test]
async fn execute_clean_merkle() {
let (previous_stage, stage_progress) = (500, 0);
// Set up the runner
let mut runner = MerkleTestRunner::default();
// set low threshold so we hash the whole storage
let input = ExecInput {
previous_stage: Some((PREV_STAGE_ID, previous_stage)),
stage_progress: Some(stage_progress),
};
runner.seed_execution(input).expect("failed to seed execution");
let rx = runner.execute(input);
// Assert the successful result
let result = rx.await.unwrap();
assert_matches!(
result,
Ok(ExecOutput { done, stage_progress })
if done && stage_progress == previous_stage
);
// Validate the stage execution
assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation");
}
/// Update small trie
#[tokio::test]
async fn execute_small_merkle() {
let (previous_stage, stage_progress) = (2, 1);
// Set up the runner
let mut runner = MerkleTestRunner::default();
let input = ExecInput {
previous_stage: Some((PREV_STAGE_ID, previous_stage)),
stage_progress: Some(stage_progress),
};
runner.seed_execution(input).expect("failed to seed execution");
let rx = runner.execute(input);
// Assert the successful result
let result = rx.await.unwrap();
assert_matches!(
result,
Ok(ExecOutput { done, stage_progress })
if done && stage_progress == previous_stage
);
// Validate the stage execution
assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation");
}
struct MerkleTestRunner {
tx: TestTransaction,
clean_threshold: u64,
}
impl Default for MerkleTestRunner {
fn default() -> Self {
Self { tx: TestTransaction::default(), clean_threshold: 10000 }
}
}
impl StageTestRunner for MerkleTestRunner {
type S = MerkleStage;
fn tx(&self) -> &TestTransaction {
&self.tx
}
fn stage(&self) -> Self::S {
Self::S::Both { clean_threshold: self.clean_threshold }
}
}
#[async_trait::async_trait]
impl ExecuteStageTestRunner for MerkleTestRunner {
type Seed = Vec<SealedBlock>;
fn seed_execution(&mut self, input: ExecInput) -> Result<Self::Seed, TestRunnerError> {
let stage_progress = input.stage_progress.unwrap_or_default();
let end = input.previous_stage_progress() + 1;
let n_accounts = 31;
let mut accounts = random_contract_account_range(&mut (0..n_accounts));
let SealedBlock { header, body, ommers } =
random_block(stage_progress, None, Some(0), None);
let mut header = header.unseal();
header.state_root = self.generate_initial_trie(&accounts)?;
let sealed_head = SealedBlock { header: header.seal(), body, ommers };
let head_hash = sealed_head.hash();
let mut blocks = vec![sealed_head];
blocks.extend(random_block_range((stage_progress + 1)..end, head_hash, 0..3));
self.tx.insert_headers(blocks.iter().map(|block| &block.header))?;
let (mut transition_id, mut tx_id) = (0, 0);
let mut storages: BTreeMap<Address, BTreeMap<H256, U256>> = BTreeMap::new();
for progress in blocks.iter() {
// Insert last progress data
self.tx.commit(|tx| {
let key: BlockNumHash = (progress.number, progress.hash()).into();
let body = StoredBlockBody {
start_tx_id: tx_id,
tx_count: progress.body.len() as u64,
};
progress.body.iter().try_for_each(|transaction| {
tx.put::<tables::TxHashNumber>(transaction.hash(), tx_id)?;
tx.put::<tables::Transactions>(tx_id, transaction.clone())?;
tx.put::<tables::TxTransitionIndex>(tx_id, transition_id)?;
// seed account changeset
let (addr, prev_acc) = accounts
.get_mut(rand::random::<usize>() % n_accounts as usize)
.unwrap();
let acc_before_tx =
AccountBeforeTx { address: *addr, info: Some(*prev_acc) };
tx.put::<tables::AccountChangeSet>(transition_id, acc_before_tx)?;
prev_acc.nonce += 1;
prev_acc.balance = prev_acc.balance.wrapping_add(U256::from(1));
let new_entry = StorageEntry {
key: keccak256([rand::random::<u8>()]),
value: U256::from(rand::random::<u8>() % 30 + 1),
};
let storage = storages.entry(*addr).or_default();
let old_value = storage.entry(new_entry.key).or_default();
tx.put::<tables::StorageChangeSet>(
(transition_id, *addr).into(),
StorageEntry { key: new_entry.key, value: *old_value },
)?;
*old_value = new_entry.value;
tx_id += 1;
transition_id += 1;
Ok(())
})?;
tx.put::<tables::BlockTransitionIndex>(key.number(), transition_id)?;
tx.put::<tables::BlockBodies>(key, body)
})?;
}
self.insert_accounts(&accounts)?;
self.insert_storages(&storages)?;
let last_numhash = self.tx.inner().get_block_numhash(end - 1).unwrap();
let root = self.state_root()?;
self.tx.commit(|tx| {
let mut last_header = tx.get::<tables::Headers>(last_numhash)?.unwrap();
last_header.state_root = root;
tx.put::<tables::Headers>(last_numhash, last_header)
})?;
Ok(blocks)
}
fn validate_execution(
&self,
input: ExecInput,
output: Option<ExecOutput>,
) -> Result<(), TestRunnerError> {
if let Some(output) = output {
let start_block = input.stage_progress.unwrap_or_default() + 1;
let end_block = output.stage_progress;
if start_block > end_block {
return Ok(())
}
}
self.check_root(input.previous_stage_progress())
}
}
impl UnwindStageTestRunner for MerkleTestRunner {
fn before_unwind(&self, input: UnwindInput) -> Result<(), TestRunnerError> {
let target_transition = self
.tx
.inner()
.get_block_transition(input.unwind_to)
.map_err(|e| TestRunnerError::Internal(Box::new(e)))
.unwrap();
self.tx
.commit(|tx| {
let mut changeset_cursor =
tx.cursor_dup_read::<tables::StorageChangeSet>().unwrap();
let mut hash_cursor = tx.cursor_dup_write::<tables::HashedStorage>().unwrap();
let mut rev_changeset_walker = changeset_cursor.walk_back(None).unwrap();
let mut tree: BTreeMap<H256, BTreeMap<H256, U256>> = BTreeMap::new();
while let Some((tid_address, entry)) =
rev_changeset_walker.next().transpose().unwrap()
{
if tid_address.transition_id() < target_transition {
break
}
tree.entry(keccak256(tid_address.address()))
.or_default()
.insert(keccak256(entry.key), entry.value);
}
for (key, val) in tree.into_iter() {
for (entry_key, entry_val) in val.into_iter() {
hash_cursor.seek_by_key_subkey(key, entry_key).unwrap();
hash_cursor.delete_current().unwrap();
if entry_val != U256::ZERO {
let storage_entry =
StorageEntry { key: entry_key, value: entry_val };
hash_cursor.append_dup(key, storage_entry).unwrap();
}
}
}
let mut changeset_cursor =
tx.cursor_dup_write::<tables::AccountChangeSet>().unwrap();
let mut rev_changeset_walker = changeset_cursor.walk_back(None).unwrap();
while let Some((transition_id, account_before_tx)) =
rev_changeset_walker.next().transpose().unwrap()
{
if transition_id < target_transition {
break
}
match account_before_tx.info {
Some(acc) => {
tx.put::<tables::PlainAccountState>(account_before_tx.address, acc)
.unwrap();
tx.put::<tables::HashedAccount>(
keccak256(account_before_tx.address),
acc,
)
.unwrap();
}
None => {
tx.delete::<tables::PlainAccountState>(
account_before_tx.address,
None,
)
.unwrap();
tx.delete::<tables::HashedAccount>(
keccak256(account_before_tx.address),
None,
)
.unwrap();
}
}
}
Ok(())
})
.unwrap();
Ok(())
}
fn validate_unwind(&self, input: UnwindInput) -> Result<(), TestRunnerError> {
self.check_root(input.unwind_to)
}
}
impl MerkleTestRunner {
fn state_root(&self) -> Result<H256, TestRunnerError> {
Ok(DBTrieLoader::default().calculate_root(&self.tx.inner()).unwrap())
}
pub(crate) fn generate_initial_trie(
&self,
accounts: &[(Address, Account)],
) -> Result<H256, TestRunnerError> {
self.insert_accounts(accounts)?;
let loader = DBTrieLoader::default();
let mut tx = self.tx.inner();
let root = loader.calculate_root(&tx).expect("couldn't create initial trie");
tx.commit()?;
Ok(root)
}
pub(crate) fn insert_accounts(
&self,
accounts: &[(Address, Account)],
) -> Result<(), TestRunnerError> {
for (addr, acc) in accounts.iter() {
self.tx.commit(|tx| {
tx.put::<tables::PlainAccountState>(*addr, *acc)?;
tx.put::<tables::HashedAccount>(keccak256(addr), *acc)?;
Ok(())
})?;
}
Ok(())
}
fn insert_storages(
&self,
storages: &BTreeMap<Address, BTreeMap<H256, U256>>,
) -> Result<(), TestRunnerError> {
self.tx
.commit(|tx| {
storages.iter().try_for_each(|(&addr, storage)| {
storage.iter().try_for_each(|(&key, &value)| {
let entry = StorageEntry { key, value };
tx.put::<tables::PlainStorageState>(addr, entry)
})
})?;
storages
.iter()
.map(|(addr, storage)| {
(
keccak256(addr),
storage
.iter()
.filter(|(_, &value)| value != U256::ZERO)
.map(|(key, value)| (keccak256(key), value)),
)
})
.collect::<BTreeMap<_, _>>()
.into_iter()
.try_for_each(|(addr, storage)| {
storage.into_iter().try_for_each(|(key, &value)| {
let entry = StorageEntry { key, value };
tx.put::<tables::HashedStorage>(addr, entry)
})
})?;
Ok(())
})
.map_err(|e| e.into())
}
fn check_root(&self, previous_stage_progress: u64) -> Result<(), TestRunnerError> {
if previous_stage_progress != 0 {
let block_root =
self.tx.inner().get_header_by_num(previous_stage_progress).unwrap().state_root;
let root = DBTrieLoader::default().calculate_root(&self.tx.inner()).unwrap();
assert_eq!(block_root, root);
}
Ok(())
}
}
}

View File

@ -60,6 +60,8 @@ macro_rules! stage_test_suite {
// Seed the database // Seed the database
runner.seed_execution(crate::stage::ExecInput::default()).expect("failed to seed"); runner.seed_execution(crate::stage::ExecInput::default()).expect("failed to seed");
runner.before_unwind(input).expect("failed to execute before_unwind hook");
// Run stage unwind // Run stage unwind
let rx = runner.unwind(input).await; let rx = runner.unwind(input).await;
assert_matches::assert_matches!( assert_matches::assert_matches!(
@ -97,12 +99,15 @@ macro_rules! stage_test_suite {
); );
assert!(runner.validate_execution(execute_input, result.ok()).is_ok(), "execution validation"); assert!(runner.validate_execution(execute_input, result.ok()).is_ok(), "execution validation");
// Run stage unwind // Run stage unwind
let unwind_input = crate::stage::UnwindInput { let unwind_input = crate::stage::UnwindInput {
unwind_to: stage_progress, stage_progress: previous_stage, bad_block: None, unwind_to: stage_progress, stage_progress: previous_stage, bad_block: None,
}; };
let rx = runner.unwind(unwind_input).await;
runner.before_unwind(unwind_input).expect("Failed to unwind state");
let rx = runner.unwind(unwind_input).await;
// Assert the successful unwind result // Assert the successful unwind result
assert_matches::assert_matches!( assert_matches::assert_matches!(
rx, rx,

View File

@ -75,4 +75,9 @@ pub(crate) trait UnwindStageTestRunner: StageTestRunner {
}); });
Box::pin(rx).await.unwrap() Box::pin(rx).await.unwrap()
} }
/// Run a hook before [Stage::unwind]. Required for MerkleStage.
fn before_unwind(&self, _input: UnwindInput) -> Result<(), TestRunnerError> {
Ok(())
}
} }

View File

@ -0,0 +1,630 @@
use crate::Transaction;
use cita_trie::{PatriciaTrie, Trie};
use hasher::HasherKeccak;
use reth_db::{
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
database::Database,
models::{AccountBeforeTx, TransitionIdAddress},
tables,
transaction::{DbTx, DbTxMut},
};
use reth_primitives::{
keccak256, proofs::EMPTY_ROOT, Account, Address, StorageEntry, StorageTrieEntry, TransitionId,
H256, KECCAK_EMPTY, U256,
};
use reth_rlp::{
encode_fixed_size, Decodable, DecodeError, Encodable, RlpDecodable, RlpEncodable,
EMPTY_STRING_CODE,
};
use std::{
collections::{BTreeMap, BTreeSet},
ops::Range,
sync::Arc,
};
use tracing::*;
#[derive(Debug, thiserror::Error)]
pub(crate) enum TrieError {
#[error("Some error occurred: {0}")]
InternalError(#[from] cita_trie::TrieError),
#[error("The root node wasn't found in the DB")]
MissingRoot(H256),
#[error("{0:?}")]
DatabaseError(#[from] reth_db::Error),
#[error("{0:?}")]
DecodeError(#[from] DecodeError),
}
/// Database wrapper implementing HashDB trait.
struct HashDatabase<'tx, 'itx, DB: Database> {
tx: &'tx Transaction<'itx, DB>,
}
impl<'tx, 'itx, DB> cita_trie::DB for HashDatabase<'tx, 'itx, DB>
where
DB: Database,
{
type Error = TrieError;
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
Ok(self.tx.get::<tables::AccountsTrie>(H256::from_slice(key))?)
}
fn contains(&self, key: &[u8]) -> Result<bool, Self::Error> {
Ok(<Self as cita_trie::DB>::get(self, key)?.is_some())
}
fn insert(&self, key: Vec<u8>, value: Vec<u8>) -> Result<(), Self::Error> {
// Caching and bulk inserting shouldn't be needed, as the data is ordered
self.tx.put::<tables::AccountsTrie>(H256::from_slice(key.as_slice()), value)?;
Ok(())
}
fn remove(&self, key: &[u8]) -> Result<(), Self::Error> {
self.tx.delete::<tables::AccountsTrie>(H256::from_slice(key), None)?;
Ok(())
}
fn flush(&self) -> Result<(), Self::Error> {
Ok(())
}
}
impl<'tx, 'itx, DB: Database> HashDatabase<'tx, 'itx, DB> {
/// Instantiates a new Database for the accounts trie, with an empty root
fn new(tx: &'tx Transaction<'itx, DB>) -> Result<Self, TrieError> {
let root = EMPTY_ROOT;
if tx.get::<tables::AccountsTrie>(root)?.is_none() {
tx.put::<tables::AccountsTrie>(root, [EMPTY_STRING_CODE].to_vec())?;
}
Ok(Self { tx })
}
/// Instantiates a new Database for the accounts trie, with an existing root
fn from_root(tx: &'tx Transaction<'itx, DB>, root: H256) -> Result<Self, TrieError> {
if root == EMPTY_ROOT {
return Self::new(tx)
}
tx.get::<tables::AccountsTrie>(root)?.ok_or(TrieError::MissingRoot(root))?;
Ok(Self { tx })
}
}
/// Database wrapper implementing HashDB trait.
struct DupHashDatabase<'tx, 'itx, DB: Database> {
tx: &'tx Transaction<'itx, DB>,
key: H256,
}
impl<'tx, 'itx, DB> cita_trie::DB for DupHashDatabase<'tx, 'itx, DB>
where
DB: Database,
{
type Error = TrieError;
fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
let mut cursor = self.tx.cursor_dup_read::<tables::StoragesTrie>()?;
Ok(cursor.seek_by_key_subkey(self.key, H256::from_slice(key))?.map(|entry| entry.node))
}
fn contains(&self, key: &[u8]) -> Result<bool, Self::Error> {
Ok(<Self as cita_trie::DB>::get(self, key)?.is_some())
}
fn insert(&self, key: Vec<u8>, value: Vec<u8>) -> Result<(), Self::Error> {
// Caching and bulk inserting shouldn't be needed, as the data is ordered
self.tx.put::<tables::StoragesTrie>(
self.key,
StorageTrieEntry { hash: H256::from_slice(key.as_slice()), node: value },
)?;
Ok(())
}
fn remove(&self, key: &[u8]) -> Result<(), Self::Error> {
let mut cursor = self.tx.cursor_dup_write::<tables::StoragesTrie>()?;
cursor
.seek_by_key_subkey(self.key, H256::from_slice(key))?
.map(|_| cursor.delete_current())
.transpose()?;
Ok(())
}
fn flush(&self) -> Result<(), Self::Error> {
Ok(())
}
}
impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> {
/// Instantiates a new Database for the storage trie, with an empty root
fn new(tx: &'tx Transaction<'itx, DB>, key: H256) -> Result<Self, TrieError> {
let root = EMPTY_ROOT;
let mut cursor = tx.cursor_dup_write::<tables::StoragesTrie>()?;
if cursor.seek_by_key_subkey(key, root)?.is_none() {
tx.put::<tables::StoragesTrie>(
key,
StorageTrieEntry { hash: root, node: [EMPTY_STRING_CODE].to_vec() },
)?;
}
Ok(Self { tx, key })
}
/// Instantiates a new Database for the storage trie, with an existing root
fn from_root(tx: &'tx Transaction<'itx, DB>, key: H256, root: H256) -> Result<Self, TrieError> {
if root == EMPTY_ROOT {
return Self::new(tx, key)
}
tx.cursor_dup_read::<tables::StoragesTrie>()?
.seek_by_key_subkey(key, root)?
.ok_or(TrieError::MissingRoot(root))?;
Ok(Self { tx, key })
}
}
/// An Ethereum account, for RLP encoding traits deriving.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)]
pub(crate) struct EthAccount {
/// Account nonce.
nonce: u64,
/// Account balance.
balance: U256,
/// Account's storage root.
storage_root: H256,
/// Hash of the account's bytecode.
code_hash: H256,
}
impl From<Account> for EthAccount {
fn from(acc: Account) -> Self {
EthAccount {
nonce: acc.nonce,
balance: acc.balance,
storage_root: EMPTY_ROOT,
code_hash: acc.bytecode_hash.unwrap_or(KECCAK_EMPTY),
}
}
}
impl EthAccount {
pub(crate) fn from_with_root(acc: Account, storage_root: H256) -> EthAccount {
Self { storage_root, ..Self::from(acc) }
}
}
#[derive(Debug, Default)]
pub(crate) struct DBTrieLoader;
impl DBTrieLoader {
/// Calculates the root of the state trie, saving intermediate hashes in the database.
pub(crate) fn calculate_root<DB: Database>(
&self,
tx: &Transaction<'_, DB>,
) -> Result<H256, TrieError> {
tx.clear::<tables::AccountsTrie>()?;
tx.clear::<tables::StoragesTrie>()?;
let mut accounts_cursor = tx.cursor_read::<tables::HashedAccount>()?;
let mut walker = accounts_cursor.walk(H256::zero())?;
let db = Arc::new(HashDatabase::new(tx)?);
let hasher = Arc::new(HasherKeccak::new());
let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher));
while let Some((hashed_address, account)) = walker.next().transpose()? {
let value = EthAccount::from_with_root(
account,
self.calculate_storage_root(tx, hashed_address)?,
);
let mut out = Vec::new();
Encodable::encode(&value, &mut out);
trie.insert(hashed_address.as_bytes().to_vec(), out)?;
}
let root = H256::from_slice(trie.root()?.as_slice());
Ok(root)
}
fn calculate_storage_root<DB: Database>(
&self,
tx: &Transaction<'_, DB>,
address: H256,
) -> Result<H256, TrieError> {
let db = Arc::new(DupHashDatabase::new(tx, address)?);
let hasher = Arc::new(HasherKeccak::new());
let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher));
let mut storage_cursor = tx.cursor_dup_read::<tables::HashedStorage>()?;
// Should be able to use walk_dup, but any call to next() causes an assert fail in mdbx.c
// let mut walker = storage_cursor.walk_dup(address, H256::zero())?;
let mut current = storage_cursor.seek_by_key_subkey(address, H256::zero())?;
while let Some(StorageEntry { key: storage_key, value }) = current {
let out = encode_fixed_size(&value).to_vec();
trie.insert(storage_key.to_vec(), out)?;
current = storage_cursor.next_dup()?.map(|(_, v)| v);
}
let root = H256::from_slice(trie.root()?.as_slice());
Ok(root)
}
/// Calculates the root of the state trie by updating an existing trie.
pub(crate) fn update_root<DB: Database>(
&self,
tx: &Transaction<'_, DB>,
root: H256,
tid_range: Range<TransitionId>,
) -> Result<H256, TrieError> {
let mut accounts_cursor = tx.cursor_read::<tables::HashedAccount>()?;
let changed_accounts = self.gather_changes(tx, tid_range)?;
let db = Arc::new(HashDatabase::from_root(tx, root)?);
let hasher = Arc::new(HasherKeccak::new());
let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?;
for (address, changed_storages) in changed_accounts {
if let Some(account) = trie.get(address.as_slice())? {
let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root;
trie.remove(address.as_bytes())?;
if let Some((_, account)) = accounts_cursor.seek_exact(address)? {
let value = EthAccount::from_with_root(
account,
self.update_storage_root(tx, storage_root, address, changed_storages)?,
);
let mut out = Vec::new();
Encodable::encode(&value, &mut out);
trie.insert(address.as_bytes().to_vec(), out)?;
}
}
}
let root = H256::from_slice(trie.root()?.as_slice());
Ok(root)
}
fn update_storage_root<DB: Database>(
&self,
tx: &Transaction<'_, DB>,
root: H256,
address: H256,
changed_storages: BTreeSet<H256>,
) -> Result<H256, TrieError> {
let db = Arc::new(DupHashDatabase::from_root(tx, address, root)?);
let hasher = Arc::new(HasherKeccak::new());
let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?;
let mut storage_cursor = tx.cursor_dup_read::<tables::HashedStorage>()?;
for key in changed_storages {
if let Some(StorageEntry { value, .. }) =
storage_cursor.seek_by_key_subkey(address, key)?
{
let out = encode_fixed_size(&value).to_vec();
trie.insert(key.as_bytes().to_vec(), out)?;
} else {
trie.remove(key.as_bytes())?;
}
}
let root = H256::from_slice(trie.root()?.as_slice());
Ok(root)
}
fn gather_changes<DB: Database>(
&self,
tx: &Transaction<'_, DB>,
tid_range: Range<TransitionId>,
) -> Result<BTreeMap<H256, BTreeSet<H256>>, TrieError> {
let mut account_cursor = tx.cursor_read::<tables::AccountChangeSet>()?;
let mut account_changes: BTreeMap<Address, BTreeSet<H256>> = BTreeMap::new();
let mut walker = account_cursor.walk_range(tid_range.clone())?;
while let Some((_, AccountBeforeTx { address, .. })) = walker.next().transpose()? {
account_changes.insert(address, Default::default());
}
let mut storage_cursor = tx.cursor_dup_read::<tables::StorageChangeSet>()?;
let start = (tid_range.start, Address::zero()).into();
let end = (tid_range.end, Address::zero()).into();
let mut walker = storage_cursor.walk_range(start..end)?;
while let Some((TransitionIdAddress((_, address)), StorageEntry { key, .. })) =
walker.next().transpose()?
{
account_changes.entry(address).or_default().insert(key);
}
let hashed_changes = account_changes
.into_iter()
.map(|(address, storage)| {
(keccak256(address), storage.into_iter().map(keccak256).collect())
})
.collect();
Ok(hashed_changes)
}
}
#[cfg(test)]
mod tests {
use super::*;
use assert_matches::assert_matches;
use reth_db::{mdbx::test_utils::create_test_rw_db, tables, transaction::DbTxMut};
use reth_primitives::{
hex_literal::hex,
keccak256,
proofs::{genesis_state_root, KeccakHasher, EMPTY_ROOT},
Address, ChainSpec,
};
use reth_staged_sync::utils::chainspec::chain_spec_value_parser;
use std::{collections::HashMap, str::FromStr};
use triehash::sec_trie_root;
#[test]
fn empty_trie() {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
assert_matches!(trie.calculate_root(&tx), Ok(got) if got == EMPTY_ROOT);
}
#[test]
fn single_account_trie() {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap();
let account = Account { nonce: 0, balance: U256::ZERO, bytecode_hash: None };
tx.put::<tables::HashedAccount>(keccak256(address), account).unwrap();
let mut encoded_account = Vec::new();
EthAccount::from(account).encode(&mut encoded_account);
let expected = H256(sec_trie_root::<KeccakHasher, _, _, _>([(address, encoded_account)]).0);
assert_matches!(
trie.calculate_root(&tx),
Ok(got) if got == expected
);
}
#[test]
fn two_accounts_trie() {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
let accounts = [
(
Address::from(hex!("9fe4abd71ad081f091bd06dd1c16f7e92927561e")),
Account { nonce: 155, balance: U256::from(414241124), bytecode_hash: None },
),
(
Address::from(hex!("f8a6edaad4a332e6e550d0915a7fd5300b0b12d1")),
Account { nonce: 3, balance: U256::from(78978), bytecode_hash: None },
),
];
for (address, account) in accounts {
tx.put::<tables::HashedAccount>(keccak256(address), account).unwrap();
}
let encoded_accounts = accounts.iter().map(|(k, v)| {
let mut out = Vec::new();
EthAccount::from(*v).encode(&mut out);
(k, out)
});
let expected = H256(sec_trie_root::<KeccakHasher, _, _, _>(encoded_accounts).0);
assert_matches!(
trie.calculate_root(&tx),
Ok(got) if got == expected
);
}
#[test]
fn single_storage_trie() {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap();
let hashed_address = keccak256(address);
let storage = Vec::from([(H256::from_low_u64_be(2), U256::from(1))]);
for (k, v) in storage.clone() {
tx.put::<tables::HashedStorage>(
hashed_address,
StorageEntry { key: keccak256(k), value: v },
)
.unwrap();
}
let encoded_storage = storage.iter().map(|(k, v)| {
let out = encode_fixed_size(v).to_vec();
(k, out)
});
let expected = H256(sec_trie_root::<KeccakHasher, _, _, _>(encoded_storage).0);
assert_matches!(
trie.calculate_storage_root(&tx, hashed_address),
Ok(got) if got == expected
);
}
#[test]
fn single_account_with_storage_trie() {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap();
let hashed_address = keccak256(address);
let storage = HashMap::from([
(H256::zero(), U256::from(3)),
(H256::from_low_u64_be(2), U256::from(1)),
]);
let code = "el buen fla";
let account = Account {
nonce: 155,
balance: U256::from(414241124u32),
bytecode_hash: Some(keccak256(code)),
};
tx.put::<tables::HashedAccount>(hashed_address, account).unwrap();
for (k, v) in storage.clone() {
tx.put::<tables::HashedStorage>(
hashed_address,
StorageEntry { key: keccak256(k), value: v },
)
.unwrap();
}
let mut out = Vec::new();
let encoded_storage = storage.iter().map(|(k, v)| {
let out = encode_fixed_size(v).to_vec();
(k, out)
});
let eth_account = EthAccount::from_with_root(
account,
H256(sec_trie_root::<KeccakHasher, _, _, _>(encoded_storage).0),
);
eth_account.encode(&mut out);
let expected = H256(sec_trie_root::<KeccakHasher, _, _, _>([(address, out)]).0);
assert_matches!(
trie.calculate_root(&tx),
Ok(got) if got == expected
);
}
#[test]
fn verify_genesis() {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let mut tx = Transaction::new(db.as_ref()).unwrap();
let ChainSpec { genesis, .. } = chain_spec_value_parser("mainnet").unwrap();
// Insert account state
for (address, account) in &genesis.alloc {
tx.put::<tables::HashedAccount>(
keccak256(address),
Account {
nonce: account.nonce.unwrap_or_default(),
balance: account.balance,
bytecode_hash: None,
},
)
.unwrap();
}
tx.commit().unwrap();
let state_root = genesis_state_root(genesis.alloc);
assert_matches!(
trie.calculate_root(&tx),
Ok(got) if got == state_root
);
}
#[test]
fn gather_changes() {
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap();
let hashed_address = keccak256(address);
let storage = HashMap::from([
(H256::zero(), U256::from(3)),
(H256::from_low_u64_be(2), U256::from(1)),
]);
let code = "el buen fla";
let account = Account {
nonce: 155,
balance: U256::from(414241124u32),
bytecode_hash: Some(keccak256(code)),
};
tx.put::<tables::HashedAccount>(hashed_address, account).unwrap();
tx.put::<tables::AccountChangeSet>(31, AccountBeforeTx { address, info: None }).unwrap();
for (k, v) in storage {
tx.put::<tables::HashedStorage>(
hashed_address,
StorageEntry { key: keccak256(k), value: v },
)
.unwrap();
tx.put::<tables::StorageChangeSet>(
(32, address).into(),
StorageEntry { key: k, value: U256::ZERO },
)
.unwrap();
}
let expected = BTreeMap::from([(
hashed_address,
BTreeSet::from([keccak256(H256::zero()), keccak256(H256::from_low_u64_be(2))]),
)]);
assert_matches!(
DBTrieLoader::default().gather_changes(&tx, 32..33),
Ok(got) if got == expected
);
}
fn test_with_accounts(accounts: BTreeMap<Address, (Account, BTreeSet<StorageEntry>)>) {
let trie = DBTrieLoader::default();
let db = create_test_rw_db();
let tx = Transaction::new(db.as_ref()).unwrap();
let encoded_accounts = accounts
.into_iter()
.map(|(address, (account, storage))| {
let hashed_address = keccak256(address);
tx.put::<tables::HashedAccount>(hashed_address, account).unwrap();
// This is to mimic real data. Only contract accounts have storage.
let storage_root = if account.has_bytecode() {
let encoded_storage = storage.into_iter().map(|StorageEntry { key, value }| {
let hashed_key = keccak256(key);
let out = encode_fixed_size(&value).to_vec();
tx.put::<tables::HashedStorage>(
hashed_address,
StorageEntry { key: hashed_key, value },
)
.unwrap();
(key, out)
});
H256(sec_trie_root::<KeccakHasher, _, _, _>(encoded_storage).0)
} else {
EMPTY_ROOT
};
let mut out = Vec::new();
EthAccount::from_with_root(account, storage_root).encode(&mut out);
(address, out)
})
.collect::<Vec<(Address, Vec<u8>)>>();
let expected = H256(sec_trie_root::<KeccakHasher, _, _, _>(encoded_accounts).0);
assert_matches!(
trie.calculate_root(&tx),
Ok(got) if got == expected
, "where expected is {expected:?}");
}
#[test]
fn arbitrary() {
proptest::proptest!(|(accounts: BTreeMap<Address, (Account, BTreeSet<StorageEntry>)>)| {
test_with_accounts(accounts);
});
}
}

View File

@ -105,7 +105,7 @@ pub trait DbCursorRW<'tx, T: Table> {
/// Read Write Cursor over DupSorted table. /// Read Write Cursor over DupSorted table.
pub trait DbDupCursorRW<'tx, T: DupSort> { pub trait DbDupCursorRW<'tx, T: DupSort> {
/// Append value to next cursor item /// Delete all duplicate entries for current key.
fn delete_current_duplicates(&mut self) -> Result<(), Error>; fn delete_current_duplicates(&mut self) -> Result<(), Error>;
/// Append duplicate value. /// Append duplicate value.

View File

@ -40,6 +40,7 @@ impl_compression_for_compact!(
Receipt, Receipt,
TxType, TxType,
StorageEntry, StorageEntry,
StorageTrieEntry,
StoredBlockBody, StoredBlockBody,
StoredBlockOmmers StoredBlockOmmers
); );

View File

@ -18,7 +18,7 @@ use crate::{
}; };
use reth_primitives::{ use reth_primitives::{
Account, Address, BlockHash, BlockNumber, Header, IntegerList, Receipt, StorageEntry, Account, Address, BlockHash, BlockNumber, Header, IntegerList, Receipt, StorageEntry,
TransactionSigned, TransitionId, TxHash, TxNumber, H256, StorageTrieEntry, TransactionSigned, TransitionId, TxHash, TxNumber, H256,
}; };
use self::models::{storage_sharded_key::StorageShardedKey, StoredBlockBody}; use self::models::{storage_sharded_key::StorageShardedKey, StoredBlockBody};
@ -33,7 +33,7 @@ pub enum TableType {
} }
/// Default tables that should be present inside database. /// Default tables that should be present inside database.
pub const TABLES: [(TableType, &str); 25] = [ pub const TABLES: [(TableType, &str); 27] = [
(TableType::Table, CanonicalHeaders::const_name()), (TableType::Table, CanonicalHeaders::const_name()),
(TableType::Table, HeaderTD::const_name()), (TableType::Table, HeaderTD::const_name()),
(TableType::Table, HeaderNumbers::const_name()), (TableType::Table, HeaderNumbers::const_name()),
@ -56,6 +56,8 @@ pub const TABLES: [(TableType, &str); 25] = [
(TableType::DupSort, StorageChangeSet::const_name()), (TableType::DupSort, StorageChangeSet::const_name()),
(TableType::Table, HashedAccount::const_name()), (TableType::Table, HashedAccount::const_name()),
(TableType::DupSort, HashedStorage::const_name()), (TableType::DupSort, HashedStorage::const_name()),
(TableType::Table, AccountsTrie::const_name()),
(TableType::DupSort, StoragesTrie::const_name()),
(TableType::Table, TxSenders::const_name()), (TableType::Table, TxSenders::const_name()),
(TableType::Table, Config::const_name()), (TableType::Table, Config::const_name()),
(TableType::Table, SyncStage::const_name()), (TableType::Table, SyncStage::const_name()),
@ -271,6 +273,16 @@ dupsort!(
( HashedStorage ) H256 | [H256] StorageEntry ( HashedStorage ) H256 | [H256] StorageEntry
); );
table!(
/// Stores the current state's Merkle Patricia Tree.
( AccountsTrie ) H256 | Vec<u8>
);
dupsort!(
/// Stores the Merkle Patricia Trees of each [`Account`]'s storage.
( StoragesTrie ) H256 | [H256] StorageTrieEntry
);
table!( table!(
/// Stores the transaction sender for each transaction. /// Stores the transaction sender for each transaction.
/// It is needed to speed up execution stage and allows fetching signer without doing /// It is needed to speed up execution stage and allows fetching signer without doing