From 1e7189d3e4f2b7e744bf59906a791784621cf6e9 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Tue, 19 Nov 2024 19:39:28 +0400 Subject: [PATCH] feat: trait-based storage API (#12616) Co-authored-by: joshie <93316087+joshieDo@users.noreply.github.com> --- Cargo.lock | 5 + bin/reth/src/commands/debug_cmd/execution.rs | 2 +- crates/cli/commands/Cargo.toml | 1 + crates/cli/commands/src/common.rs | 26 ++++- crates/cli/commands/src/import.rs | 2 +- crates/cli/commands/src/stage/unwind.rs | 2 +- crates/e2e-test-utils/Cargo.toml | 2 + crates/e2e-test-utils/src/lib.rs | 15 ++- crates/ethereum/node/src/node.rs | 11 +- crates/exex/test-utils/src/lib.rs | 16 ++- crates/node/builder/src/builder/mod.rs | 29 +++-- crates/node/builder/src/launch/common.rs | 29 +++-- crates/node/builder/src/launch/engine.rs | 6 +- crates/node/builder/src/launch/mod.rs | 10 +- crates/node/builder/src/node.rs | 2 + crates/node/builder/src/setup.rs | 9 +- crates/node/types/src/lib.rs | 106 +++++++++++++----- .../cli/src/commands/build_pipeline.rs | 3 +- crates/optimism/node/Cargo.toml | 1 + crates/optimism/node/src/node.rs | 61 +++++++++- crates/primitives-traits/src/block/mod.rs | 4 +- crates/primitives/src/lib.rs | 9 ++ crates/stages/stages/src/stages/bodies.rs | 5 +- .../stages/stages/src/test_utils/test_db.rs | 4 +- crates/storage/provider/Cargo.toml | 3 + .../provider/src/providers/database/chain.rs | 26 +++++ .../provider/src/providers/database/mod.rs | 22 +++- .../src/providers/database/provider.rs | 74 ++++++------ crates/storage/provider/src/providers/mod.rs | 34 +++++- .../storage/provider/src/test_utils/mock.rs | 11 +- crates/storage/provider/src/test_utils/mod.rs | 12 +- crates/storage/provider/src/traits/block.rs | 2 +- crates/storage/storage-api/Cargo.toml | 1 + crates/storage/storage-api/src/chain.rs | 72 ++++++++++++ crates/storage/storage-api/src/lib.rs | 3 + examples/custom-engine-types/src/main.rs | 15 ++- 36 files changed, 486 insertions(+), 149 deletions(-) create mode 100644 crates/storage/provider/src/providers/database/chain.rs create mode 100644 crates/storage/storage-api/src/chain.rs diff --git a/Cargo.lock b/Cargo.lock index adacf448f..1b38e0566 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6683,6 +6683,7 @@ dependencies = [ "reth-network", "reth-network-p2p", "reth-network-peers", + "reth-node-api", "reth-node-builder", "reth-node-core", "reth-node-events", @@ -7089,10 +7090,12 @@ dependencies = [ "reth-db", "reth-engine-local", "reth-network-peers", + "reth-node-api", "reth-node-builder", "reth-payload-builder", "reth-payload-builder-primitives", "reth-payload-primitives", + "reth-primitives", "reth-provider", "reth-rpc-layer", "reth-stages-types", @@ -8626,6 +8629,7 @@ dependencies = [ "reth-node-types", "reth-optimism-primitives", "reth-primitives", + "reth-primitives-traits", "reth-prune-types", "reth-stages-types", "reth-storage-api", @@ -9189,6 +9193,7 @@ dependencies = [ "reth-db-models", "reth-execution-types", "reth-primitives", + "reth-primitives-traits", "reth-prune-types", "reth-stages-types", "reth-storage-errors", diff --git a/bin/reth/src/commands/debug_cmd/execution.rs b/bin/reth/src/commands/debug_cmd/execution.rs index da928645b..a6203ea2a 100644 --- a/bin/reth/src/commands/debug_cmd/execution.rs +++ b/bin/reth/src/commands/debug_cmd/execution.rs @@ -59,7 +59,7 @@ pub struct Command { } impl> Command { - fn build_pipeline, Client>( + fn build_pipeline + CliNodeTypes, Client>( &self, config: &Config, client: Client, diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index 7e27d9b4e..90acb82d7 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -32,6 +32,7 @@ reth-fs-util.workspace = true reth-network = { workspace = true, features = ["serde"] } reth-network-p2p.workspace = true reth-network-peers = { workspace = true, features = ["secp256k1"] } +reth-node-api.workspace = true reth-node-builder.workspace = true reth-node-core.workspace = true reth-node-events.workspace = true diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 0e4eb2723..e557f15da 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -10,12 +10,16 @@ use reth_db::{init_db, open_db_read_only, DatabaseEnv}; use reth_db_common::init::init_genesis; use reth_downloaders::{bodies::noop::NoopBodiesDownloader, headers::noop::NoopHeaderDownloader}; use reth_evm::noop::NoopBlockExecutorProvider; +use reth_node_api::FullNodePrimitives; use reth_node_builder::{NodeTypesWithDBAdapter, NodeTypesWithEngine}; use reth_node_core::{ args::{DatabaseArgs, DatadirArgs}, dirs::{ChainPath, DataDirPath}, }; -use reth_provider::{providers::StaticFileProvider, ProviderFactory, StaticFileProviderFactory}; +use reth_provider::{ + providers::{NodeTypesForProvider, StaticFileProvider}, + ProviderFactory, StaticFileProviderFactory, +}; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc}; @@ -191,5 +195,21 @@ impl AccessRights { /// Helper trait with a common set of requirements for the /// [`NodeTypes`](reth_node_builder::NodeTypes) in CLI. -pub trait CliNodeTypes: NodeTypesWithEngine {} -impl CliNodeTypes for N where N: NodeTypesWithEngine {} +pub trait CliNodeTypes: + NodeTypesWithEngine + + NodeTypesForProvider< + Primitives: FullNodePrimitives< + Block: reth_node_api::Block, + >, + > +{ +} +impl CliNodeTypes for N where + N: NodeTypesWithEngine + + NodeTypesForProvider< + Primitives: FullNodePrimitives< + Block: reth_node_api::Block, + >, + > +{ +} diff --git a/crates/cli/commands/src/import.rs b/crates/cli/commands/src/import.rs index 539211a22..c1f6408b4 100644 --- a/crates/cli/commands/src/import.rs +++ b/crates/cli/commands/src/import.rs @@ -167,7 +167,7 @@ pub fn build_import_pipeline( executor: E, ) -> eyre::Result<(Pipeline, impl Stream)> where - N: ProviderNodeTypes, + N: ProviderNodeTypes + CliNodeTypes, C: Consensus + 'static, E: BlockExecutorProvider, { diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index e71861a98..4f47a70b0 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -113,7 +113,7 @@ impl> Command Ok(()) } - fn build_pipeline>( + fn build_pipeline + CliNodeTypes>( self, config: Config, provider_factory: ProviderFactory, diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index c4c74ebcd..9c40e2ba9 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -19,7 +19,9 @@ reth-rpc-layer.workspace = true reth-payload-builder = { workspace = true, features = ["test-utils"] } reth-payload-builder-primitives.workspace = true reth-payload-primitives.workspace = true +reth-primitives.workspace = true reth-provider.workspace = true +reth-node-api.workspace = true reth-node-builder = { workspace = true, features = ["test-utils"] } reth-tokio-util.workspace = true reth-stages-types.workspace = true diff --git a/crates/e2e-test-utils/src/lib.rs b/crates/e2e-test-utils/src/lib.rs index 1e9b39058..df459f641 100644 --- a/crates/e2e-test-utils/src/lib.rs +++ b/crates/e2e-test-utils/src/lib.rs @@ -5,12 +5,12 @@ use std::sync::Arc; use node::NodeTestContext; use reth::{ args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}, - builder::{NodeBuilder, NodeConfig, NodeHandle}, + builder::{FullNodePrimitives, NodeBuilder, NodeConfig, NodeHandle}, network::PeersHandleProvider, rpc::server_types::RpcModuleSelection, tasks::TaskManager, }; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_chainspec::EthChainSpec; use reth_db::{test_utils::TempDatabase, DatabaseEnv}; use reth_engine_local::LocalPayloadAttributesBuilder; use reth_node_builder::{ @@ -18,7 +18,7 @@ use reth_node_builder::{ FullNodeTypesAdapter, Node, NodeAdapter, NodeComponents, NodeTypesWithDBAdapter, NodeTypesWithEngine, PayloadAttributesBuilder, PayloadTypes, }; -use reth_provider::providers::{BlockchainProvider, BlockchainProvider2}; +use reth_provider::providers::{BlockchainProvider, BlockchainProvider2, NodeTypesForProvider}; use tracing::{span, Level}; use wallet::Wallet; @@ -53,12 +53,14 @@ pub async fn setup( attributes_generator: impl Fn(u64) -> <::Engine as PayloadTypes>::PayloadBuilderAttributes + Copy + 'static, ) -> eyre::Result<(Vec>, TaskManager, Wallet)> where - N: Default + Node> + NodeTypesWithEngine, + N: Default + Node> + NodeTypesForProvider + NodeTypesWithEngine, N::ComponentsBuilder: NodeComponentsBuilder< TmpNodeAdapter, Components: NodeComponents, Network: PeersHandleProvider>, >, N::AddOns: RethRpcAddOns>, + N::Primitives: + FullNodePrimitives>, { let tasks = TaskManager::current(); let exec = tasks.executor(); @@ -120,7 +122,8 @@ pub async fn setup_engine( where N: Default + Node>>> - + NodeTypesWithEngine, + + NodeTypesWithEngine + + NodeTypesForProvider, N::ComponentsBuilder: NodeComponentsBuilder< TmpNodeAdapter>>, Components: NodeComponents< @@ -132,6 +135,8 @@ where LocalPayloadAttributesBuilder: PayloadAttributesBuilder< <::Engine as PayloadTypes>::PayloadAttributes, >, + N::Primitives: + FullNodePrimitives>, { let tasks = TaskManager::current(); let exec = tasks.executor(); diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 1615ef0e6..a2ae2374b 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -26,7 +26,7 @@ use reth_node_builder::{ }; use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; use reth_primitives::EthPrimitives; -use reth_provider::CanonStateSubscriptions; +use reth_provider::{CanonStateSubscriptions, EthStorage}; use reth_rpc::EthApi; use reth_tracing::tracing::{debug, info}; use reth_transaction_pool::{ @@ -74,6 +74,7 @@ impl NodeTypes for EthereumNode { type Primitives = EthPrimitives; type ChainSpec = ChainSpec; type StateCommitment = MerklePatriciaTrie; + type Storage = EthStorage; } impl NodeTypesWithEngine for EthereumNode { @@ -94,7 +95,13 @@ pub type EthereumAddOns = RpcAddOns< impl Node for EthereumNode where - Types: NodeTypesWithDB + NodeTypesWithEngine, + Types: NodeTypesWithDB + + NodeTypesWithEngine< + Engine = EthEngineTypes, + ChainSpec = ChainSpec, + Primitives = EthPrimitives, + Storage = EthStorage, + >, N: FullNodeTypes, { type ComponentsBuilder = ComponentsBuilder< diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index 5c3468a3c..5b2267505 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -45,10 +45,10 @@ use reth_node_ethereum::{ EthEngineTypes, EthEvmConfig, }; use reth_payload_builder::noop::NoopPayloadBuilderService; -use reth_primitives::{Head, SealedBlockWithSenders}; +use reth_primitives::{EthPrimitives, Head, SealedBlockWithSenders}; use reth_provider::{ providers::{BlockchainProvider, StaticFileProvider}, - BlockReader, ProviderFactory, + BlockReader, EthStorage, ProviderFactory, }; use reth_tasks::TaskManager; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; @@ -118,9 +118,10 @@ where pub struct TestNode; impl NodeTypes for TestNode { - type Primitives = (); + type Primitives = EthPrimitives; type ChainSpec = ChainSpec; type StateCommitment = reth_trie_db::MerklePatriciaTrie; + type Storage = EthStorage; } impl NodeTypesWithEngine for TestNode { @@ -129,7 +130,14 @@ impl NodeTypesWithEngine for TestNode { impl Node for TestNode where - N: FullNodeTypes>, + N: FullNodeTypes< + Types: NodeTypesWithEngine< + Engine = EthEngineTypes, + ChainSpec = ChainSpec, + Primitives = EthPrimitives, + Storage = EthStorage, + >, + >, { type ComponentsBuilder = ComponentsBuilder< N, diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 89892ed59..3ad90a493 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -22,8 +22,8 @@ use reth_network::{ NetworkHandle, NetworkManager, }; use reth_node_api::{ - FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter, - NodeTypesWithEngine, + FullNodePrimitives, FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, + NodeTypesWithDBAdapter, NodeTypesWithEngine, }; use reth_node_core::{ cli::config::{PayloadBuilderConfig, RethTransactionPoolConfig}, @@ -31,7 +31,10 @@ use reth_node_core::{ node_config::NodeConfig, primitives::Head, }; -use reth_provider::{providers::BlockchainProvider, ChainSpecProvider, FullProvider}; +use reth_provider::{ + providers::{BlockchainProvider, NodeTypesForProvider}, + ChainSpecProvider, FullProvider, +}; use reth_tasks::TaskExecutor; use reth_transaction_pool::{PoolConfig, TransactionPool}; use revm_primitives::EnvKzgSettings; @@ -240,7 +243,7 @@ where /// Configures the types of the node. pub fn with_types(self) -> NodeBuilderWithTypes> where - T: NodeTypesWithEngine, + T: NodeTypesWithEngine + NodeTypesForProvider, { self.with_types_and_provider() } @@ -250,7 +253,7 @@ where self, ) -> NodeBuilderWithTypes, P>> where - T: NodeTypesWithEngine, + T: NodeTypesWithEngine + NodeTypesForProvider, P: FullProvider>, { NodeBuilderWithTypes::new(self.config, self.database) @@ -264,7 +267,7 @@ where node: N, ) -> NodeBuilderWithComponents, N::ComponentsBuilder, N::AddOns> where - N: Node, ChainSpec = ChainSpec>, + N: Node, ChainSpec = ChainSpec> + NodeTypesForProvider, { self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons()) } @@ -301,7 +304,7 @@ where /// Configures the types of the node. pub fn with_types(self) -> WithLaunchContext>> where - T: NodeTypesWithEngine, + T: NodeTypesWithEngine + NodeTypesForProvider, { WithLaunchContext { builder: self.builder.with_types(), task_executor: self.task_executor } } @@ -313,7 +316,7 @@ where NodeBuilderWithTypes, P>>, > where - T: NodeTypesWithEngine, + T: NodeTypesWithEngine + NodeTypesForProvider, P: FullProvider>, { WithLaunchContext { @@ -332,7 +335,7 @@ where NodeBuilderWithComponents, N::ComponentsBuilder, N::AddOns>, > where - N: Node, ChainSpec = ChainSpec>, + N: Node, ChainSpec = ChainSpec> + NodeTypesForProvider, { self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons()) } @@ -355,13 +358,15 @@ where >, > where - N: Node, ChainSpec = ChainSpec>, + N: Node, ChainSpec = ChainSpec> + NodeTypesForProvider, N::AddOns: RethRpcAddOns< NodeAdapter< RethFullAdapter, >>::Components, >, >, + N::Primitives: + FullNodePrimitives>, { self.node(node).launch().await } @@ -549,9 +554,11 @@ where impl WithLaunchContext, CB, AO>> where DB: Database + DatabaseMetrics + DatabaseMetadata + Clone + Unpin + 'static, - T: NodeTypesWithEngine, + T: NodeTypesWithEngine + NodeTypesForProvider, CB: NodeComponentsBuilder>, AO: RethRpcAddOns, CB::Components>>, + T::Primitives: + FullNodePrimitives>, { /// Launches the node with the [`DefaultNodeLauncher`] that sets up engine API consensus and rpc pub async fn launch( diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index e01d117e7..903b09803 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -26,7 +26,7 @@ use reth_evm::noop::NoopBlockExecutorProvider; use reth_fs_util as fs; use reth_invalid_block_hooks::InvalidBlockWitnessHook; use reth_network_p2p::headers::client::HeadersClient; -use reth_node_api::{FullNodeTypes, NodeTypes, NodeTypesWithDB}; +use reth_node_api::{FullNodePrimitives, FullNodeTypes, NodeTypes, NodeTypesWithDB}; use reth_node_core::{ args::InvalidBlockHookType, dirs::{ChainPath, DataDirPath}, @@ -404,9 +404,12 @@ where /// Returns the [`ProviderFactory`] for the attached storage after executing a consistent check /// between the database and static files. **It may execute a pipeline unwind if it fails this /// check.** - pub async fn create_provider_factory>( - &self, - ) -> eyre::Result> { + pub async fn create_provider_factory(&self) -> eyre::Result> + where + N: ProviderNodeTypes, + N::Primitives: + FullNodePrimitives>, + { let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), @@ -467,9 +470,14 @@ where } /// Creates a new [`ProviderFactory`] and attaches it to the launch context. - pub async fn with_provider_factory>( + pub async fn with_provider_factory( self, - ) -> eyre::Result, ProviderFactory>>> { + ) -> eyre::Result, ProviderFactory>>> + where + N: ProviderNodeTypes, + N::Primitives: + FullNodePrimitives>, + { let factory = self.create_provider_factory().await?; let ctx = LaunchContextWith { inner: self.inner, @@ -482,7 +490,7 @@ where impl LaunchContextWith, ProviderFactory>> where - T: NodeTypesWithDB, + T: ProviderNodeTypes, { /// Returns access to the underlying database. pub const fn database(&self) -> &T::DB { @@ -748,10 +756,7 @@ impl Attached::ChainSpec>, WithComponents>, > where - T: FullNodeTypes< - Provider: WithTree, - Types: NodeTypes, - >, + T: FullNodeTypes, CB: NodeComponentsBuilder, { /// Returns the configured `ProviderFactory`. @@ -913,7 +918,7 @@ impl where T: FullNodeTypes< Provider: WithTree + StateProviderFactory + ChainSpecProvider, - Types: NodeTypes, + Types: ProviderNodeTypes, >, CB: NodeComponentsBuilder, { diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 86ab0b9a3..5a8405047 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -19,8 +19,8 @@ use reth_exex::ExExManagerHandle; use reth_network::{NetworkSyncUpdater, SyncState}; use reth_network_api::{BlockDownloaderProvider, NetworkEventListenerProvider}; use reth_node_api::{ - BuiltPayload, FullNodeTypes, NodeTypesWithEngine, PayloadAttributesBuilder, PayloadBuilder, - PayloadTypes, + BuiltPayload, FullNodePrimitives, FullNodeTypes, NodeTypesWithEngine, PayloadAttributesBuilder, + PayloadBuilder, PayloadTypes, }; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, @@ -77,6 +77,8 @@ where LocalPayloadAttributesBuilder: PayloadAttributesBuilder< <::Engine as PayloadTypes>::PayloadAttributes, >, + Types::Primitives: + FullNodePrimitives>, { type Node = NodeHandle, AO>; diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index 4f9e850c9..c4146f483 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -17,18 +17,18 @@ use reth_beacon_consensus::{ BeaconConsensusEngine, }; use reth_blockchain_tree::{noop::NoopBlockchainTree, BlockchainTreeConfig}; -use reth_chainspec::{EthChainSpec, EthereumHardforks}; +use reth_chainspec::EthChainSpec; use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider, RpcBlockProvider}; use reth_engine_util::EngineMessageStreamExt; use reth_exex::ExExManagerHandle; use reth_network::{BlockDownloaderProvider, NetworkEventListenerProvider}; -use reth_node_api::{AddOnsContext, FullNodeTypes, NodeTypesWithDB, NodeTypesWithEngine}; +use reth_node_api::{AddOnsContext, FullNodePrimitives, FullNodeTypes, NodeTypesWithEngine}; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, exit::NodeExitFuture, }; use reth_node_events::{cl::ConsensusLayerHealthEvents, node}; -use reth_provider::providers::BlockchainProvider; +use reth_provider::providers::{BlockchainProvider, ProviderNodeTypes}; use reth_rpc::eth::RpcNodeCore; use reth_tasks::TaskExecutor; use reth_tracing::tracing::{debug, info}; @@ -98,10 +98,12 @@ impl DefaultNodeLauncher { impl LaunchNode> for DefaultNodeLauncher where - Types: NodeTypesWithDB + NodeTypesWithEngine, + Types: ProviderNodeTypes + NodeTypesWithEngine, T: FullNodeTypes, Types = Types>, CB: NodeComponentsBuilder, AO: RethRpcAddOns>, + Types::Primitives: + FullNodePrimitives>, { type Node = NodeHandle, AO>; diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index 62c710ea8..ce7d12fee 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -71,6 +71,8 @@ where type ChainSpec = ::ChainSpec; type StateCommitment = ::StateCommitment; + + type Storage = ::Storage; } impl NodeTypesWithEngine for AnyNode diff --git a/crates/node/builder/src/setup.rs b/crates/node/builder/src/setup.rs index db188402c..337e37eee 100644 --- a/crates/node/builder/src/setup.rs +++ b/crates/node/builder/src/setup.rs @@ -14,6 +14,7 @@ use reth_exex::ExExManagerHandle; use reth_network_p2p::{ bodies::downloader::BodyDownloader, headers::downloader::HeaderDownloader, EthBlockClient, }; +use reth_node_api::FullNodePrimitives; use reth_provider::{providers::ProviderNodeTypes, ProviderFactory}; use reth_stages::{prelude::DefaultStages, stages::ExecutionStage, Pipeline, StageSet}; use reth_static_file::StaticFileProducer; @@ -40,6 +41,8 @@ where N: ProviderNodeTypes, Client: EthBlockClient + 'static, Executor: BlockExecutorProvider, + N::Primitives: + FullNodePrimitives>, { // building network downloaders using the fetch client let header_downloader = ReverseHeadersDownloaderBuilder::new(config.headers) @@ -85,8 +88,12 @@ pub fn build_pipeline( where N: ProviderNodeTypes, H: HeaderDownloader
+ 'static, - B: BodyDownloader + 'static, + B: BodyDownloader< + Body = <::Block as reth_node_api::Block>::Body, + > + 'static, Executor: BlockExecutorProvider, + N::Primitives: + FullNodePrimitives>, { let mut builder = Pipeline::::builder(); diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index f8770a3c0..2da8180a9 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -9,12 +9,11 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![cfg_attr(not(feature = "std"), no_std)] +use core::{fmt::Debug, marker::PhantomData}; pub use reth_primitives_traits::{ Block, BlockBody, FullBlock, FullNodePrimitives, FullReceipt, FullSignedTx, NodePrimitives, }; -use core::marker::PhantomData; - use reth_chainspec::EthChainSpec; use reth_db_api::{ database_metrics::{DatabaseMetadata, DatabaseMetrics}, @@ -35,6 +34,8 @@ pub trait NodeTypes: Send + Sync + Unpin + 'static { type ChainSpec: EthChainSpec; /// The type used to perform state commitment operations. type StateCommitment: StateCommitment; + /// The type responsible for writing chain primitives to storage. + type Storage: Default + Send + Sync + Unpin + Debug + 'static; } /// The type that configures an Ethereum-like node with an engine for consensus. @@ -86,6 +87,7 @@ where type Primitives = Types::Primitives; type ChainSpec = Types::ChainSpec; type StateCommitment = Types::StateCommitment; + type Storage = Types::Storage; } impl NodeTypesWithEngine for NodeTypesWithDBAdapter @@ -105,86 +107,128 @@ where } /// A [`NodeTypes`] type builder. -#[derive(Default, Debug)] -pub struct AnyNodeTypes

(PhantomData

, PhantomData, PhantomData); +#[derive(Debug)] +pub struct AnyNodeTypes

( + PhantomData

, + PhantomData, + PhantomData, + PhantomData, +); + +impl Default for AnyNodeTypes { + fn default() -> Self { + Self::new() + } +} + +impl AnyNodeTypes { + /// Creates a new instance of [`AnyNodeTypes`]. + pub const fn new() -> Self { + Self(PhantomData, PhantomData, PhantomData, PhantomData) + } -impl AnyNodeTypes { /// Sets the `Primitives` associated type. - pub const fn primitives(self) -> AnyNodeTypes { - AnyNodeTypes::(PhantomData::, PhantomData::, PhantomData::) + pub const fn primitives(self) -> AnyNodeTypes { + AnyNodeTypes::new() } /// Sets the `ChainSpec` associated type. - pub const fn chain_spec(self) -> AnyNodeTypes { - AnyNodeTypes::(PhantomData::

, PhantomData::, PhantomData::) + pub const fn chain_spec(self) -> AnyNodeTypes { + AnyNodeTypes::new() } /// Sets the `StateCommitment` associated type. - pub const fn state_commitment(self) -> AnyNodeTypes { - AnyNodeTypes::(PhantomData::

, PhantomData::, PhantomData::) + pub const fn state_commitment(self) -> AnyNodeTypes { + AnyNodeTypes::new() + } + + /// Sets the `Storage` associated type. + pub const fn storage(self) -> AnyNodeTypes { + AnyNodeTypes::new() } } -impl NodeTypes for AnyNodeTypes +impl NodeTypes for AnyNodeTypes where P: NodePrimitives + Send + Sync + Unpin + 'static, C: EthChainSpec + 'static, - S: StateCommitment, + SC: StateCommitment, + S: Default + Send + Sync + Unpin + Debug + 'static, { type Primitives = P; type ChainSpec = C; - type StateCommitment = S; + type StateCommitment = SC; + type Storage = S; } /// A [`NodeTypesWithEngine`] type builder. -#[derive(Default, Debug)] -pub struct AnyNodeTypesWithEngine

{ +#[derive(Debug)] +pub struct AnyNodeTypesWithEngine

{ /// Embedding the basic node types. - base: AnyNodeTypes, + _base: AnyNodeTypes, /// Phantom data for the engine. _engine: PhantomData, } -impl AnyNodeTypesWithEngine { +impl Default for AnyNodeTypesWithEngine { + fn default() -> Self { + Self::new() + } +} + +impl AnyNodeTypesWithEngine { + /// Creates a new instance of [`AnyNodeTypesWithEngine`]. + pub const fn new() -> Self { + Self { _base: AnyNodeTypes::new(), _engine: PhantomData } + } + /// Sets the `Primitives` associated type. - pub const fn primitives(self) -> AnyNodeTypesWithEngine { - AnyNodeTypesWithEngine { base: self.base.primitives::(), _engine: PhantomData } + pub const fn primitives(self) -> AnyNodeTypesWithEngine { + AnyNodeTypesWithEngine::new() } /// Sets the `Engine` associated type. - pub const fn engine(self) -> AnyNodeTypesWithEngine { - AnyNodeTypesWithEngine { base: self.base, _engine: PhantomData:: } + pub const fn engine(self) -> AnyNodeTypesWithEngine { + AnyNodeTypesWithEngine::new() } /// Sets the `ChainSpec` associated type. - pub const fn chain_spec(self) -> AnyNodeTypesWithEngine { - AnyNodeTypesWithEngine { base: self.base.chain_spec::(), _engine: PhantomData } + pub const fn chain_spec(self) -> AnyNodeTypesWithEngine { + AnyNodeTypesWithEngine::new() } /// Sets the `StateCommitment` associated type. - pub const fn state_commitment(self) -> AnyNodeTypesWithEngine { - AnyNodeTypesWithEngine { base: self.base.state_commitment::(), _engine: PhantomData } + pub const fn state_commitment(self) -> AnyNodeTypesWithEngine { + AnyNodeTypesWithEngine::new() + } + + /// Sets the `Storage` associated type. + pub const fn storage(self) -> AnyNodeTypesWithEngine { + AnyNodeTypesWithEngine::new() } } -impl NodeTypes for AnyNodeTypesWithEngine +impl NodeTypes for AnyNodeTypesWithEngine where P: NodePrimitives + Send + Sync + Unpin + 'static, E: EngineTypes + Send + Sync + Unpin, C: EthChainSpec + 'static, - S: StateCommitment, + SC: StateCommitment, + S: Default + Send + Sync + Unpin + Debug + 'static, { type Primitives = P; type ChainSpec = C; - type StateCommitment = S; + type StateCommitment = SC; + type Storage = S; } -impl NodeTypesWithEngine for AnyNodeTypesWithEngine +impl NodeTypesWithEngine for AnyNodeTypesWithEngine where P: NodePrimitives + Send + Sync + Unpin + 'static, E: EngineTypes + Send + Sync + Unpin, C: EthChainSpec + 'static, - S: StateCommitment, + SC: StateCommitment, + S: Default + Send + Sync + Unpin + Debug + 'static, { type Engine = E; } diff --git a/crates/optimism/cli/src/commands/build_pipeline.rs b/crates/optimism/cli/src/commands/build_pipeline.rs index 88dc09897..8ebefdcc0 100644 --- a/crates/optimism/cli/src/commands/build_pipeline.rs +++ b/crates/optimism/cli/src/commands/build_pipeline.rs @@ -1,5 +1,6 @@ use alloy_primitives::B256; use futures_util::{Stream, StreamExt}; +use reth_cli_commands::common::CliNodeTypes; use reth_config::Config; use reth_consensus::Consensus; use reth_downloaders::{ @@ -38,7 +39,7 @@ pub(crate) async fn build_import_pipeline( disable_exec: bool, ) -> eyre::Result<(Pipeline, impl Stream)> where - N: ProviderNodeTypes, + N: CliNodeTypes + ProviderNodeTypes, C: Consensus + 'static, { if !file_client.has_canonical_blocks() { diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 03ea75a26..2e3e9fb4f 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] # reth reth-chainspec.workspace = true +reth-db.workspace = true reth-engine-local.workspace = true reth-primitives.workspace = true reth-payload-builder.workspace = true diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 70f32c01f..6cdffd090 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -5,10 +5,12 @@ use std::sync::Arc; use alloy_consensus::Header; use reth_basic_payload_builder::{BasicPayloadJobGenerator, BasicPayloadJobGeneratorConfig}; use reth_chainspec::{EthChainSpec, Hardforks}; +use reth_db::transaction::{DbTx, DbTxMut}; use reth_evm::{execute::BasicBlockExecutorProvider, ConfigureEvm}; use reth_network::{NetworkConfig, NetworkHandle, NetworkManager, PeersInfo}; use reth_node_api::{ - AddOnsContext, EngineValidator, FullNodeComponents, NodeAddOns, PayloadBuilder, + AddOnsContext, EngineValidator, FullNodeComponents, FullNodePrimitives, NodeAddOns, + PayloadBuilder, }; use reth_node_builder::{ components::{ @@ -28,7 +30,11 @@ use reth_optimism_rpc::{ OpEthApi, }; use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; -use reth_provider::CanonStateSubscriptions; +use reth_primitives::{Block, BlockBody, Receipt, TransactionSigned, TxType}; +use reth_provider::{ + providers::ChainStorage, BlockBodyWriter, CanonStateSubscriptions, DBProvider, EthStorage, + ProviderResult, +}; use reth_rpc_server_types::RethRpcModule; use reth_tracing::tracing::{debug, info}; use reth_transaction_pool::{ @@ -43,7 +49,42 @@ use crate::{ txpool::{OpTransactionPool, OpTransactionValidator}, OpEngineTypes, }; +/// Optimism primitive types. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct OpPrimitives; +impl FullNodePrimitives for OpPrimitives { + type Block = Block; + type SignedTx = TransactionSigned; + type TxType = TxType; + type Receipt = Receipt; +} + +/// Storage implementation for Optimism. +#[derive(Debug, Default, Clone)] +pub struct OpStorage(EthStorage); + +impl> BlockBodyWriter for OpStorage { + fn write_block_bodies( + &self, + provider: &Provider, + bodies: Vec<(u64, Option)>, + ) -> ProviderResult<()> { + self.0.write_block_bodies(provider, bodies) + } +} + +impl ChainStorage for OpStorage { + fn writer( + &self, + ) -> impl reth_provider::ChainStorageWriter, OpPrimitives> + where + TX: DbTxMut + DbTx + 'static, + Types: NodeTypes, + { + self + } +} /// Type configuration for a regular Optimism node. #[derive(Debug, Default, Clone)] #[non_exhaustive] @@ -90,7 +131,14 @@ impl OpNode { impl Node for OpNode where - N: FullNodeTypes>, + N: FullNodeTypes< + Types: NodeTypesWithEngine< + Engine = OpEngineTypes, + ChainSpec = OpChainSpec, + Primitives = OpPrimitives, + Storage = OpStorage, + >, + >, { type ComponentsBuilder = ComponentsBuilder< N, @@ -115,9 +163,10 @@ where } impl NodeTypes for OpNode { - type Primitives = reth_primitives::EthPrimitives; // todo: replace with OpPrimitives when EthPrimitives is only used in reth-ethereum-* crates + type Primitives = OpPrimitives; type ChainSpec = OpChainSpec; type StateCommitment = MerklePatriciaTrie; + type Storage = OpStorage; } impl NodeTypesWithEngine for OpNode { @@ -144,7 +193,7 @@ impl OpAddOns { impl NodeAddOns for OpAddOns where N: FullNodeComponents< - Types: NodeTypes, + Types: NodeTypes, PayloadBuilder: PayloadBuilder::Engine>, >, OpEngineValidator: EngineValidator<::Engine>, @@ -172,7 +221,7 @@ where impl RethRpcAddOns for OpAddOns where N: FullNodeComponents< - Types: NodeTypes, + Types: NodeTypes, PayloadBuilder: PayloadBuilder::Engine>, >, OpEngineValidator: EngineValidator<::Engine>, diff --git a/crates/primitives-traits/src/block/mod.rs b/crates/primitives-traits/src/block/mod.rs index 5b1faeafb..67658c39e 100644 --- a/crates/primitives-traits/src/block/mod.rs +++ b/crates/primitives-traits/src/block/mod.rs @@ -10,9 +10,9 @@ use reth_codecs::Compact; use crate::{BlockHeader, FullBlockHeader, InMemorySize, MaybeSerde}; /// Helper trait that unifies all behaviour required by block to support full node operations. -pub trait FullBlock: Block + Compact {} +pub trait FullBlock: Block {} -impl FullBlock for T where T: Block + Compact {} +impl FullBlock for T where T: Block {} /// Abstraction of block data type. // todo: make sealable super-trait, depends on diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index b2f438920..c3682ecba 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -79,6 +79,15 @@ pub mod serde_bincode_compat { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct EthPrimitives; +#[cfg(feature = "reth-codec")] +impl reth_primitives_traits::FullNodePrimitives for EthPrimitives { + type Block = crate::Block; + type SignedTx = crate::TransactionSigned; + type TxType = crate::TxType; + type Receipt = crate::Receipt; +} + +#[cfg(not(feature = "reth-codec"))] impl NodePrimitives for EthPrimitives { type Block = crate::Block; type SignedTx = crate::TransactionSigned; diff --git a/crates/stages/stages/src/stages/bodies.rs b/crates/stages/stages/src/stages/bodies.rs index 07b975749..80185eade 100644 --- a/crates/stages/stages/src/stages/bodies.rs +++ b/crates/stages/stages/src/stages/bodies.rs @@ -198,7 +198,10 @@ where // Write bodies to database. This will NOT write transactions to database as we've already // written them directly to static files. provider.append_block_bodies( - buffer.into_iter().map(|response| (response.block_number(), response.into_body())), + buffer + .into_iter() + .map(|response| (response.block_number(), response.into_body())) + .collect(), )?; // The stage is "done" if: diff --git a/crates/stages/stages/src/test_utils/test_db.rs b/crates/stages/stages/src/test_utils/test_db.rs index 772e9cb78..2f9712f84 100644 --- a/crates/stages/stages/src/test_utils/test_db.rs +++ b/crates/stages/stages/src/test_utils/test_db.rs @@ -15,7 +15,7 @@ use reth_db_api::{ DatabaseError as DbError, }; use reth_primitives::{ - Account, Receipt, SealedBlock, SealedHeader, StaticFileSegment, StorageEntry, + Account, EthPrimitives, Receipt, SealedBlock, SealedHeader, StaticFileSegment, StorageEntry, }; use reth_provider::{ providers::{StaticFileProvider, StaticFileProviderRWRefMut, StaticFileWriter}, @@ -142,7 +142,7 @@ impl TestStageDB { /// Insert header to static file if `writer` exists, otherwise to DB. pub fn insert_header( - writer: Option<&mut StaticFileProviderRWRefMut<'_, ()>>, + writer: Option<&mut StaticFileProviderRWRefMut<'_, EthPrimitives>>, tx: &TX, header: &SealedHeader, td: U256, diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index eff054063..674f02ada 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -17,6 +17,7 @@ reth-chainspec.workspace = true reth-blockchain-tree-api.workspace = true reth-execution-types.workspace = true reth-primitives = { workspace = true, features = ["reth-codec", "secp256k1"] } +reth-primitives-traits.workspace = true reth-fs-util.workspace = true reth-errors.workspace = true reth-storage-errors.workspace = true @@ -111,6 +112,7 @@ serde = [ "revm/serde", "reth-codecs/serde", "reth-optimism-primitives?/serde", + "reth-primitives-traits/serde", ] test-utils = [ "reth-db/test-utils", @@ -122,6 +124,7 @@ test-utils = [ "reth-evm/test-utils", "reth-network-p2p/test-utils", "reth-primitives/test-utils", + "reth-primitives-traits/test-utils", "reth-codecs/test-utils", "reth-db-api/test-utils", "reth-trie-db/test-utils", diff --git a/crates/storage/provider/src/providers/database/chain.rs b/crates/storage/provider/src/providers/database/chain.rs new file mode 100644 index 000000000..8f9a6395a --- /dev/null +++ b/crates/storage/provider/src/providers/database/chain.rs @@ -0,0 +1,26 @@ +use crate::{providers::NodeTypes, DatabaseProvider}; +use reth_db::transaction::{DbTx, DbTxMut}; +use reth_node_types::FullNodePrimitives; +use reth_primitives::EthPrimitives; +use reth_storage_api::{ChainStorageWriter, EthStorage}; + +/// Trait that provides access to implementations of [`ChainStorage`] +pub trait ChainStorage: Send + Sync { + /// Provides access to the chain writer. + fn writer(&self) -> impl ChainStorageWriter, Primitives> + where + TX: DbTxMut + DbTx + 'static, + Types: NodeTypes; +} + +impl ChainStorage for EthStorage { + fn writer( + &self, + ) -> impl ChainStorageWriter, EthPrimitives> + where + TX: DbTxMut + DbTx + 'static, + Types: NodeTypes, + { + self + } +} diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 94c83bbb4..a64bb2578 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -44,6 +44,9 @@ use super::ProviderNodeTypes; mod metrics; +mod chain; +pub use chain::*; + /// A common provider that fetches data from a database or static file. /// /// This provider implements most provider or provider factory traits. @@ -56,19 +59,22 @@ pub struct ProviderFactory { static_file_provider: StaticFileProvider, /// Optional pruning configuration prune_modes: PruneModes, + /// The node storage handler. + storage: Arc, } impl fmt::Debug for ProviderFactory where - N: NodeTypesWithDB, + N: NodeTypesWithDB, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { db, chain_spec, static_file_provider, prune_modes } = self; + let Self { db, chain_spec, static_file_provider, prune_modes, storage } = self; f.debug_struct("ProviderFactory") .field("db", &db) .field("chain_spec", &chain_spec) .field("static_file_provider", &static_file_provider) .field("prune_modes", &prune_modes) + .field("storage", &storage) .finish() } } @@ -80,7 +86,13 @@ impl ProviderFactory { chain_spec: Arc, static_file_provider: StaticFileProvider, ) -> Self { - Self { db, chain_spec, static_file_provider, prune_modes: PruneModes::none() } + Self { + db, + chain_spec, + static_file_provider, + prune_modes: PruneModes::none(), + storage: Default::default(), + } } /// Enables metrics on the static file provider. @@ -121,6 +133,7 @@ impl>> ProviderFactory { chain_spec, static_file_provider, prune_modes: PruneModes::none(), + storage: Default::default(), }) } } @@ -139,6 +152,7 @@ impl ProviderFactory { self.chain_spec.clone(), self.static_file_provider.clone(), self.prune_modes.clone(), + self.storage.clone(), )) } @@ -153,6 +167,7 @@ impl ProviderFactory { self.chain_spec.clone(), self.static_file_provider.clone(), self.prune_modes.clone(), + self.storage.clone(), ))) } @@ -617,6 +632,7 @@ impl Clone for ProviderFactory { chain_spec: self.chain_spec.clone(), static_file_provider: self.static_file_provider.clone(), prune_modes: self.prune_modes.clone(), + storage: self.storage.clone(), } } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index b93112e70..4690e2782 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,19 +1,24 @@ use crate::{ bundle_state::StorageRevertsIter, - providers::{database::metrics, static_file::StaticFileWriter, StaticFileProvider}, + providers::{ + database::{chain::ChainStorage, metrics}, + static_file::StaticFileWriter, + ProviderNodeTypes, StaticFileProvider, + }, to_range, traits::{ AccountExtReader, BlockSource, ChangeSetReader, ReceiptProvider, StageCheckpointWriter, }, writer::UnifiedStorageWriter, - AccountReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter, - BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, DBProvider, EvmEnvProvider, - HashingWriter, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, HistoricalStateProvider, - HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, LatestStateProviderRef, - OriginalValuesKnown, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, - StageCheckpointReader, StateChangeWriter, StateProviderBox, StateReader, StateWriter, - StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, TrieWriter, WithdrawalsProvider, + AccountReader, BlockBodyWriter, BlockExecutionWriter, BlockHashReader, BlockNumReader, + BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, + DBProvider, EvmEnvProvider, HashingWriter, HeaderProvider, HeaderSyncGap, + HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, + LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, + PruneCheckpointReader, PruneCheckpointWriter, RevertsInit, StageCheckpointReader, + StateChangeWriter, StateProviderBox, StateReader, StateWriter, StaticFileProviderFactory, + StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, TransactionsProvider, + TransactionsProviderExt, TrieWriter, WithdrawalsProvider, }; use alloy_consensus::Header; use alloy_eips::{ @@ -47,6 +52,7 @@ use reth_primitives::{ SealedBlockWithSenders, SealedHeader, StaticFileSegment, StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedEcRecovered, TransactionSignedNoHash, }; +use reth_primitives_traits::{BlockBody as _, FullNodePrimitives}; use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{StateProvider, StorageChangeSetReader, TryIntoHistoricalStateProvider}; @@ -138,6 +144,8 @@ pub struct DatabaseProvider { static_file_provider: StaticFileProvider, /// Pruning configuration prune_modes: PruneModes, + /// Node storage handler. + storage: Arc, } impl DatabaseProvider { @@ -224,8 +232,9 @@ impl DatabaseProvider { chain_spec: Arc, static_file_provider: StaticFileProvider, prune_modes: PruneModes, + storage: Arc, ) -> Self { - Self { tx, chain_spec, static_file_provider, prune_modes } + Self { tx, chain_spec, static_file_provider, prune_modes, storage } } } @@ -277,9 +286,7 @@ impl TryIntoHistoricalStateProvider for Databa } } -impl + 'static> - DatabaseProvider -{ +impl DatabaseProvider { // TODO: uncomment below, once `reth debug_cmd` has been feature gated with dev. // #[cfg(any(test, feature = "test-utils"))] /// Inserts an historical block. **Used for setting up test environments** @@ -367,8 +374,9 @@ impl DatabaseProvider { chain_spec: Arc, static_file_provider: StaticFileProvider, prune_modes: PruneModes, + storage: Arc, ) -> Self { - Self { tx, chain_spec, static_file_provider, prune_modes } + Self { tx, chain_spec, static_file_provider, prune_modes, storage } } /// Consume `DbTx` or `DbTxMut`. @@ -2899,8 +2907,8 @@ impl StateReader for DatabaseProvider { } } -impl + 'static> - BlockExecutionWriter for DatabaseProvider +impl BlockExecutionWriter + for DatabaseProvider { fn take_block_and_execution_range( &self, @@ -3101,10 +3109,11 @@ impl + } } -impl + 'static> BlockWriter +impl BlockWriter for DatabaseProvider { - type Body = BlockBody; + type Body = + <::Block as reth_primitives_traits::Block>::Body; /// Inserts the block into the database, always modifying the following tables: /// * [`CanonicalHeaders`](tables::CanonicalHeaders) @@ -3266,45 +3275,32 @@ impl + fn append_block_bodies( &self, - bodies: impl Iterator)>, + bodies: Vec<(BlockNumber, Option)>, ) -> ProviderResult<()> { let mut block_indices_cursor = self.tx.cursor_write::()?; let mut tx_block_cursor = self.tx.cursor_write::()?; - let mut ommers_cursor = self.tx.cursor_write::()?; - let mut withdrawals_cursor = self.tx.cursor_write::()?; // Get id for the next tx_num of zero if there are no transactions. let mut next_tx_num = tx_block_cursor.last()?.map(|(id, _)| id + 1).unwrap_or_default(); - for (block_number, body) in bodies { - let tx_count = body.as_ref().map(|b| b.transactions.len() as u64).unwrap_or_default(); + for (block_number, body) in &bodies { + let tx_count = body.as_ref().map(|b| b.transactions().len() as u64).unwrap_or_default(); let block_indices = StoredBlockBodyIndices { first_tx_num: next_tx_num, tx_count }; // insert block meta - block_indices_cursor.append(block_number, block_indices)?; + block_indices_cursor.append(*block_number, block_indices)?; next_tx_num += tx_count; let Some(body) = body else { continue }; // write transaction block index - if !body.transactions.is_empty() { - tx_block_cursor.append(block_indices.last_tx_num(), block_number)?; - } - - // Write ommers if any - if !body.ommers.is_empty() { - ommers_cursor.append(block_number, StoredBlockOmmers { ommers: body.ommers })?; - } - - // Write withdrawals if any - if let Some(withdrawals) = body.withdrawals { - if !withdrawals.is_empty() { - withdrawals_cursor - .append(block_number, StoredBlockWithdrawals { withdrawals })?; - } + if !body.transactions().is_empty() { + tx_block_cursor.append(block_indices.last_tx_num(), *block_number)?; } } + self.storage.writer().write_block_bodies(self, bodies)?; + Ok(()) } diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 3bf3e7b24..d04924337 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -22,7 +22,7 @@ use reth_chain_state::{ChainInfoTracker, ForkChoiceNotifications, ForkChoiceSubs use reth_chainspec::{ChainInfo, EthereumHardforks}; use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_evm::ConfigureEvmEnv; -use reth_node_types::NodeTypesWithDB; +use reth_node_types::{FullNodePrimitives, NodeTypes, NodeTypesWithDB}; use reth_primitives::{ Account, Block, BlockWithSenders, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, TransactionMeta, TransactionSigned, TransactionSignedNoHash, @@ -37,6 +37,7 @@ use std::{ sync::Arc, time::Instant, }; + use tracing::trace; mod database; @@ -67,10 +68,35 @@ pub use blockchain_provider::BlockchainProvider2; mod consistent; pub use consistent::ConsistentProvider; -/// Helper trait keeping common requirements of providers for [`NodeTypesWithDB`]. -pub trait ProviderNodeTypes: NodeTypesWithDB {} +/// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy +/// [`ProviderNodeTypes`]. +pub trait NodeTypesForProvider +where + Self: NodeTypes< + ChainSpec: EthereumHardforks, + Storage: ChainStorage, + Primitives: FullNodePrimitives, + >, +{ +} -impl ProviderNodeTypes for T where T: NodeTypesWithDB {} +impl NodeTypesForProvider for T where + T: NodeTypes< + ChainSpec: EthereumHardforks, + Storage: ChainStorage, + Primitives: FullNodePrimitives, + > +{ +} + +/// Helper trait keeping common requirements of providers for [`NodeTypesWithDB`]. +pub trait ProviderNodeTypes +where + Self: NodeTypesForProvider + NodeTypesWithDB, +{ +} + +impl ProviderNodeTypes for T where T: NodeTypesForProvider + NodeTypesWithDB {} /// The main type for interacting with the blockchain. /// diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 9661ab205..43bb1e809 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1,9 +1,9 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, - ChainSpecProvider, ChangeSetReader, DatabaseProvider, EvmEnvProvider, HeaderProvider, - ReceiptProviderIdExt, StateProvider, StateProviderBox, StateProviderFactory, StateReader, - StateRootProvider, TransactionVariant, TransactionsProvider, WithdrawalsProvider, + ChainSpecProvider, ChangeSetReader, DatabaseProvider, EthStorage, EvmEnvProvider, + HeaderProvider, ReceiptProviderIdExt, StateProvider, StateProviderBox, StateProviderFactory, + StateReader, StateRootProvider, TransactionVariant, TransactionsProvider, WithdrawalsProvider, }; use alloy_consensus::{constants::EMPTY_ROOT_HASH, Header}; use alloy_eips::{ @@ -23,7 +23,7 @@ use reth_evm::ConfigureEvmEnv; use reth_execution_types::ExecutionOutcome; use reth_node_types::NodeTypes; use reth_primitives::{ - Account, Block, BlockWithSenders, Bytecode, GotExpected, Receipt, SealedBlock, + Account, Block, BlockWithSenders, Bytecode, EthPrimitives, GotExpected, Receipt, SealedBlock, SealedBlockWithSenders, SealedHeader, TransactionMeta, TransactionSigned, TransactionSignedNoHash, }; @@ -158,9 +158,10 @@ impl MockEthProvider { pub struct MockNode; impl NodeTypes for MockNode { - type Primitives = (); + type Primitives = EthPrimitives; type ChainSpec = ChainSpec; type StateCommitment = MerklePatriciaTrie; + type Storage = EthStorage; } impl DatabaseProviderFactory for MockEthProvider { diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index c0e80930b..2c3795573 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -1,4 +1,7 @@ -use crate::{providers::StaticFileProvider, HashingWriter, ProviderFactory, TrieWriter}; +use crate::{ + providers::{ProviderNodeTypes, StaticFileProvider}, + HashingWriter, ProviderFactory, TrieWriter, +}; use alloy_primitives::B256; use reth_chainspec::{ChainSpec, MAINNET}; use reth_db::{ @@ -6,7 +9,7 @@ use reth_db::{ DatabaseEnv, }; use reth_errors::ProviderResult; -use reth_node_types::{NodeTypesWithDB, NodeTypesWithDBAdapter}; +use reth_node_types::NodeTypesWithDBAdapter; use reth_primitives::{Account, StorageEntry}; use reth_trie::StateRoot; use reth_trie_db::DatabaseStateRoot; @@ -22,10 +25,11 @@ pub use reth_chain_state::test_utils::TestCanonStateSubscriptions; /// Mock [`reth_node_types::NodeTypes`] for testing. pub type MockNodeTypes = reth_node_types::AnyNodeTypesWithEngine< - (), + reth_primitives::EthPrimitives, reth_ethereum_engine_primitives::EthEngineTypes, reth_chainspec::ChainSpec, reth_trie_db::MerklePatriciaTrie, + crate::EthStorage, >; /// Mock [`reth_node_types::NodeTypesWithDB`] for testing. @@ -51,7 +55,7 @@ pub fn create_test_provider_factory_with_chain_spec( } /// Inserts the genesis alloc from the provided chain spec into the trie. -pub fn insert_genesis>( +pub fn insert_genesis>( provider_factory: &ProviderFactory, chain_spec: Arc, ) -> ProviderResult { diff --git a/crates/storage/provider/src/traits/block.rs b/crates/storage/provider/src/traits/block.rs index 50fb03292..a0dae1783 100644 --- a/crates/storage/provider/src/traits/block.rs +++ b/crates/storage/provider/src/traits/block.rs @@ -50,7 +50,7 @@ pub trait BlockWriter: Send + Sync { /// Bodies are passed as [`Option`]s, if body is `None` the corresponding block is empty. fn append_block_bodies( &self, - bodies: impl Iterator)>, + bodies: Vec<(BlockNumber, Option)>, ) -> ProviderResult<()>; /// Appends a batch of sealed blocks to the blockchain, including sender information, and diff --git a/crates/storage/storage-api/Cargo.toml b/crates/storage/storage-api/Cargo.toml index 2b13f6332..c059eb0d6 100644 --- a/crates/storage/storage-api/Cargo.toml +++ b/crates/storage/storage-api/Cargo.toml @@ -18,6 +18,7 @@ reth-db-models.workspace = true reth-db-api.workspace = true reth-execution-types.workspace = true reth-primitives.workspace = true +reth-primitives-traits.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true reth-storage-errors.workspace = true diff --git a/crates/storage/storage-api/src/chain.rs b/crates/storage/storage-api/src/chain.rs new file mode 100644 index 000000000..099f61f1b --- /dev/null +++ b/crates/storage/storage-api/src/chain.rs @@ -0,0 +1,72 @@ +use crate::DBProvider; +use alloy_primitives::BlockNumber; +use reth_db::{ + cursor::DbCursorRW, + models::{StoredBlockOmmers, StoredBlockWithdrawals}, + tables, + transaction::DbTxMut, +}; +use reth_primitives_traits::{Block, BlockBody, FullNodePrimitives}; +use reth_storage_errors::provider::ProviderResult; + +/// Trait that implements how block bodies are written to the storage. +/// +/// Note: Within the current abstraction, this should only write to tables unrelated to +/// transactions. Writing of transactions is handled separately. +#[auto_impl::auto_impl(&, Arc)] +pub trait BlockBodyWriter { + /// Writes a set of block bodies to the storage. + fn write_block_bodies( + &self, + provider: &Provider, + bodies: Vec<(BlockNumber, Option)>, + ) -> ProviderResult<()>; +} + +/// Trait that implements how chain-specific types are written to the storage. +pub trait ChainStorageWriter: + BlockBodyWriter::Body> +{ +} +impl ChainStorageWriter for T where + T: BlockBodyWriter::Body> +{ +} + +/// Ethereum storage implementation. +#[derive(Debug, Default, Clone, Copy)] +pub struct EthStorage; + +impl BlockBodyWriter for EthStorage +where + Provider: DBProvider, +{ + fn write_block_bodies( + &self, + provider: &Provider, + bodies: Vec<(u64, Option)>, + ) -> ProviderResult<()> { + let mut ommers_cursor = provider.tx_ref().cursor_write::()?; + let mut withdrawals_cursor = + provider.tx_ref().cursor_write::()?; + + for (block_number, body) in bodies { + let Some(body) = body else { continue }; + + // Write ommers if any + if !body.ommers.is_empty() { + ommers_cursor.append(block_number, StoredBlockOmmers { ommers: body.ommers })?; + } + + // Write withdrawals if any + if let Some(withdrawals) = body.withdrawals { + if !withdrawals.is_empty() { + withdrawals_cursor + .append(block_number, StoredBlockWithdrawals { withdrawals })?; + } + } + } + + Ok(()) + } +} diff --git a/crates/storage/storage-api/src/lib.rs b/crates/storage/storage-api/src/lib.rs index 7b7ad7614..be52a817e 100644 --- a/crates/storage/storage-api/src/lib.rs +++ b/crates/storage/storage-api/src/lib.rs @@ -22,6 +22,9 @@ pub use block_id::*; mod block_hash; pub use block_hash::*; +mod chain; +pub use chain::*; + mod header; pub use header::*; diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 704ecb7e3..c21e893e0 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -42,7 +42,8 @@ use reth::{ PayloadBuilderConfig, }, network::NetworkHandle, - providers::{CanonStateSubscriptions, StateProviderFactory}, + primitives::EthPrimitives, + providers::{CanonStateSubscriptions, EthStorage, StateProviderFactory}, rpc::eth::EthApi, tasks::TaskManager, transaction_pool::TransactionPool, @@ -227,9 +228,10 @@ struct MyCustomNode; /// Configure the node types impl NodeTypes for MyCustomNode { - type Primitives = (); + type Primitives = EthPrimitives; type ChainSpec = ChainSpec; type StateCommitment = MerklePatriciaTrie; + type Storage = EthStorage; } /// Configure the node types with the custom engine types @@ -254,7 +256,14 @@ pub type MyNodeAddOns = RpcAddOns< /// This provides a preset configuration for the node impl Node for MyCustomNode where - N: FullNodeTypes>, + N: FullNodeTypes< + Types: NodeTypesWithEngine< + Engine = CustomEngineTypes, + ChainSpec = ChainSpec, + Primitives = EthPrimitives, + Storage = EthStorage, + >, + >, { type ComponentsBuilder = ComponentsBuilder< N,