diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 5c5879983..be56462b1 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -22,7 +22,7 @@ use reth_node_builder::{ BuilderContext, Node, NodeAdapter, NodeComponentsBuilder, PayloadTypes, }; use reth_primitives::{EthPrimitives, PooledTransaction}; -use reth_provider::{CanonStateSubscriptions, EthStorage}; +use reth_provider::{providers::ProviderFactoryBuilder, CanonStateSubscriptions, EthStorage}; use reth_rpc::EthApi; use reth_tracing::tracing::{debug, info}; use reth_transaction_pool::{ @@ -63,6 +63,41 @@ impl EthereumNode { .executor(EthereumExecutorBuilder::default()) .consensus(EthereumConsensusBuilder::default()) } + + /// Instantiates the [`ProviderFactoryBuilder`] for an ethereum node. + /// + /// # Open a Providerfactory in read-only mode from a datadir + /// + /// See also: [`ProviderFactoryBuilder`] and + /// [`ReadOnlyConfig`](reth_provider::providers::ReadOnlyConfig). + /// + /// ```no_run + /// use reth_chainspec::MAINNET; + /// use reth_node_ethereum::EthereumNode; + /// + /// let factory = EthereumNode::provider_factory_builder() + /// .open_read_only(MAINNET.clone(), "datadir") + /// .unwrap(); + /// ``` + /// + /// # Open a Providerfactory manually with with all required componets + /// + /// ```no_run + /// use reth_chainspec::ChainSpecBuilder; + /// use reth_db::open_db_read_only; + /// use reth_node_ethereum::EthereumNode; + /// use reth_provider::providers::StaticFileProvider; + /// use std::{path::Path, sync::Arc}; + /// + /// let factory = EthereumNode::provider_factory_builder() + /// .db(Arc::new(open_db_read_only(Path::new("db"), Default::default()).unwrap())) + /// .chainspec(ChainSpecBuilder::mainnet().build().into()) + /// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap()) + /// .build_provider_factory(); + /// ``` + pub fn provider_factory_builder() -> ProviderFactoryBuilder { + ProviderFactoryBuilder::default() + } } impl NodeTypes for EthereumNode { diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 405bebf3d..a728ef71f 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -58,6 +58,7 @@ notify = { workspace = true, default-features = false, features = ["macos_fseven parking_lot.workspace = true dashmap = { workspace = true, features = ["inline"] } strum.workspace = true +eyre.workspace = true # test-utils reth-ethereum-engine-primitives = { workspace = true, optional = true } diff --git a/crates/storage/provider/src/providers/database/builder.rs b/crates/storage/provider/src/providers/database/builder.rs new file mode 100644 index 000000000..dd6bdfca1 --- /dev/null +++ b/crates/storage/provider/src/providers/database/builder.rs @@ -0,0 +1,280 @@ +//! Helper builder entrypoint to instantiate a [`ProviderFactory`]. +//! +//! This also includes general purpose staging types that provide builder style functions that lead +//! up to the intended build target. + +use crate::{providers::StaticFileProvider, ProviderFactory}; +use reth_db::{mdbx::DatabaseArguments, open_db_read_only, DatabaseEnv}; +use reth_db_api::{database_metrics::DatabaseMetrics, Database}; +use reth_node_types::{NodeTypes, NodeTypesWithDBAdapter}; +use std::{ + marker::PhantomData, + path::{Path, PathBuf}, + sync::Arc, +}; + +/// Helper type to create a [`ProviderFactory`]. +/// +/// This type is the entry point for a stage based builder. +/// +/// The intended staging is: +/// 1. Configure the database: [`ProviderFactoryBuilder::db`] +/// 2. Configure the chainspec: `chainspec` +/// 3. Configure the [`StaticFileProvider`]: `static_file` +#[derive(Debug)] +pub struct ProviderFactoryBuilder { + _types: PhantomData, +} + +impl ProviderFactoryBuilder { + /// Maps the [`NodeTypes`] of this builder. + pub fn types(self) -> ProviderFactoryBuilder { + ProviderFactoryBuilder::default() + } + + /// Configures the database. + pub fn db(self, db: DB) -> TypesAnd1 { + TypesAnd1::new(db) + } + + /// Opens the database with the given chainspec and [`ReadOnlyConfig`]. + /// + /// # Open a monitored instance + /// + /// This is recommended when the new read-only instance is used with an active node. + /// + /// ```no_run + /// use reth_chainspec::MAINNET; + /// use reth_node_types::NodeTypes; + /// use reth_provider::providers::ProviderFactoryBuilder; + /// + /// fn demo>() { + /// let provider_factory = ProviderFactoryBuilder::::default() + /// .open_read_only(MAINNET.clone(), "datadir") + /// .unwrap(); + /// } + /// ``` + /// + /// # Open an unmonitored instace + /// + /// This is recommended when no changes to the static files are expected (e.g. no active node) + /// + /// ```no_run + /// use reth_chainspec::MAINNET; + /// use reth_node_types::NodeTypes; + /// /// + /// use reth_provider::providers::{ProviderFactoryBuilder, ReadOnlyConfig}; + /// + /// fn demo>() { + /// let provider_factory = ProviderFactoryBuilder::::default() + /// .open_read_only(MAINNET.clone(), ReadOnlyConfig::from_datadir("datadir").no_watch()) + /// .unwrap(); + /// } + /// ``` + pub fn open_read_only( + self, + chainspec: Arc, + config: impl Into, + ) -> eyre::Result>>> + where + N: NodeTypes, + { + let ReadOnlyConfig { db_dir, db_args, static_files_dir, watch_static_files } = + config.into(); + Ok(self + .db(Arc::new(open_db_read_only(db_dir, db_args)?)) + .chainspec(chainspec) + .static_file(StaticFileProvider::read_only(static_files_dir, watch_static_files)?) + .build_provider_factory()) + } +} + +impl Default for ProviderFactoryBuilder { + fn default() -> Self { + Self { _types: Default::default() } + } +} + +/// Settings for how to open the database and static files. +/// +/// The default derivation from a path assumes the path is the datadir: +/// [`ReadOnlyConfig::from_datadir`] +#[derive(Debug, Clone)] +pub struct ReadOnlyConfig { + /// The path to the database directory. + pub db_dir: PathBuf, + /// How to open the database + pub db_args: DatabaseArguments, + /// The path to the static file dir + pub static_files_dir: PathBuf, + /// Whether the static files should be watched for changes. + pub watch_static_files: bool, +} + +impl ReadOnlyConfig { + /// Derives the [`ReadOnlyConfig`] from the datadir. + /// + /// By default this assumes the following datadir layour: + /// + /// ```text + /// -`datadir` + /// |__db + /// |__static_files + /// ``` + /// + /// By default this watches the static file directory for changes, see also + /// [`StaticFileProvider::read_only`] + pub fn from_datadir(datadir: impl AsRef) -> Self { + let datadir = datadir.as_ref(); + let db_path = datadir.join("db"); + Self::from_db_dir(db_path) + } + + /// Derives the [`ReadOnlyConfig`] from the database dir. + /// + /// By default this assumes the following datadir layour: + /// + /// ```text + /// - db + /// |__static_files + /// ``` + /// + /// By default this watches the static file directory for changes, see also + /// [`StaticFileProvider::read_only`] + pub fn from_db_dir(db_dir: impl AsRef) -> Self { + let db_dir = db_dir.as_ref(); + Self { + static_files_dir: db_dir.join("static_files"), + db_dir: db_dir.into(), + db_args: Default::default(), + watch_static_files: true, + } + } + + /// Configures the db arguments used when opening the database. + pub fn with_db_args(mut self, db_args: impl Into) -> Self { + self.db_args = db_args.into(); + self + } + + /// Configures the db directory. + pub fn with_db_dir(mut self, db_dir: impl Into) -> Self { + self.db_dir = db_dir.into(); + self + } + + /// Configures the static file directory. + pub fn with_static_file_dir(mut self, static_file_dir: impl Into) -> Self { + self.static_files_dir = static_file_dir.into(); + self + } + + /// Whether the static file directory should be watches for changes, see also + /// [`StaticFileProvider::read_only`] + pub fn set_watch_static_files(&mut self, watch_static_files: bool) { + self.watch_static_files = watch_static_files; + } + + /// Don't watch the static files for changes. + /// + /// This is only recommended if this is used without a running node instance that modifies + /// static files. + pub fn no_watch(mut self) -> Self { + self.set_watch_static_files(false); + self + } +} + +impl From for ReadOnlyConfig +where + T: AsRef, +{ + fn from(value: T) -> Self { + Self::from_datadir(value.as_ref()) + } +} + +/// This is staging type that contains the configured types and _one_ value. +#[derive(Debug)] +pub struct TypesAnd1 { + _types: PhantomData, + val_1: Val1, +} + +impl TypesAnd1 { + /// Creates a new instance with the given types and one value. + pub fn new(val_1: Val1) -> Self { + Self { _types: Default::default(), val_1 } + } + + /// Configures the chainspec. + pub fn chainspec(self, chainspec: Arc) -> TypesAnd2> { + TypesAnd2::new(self.val_1, chainspec) + } +} + +/// This is staging type that contains the configured types and _two_ values. +#[derive(Debug)] +pub struct TypesAnd2 { + _types: PhantomData, + val_1: Val1, + val_2: Val2, +} + +impl TypesAnd2 { + /// Creates a new instance with the given types and two values. + pub fn new(val_1: Val1, val_2: Val2) -> Self { + Self { _types: Default::default(), val_1, val_2 } + } + + /// Returns the first value. + pub const fn val_1(&self) -> &Val1 { + &self.val_1 + } + + /// Returns the second value. + pub const fn val_2(&self) -> &Val2 { + &self.val_2 + } + + /// Configures the [`StaticFileProvider`]. + pub fn static_file( + self, + static_file_provider: StaticFileProvider, + ) -> TypesAnd3> + where + N: NodeTypes, + { + TypesAnd3::new(self.val_1, self.val_2, static_file_provider) + } + + // TODO: add helper fns for opening static file provider +} + +/// This is staging type that contains the configured types and _three_ values. +#[derive(Debug)] +pub struct TypesAnd3 { + _types: PhantomData, + val_1: Val1, + val_2: Val2, + val_3: Val3, +} + +impl TypesAnd3 { + /// Creates a new instance with the given types and three values. + pub fn new(val_1: Val1, val_2: Val2, val_3: Val3) -> Self { + Self { _types: Default::default(), val_1, val_2, val_3 } + } +} + +impl TypesAnd3, StaticFileProvider> +where + N: NodeTypes, + DB: Database + DatabaseMetrics + Clone + Unpin + 'static, +{ + /// Creates the [`ProviderFactory`]. + pub fn build_provider_factory(self) -> ProviderFactory> { + let Self { _types, val_1, val_2, val_3 } = self; + ProviderFactory::new(val_1, val_2, val_3) + } +} diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 157922b50..9c2e5031f 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -15,7 +15,9 @@ use reth_chainspec::{ChainInfo, EthereumHardforks}; use reth_db::{init_db, mdbx::DatabaseArguments, DatabaseEnv}; use reth_db_api::{database::Database, models::StoredBlockBodyIndices}; use reth_errors::{RethError, RethResult}; -use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; +use reth_node_types::{ + BlockTy, HeaderTy, NodeTypes, NodeTypesWithDB, NodeTypesWithDBAdapter, ReceiptTy, TxTy, +}; use reth_primitives::{RecoveredBlock, SealedBlock, SealedHeader, StaticFileSegment}; use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; @@ -40,6 +42,9 @@ pub use provider::{DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW}; use super::ProviderNodeTypes; +mod builder; +pub use builder::{ProviderFactoryBuilder, ReadOnlyConfig}; + mod metrics; mod chain; @@ -49,7 +54,7 @@ pub use chain::*; /// /// This provider implements most provider or provider factory traits. pub struct ProviderFactory { - /// Database + /// Database instance db: N::DB, /// Chain spec chain_spec: Arc, @@ -61,19 +66,10 @@ pub struct ProviderFactory { storage: Arc, } -impl fmt::Debug for ProviderFactory -where - N: NodeTypesWithDB, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let Self { db, chain_spec, static_file_provider, prune_modes, storage } = self; - f.debug_struct("ProviderFactory") - .field("db", &db) - .field("chain_spec", &chain_spec) - .field("static_file_provider", &static_file_provider) - .field("prune_modes", &prune_modes) - .field("storage", &storage) - .finish() +impl ProviderFactory>> { + /// Instantiates the builder for this type + pub fn builder() -> ProviderFactoryBuilder { + ProviderFactoryBuilder::default() } } @@ -630,6 +626,22 @@ impl HashedPostStateProvider for ProviderFactory { } } +impl fmt::Debug for ProviderFactory +where + N: NodeTypesWithDB, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Self { db, chain_spec, static_file_provider, prune_modes, storage } = self; + f.debug_struct("ProviderFactory") + .field("db", &db) + .field("chain_spec", &chain_spec) + .field("static_file_provider", &static_file_provider) + .field("prune_modes", &prune_modes) + .field("storage", &storage) + .finish() + } +} + impl Clone for ProviderFactory { fn clone(&self) -> Self { Self { diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 3dbfbf331..54a191177 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -79,6 +79,13 @@ impl StaticFileAccess { } /// [`StaticFileProvider`] manages all existing [`StaticFileJarProvider`]. +/// +/// "Static files" contain immutable chain history data, such as: +/// - transactions +/// - headers +/// - receipts +/// +/// This provider type is responsible for reading and writing to static files. #[derive(Debug)] pub struct StaticFileProvider(pub(crate) Arc>); @@ -89,7 +96,7 @@ impl Clone for StaticFileProvider { } impl StaticFileProvider { - /// Creates a new [`StaticFileProvider`]. + /// Creates a new [`StaticFileProvider`] with the given [`StaticFileAccess`]. fn new(path: impl AsRef, access: StaticFileAccess) -> ProviderResult { let provider = Self(Arc::new(StaticFileProviderInner::new(path, access)?)); provider.initialize_index()?; @@ -100,6 +107,11 @@ impl StaticFileProvider { /// /// Set `watch_directory` to `true` to track the most recent changes in static files. Otherwise, /// new data won't be detected or queryable. + /// + /// Watching is recommended if the read-only provider is used on a directory that an active node + /// instance is modifying. + /// + /// See also [`StaticFileProvider::watch_directory`]. pub fn read_only(path: impl AsRef, watch_directory: bool) -> ProviderResult { let provider = Self::new(path, StaticFileAccess::RO)?; diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index 0b5ca570a..dc871567a 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -2,16 +2,13 @@ use alloy_consensus::BlockHeader; use alloy_primitives::{Address, B256}; use alloy_rpc_types_eth::{Filter, FilteredParams}; use reth_chainspec::ChainSpecBuilder; -use reth_db::{open_db_read_only, DatabaseEnv}; use reth_node_ethereum::EthereumNode; -use reth_node_types::NodeTypesWithDBAdapter; use reth_primitives::{SealedBlock, SealedHeader, TransactionSigned}; use reth_primitives_traits::transaction::signed::SignedTransaction; use reth_provider::{ - providers::StaticFileProvider, AccountReader, BlockReader, BlockSource, HeaderProvider, - ProviderFactory, ReceiptProvider, StateProvider, TransactionsProvider, + providers::ReadOnlyConfig, AccountReader, BlockReader, BlockSource, HeaderProvider, + ReceiptProvider, StateProvider, TransactionsProvider, }; -use std::{path::Path, sync::Arc}; // Providers are zero cost abstractions on top of an opened MDBX Transaction // exposing a familiar API to query the chain's information without requiring knowledge @@ -22,17 +19,11 @@ use std::{path::Path, sync::Arc}; fn main() -> eyre::Result<()> { // Opens a RO handle to the database file. let db_path = std::env::var("RETH_DB_PATH")?; - let db_path = Path::new(&db_path); - let db = open_db_read_only(db_path.join("db").as_path(), Default::default())?; - // Instantiate a provider factory for Ethereum mainnet using the provided DB. - // TODO: Should the DB version include the spec so that you do not need to specify it here? + // Instantiate a provider factory for Ethereum mainnet using the provided DB path. let spec = ChainSpecBuilder::mainnet().build(); - let factory = ProviderFactory::>>::new( - db.into(), - spec.into(), - StaticFileProvider::read_only(db_path.join("static_files"), true)?, - ); + let factory = EthereumNode::provider_factory_builder() + .open_read_only(spec.into(), ReadOnlyConfig::from_db_dir(db_path))?; // This call opens a RO transaction on the database. To write to the DB you'd need to call // the `provider_rw` function and look for the `Writer` variants of the traits.