Move reth-primitives::fs module to fs-utils crate (#8286)

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

crates/fs-util/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "reth-fs-util"
+version.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+description = "Commonly used fs utils in reth."
+
+[lints]
+workspace = true
+
+[dependencies]
+
+# misc
+serde_json.workspace = true
+thiserror.workspace = true

crates/fs-util/src/lib.rs

@@ -1,11 +1,21 @@
 //! Wrapper for `std::fs` methods
 
+#![doc(
+    html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
+    html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
+    issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
+)]
+#![cfg_attr(not(test), warn(unused_crate_dependencies))]
+
 use std::{
     fs::{self, ReadDir},
     io,
     path::{Path, PathBuf},
 };
 
+/// Result alias for [FsPathError].
+pub type Result<T> = std::result::Result<T, FsPathError>;
+
 /// Various error variants for `std::fs` operations that serve as an addition to the io::Error which
 /// does not provide any information about the path.
 #[derive(Debug, thiserror::Error)]
@@ -187,8 +197,6 @@ impl FsPathError {
     }
 }
 
-type Result<T> = std::result::Result<T, FsPathError>;
-
 /// Wrapper for `std::fs::read_to_string`
 pub fn read_to_string(path: impl AsRef<Path>) -> Result<String> {
     let path = path.as_ref();

crates/interfaces/Cargo.toml

@@ -12,6 +12,7 @@ workspace = true
 
 [dependencies]
 reth-primitives.workspace = true
+reth-fs-util.workspace = true
 reth-network-api.workspace = true
 reth-eth-wire-types.workspace = true
 reth-consensus.workspace = true

crates/interfaces/src/error.rs

@@ -5,8 +5,8 @@ use crate::{
     provider::ProviderError,
 };
 use reth_consensus::ConsensusError;
+use reth_fs_util::FsPathError;
 use reth_network_api::NetworkError;
-use reth_primitives::fs::FsPathError;
 
 /// Result alias for [`RethError`].
 pub type RethResult<T> = Result<T, RethError>;

crates/interfaces/src/provider.rs

@@ -130,8 +130,8 @@ pub enum ProviderError {
     ConsistentView(Box<ConsistentViewError>),
 }
 
-impl From<reth_primitives::fs::FsPathError> for ProviderError {
-    fn from(err: reth_primitives::fs::FsPathError) -> Self {
+impl From<reth_fs_util::FsPathError> for ProviderError {
+    fn from(err: reth_fs_util::FsPathError) -> Self {
         ProviderError::FsPathError(err.to_string())
     }
 }

crates/node-core/Cargo.toml

@@ -13,6 +13,7 @@ workspace = true
 [dependencies]
 # reth
 reth-primitives.workspace = true
+reth-fs-util.workspace = true
 reth-db = { workspace = true, features = ["mdbx"] }
 reth-interfaces = { workspace = true, features = ["clap"] }
 reth-provider.workspace = true

crates/node-core/src/args/secret_key.rs

@@ -1,5 +1,6 @@
+use reth_fs_util::{self as fs, FsPathError};
 use reth_network::config::rng_secret_key;
-use reth_primitives::{fs, fs::FsPathError, hex::encode as hex_encode};
+use reth_primitives::hex::encode as hex_encode;
 use secp256k1::{Error as SecretKeyBaseError, SecretKey};
 use std::{
     io,

crates/node-core/src/args/utils.rs

@@ -1,6 +1,7 @@
 //! Clap parser utilities
 
-use reth_primitives::{fs, AllGenesisFormats, BlockHashOrNumber, ChainSpec, B256};
+use reth_fs_util as fs;
+use reth_primitives::{AllGenesisFormats, BlockHashOrNumber, ChainSpec, B256};
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs},
     path::PathBuf,

crates/node-core/src/engine/engine_store.rs

@@ -3,7 +3,7 @@
 use futures::{Stream, StreamExt};
 use reth_beacon_consensus::BeaconEngineMessage;
 use reth_engine_primitives::EngineTypes;
-use reth_primitives::fs;
+use reth_fs_util as fs;
 use reth_rpc_types::{
     engine::{CancunPayloadFields, ForkchoiceState},
     ExecutionPayload,

crates/node-core/src/utils.rs

@@ -3,15 +3,14 @@
 
 use eyre::Result;
 use reth_consensus_common::validation::validate_block_standalone;
+use reth_fs_util as fs;
 use reth_interfaces::p2p::{
     bodies::client::BodiesClient,
     headers::client::{HeadersClient, HeadersRequest},
     priority::Priority,
 };
 use reth_network::NetworkManager;
-use reth_primitives::{
-    fs, BlockHashOrNumber, ChainSpec, HeadersDirection, SealedBlock, SealedHeader,
-};
+use reth_primitives::{BlockHashOrNumber, ChainSpec, HeadersDirection, SealedBlock, SealedHeader};
 use reth_provider::BlockReader;
 use reth_rpc_layer::{JwtError, JwtSecret};
 use std::{

crates/primitives/Cargo.toml

@@ -30,7 +30,11 @@ alloy-eips = { workspace = true, features = ["serde"] }
 nybbles = { workspace = true, features = ["serde", "rlp"] }
 
 # crypto
-secp256k1 = { workspace = true, features = ["global-context", "recovery", "rand"] }
+secp256k1 = { workspace = true, features = [
+    "global-context",
+    "recovery",
+    "rand",
+] }
 # for eip-4844
 c-kzg = { workspace = true, features = ["serde"], optional = true }
 
@@ -83,7 +87,11 @@ plain_hasher = "0.2"
 sucds = "0.8.1"
 
 criterion.workspace = true
-pprof = { workspace = true, features = ["flamegraph", "frame-pointer", "criterion"] }
+pprof = { workspace = true, features = [
+    "flamegraph",
+    "frame-pointer",
+    "criterion",
+] }
 
 [features]
 default = ["c-kzg", "zstd-codec"]
@@ -100,7 +108,13 @@ arbitrary = [
     "dep:proptest-derive",
     "zstd-codec",
 ]
-c-kzg = ["dep:c-kzg", "revm/c-kzg", "revm-primitives/c-kzg", "dep:tempfile", "alloy-eips/kzg"]
+c-kzg = [
+    "dep:c-kzg",
+    "revm/c-kzg",
+    "revm-primitives/c-kzg",
+    "dep:tempfile",
+    "alloy-eips/kzg",
+]
 zstd-codec = ["dep:zstd"]
 clap = ["dep:clap"]
 optimism = [

crates/primitives/src/lib.rs

@@ -30,7 +30,6 @@ pub mod constants;
 pub mod eip4844;
 mod error;
 mod exex;
-pub mod fs;
 pub mod genesis;
 mod header;
 mod integer_list;

crates/rpc/rpc-layer/Cargo.toml

@@ -12,6 +12,7 @@ workspace = true
 
 [dependencies]
 reth-primitives.workspace = true
+reth-fs-util.workspace = true
 
 http.workspace = true
 hyper.workspace = true

crates/rpc/rpc-layer/src/jwt_secret.rs

@@ -1,10 +1,7 @@
 use jsonwebtoken::{decode, errors::ErrorKind, Algorithm, DecodingKey, Validation};
 use rand::Rng;
-use reth_primitives::{
-    fs,
-    fs::FsPathError,
-    hex::{self, encode as hex_encode},
-};
+use reth_fs_util::{self as fs, FsPathError};
+use reth_primitives::hex::{self, encode as hex_encode};
 use serde::{Deserialize, Serialize};
 use std::{
     path::Path,

crates/stages/benches/setup/account_hashing.rs

@@ -4,12 +4,12 @@ use super::constants;
 use reth_db::{
     cursor::DbCursorRO, database::Database, tables, transaction::DbTx, DatabaseError as DbError,
 };
-use reth_primitives::{fs, stage::StageCheckpoint, BlockNumber};
+use reth_primitives::{stage::StageCheckpoint, BlockNumber};
 use reth_stages::{
     stages::{AccountHashingStage, SeedOpts},
     test_utils::TestStageDB,
 };
-use std::{ops::RangeInclusive, path::Path};
+use std::{fs, ops::RangeInclusive, path::Path};
 
 /// Prepares a database for [`AccountHashingStage`]
 /// If the environment variable [`constants::ACCOUNT_HASHING_DB`] is set, it will use that one and

crates/stages/benches/setup/mod.rs

@@ -14,13 +14,13 @@ use reth_interfaces::test_utils::{
         random_eoa_accounts,
     },
 };
-use reth_primitives::{fs, Account, Address, SealedBlock, B256, U256};
+use reth_primitives::{Account, Address, SealedBlock, B256, U256};
 use reth_stages::{
     stages::{AccountHashingStage, StorageHashingStage},
     test_utils::{StorageKind, TestStageDB},
 };
 use reth_trie::StateRoot;
-use std::{collections::BTreeMap, path::Path, sync::Arc};
+use std::{collections::BTreeMap, fs, path::Path, sync::Arc};
 use tokio::runtime::Handle;
 
 mod constants;

crates/storage/db/Cargo.toml

@@ -14,6 +14,7 @@ workspace = true
 [dependencies]
 # reth
 reth-primitives.workspace = true
+reth-fs-util.workspace = true
 reth-interfaces.workspace = true
 reth-codecs.workspace = true
 reth-libmdbx = { workspace = true, optional = true, features = [

crates/storage/db/benches/utils.rs

@@ -5,7 +5,8 @@ use reth_db::{
     transaction::DbTxMut,
     DatabaseEnv,
 };
-use reth_primitives::{fs, Bytes};
+use reth_fs_util as fs;
+use reth_primitives::Bytes;
 use std::{path::Path, sync::Arc};
 
 /// Path where the DB is initialized for benchmarks.

crates/storage/db/src/lib.rs

@@ -99,7 +99,7 @@ pub fn create_db<P: AsRef<Path>>(path: P, args: DatabaseArguments) -> eyre::Resu
 
     let rpath = path.as_ref();
     if is_database_empty(rpath) {
-        reth_primitives::fs::create_dir_all(rpath)
+        reth_fs_util::create_dir_all(rpath)
             .wrap_err_with(|| format!("Could not create database directory {}", rpath.display()))?;
         create_db_version_file(rpath)?;
     } else {
@@ -175,8 +175,8 @@ pub mod test_utils {
         database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics},
         models::client_version::ClientVersion,
     };
+    use reth_fs_util;
     use reth_libmdbx::MaxReadTransactionDuration;
-    use reth_primitives::fs;
     use std::{path::PathBuf, sync::Arc};
     use tempfile::TempDir;
 
@@ -202,7 +202,7 @@ pub mod test_utils {
         fn drop(&mut self) {
             if let Some(db) = self.db.take() {
                 drop(db);
-                let _ = fs::remove_dir_all(&self.path);
+                let _ = reth_fs_util::remove_dir_all(&self.path);
             }
         }
     }
@@ -318,7 +318,6 @@ mod tests {
     };
     use assert_matches::assert_matches;
     use reth_libmdbx::MaxReadTransactionDuration;
-    use reth_primitives::fs;
     use tempfile::tempdir;
 
     #[test]
@@ -342,7 +341,8 @@ mod tests {
 
         // Database is not empty, version file is malformed
         {
-            fs::write(path.path().join(db_version_file_path(&path)), "invalid-version").unwrap();
+            reth_fs_util::write(path.path().join(db_version_file_path(&path)), "invalid-version")
+                .unwrap();
             let db = init_db(&path, args.clone());
             assert!(db.is_err());
             assert_matches!(
@@ -353,7 +353,7 @@ mod tests {
 
         // Database is not empty, version file contains not matching version
         {
-            fs::write(path.path().join(db_version_file_path(&path)), "0").unwrap();
+            reth_fs_util::write(path.path().join(db_version_file_path(&path)), "0").unwrap();
             let db = init_db(&path, args);
             assert!(db.is_err());
             assert_matches!(

crates/storage/db/src/static_file/mod.rs

@@ -31,12 +31,11 @@ type SortedStaticFiles =
 pub fn iter_static_files(path: impl AsRef<Path>) -> Result<SortedStaticFiles, NippyJarError> {
     let path = path.as_ref();
     if !path.exists() {
-        reth_primitives::fs::create_dir_all(path)
-            .map_err(|err| NippyJarError::Custom(err.to_string()))?;
+        reth_fs_util::create_dir_all(path).map_err(|err| NippyJarError::Custom(err.to_string()))?;
     }
 
     let mut static_files = SortedStaticFiles::default();
-    let entries = reth_primitives::fs::read_dir(path)
+    let entries = reth_fs_util::read_dir(path)
         .map_err(|err| NippyJarError::Custom(err.to_string()))?
         .filter_map(Result::ok)
         .collect::<Vec<_>>();

crates/storage/nippy-jar/Cargo.toml

@@ -16,7 +16,7 @@ name = "reth_nippy_jar"
 
 [dependencies]
 # reth
-reth-primitives.workspace = true
+reth-fs-util.workspace = true
 
 # filter
 ph = "0.8.0"

crates/storage/nippy-jar/src/error.rs

@@ -8,7 +8,7 @@ pub enum NippyJarError {
     #[error(transparent)]
     Disconnect(#[from] std::io::Error),
     #[error(transparent)]
-    FileSystem(#[from] reth_primitives::fs::FsPathError),
+    FileSystem(#[from] reth_fs_util::FsPathError),
     #[error("{0}")]
     Custom(String),
     #[error(transparent)]

crates/storage/nippy-jar/src/lib.rs

@@ -245,7 +245,7 @@ impl<H: NippyJarHeader> NippyJar<H> {
         // Read [`Self`] located at the data file.
         let config_path = path.with_extension(CONFIG_FILE_EXTENSION);
         let config_file = File::open(&config_path)
-            .map_err(|err| reth_primitives::fs::FsPathError::open(err, config_path))?;
+            .map_err(|err| reth_fs_util::FsPathError::open(err, config_path))?;
 
         let mut obj: Self = bincode::deserialize_from(&config_file)?;
         obj.path = path.to_path_buf();
@@ -290,7 +290,7 @@ impl<H: NippyJarHeader> NippyJar<H> {
             [self.data_path().into(), self.index_path(), self.offsets_path(), self.config_path()]
         {
             if path.exists() {
-                reth_primitives::fs::remove_file(path)?;
+                reth_fs_util::remove_file(path)?;
             }
         }
 

crates/storage/provider/Cargo.toml

@@ -14,6 +14,7 @@ workspace = true
 [dependencies]
 # reth
 reth-primitives.workspace = true
+reth-fs-util.workspace = true
 reth-interfaces.workspace = true
 reth-db.workspace = true
 reth-trie = { workspace = true, features = ["metrics"] }

crates/storage/provider/src/providers/static_file/manager.rs

@@ -132,16 +132,16 @@ impl StaticFileProvider {
 
                 entries += jar_provider.rows();
 
-                let data_size = reth_primitives::fs::metadata(jar_provider.data_path())
+                let data_size = reth_fs_util::metadata(jar_provider.data_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
-                let index_size = reth_primitives::fs::metadata(jar_provider.index_path())
+                let index_size = reth_fs_util::metadata(jar_provider.index_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
-                let offsets_size = reth_primitives::fs::metadata(jar_provider.offsets_path())
+                let offsets_size = reth_fs_util::metadata(jar_provider.offsets_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
-                let config_size = reth_primitives::fs::metadata(jar_provider.config_path())
+                let config_size = reth_fs_util::metadata(jar_provider.config_path())
                     .map(|metadata| metadata.len())
                     .unwrap_or_default();
 

crates/transaction-pool/Cargo.toml

@@ -15,6 +15,7 @@ workspace = true
 # reth
 reth-eth-wire.workspace = true
 reth-primitives.workspace = true
+reth-fs-util.workspace = true
 reth-provider.workspace = true
 reth-tasks.workspace = true
 revm.workspace = true

crates/transaction-pool/src/maintain.rs

@@ -11,10 +11,11 @@ use futures_util::{
     future::{BoxFuture, Fuse, FusedFuture},
     FutureExt, Stream, StreamExt,
 };
+use reth_fs_util::FsPathError;
 use reth_primitives::{
-    fs::FsPathError, Address, BlockHash, BlockNumber, BlockNumberOrTag,
-    FromRecoveredPooledTransaction, IntoRecoveredTransaction, PooledTransactionsElementEcRecovered,
-    TransactionSigned, TryFromRecoveredTransaction,
+    Address, BlockHash, BlockNumber, BlockNumberOrTag, FromRecoveredPooledTransaction,
+    IntoRecoveredTransaction, PooledTransactionsElementEcRecovered, TransactionSigned,
+    TryFromRecoveredTransaction,
 };
 use reth_provider::{
     BlockReaderIdExt, BundleStateWithReceipts, CanonStateNotification, ChainSpecProvider,
@@ -578,7 +579,7 @@ where
     }
 
     debug!(target: "txpool", txs_file =?file_path, "Check local persistent storage for saved transactions");
-    let data = reth_primitives::fs::read(file_path)?;
+    let data = reth_fs_util::read(file_path)?;
 
     if data.is_empty() {
         return Ok(())
@@ -598,7 +599,7 @@ where
     let outcome = pool.add_transactions(crate::TransactionOrigin::Local, pool_transactions).await;
 
     info!(target: "txpool", txs_file =?file_path, num_txs=%outcome.len(), "Successfully reinserted local transactions from file");
-    reth_primitives::fs::remove_file(file_path)?;
+    reth_fs_util::remove_file(file_path)?;
     Ok(())
 }
 
@@ -623,7 +624,7 @@ where
     info!(target: "txpool", txs_file =?file_path, num_txs=%num_txs, "Saving current local transactions");
     let parent_dir = file_path.parent().map(std::fs::create_dir_all).transpose();
 
-    match parent_dir.map(|_| reth_primitives::fs::write(file_path, buf)) {
+    match parent_dir.map(|_| reth_fs_util::write(file_path, buf)) {
         Ok(_) => {
             info!(target: "txpool", txs_file=?file_path, "Wrote local transactions to file");
         }
@@ -680,7 +681,8 @@ mod tests {
         blobstore::InMemoryBlobStore, validate::EthTransactionValidatorBuilder,
         CoinbaseTipOrdering, EthPooledTransaction, Pool, PoolTransaction, TransactionOrigin,
     };
-    use reth_primitives::{fs, hex, PooledTransactionsElement, MAINNET, U256};
+    use reth_fs_util as fs;
+    use reth_primitives::{hex, PooledTransactionsElement, MAINNET, U256};
     use reth_provider::test_utils::{ExtendedAccount, MockEthProvider};
     use reth_tasks::TaskManager;