jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move DbTool type to db-common (#9119)

Browse Source
This commit is contained in:
Thomas Coratger
2024-06-26 16:05:18 +02:00
committed by GitHub
parent 1d1fb797e2
commit bdabe66426
19 changed files with 218 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@ -6233,7 +6233,6 @@ dependencies = [
"arbitrary",
"assert_matches",
"backon",
"boyer-moore-magiclen",
"clap",
"comfy-table",
"confy",
@ -6700,6 +6699,7 @@ name = "reth-db-common"
version = "1.0.0"
dependencies = [
"alloy-genesis",
"boyer-moore-magiclen",
"eyre",
"reth-chainspec",
"reth-codecs",
@ -6707,6 +6707,7 @@ dependencies = [
"reth-db",
"reth-db-api",
"reth-etl",
"reth-fs-util",
"reth-primitives",
"reth-primitives-traits",
"reth-provider",

1
Cargo.toml
View File

@ -439,6 +439,7 @@ sha2 = { version = "0.10", default-features = false }
paste = "1.0"
url = "2.3"
backon = "0.4"
boyer-moore-magiclen = "0.2.16"
# metrics
metrics = "0.23.0"

1
bin/reth/Cargo.toml
View File

@ -116,7 +116,6 @@ backon.workspace = true
similar-asserts.workspace = true
itertools.workspace = true
rayon.workspace = true
boyer-moore-magiclen = "0.2.16"
ahash = "0.8"
# p2p

6
bin/reth/src/commands/db/checksum.rs
View File

@ -1,11 +1,9 @@
use crate::{
commands::db::get::{maybe_json_value_parser, table_key},
utils::DbTool,
};
use crate::commands::db::get::{maybe_json_value_parser, table_key};
use ahash::RandomState;
use clap::Parser;
use reth_db::{DatabaseEnv, RawKey, RawTable, RawValue, TableViewer, Tables};
use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx};
use reth_db_common::DbTool;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,

2
bin/reth/src/commands/db/diff.rs
View File

@ -1,11 +1,11 @@
use crate::{
args::DatabaseArgs,
dirs::{DataDirPath, PlatformPath},
utils::DbTool,
};
use clap::Parser;
use reth_db::{open_db_read_only, tables_to_generic, DatabaseEnv, Tables};
use reth_db_api::{cursor::DbCursorRO, database::Database, table::Table, transaction::DbTx};
use reth_db_common::DbTool;
use std::{
collections::HashMap,
fmt::Debug,

2
bin/reth/src/commands/db/get.rs
View File

@ -1,4 +1,3 @@
use crate::utils::DbTool;
use clap::Parser;
use reth_db::{
static_file::{ColumnSelectorOne, ColumnSelectorTwo, HeaderMask, ReceiptMask, TransactionMask},
@ -8,6 +7,7 @@ use reth_db_api::{
database::Database,
table::{Decompress, DupSort, Table},
};
use reth_db_common::DbTool;
use reth_primitives::{BlockHash, Header};
use reth_provider::StaticFileProviderFactory;
use reth_static_file_types::StaticFileSegment;

2
bin/reth/src/commands/db/list.rs
View File

@ -1,9 +1,9 @@
use super::tui::DbListTUI;
use crate::utils::{DbTool, ListFilter};
use clap::Parser;
use eyre::WrapErr;
use reth_db::{DatabaseEnv, RawValue, TableViewer, Tables};
use reth_db_api::{database::Database, table::Table};
use reth_db_common::{DbTool, ListFilter};
use reth_primitives::hex;
use std::{cell::RefCell, sync::Arc};
use tracing::error;

6
bin/reth/src/commands/db/mod.rs
View File

@ -1,11 +1,9 @@
//! Database debugging tool
use crate::{
commands::common::{AccessRights, Environment, EnvironmentArgs},
utils::DbTool,
};
use crate::commands::common::{AccessRights, Environment, EnvironmentArgs};
use clap::{Parser, Subcommand};
use reth_db::version::{get_db_version, DatabaseVersionError, DB_VERSION};
use reth_db_common::DbTool;
use std::io::{self, Write};
mod checksum;

3
bin/reth/src/commands/db/stats.rs
View File

@ -1,4 +1,4 @@
use crate::{commands::db::checksum::ChecksumViewer, utils::DbTool};
use crate::commands::db::checksum::ChecksumViewer;
use clap::Parser;
use comfy_table::{Cell, Row, Table as ComfyTable};
use eyre::WrapErr;
@ -6,6 +6,7 @@ use human_bytes::human_bytes;
use itertools::Itertools;
use reth_db::{mdbx, static_file::iter_static_files, DatabaseEnv, TableViewer, Tables};
use reth_db_api::database::Database;
use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::providers::StaticFileProvider;

6
bin/reth/src/commands/stage/drop.rs
View File

@ -3,13 +3,15 @@
use crate::{
args::StageEnum,
commands::common::{AccessRights, Environment, EnvironmentArgs},
utils::DbTool,
};
use clap::Parser;
use itertools::Itertools;
use reth_db::{static_file::iter_static_files, tables, DatabaseEnv};
use reth_db_api::transaction::DbTxMut;
use reth_db_common::init::{insert_genesis_header, insert_genesis_history, insert_genesis_state};
use reth_db_common::{
init::{insert_genesis_header, insert_genesis_history, insert_genesis_state},
DbTool,
};
use reth_provider::{providers::StaticFileWriter, StaticFileProviderFactory};
use reth_stages::StageId;
use reth_static_file_types::{find_fixed_range, StaticFileSegment};

3
bin/reth/src/commands/stage/dump/execution.rs
View File

@ -1,9 +1,10 @@
use super::setup;
use crate::{macros::block_executor, utils::DbTool};
use crate::macros::block_executor;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, database::Database, table::TableImporter, transaction::DbTx,
};
use reth_db_common::DbTool;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::{providers::StaticFileProvider, ChainSpecProvider, ProviderFactory};
use reth_stages::{stages::ExecutionStage, Stage, StageCheckpoint, UnwindInput};

2
bin/reth/src/commands/stage/dump/hashing_account.rs
View File

@ -1,8 +1,8 @@
use super::setup;
use crate::utils::DbTool;
use eyre::Result;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::{database::Database, table::TableImporter};
use reth_db_common::DbTool;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::BlockNumber;
use reth_provider::{providers::StaticFileProvider, ProviderFactory};

2
bin/reth/src/commands/stage/dump/hashing_storage.rs
View File

@ -1,8 +1,8 @@
use super::setup;
use crate::utils::DbTool;
use eyre::Result;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::{database::Database, table::TableImporter};
use reth_db_common::DbTool;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::{providers::StaticFileProvider, ProviderFactory};
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};

3
bin/reth/src/commands/stage/dump/merkle.rs
View File

@ -1,9 +1,10 @@
use super::setup;
use crate::{macros::block_executor, utils::DbTool};
use crate::macros::block_executor;
use eyre::Result;
use reth_config::config::EtlConfig;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::{database::Database, table::TableImporter};
use reth_db_common::DbTool;
use reth_exex::ExExManagerHandle;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_primitives::BlockNumber;

5
bin/reth/src/commands/stage/dump/mod.rs
View File

@ -1,18 +1,17 @@
//! Database debugging tool
use crate::{
args::DatadirArgs,
commands::common::{AccessRights, Environment, EnvironmentArgs},
dirs::DataDirPath,
utils::DbTool,
};
use crate::args::DatadirArgs;
use clap::Parser;
use reth_db::{init_db, mdbx::DatabaseArguments, tables, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, database::Database, models::ClientVersion, table::TableImporter,
transaction::DbTx,
};
use reth_db_common::DbTool;
use reth_node_core::dirs::PlatformPath;
use std::path::PathBuf;
use tracing::info;

188
bin/reth/src/utils.rs
View File

@ -1,21 +1,5 @@
//! Common CLI utility functions.
use boyer_moore_magiclen::BMByte;
use eyre::Result;
use reth_chainspec::ChainSpec;
use reth_db::{RawTable, TableRawRow};
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRO},
database::Database,
table::{Decode, Decompress, DupSort, Table, TableRow},
transaction::{DbTx, DbTxMut},
DatabaseError,
};
use reth_fs_util as fs;
use reth_provider::{ChainSpecProvider, ProviderFactory};
use std::{path::Path, rc::Rc, sync::Arc};
use tracing::info;
/// Exposing `open_db_read_only` function
pub mod db {
pub use reth_db::open_db_read_only;
@ -24,175 +8,3 @@ pub mod db {
/// Re-exported from `reth_node_core`, also to prevent a breaking change. See the comment on
/// the `reth_node_core::args` re-export for more details.
pub use reth_node_core::utils::*;
/// Wrapper over DB that implements many useful DB queries.
#[derive(Debug)]
pub struct DbTool<DB: Database> {
/// The provider factory that the db tool will use.
pub provider_factory: ProviderFactory<DB>,
}
impl<DB: Database> DbTool<DB> {
/// Takes a DB where the tables have already been created.
pub fn new(provider_factory: ProviderFactory<DB>) -> eyre::Result<Self> {
// Disable timeout because we are entering a TUI which might read for a long time. We
// disable on the [`DbTool`] level since it's only used in the CLI.
provider_factory.provider()?.disable_long_read_transaction_safety();
Ok(Self { provider_factory })
}
/// Get an [`Arc`] to the [`ChainSpec`].
pub fn chain(&self) -> Arc<ChainSpec> {
self.provider_factory.chain_spec()
}
/// Grabs the contents of the table within a certain index range and places the
/// entries into a [`HashMap`][std::collections::HashMap].
///
/// [`ListFilter`] can be used to further
/// filter down the desired results. (eg. List only rows which include `0xd3adbeef`)
pub fn list<T: Table>(&self, filter: &ListFilter) -> Result<(Vec<TableRow<T>>, usize)> {
let bmb = Rc::new(BMByte::from(&filter.search));
if bmb.is_none() && filter.has_search() {
eyre::bail!("Invalid search.")
}
let mut hits = 0;
let data = self.provider_factory.db_ref().view(|tx| {
let mut cursor =
tx.cursor_read::<RawTable<T>>().expect("Was not able to obtain a cursor.");
let map_filter = |row: Result<TableRawRow<T>, _>| {
if let Ok((k, v)) = row {
let (key, value) = (k.into_key(), v.into_value());
if key.len() + value.len() < filter.min_row_size {
return None
}
if key.len() < filter.min_key_size {
return None
}
if value.len() < filter.min_value_size {
return None
}
let result = || {
if filter.only_count {
return None
}
Some((
<T as Table>::Key::decode(&key).unwrap(),
<T as Table>::Value::decompress(&value).unwrap(),
))
};
match &*bmb {
Some(searcher) => {
if searcher.find_first_in(&value).is_some() ||
searcher.find_first_in(&key).is_some()
{
hits += 1;
return result()
}
}
None => {
hits += 1;
return result()
}
}
}
None
};
if filter.reverse {
Ok(cursor
.walk_back(None)?
.skip(filter.skip)
.filter_map(map_filter)
.take(filter.len)
.collect::<Vec<(_, _)>>())
} else {
Ok(cursor
.walk(None)?
.skip(filter.skip)
.filter_map(map_filter)
.take(filter.len)
.collect::<Vec<(_, _)>>())
}
})?;
Ok((data.map_err(|e: DatabaseError| eyre::eyre!(e))?, hits))
}
/// Grabs the content of the table for the given key
pub fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>> {
self.provider_factory.db_ref().view(|tx| tx.get::<T>(key))?.map_err(|e| eyre::eyre!(e))
}
/// Grabs the content of the `DupSort` table for the given key and subkey
pub fn get_dup<T: DupSort>(&self, key: T::Key, subkey: T::SubKey) -> Result<Option<T::Value>> {
self.provider_factory
.db_ref()
.view(|tx| tx.cursor_dup_read::<T>()?.seek_by_key_subkey(key, subkey))?
.map_err(|e| eyre::eyre!(e))
}
/// Drops the database and the static files at the given path.
pub fn drop(
&self,
db_path: impl AsRef<Path>,
static_files_path: impl AsRef<Path>,
) -> Result<()> {
let db_path = db_path.as_ref();
info!(target: "reth::cli", "Dropping database at {:?}", db_path);
fs::remove_dir_all(db_path)?;
let static_files_path = static_files_path.as_ref();
info!(target: "reth::cli", "Dropping static files at {:?}", static_files_path);
fs::remove_dir_all(static_files_path)?;
fs::create_dir_all(static_files_path)?;
Ok(())
}
/// Drops the provided table from the database.
pub fn drop_table<T: Table>(&self) -> Result<()> {
self.provider_factory.db_ref().update(|tx| tx.clear::<T>())??;
Ok(())
}
}
/// Filters the results coming from the database.
#[derive(Debug)]
pub struct ListFilter {
/// Skip first N entries.
pub skip: usize,
/// Take N entries.
pub len: usize,
/// Sequence of bytes that will be searched on values and keys from the database.
pub search: Vec<u8>,
/// Minimum row size.
pub min_row_size: usize,
/// Minimum key size.
pub min_key_size: usize,
/// Minimum value size.
pub min_value_size: usize,
/// Reverse order of entries.
pub reverse: bool,
/// Only counts the number of filtered entries without decoding and returning them.
pub only_count: bool,
}
impl ListFilter {
/// If `search` has a list of bytes, then filter for rows that have this sequence.
pub fn has_search(&self) -> bool {
!self.search.is_empty()
}
/// Updates the page with new `skip` and `len` values.
pub fn update_page(&mut self, skip: usize, len: usize) {
self.skip = skip;
self.len = len;
}
}

2
crates/storage/db-common/Cargo.toml
View File

@ -19,6 +19,7 @@ reth-trie.workspace = true
reth-etl.workspace = true
reth-codecs.workspace = true
reth-stages-types.workspace = true
reth-fs-util.workspace = true
# eth
alloy-genesis.workspace = true
@ -26,6 +27,7 @@ alloy-genesis.workspace = true
# misc
eyre.workspace = true
thiserror.workspace = true
boyer-moore-magiclen.workspace = true
# io
serde.workspace = true

189
crates/storage/db-common/src/db_tool/mod.rs Normal file
View File

@ -0,0 +1,189 @@
//! Common db operations
use boyer_moore_magiclen::BMByte;
use eyre::Result;
use reth_chainspec::ChainSpec;
use reth_db::{RawTable, TableRawRow};
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRO},
database::Database,
table::{Decode, Decompress, DupSort, Table, TableRow},
transaction::{DbTx, DbTxMut},
DatabaseError,
};
use reth_fs_util as fs;
use reth_provider::{ChainSpecProvider, ProviderFactory};
use std::{path::Path, rc::Rc, sync::Arc};
use tracing::info;
/// Wrapper over DB that implements many useful DB queries.
#[derive(Debug)]
pub struct DbTool<DB: Database> {
/// The provider factory that the db tool will use.
pub provider_factory: ProviderFactory<DB>,
}
impl<DB: Database> DbTool<DB> {
/// Takes a DB where the tables have already been created.
pub fn new(provider_factory: ProviderFactory<DB>) -> eyre::Result<Self> {
// Disable timeout because we are entering a TUI which might read for a long time. We
// disable on the [`DbTool`] level since it's only used in the CLI.
provider_factory.provider()?.disable_long_read_transaction_safety();
Ok(Self { provider_factory })
}
/// Get an [`Arc`] to the [`ChainSpec`].
pub fn chain(&self) -> Arc<ChainSpec> {
self.provider_factory.chain_spec()
}
/// Grabs the contents of the table within a certain index range and places the
/// entries into a [`HashMap`][std::collections::HashMap].
///
/// [`ListFilter`] can be used to further
/// filter down the desired results. (eg. List only rows which include `0xd3adbeef`)
pub fn list<T: Table>(&self, filter: &ListFilter) -> Result<(Vec<TableRow<T>>, usize)> {
let bmb = Rc::new(BMByte::from(&filter.search));
if bmb.is_none() && filter.has_search() {
eyre::bail!("Invalid search.")
}
let mut hits = 0;
let data = self.provider_factory.db_ref().view(|tx| {
let mut cursor =
tx.cursor_read::<RawTable<T>>().expect("Was not able to obtain a cursor.");
let map_filter = |row: Result<TableRawRow<T>, _>| {
if let Ok((k, v)) = row {
let (key, value) = (k.into_key(), v.into_value());
if key.len() + value.len() < filter.min_row_size {
return None
}
if key.len() < filter.min_key_size {
return None
}
if value.len() < filter.min_value_size {
return None
}
let result = || {
if filter.only_count {
return None
}
Some((
<T as Table>::Key::decode(&key).unwrap(),
<T as Table>::Value::decompress(&value).unwrap(),
))
};
match &*bmb {
Some(searcher) => {
if searcher.find_first_in(&value).is_some() ||
searcher.find_first_in(&key).is_some()
{
hits += 1;
return result()
}
}
None => {
hits += 1;
return result()
}
}
}
None
};
if filter.reverse {
Ok(cursor
.walk_back(None)?
.skip(filter.skip)
.filter_map(map_filter)
.take(filter.len)
.collect::<Vec<(_, _)>>())
} else {
Ok(cursor
.walk(None)?
.skip(filter.skip)
.filter_map(map_filter)
.take(filter.len)
.collect::<Vec<(_, _)>>())
}
})?;
Ok((data.map_err(|e: DatabaseError| eyre::eyre!(e))?, hits))
}
/// Grabs the content of the table for the given key
pub fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>> {
self.provider_factory.db_ref().view(|tx| tx.get::<T>(key))?.map_err(|e| eyre::eyre!(e))
}
/// Grabs the content of the `DupSort` table for the given key and subkey
pub fn get_dup<T: DupSort>(&self, key: T::Key, subkey: T::SubKey) -> Result<Option<T::Value>> {
self.provider_factory
.db_ref()
.view(|tx| tx.cursor_dup_read::<T>()?.seek_by_key_subkey(key, subkey))?
.map_err(|e| eyre::eyre!(e))
}
/// Drops the database and the static files at the given path.
pub fn drop(
&self,
db_path: impl AsRef<Path>,
static_files_path: impl AsRef<Path>,
) -> Result<()> {
let db_path = db_path.as_ref();
info!(target: "reth::cli", "Dropping database at {:?}", db_path);
fs::remove_dir_all(db_path)?;
let static_files_path = static_files_path.as_ref();
info!(target: "reth::cli", "Dropping static files at {:?}", static_files_path);
fs::remove_dir_all(static_files_path)?;
fs::create_dir_all(static_files_path)?;
Ok(())
}
/// Drops the provided table from the database.
pub fn drop_table<T: Table>(&self) -> Result<()> {
self.provider_factory.db_ref().update(|tx| tx.clear::<T>())??;
Ok(())
}
}
/// Filters the results coming from the database.
#[derive(Debug)]
pub struct ListFilter {
/// Skip first N entries.
pub skip: usize,
/// Take N entries.
pub len: usize,
/// Sequence of bytes that will be searched on values and keys from the database.
pub search: Vec<u8>,
/// Minimum row size.
pub min_row_size: usize,
/// Minimum key size.
pub min_key_size: usize,
/// Minimum value size.
pub min_value_size: usize,
/// Reverse order of entries.
pub reverse: bool,
/// Only counts the number of filtered entries without decoding and returning them.
pub only_count: bool,
}
impl ListFilter {
/// If `search` has a list of bytes, then filter for rows that have this sequence.
pub fn has_search(&self) -> bool {
!self.search.is_empty()
}
/// Updates the page with new `skip` and `len` values.
pub fn update_page(&mut self, skip: usize, len: usize) {
self.skip = skip;
self.len = len;
}
}

3
crates/storage/db-common/src/lib.rs
View File

@ -9,3 +9,6 @@
#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
pub mod init;
mod db_tool;
pub use db_tool::*;