From dff3936b292c34bc7c4cfcddf1941bfbc8a6dab9 Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Tue, 13 Dec 2022 18:57:37 +0200 Subject: [PATCH] feat(cli): db cmd scaffold (#405) * feat(cli): db cmd scaffold * feat(cli): add methods for seeding/listing tables * feat(cli): add reth db stats * chore: docs / cleanup * chore: remove ethers Co-authored-by: Oliver Nordbjerg --- Cargo.lock | 1 + bin/reth/Cargo.toml | 3 +- bin/reth/src/cli.rs | 8 +- bin/reth/src/db/mod.rs | 169 +++++++++++++++++++++++++++++ bin/reth/src/lib.rs | 1 + crates/stages/src/db.rs | 5 + crates/stages/src/lib.rs | 1 + crates/storage/provider/src/lib.rs | 1 + 8 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 bin/reth/src/db/mod.rs diff --git a/Cargo.lock b/Cargo.lock index d049b76be..7f7f199cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3177,6 +3177,7 @@ dependencies = [ "reth-db", "reth-interfaces", "reth-primitives", + "reth-provider", "reth-rpc", "reth-stages", "reth-transaction-pool", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 5fde2b4b8..09ab265ad 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -10,8 +10,9 @@ readme = "README.md" # reth reth-primitives = { path = "../../crates/primitives" } reth-db = {path = "../../crates/storage/db", features = ["mdbx"]} +reth-provider = {path = "../../crates/storage/provider" } reth-stages = {path = "../../crates/stages"} -reth-interfaces = {path = "../../crates/interfaces"} +reth-interfaces = {path = "../../crates/interfaces", features = ["test-utils"] } reth-transaction-pool = {path = "../../crates/transaction-pool"} reth-consensus = {path = "../../crates/consensus"} reth-rpc = {path = "../../crates/net/rpc"} diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index 2643f7190..47a32c059 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -4,7 +4,7 @@ use clap::{ArgAction, Parser, Subcommand}; use tracing_subscriber::util::SubscriberInitExt; use crate::{ - node, test_eth_chain, + db, node, test_eth_chain, util::reth_tracing::{self, TracingMode}, }; @@ -19,6 +19,7 @@ pub async fn run() -> eyre::Result<()> { match opt.command { Commands::Node(command) => command.execute().await, Commands::TestEthChain(command) => command.execute().await, + Commands::Db(command) => command.execute().await, } } @@ -31,6 +32,9 @@ pub enum Commands { /// Runs Ethereum blockchain tests #[command(name = "test-chain")] TestEthChain(test_eth_chain::Command), + /// DB Debugging utilities + #[command(name = "db")] + Db(db::Command), } #[derive(Parser)] @@ -45,6 +49,6 @@ struct Cli { verbose: u8, /// Silence all output - #[clap(short, long, global = true)] + #[clap(long, global = true)] silent: bool, } diff --git a/bin/reth/src/db/mod.rs b/bin/reth/src/db/mod.rs new file mode 100644 index 000000000..69b839418 --- /dev/null +++ b/bin/reth/src/db/mod.rs @@ -0,0 +1,169 @@ +// TODO: Remove +//! Main db command +//! +//! Database debugging tool + +use clap::{Parser, Subcommand}; +use eyre::Result; +use reth_db::{ + cursor::{DbCursorRO, Walker}, + database::Database, + table::Table, + tables, + transaction::DbTx, +}; +use reth_interfaces::test_utils::generators::random_block_range; +use reth_provider::insert_canonical_block; +use reth_stages::StageDB; +use std::path::Path; +use tracing::info; + +/// `reth db` command +#[derive(Debug, Parser)] +pub struct Command { + /// Path to database folder + #[arg(long, default_value = "~/.reth/db")] + db: String, + + #[clap(subcommand)] + command: Subcommands, +} + +const DEFAULT_NUM_ITEMS: &str = "5"; + +#[derive(Subcommand, Debug)] +/// `reth db` subcommands +pub enum Subcommands { + /// Lists all the table names, number of entries, and size in KB + Stats, + /// Lists the contents of a table + List(ListArgs), + /// Seeds the block db with random blocks on top of each other + Seed { + /// How many blocks to generate + #[arg(default_value = DEFAULT_NUM_ITEMS)] + len: u64, + }, +} + +#[derive(Parser, Debug)] +/// The arguments for the `reth db list` command +pub struct ListArgs { + /// The table name + table: String, // TODO: Convert to enum + /// Where to start iterating + #[arg(long, short, default_value = "0")] + start: usize, + /// How many items to take from the walker + #[arg(long, short, default_value = DEFAULT_NUM_ITEMS)] + len: usize, +} + +impl Command { + /// Execute `node` command + pub async fn execute(&self) -> eyre::Result<()> { + let path = shellexpand::full(&self.db)?.into_owned(); + let expanded_db_path = Path::new(&path); + std::fs::create_dir_all(expanded_db_path)?; + + // TODO: Auto-impl for Database trait + let db = reth_db::mdbx::Env::::open( + expanded_db_path, + reth_db::mdbx::EnvKind::RW, + )?; + db.create_tables()?; + + let mut tool = DbTool::new(&db)?; + match &self.command { + // TODO: We'll need to add this on the DB trait. + Subcommands::Stats { .. } => { + // Get the env from MDBX + let env = &tool.db.inner().inner; + let tx = env.begin_ro_txn()?; + for table in tables::TABLES.iter().map(|(_, name)| name) { + let table_db = tx.open_db(Some(table))?; + let stats = tx.db_stat(&table_db)?; + + // Defaults to 16KB right now but we should + // re-evaluate depending on the DB we end up using + // (e.g. REDB does not have these options as configurable intentionally) + let page_size = stats.page_size() as usize; + let leaf_pages = stats.leaf_pages(); + let branch_pages = stats.branch_pages(); + let overflow_pages = stats.overflow_pages(); + let num_pages = leaf_pages + branch_pages + overflow_pages; + let table_size = page_size * num_pages; + tracing::info!( + "Table {} has {} entries (total size: {} KB)", + table, + stats.entries(), + table_size / 1024 + ); + } + } + Subcommands::Seed { len } => { + tool.seed(*len)?; + } + Subcommands::List(args) => { + tool.list(args)?; + } + } + + Ok(()) + } +} + +/// Abstraction over StageDB for writing/reading from/to the DB +/// Wraps over the StageDB and derefs to a transaction. +struct DbTool<'a, DB: Database> { + // TODO: StageDB derefs to Tx, is this weird or not? + pub(crate) db: StageDB<'a, DB>, +} + +impl<'a, DB: Database> DbTool<'a, DB> { + /// Takes a DB where the tables have already been created + fn new(db: &'a DB) -> eyre::Result { + Ok(Self { db: StageDB::new(db)? }) + } + + /// Seeds the database with some random data, only used for testing + fn seed(&mut self, len: u64) -> Result<()> { + tracing::info!("generating random block range from 0 to {len}"); + let chain = random_block_range(0..len, Default::default()); + chain.iter().try_for_each(|block| { + insert_canonical_block(&*self.db, block, true)?; + Ok::<_, eyre::Error>(()) + })?; + self.db.commit()?; + info!("Database committed with {len} blocks"); + + Ok(()) + } + + /// Lists the given table data + fn list(&mut self, args: &ListArgs) -> Result<()> { + match args.table.as_str() { + "headers" => self.list_table::(args.start, args.len)?, + "txs" => self.list_table::(args.start, args.len)?, + _ => panic!(), + }; + Ok(()) + } + + fn list_table(&mut self, start: usize, len: usize) -> Result<()> { + let mut cursor = self.db.cursor::()?; + + // TODO: Upstream this in the DB trait. + let start_walker = cursor.current().transpose(); + let walker = Walker { + cursor: &mut cursor, + start: start_walker, + _tx_phantom: std::marker::PhantomData, + }; + + let data = walker.skip(start).take(len).collect::>(); + dbg!(&data); + + Ok(()) + } +} diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 208bba77a..753ab58cc 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -7,6 +7,7 @@ //! Rust Ethereum (reth) binary executable. pub mod cli; +pub mod db; pub mod node; pub mod test_eth_chain; pub mod util; diff --git a/crates/stages/src/db.rs b/crates/stages/src/db.rs index bb278b947..c65318e64 100644 --- a/crates/stages/src/db.rs +++ b/crates/stages/src/db.rs @@ -70,6 +70,11 @@ where Ok(Self { db, tx: Some(db.tx_mut()?) }) } + /// Accessor to the internal Database + pub fn inner(&self) -> &'this DB { + self.db + } + /// Commit the current inner transaction and open a new one. /// /// # Panics diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index 524617ae5..23e10ea61 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -15,6 +15,7 @@ //! - `stage.progress{stage}`: The block number each stage has currently reached. mod db; +pub use db::StageDB; mod error; mod id; mod pipeline; diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 523fa34d9..e3d7352dd 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -8,6 +8,7 @@ //! mod block; + pub mod db_provider; mod state;