From a8b7a089b65d678f1921b7a505bbfbb9baef82cb Mon Sep 17 00:00:00 2001 From: clabby Date: Wed, 18 Jan 2023 18:59:09 +0100 Subject: [PATCH] feat(bin): Format `db list` & `db status` subcommands (#667) Co-authored-by: Georgios Konstantopoulos --- Cargo.lock | 80 +++++++ bin/reth/Cargo.toml | 3 + bin/reth/src/db/mod.rs | 124 +++++++---- bin/reth/src/db/tui.rs | 209 ++++++++++++++++++ crates/primitives/src/storage.rs | 3 +- crates/storage/db/src/abstraction/database.rs | 4 +- crates/storage/db/src/abstraction/table.rs | 9 +- crates/storage/db/src/tables/mod.rs | 6 +- .../storage/db/src/tables/models/accounts.rs | 4 +- crates/storage/db/src/tables/models/blocks.rs | 2 +- .../db/src/tables/models/sharded_key.rs | 2 +- docs/repo/crates/db.md | 48 ++-- 12 files changed, 416 insertions(+), 78 deletions(-) create mode 100644 bin/reth/src/db/tui.rs diff --git a/Cargo.lock b/Cargo.lock index 7ab5b5927..1fe09d2af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -517,6 +517,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "cassowary" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" + [[package]] name = "cast" version = "0.3.0" @@ -756,6 +762,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "comfy-table" +version = "6.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d" +dependencies = [ + "crossterm", + "strum", + "strum_macros", + "unicode-width", +] + [[package]] name = "confy" version = "0.5.1" @@ -908,6 +926,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossterm" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.1", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -3917,7 +3960,9 @@ version = "0.1.0" dependencies = [ "backon", "clap 4.0.32", + "comfy-table", "confy", + "crossterm", "dirs-next", "eyre", "fdlimit", @@ -3949,6 +3994,7 @@ dependencies = [ "tokio", "tokio-stream", "tracing", + "tui", "walkdir", ] @@ -5213,6 +5259,27 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -5989,6 +6056,19 @@ dependencies = [ "toml", ] +[[package]] +name = "tui" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1" +dependencies = [ + "bitflags", + "cassowary", + "crossterm", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "typenum" version = "1.16.0" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 7772bb52a..51cae5966 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -53,3 +53,6 @@ tokio-stream = "0.1" futures = "0.3.25" tempfile = { version = "3.3.0" } backon = "0.2.0" +comfy-table = "6.1.4" +crossterm = "0.25.0" +tui = "0.19.0" diff --git a/bin/reth/src/db/mod.rs b/bin/reth/src/db/mod.rs index 003c3502f..19d64fca0 100644 --- a/bin/reth/src/db/mod.rs +++ b/bin/reth/src/db/mod.rs @@ -1,6 +1,7 @@ //! Database debugging tool use crate::dirs::{DbPath, PlatformPath}; use clap::{Parser, Subcommand}; +use comfy_table::{Cell, Row, Table as ComfyTable}; use eyre::{Result, WrapErr}; use reth_db::{ cursor::{DbCursorRO, Walker}, @@ -11,7 +12,11 @@ use reth_db::{ }; use reth_interfaces::test_utils::generators::random_block_range; use reth_provider::insert_canonical_block; -use tracing::info; +use std::collections::BTreeMap; +use tracing::{error, info}; + +/// DB List TUI +mod tui; /// `reth db` command #[derive(Debug, Parser)] @@ -78,6 +83,17 @@ impl Command { match &self.command { // TODO: We'll need to add this on the DB trait. Subcommands::Stats { .. } => { + let mut stats_table = ComfyTable::new(); + stats_table.load_preset(comfy_table::presets::ASCII_MARKDOWN); + stats_table.set_header([ + "Table Name", + "# Entries", + "Branch Pages", + "Leaf Pages", + "Overflow Pages", + "Total Size (KB)", + ]); + tool.db.view(|tx| { for table in tables::TABLES.iter().map(|(_, name)| name) { let table_db = @@ -97,22 +113,69 @@ impl Command { let overflow_pages = stats.overflow_pages(); let num_pages = leaf_pages + branch_pages + overflow_pages; let table_size = page_size * num_pages; - info!( - target: "reth::cli", - "Table {} has {} entries (total size: {} KB)", - table, - stats.entries(), - table_size / 1024 - ); + + let mut row = Row::new(); + row.add_cell(Cell::new(table)) + .add_cell(Cell::new(stats.entries())) + .add_cell(Cell::new(branch_pages)) + .add_cell(Cell::new(leaf_pages)) + .add_cell(Cell::new(overflow_pages)) + .add_cell(Cell::new(table_size / 1024)); + stats_table.add_row(row); } Ok::<(), eyre::Report>(()) })??; + + println!("{stats_table}"); } Subcommands::Seed { len } => { tool.seed(*len)?; } Subcommands::List(args) => { - tool.list(args)?; + macro_rules! table_tui { + ($arg:expr, $start:expr, $len:expr => [$($table:ident),*]) => { + match $arg { + $(stringify!($table) => { + tool.db.view(|tx| { + let table_db = tx.inner.open_db(Some(stringify!($table))).wrap_err("Could not open db.")?; + let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?; + let total_entries = stats.entries(); + if $start > total_entries - 1 { + error!( + target: "reth::cli", + "Start index {start} is greater than the final entry index ({final_entry_idx}) in the table {table}", + start = $start, + final_entry_idx = total_entries - 1, + table = stringify!($table) + ); + return Ok(()); + } + let map = tool.list::($start, $len)?; + tui::DbListTUI::::show_tui(map, $start, total_entries) + })?? + },)* + _ => { + error!(target: "reth::cli", "Unknown table."); + return Ok(()); + } + } + } + } + + table_tui!(args.table.as_str(), args.start, args.len => [ + CanonicalHeaders, + HeaderTD, + HeaderNumbers, + Headers, + BlockBodies, + BlockOmmers, + TxHashNumber, + PlainAccountState, + BlockTransitionIndex, + TxTransitionIndex, + SyncStage, + Transactions + ]); } Subcommands::Drop => { tool.drop(&self.db)?; @@ -150,41 +213,9 @@ impl<'a, DB: Database> DbTool<'a, DB> { Ok(()) } - /// Lists the given table data - fn list(&mut self, args: &ListArgs) -> Result<()> { - macro_rules! list_tables { - ($arg:expr, $start:expr, $len:expr => [$($table:ident,)*]) => { - match $arg { - $(stringify!($table) => { - self.list_table::($start, $len)? - },)* - _ => { - tracing::error!(target: "reth::cli", "Unknown table."); - return Ok(()) - } - } - }; - } - - list_tables!(args.table.as_str(), args.start, args.len => [ - CanonicalHeaders, - HeaderTD, - HeaderNumbers, - Headers, - BlockBodies, - BlockOmmers, - TxHashNumber, - PlainAccountState, - BlockTransitionIndex, - TxTransitionIndex, - SyncStage, - Transactions, - ]); - - Ok(()) - } - - fn list_table(&mut self, start: usize, len: usize) -> Result<()> { + /// Grabs the contents of the table within a certain index range and places the + /// entries into a [HashMap]. + fn list(&mut self, start: usize, len: usize) -> Result> { let data = self.db.view(|tx| { let mut cursor = tx.cursor_read::().expect("Was not able to obtain a cursor."); @@ -195,8 +226,9 @@ impl<'a, DB: Database> DbTool<'a, DB> { walker.skip(start).take(len).collect::>() })?; - println!("{data:?}"); - Ok(()) + data.into_iter() + .collect::, _>>() + .map_err(|e| eyre::eyre!(e)) } fn drop(&mut self, path: &PlatformPath) -> Result<()> { diff --git a/bin/reth/src/db/tui.rs b/bin/reth/src/db/tui.rs new file mode 100644 index 000000000..e2361542b --- /dev/null +++ b/bin/reth/src/db/tui.rs @@ -0,0 +1,209 @@ +use crossterm::{ + event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode, MouseEventKind}, + execute, + terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}, +}; +use reth_db::table::Table; +use std::{ + collections::BTreeMap, + io, + time::{Duration, Instant}, +}; +use tracing::error; +use tui::{ + backend::{Backend, CrosstermBackend}, + layout::{Alignment, Constraint, Corner, Direction, Layout}, + style::{Color, Modifier, Style}, + widgets::{Block, Borders, List, ListItem, ListState, Paragraph, Wrap}, + Frame, Terminal, +}; + +/// Available keybindings for the [DbListTUI] +static CMDS: [(&str, &str); 3] = [("q", "Quit"), ("up", "Entry Above"), ("down", "Entry Below")]; + +/// Modified version of the [ListState] struct that exposes the `offset` field. +/// Used to make the [DbListTUI] keys clickable. +struct ExpListState { + pub(crate) offset: usize, +} + +#[derive(Default)] +pub(crate) struct DbListTUI { + /// The state of the key list. + pub(crate) state: ListState, + /// The starting index of the key list in the DB. + pub(crate) start: usize, + /// The total number of entries in the database + pub(crate) total_entries: usize, + /// Entries to show in the TUI. + pub(crate) entries: BTreeMap, +} + +impl DbListTUI { + fn new(entries: BTreeMap, start: usize, total_entries: usize) -> Self { + Self { state: ListState::default(), start, total_entries, entries } + } + + /// Move to the next list selection + fn next(&mut self) { + let i = match self.state.selected() { + Some(i) => { + if i >= self.entries.len() - 1 { + 0 + } else { + i + 1 + } + } + None => 0, + }; + self.state.select(Some(i)); + } + + /// Move to the previous list selection + fn previous(&mut self) { + let i = match self.state.selected() { + Some(i) => { + if i == 0 { + self.entries.len() - 1 + } else { + i - 1 + } + } + None => 0, + }; + self.state.select(Some(i)); + } + + /// Show the [DbListTUI] in the terminal. + pub(crate) fn show_tui( + entries: BTreeMap, + start: usize, + total_entries: usize, + ) -> eyre::Result<()> { + // setup terminal + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?; + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + // create app and run it + let tick_rate = Duration::from_millis(250); + let mut app = DbListTUI::::new(entries, start, total_entries); + app.state.select(Some(0)); + let res = run(&mut terminal, app, tick_rate); + + // restore terminal + disable_raw_mode()?; + execute!(terminal.backend_mut(), LeaveAlternateScreen, DisableMouseCapture)?; + terminal.show_cursor()?; + + if let Err(err) = res { + error!("{:?}", err) + } + + Ok(()) + } +} + +fn run( + terminal: &mut Terminal, + mut app: DbListTUI, + tick_rate: Duration, +) -> io::Result<()> { + let mut last_tick = Instant::now(); + loop { + terminal.draw(|f| ui(f, &mut app))?; + + let timeout = + tick_rate.checked_sub(last_tick.elapsed()).unwrap_or_else(|| Duration::from_secs(0)); + if crossterm::event::poll(timeout)? { + match event::read()? { + Event::Key(key) => match key.code { + KeyCode::Char('q') | KeyCode::Char('Q') => return Ok(()), + KeyCode::Down => app.next(), + KeyCode::Up => app.previous(), + _ => {} + }, + Event::Mouse(e) => match e.kind { + MouseEventKind::ScrollDown => app.next(), + MouseEventKind::ScrollUp => app.previous(), + // TODO: This click event can be triggered outside of the list widget. + MouseEventKind::Down(_) => { + // SAFETY: The pointer to the app's state will always be valid for + // reads here, and the source is larger than the destination. + // + // This is technically unsafe, but because the alignment requirements + // in both the source and destination are the same and we can ensure + // that the pointer to `app.state` is valid for reads, this is safe. + let state: ExpListState = unsafe { std::mem::transmute_copy(&app.state) }; + let new_idx = (e.row as usize + state.offset).saturating_sub(1); + if new_idx < app.entries.len() { + app.state.select(Some(new_idx)); + } + } + _ => {} + }, + _ => {} + } + } + if last_tick.elapsed() >= tick_rate { + last_tick = Instant::now(); + } + } +} + +fn ui(f: &mut Frame<'_, B>, app: &mut DbListTUI) { + let outer_chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Percentage(95), Constraint::Percentage(5)].as_ref()) + .split(f.size()); + + // Columns + { + let inner_chunks = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Percentage(50), Constraint::Percentage(50)]) + .split(outer_chunks[0]); + + let formatted_keys = app + .entries + .keys() + .enumerate() + .map(|(i, k)| ListItem::new(format!("[{}] - {k:?}", i + app.start))) + .collect::>>(); + + let key_list = List::new(formatted_keys) + .block(Block::default().borders(Borders::ALL).title(format!( + "Keys (Showing range [{}, {}] out of {} entries)", + app.start, + app.start + app.entries.len() - 1, + app.total_entries + ))) + .style(Style::default().fg(Color::White)) + .highlight_style(Style::default().fg(Color::Cyan).add_modifier(Modifier::ITALIC)) + .highlight_symbol("➜ ") + .start_corner(Corner::TopLeft); + f.render_stateful_widget(key_list, inner_chunks[0], &mut app.state); + + let value_display = Paragraph::new( + serde_json::to_string_pretty( + &app.entries.values().collect::>()[app.state.selected().unwrap_or(0)], + ) + .unwrap_or(String::from("Error serializing value!")), + ) + .block(Block::default().borders(Borders::ALL).title("Value (JSON)")) + .wrap(Wrap { trim: false }) + .alignment(Alignment::Left); + f.render_widget(value_display, inner_chunks[1]); + } + + // Footer + let footer = Paragraph::new( + CMDS.iter().map(|(k, v)| format!("[{k}] {v}")).collect::>().join(" | "), + ) + .block(Block::default().borders(Borders::ALL)) + .alignment(Alignment::Center) + .style(Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD)); + f.render_widget(footer, outer_chunks[1]); +} diff --git a/crates/primitives/src/storage.rs b/crates/primitives/src/storage.rs index 1b5ad8b9f..8482dea01 100644 --- a/crates/primitives/src/storage.rs +++ b/crates/primitives/src/storage.rs @@ -1,8 +1,9 @@ use super::{H256, U256}; use reth_codecs::Compact; +use serde::Serialize; /// Account storage entry. -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] pub struct StorageEntry { /// Storage key. pub key: H256, diff --git a/crates/storage/db/src/abstraction/database.rs b/crates/storage/db/src/abstraction/database.rs index 89a602de8..5e47bc456 100644 --- a/crates/storage/db/src/abstraction/database.rs +++ b/crates/storage/db/src/abstraction/database.rs @@ -25,9 +25,9 @@ pub trait Database: for<'a> DatabaseGAT<'a> { /// Takes a function and passes a read-only transaction into it, making sure it's closed in the /// end of the execution. - fn view(&self, f: F) -> Result + fn view(&self, mut f: F) -> Result where - F: Fn(&>::TX) -> T, + F: FnMut(&>::TX) -> T, { let tx = self.tx()?; diff --git a/crates/storage/db/src/abstraction/table.rs b/crates/storage/db/src/abstraction/table.rs index deba47b5b..35e3dc6c6 100644 --- a/crates/storage/db/src/abstraction/table.rs +++ b/crates/storage/db/src/abstraction/table.rs @@ -1,5 +1,6 @@ use crate::Error; use bytes::Bytes; +use serde::Serialize; use std::{ fmt::Debug, marker::{Send, Sync}, @@ -36,14 +37,14 @@ pub trait Decode: Send + Sync + Sized + Debug { } /// Generic trait that enforces the database key to implement [`Encode`] and [`Decode`]. -pub trait Key: Encode + Decode {} +pub trait Key: Encode + Decode + Ord {} -impl Key for T where T: Encode + Decode {} +impl Key for T where T: Encode + Decode + Ord {} /// Generic trait that enforces the database value to implement [`Compress`] and [`Decompress`]. -pub trait Value: Compress + Decompress {} +pub trait Value: Compress + Decompress + Serialize {} -impl Value for T where T: Compress + Decompress {} +impl Value for T where T: Compress + Decompress + Serialize {} /// Generic trait that a database table should follow. /// diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 28ca65177..69b4db247 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -78,12 +78,14 @@ macro_rules! table { impl $table_name { #[doc=concat!("Return ", stringify!($table_name), " as it is present inside the database.")] pub const fn const_name() -> &'static str { - stringify!($table_name) } + stringify!($table_name) + } } impl std::fmt::Display for $table_name { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", stringify!($table_name)) } + write!(f, "{}", stringify!($table_name)) + } } }; } diff --git a/crates/storage/db/src/tables/models/accounts.rs b/crates/storage/db/src/tables/models/accounts.rs index 0b53c0de1..d4cbd466b 100644 --- a/crates/storage/db/src/tables/models/accounts.rs +++ b/crates/storage/db/src/tables/models/accounts.rs @@ -11,7 +11,7 @@ use reth_primitives::{Account, Address, TransitionId}; use serde::{Deserialize, Serialize}; /// Account as it is saved inside [`AccountChangeSet`]. [`Address`] is the subkey. -#[derive(Debug, Default, Clone, Eq, PartialEq)] +#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)] pub struct AccountBeforeTx { /// Address for the account. Acts as `DupSort::SubKey`. pub address: Address, @@ -42,7 +42,7 @@ impl Compact for AccountBeforeTx { /// [`TxNumber`] concatenated with [`Address`]. Used as a key for [`StorageChangeSet`] /// /// Since it's used as a key, it isn't compressed when encoding it. -#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd)] pub struct TransitionIdAddress(pub (TransitionId, Address)); impl TransitionIdAddress { diff --git a/crates/storage/db/src/tables/models/blocks.rs b/crates/storage/db/src/tables/models/blocks.rs index ba152c6c8..3433249d8 100644 --- a/crates/storage/db/src/tables/models/blocks.rs +++ b/crates/storage/db/src/tables/models/blocks.rs @@ -65,7 +65,7 @@ pub type HeaderHash = H256; /// element as BlockNumber, helps out with querying/sorting. /// /// Since it's used as a key, the `BlockNumber` is not compressed when encoding it. -#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, Ord, PartialOrd)] pub struct BlockNumHash(pub (BlockNumber, BlockHash)); impl std::fmt::Debug for BlockNumHash { diff --git a/crates/storage/db/src/tables/models/sharded_key.rs b/crates/storage/db/src/tables/models/sharded_key.rs index 518bebf9e..859c6b7f7 100644 --- a/crates/storage/db/src/tables/models/sharded_key.rs +++ b/crates/storage/db/src/tables/models/sharded_key.rs @@ -12,7 +12,7 @@ use reth_primitives::TxNumber; /// `Address | 200` -> data is from transaction 0 to 200. /// /// `Address | 300` -> data is from transaction 201 to 300. -#[derive(Debug, Default, Clone, Eq, PartialEq)] +#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct ShardedKey { /// The key for this type. pub key: T, diff --git a/docs/repo/crates/db.md b/docs/repo/crates/db.md index 133ec3c9a..784774b31 100644 --- a/docs/repo/crates/db.md +++ b/docs/repo/crates/db.md @@ -9,6 +9,7 @@ The database is a central component to Reth, enabling persistent storage for dat Within Reth, the database is organized via "tables". A table is any struct that implements the `Table` trait. [File: crates/storage/db/src/abstraction/table.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/table.rs#L56-L65) + ```rust ignore pub trait Table: Send + Sync + Debug + 'static { /// Return table name as it is present inside the MDBX. @@ -22,10 +23,10 @@ pub trait Table: Send + Sync + Debug + 'static { } //--snip-- -pub trait Key: Encode + Decode {} +pub trait Key: Encode + Decode + Ord {} //--snip-- -pub trait Value: Compress + Decompress {} +pub trait Value: Compress + Decompress + Serialize {} ``` @@ -61,9 +62,10 @@ There are many tables within the node, all used to store different types of data ## Database -Reth's database design revolves around it's main [Database trait](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/mod.rs#L33), which takes advantage of [generic associated types](https://blog.rust-lang.org/2022/10/28/gats-stabilization.html) and [a few design tricks](https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats) to implement the database's functionality across many types. Let's take a quick look at the `Database` trait and how it works. +Reth's database design revolves around it's main [Database trait](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/mod.rs#L33), which takes advantage of [generic associated types](https://blog.rust-lang.org/2022/10/28/gats-stabilization.html) and [a few design tricks](https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats) to implement the database's functionality across many types. Let's take a quick look at the `Database` trait and how it works. [File: crates/storage/db/src/abstraction/database.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/database.rs#L19) + ```rust ignore /// Main Database trait that spawns transactions to be executed. pub trait Database: for<'a> DatabaseGAT<'a> { @@ -102,10 +104,11 @@ pub trait Database: for<'a> DatabaseGAT<'a> { } } ``` -Any type that implements the `Database` trait can create a database transaction, as well as view or update existing transactions. As an example, lets revisit the `Transaction` struct from the `stages` crate. This struct contains a field named `db` which is a reference to a generic type `DB` that implements the `Database` trait. The `Transaction` struct can use the `db` field to store new headers, bodies and senders in the database. In the code snippet below, you can see the `Transaction::open()` method, which uses the `Database::tx_mut()` function to create a mutable transaction. +Any type that implements the `Database` trait can create a database transaction, as well as view or update existing transactions. As an example, lets revisit the `Transaction` struct from the `stages` crate. This struct contains a field named `db` which is a reference to a generic type `DB` that implements the `Database` trait. The `Transaction` struct can use the `db` field to store new headers, bodies and senders in the database. In the code snippet below, you can see the `Transaction::open()` method, which uses the `Database::tx_mut()` function to create a mutable transaction. [File: crates/stages/src/db.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/db.rs#L95-L98) + ```rust ignore pub struct Transaction<'this, DB: Database> { /// A handle to the DB. @@ -131,6 +134,7 @@ where The `Database` trait also implements the `DatabaseGAT` trait which defines two associated types `TX` and `TXMut`. [File: crates/storage/db/src/abstraction/database.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/database.rs#L11) + ```rust ignore /// Implements the GAT method from: /// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. @@ -144,13 +148,14 @@ pub trait DatabaseGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + S } ``` -In Rust, associated types are like generics in that they can be any type fitting the generic's definition, with the difference being that associated types are associated with a trait and can only be used in the context of that trait. +In Rust, associated types are like generics in that they can be any type fitting the generic's definition, with the difference being that associated types are associated with a trait and can only be used in the context of that trait. -In the code snippet above, the `DatabaseGAT` trait has two associated types, `TX` and `TXMut`. +In the code snippet above, the `DatabaseGAT` trait has two associated types, `TX` and `TXMut`. -The `TX` type can be any type that implements the `DbTx` trait, which provides a set of functions to interact with read only transactions. +The `TX` type can be any type that implements the `DbTx` trait, which provides a set of functions to interact with read only transactions. [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L36) + ```rust ignore /// Read only transaction pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> { @@ -169,6 +174,7 @@ pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> { The `TXMut` type can be any type that implements the `DbTxMut` trait, which provides a set of functions to interact with read/write transactions. [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L49) + ```rust ignore /// Read write transaction that allows writing to database pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> { @@ -178,9 +184,9 @@ pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> { fn delete(&self, key: T::Key, value: Option) -> Result; /// Clears database. fn clear(&self) -> Result<(), Error>; - /// Cursor for writing + /// Cursor for writing fn cursor_write(&self) -> Result<>::CursorMut, Error>; - /// DupCursor for writing + /// DupCursor for writing fn cursor_dup_write( &self, ) -> Result<>::DupCursorMut, Error>; @@ -190,6 +196,7 @@ pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> { Lets take a look at the `DbTx` and `DbTxMut` traits in action. Revisiting the `Transaction` struct as an example, the `Transaction::get_block_hash()` method uses the `DbTx::get()` function to get a block header hash in the form of `self.get::(number)`. [File: crates/stages/src/db.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/db.rs#L106) + ```rust ignore impl<'this, DB> Transaction<'this, DB> @@ -219,20 +226,21 @@ impl<'a, DB: Database> Deref for Transaction<'a, DB> { The `Transaction` struct implements the `Deref` trait, which returns a reference to its `tx` field, which is a `TxMut`. Recall that `TxMut` is a generic type on the `DatabaseGAT` trait, which is defined as `type TXMut: DbTxMut<'a> + DbTx<'a> + Send + Sync;`, giving it access to all of the functions available to `DbTx`, including the `DbTx::get()` function. -Notice that the function uses a [turbofish](https://techblog.tonsser.com/posts/what-is-rusts-turbofish) to define which table to use when passing in the `key` to the `DbTx::get()` function. Taking a quick look at the function definition, a generic `T` is defined that implements the `Table` trait mentioned at the beginning of this chapter. - +Notice that the function uses a [turbofish](https://techblog.tonsser.com/posts/what-is-rusts-turbofish) to define which table to use when passing in the `key` to the `DbTx::get()` function. Taking a quick look at the function definition, a generic `T` is defined that implements the `Table` trait mentioned at the beginning of this chapter. [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L38) + ```rust ignore fn get(&self, key: T::Key) -> Result, Error>; ``` -This design pattern is very powerful and allows Reth to use the methods available to the `DbTx` and `DbTxMut` traits without having to define implementation blocks for each table within the database. +This design pattern is very powerful and allows Reth to use the methods available to the `DbTx` and `DbTxMut` traits without having to define implementation blocks for each table within the database. Lets take a look at a couple examples before moving on. In the snippet below, the `DbTxMut::put()` method is used to insert values into the `CanonicalHeaders`, `Headers` and `HeaderNumbers` tables. [File: crates/storage/provider/src/block.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/provider/src/block.rs#L121-L125) -```rust ignore + +```rust ignore let block_num_hash = BlockNumHash((block.number, block.hash())); tx.put::(block.number, block.hash())?; // Put header with canonical hashes. @@ -240,12 +248,11 @@ Lets take a look at a couple examples before moving on. In the snippet below, th tx.put::(block.hash(), block.number)?; ``` - This next example uses the `DbTx::cursor()` method to get a `Cursor`. The `Cursor` type provides a way to traverse through rows in a database table, one row at a time. A cursor enables the program to perform an operation (updating, deleting, etc) on each row in the table individually. The following code snippet gets a cursor for a few different tables in the database. - [File: crates/stages/src/stages/execution.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/stages/execution.rs#L93-L101) -```rust ignore + +```rust ignore // Get next canonical block hashes to execute. let mut canonicals = db_tx.cursor_read::()?; // Get header with canonical hashes. @@ -262,6 +269,7 @@ This next example uses the `DbTx::cursor()` method to get a `Cursor`. The `Curso We are almost at the last stop in the tour of the `db` crate. In addition to the methods provided by the `DbTx` and `DbTxMut` traits, `DbTx` also inherits the `DbTxGAT` trait, while `DbTxMut` inherits `DbTxMutGAT`. These next two traits provide various associated types related to cursors as well as methods to utilize the cursor types. [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L12-L17) + ```rust ignore pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { /// Cursor GAT @@ -271,9 +279,10 @@ pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync } ``` -Lets look at an examples of how cursors are used. The code snippet below contains the `unwind` method from the `BodyStage` defined in the `stages` crate. This function is responsible for unwinding any changes to the database if there is an error when executing the body stage within the Reth pipeline. +Lets look at an examples of how cursors are used. The code snippet below contains the `unwind` method from the `BodyStage` defined in the `stages` crate. This function is responsible for unwinding any changes to the database if there is an error when executing the body stage within the Reth pipeline. [File: crates/stages/src/stages/bodies.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/stages/bodies.rs#L205-L238) + ```rust ignore /// Unwind the stage. async fn unwind( @@ -311,7 +320,7 @@ Lets look at an examples of how cursors are used. The code snippet below contain ``` -This function first grabs a mutable cursor for the `CumulativeTxCount`, `BlockOmmers` and `Transactions` tables. +This function first grabs a mutable cursor for the `CumulativeTxCount`, `BlockOmmers` and `Transactions` tables. The `tx_count_cursor` is used to get the last key value pair written to the `CumulativeTxCount` table and delete key value pair where the cursor is currently pointing. @@ -324,10 +333,11 @@ While this is a brief look at how cursors work in the context of database tables
## Summary + This chapter was packed with information, so lets do a quick review. The database is comprised of tables, with each table being a collection of key-value pairs representing various pieces of data in the blockchain. Any struct that implements the `Database` trait can view, update or delete entries in the various tables. The database design leverages nested traits and generic associated types to provide methods to interact with each table in the database.
# Next Chapter -[Next Chapter]() \ No newline at end of file +[Next Chapter]()