feat(bin): Format db list & db status subcommands (#667)

Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
This commit is contained in:
clabby
2023-01-18 18:59:09 +01:00
committed by GitHub
parent 161fe0d957
commit a8b7a089b6
12 changed files with 416 additions and 78 deletions

80
Cargo.lock generated
View File

@ -517,6 +517,12 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "cassowary"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
[[package]] [[package]]
name = "cast" name = "cast"
version = "0.3.0" version = "0.3.0"
@ -756,6 +762,18 @@ dependencies = [
"thiserror", "thiserror",
] ]
[[package]]
name = "comfy-table"
version = "6.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d"
dependencies = [
"crossterm",
"strum",
"strum_macros",
"unicode-width",
]
[[package]] [[package]]
name = "confy" name = "confy"
version = "0.5.1" version = "0.5.1"
@ -908,6 +926,31 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crossterm"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
dependencies = [
"bitflags",
"crossterm_winapi",
"libc",
"mio",
"parking_lot 0.12.1",
"signal-hook",
"signal-hook-mio",
"winapi",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "crunchy" name = "crunchy"
version = "0.2.2" version = "0.2.2"
@ -3917,7 +3960,9 @@ version = "0.1.0"
dependencies = [ dependencies = [
"backon", "backon",
"clap 4.0.32", "clap 4.0.32",
"comfy-table",
"confy", "confy",
"crossterm",
"dirs-next", "dirs-next",
"eyre", "eyre",
"fdlimit", "fdlimit",
@ -3949,6 +3994,7 @@ dependencies = [
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"tracing", "tracing",
"tui",
"walkdir", "walkdir",
] ]
@ -5213,6 +5259,27 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
[[package]]
name = "signal-hook"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
dependencies = [
"libc",
"signal-hook-registry",
]
[[package]]
name = "signal-hook-mio"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
dependencies = [
"libc",
"mio",
"signal-hook",
]
[[package]] [[package]]
name = "signal-hook-registry" name = "signal-hook-registry"
version = "1.4.0" version = "1.4.0"
@ -5989,6 +6056,19 @@ dependencies = [
"toml", "toml",
] ]
[[package]]
name = "tui"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccdd26cbd674007e649a272da4475fb666d3aa0ad0531da7136db6fab0e5bad1"
dependencies = [
"bitflags",
"cassowary",
"crossterm",
"unicode-segmentation",
"unicode-width",
]
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.16.0" version = "1.16.0"

View File

@ -53,3 +53,6 @@ tokio-stream = "0.1"
futures = "0.3.25" futures = "0.3.25"
tempfile = { version = "3.3.0" } tempfile = { version = "3.3.0" }
backon = "0.2.0" backon = "0.2.0"
comfy-table = "6.1.4"
crossterm = "0.25.0"
tui = "0.19.0"

View File

@ -1,6 +1,7 @@
//! Database debugging tool //! Database debugging tool
use crate::dirs::{DbPath, PlatformPath}; use crate::dirs::{DbPath, PlatformPath};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use comfy_table::{Cell, Row, Table as ComfyTable};
use eyre::{Result, WrapErr}; use eyre::{Result, WrapErr};
use reth_db::{ use reth_db::{
cursor::{DbCursorRO, Walker}, cursor::{DbCursorRO, Walker},
@ -11,7 +12,11 @@ use reth_db::{
}; };
use reth_interfaces::test_utils::generators::random_block_range; use reth_interfaces::test_utils::generators::random_block_range;
use reth_provider::insert_canonical_block; use reth_provider::insert_canonical_block;
use tracing::info; use std::collections::BTreeMap;
use tracing::{error, info};
/// DB List TUI
mod tui;
/// `reth db` command /// `reth db` command
#[derive(Debug, Parser)] #[derive(Debug, Parser)]
@ -78,6 +83,17 @@ impl Command {
match &self.command { match &self.command {
// TODO: We'll need to add this on the DB trait. // TODO: We'll need to add this on the DB trait.
Subcommands::Stats { .. } => { Subcommands::Stats { .. } => {
let mut stats_table = ComfyTable::new();
stats_table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
stats_table.set_header([
"Table Name",
"# Entries",
"Branch Pages",
"Leaf Pages",
"Overflow Pages",
"Total Size (KB)",
]);
tool.db.view(|tx| { tool.db.view(|tx| {
for table in tables::TABLES.iter().map(|(_, name)| name) { for table in tables::TABLES.iter().map(|(_, name)| name) {
let table_db = let table_db =
@ -97,22 +113,69 @@ impl Command {
let overflow_pages = stats.overflow_pages(); let overflow_pages = stats.overflow_pages();
let num_pages = leaf_pages + branch_pages + overflow_pages; let num_pages = leaf_pages + branch_pages + overflow_pages;
let table_size = page_size * num_pages; let table_size = page_size * num_pages;
info!(
target: "reth::cli", let mut row = Row::new();
"Table {} has {} entries (total size: {} KB)", row.add_cell(Cell::new(table))
table, .add_cell(Cell::new(stats.entries()))
stats.entries(), .add_cell(Cell::new(branch_pages))
table_size / 1024 .add_cell(Cell::new(leaf_pages))
); .add_cell(Cell::new(overflow_pages))
.add_cell(Cell::new(table_size / 1024));
stats_table.add_row(row);
} }
Ok::<(), eyre::Report>(()) Ok::<(), eyre::Report>(())
})??; })??;
println!("{stats_table}");
} }
Subcommands::Seed { len } => { Subcommands::Seed { len } => {
tool.seed(*len)?; tool.seed(*len)?;
} }
Subcommands::List(args) => { Subcommands::List(args) => {
tool.list(args)?; macro_rules! table_tui {
($arg:expr, $start:expr, $len:expr => [$($table:ident),*]) => {
match $arg {
$(stringify!($table) => {
tool.db.view(|tx| {
let table_db = tx.inner.open_db(Some(stringify!($table))).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", stringify!($table)))?;
let total_entries = stats.entries();
if $start > total_entries - 1 {
error!(
target: "reth::cli",
"Start index {start} is greater than the final entry index ({final_entry_idx}) in the table {table}",
start = $start,
final_entry_idx = total_entries - 1,
table = stringify!($table)
);
return Ok(());
}
let map = tool.list::<tables::$table>($start, $len)?;
tui::DbListTUI::<tables::$table>::show_tui(map, $start, total_entries)
})??
},)*
_ => {
error!(target: "reth::cli", "Unknown table.");
return Ok(());
}
}
}
}
table_tui!(args.table.as_str(), args.start, args.len => [
CanonicalHeaders,
HeaderTD,
HeaderNumbers,
Headers,
BlockBodies,
BlockOmmers,
TxHashNumber,
PlainAccountState,
BlockTransitionIndex,
TxTransitionIndex,
SyncStage,
Transactions
]);
} }
Subcommands::Drop => { Subcommands::Drop => {
tool.drop(&self.db)?; tool.drop(&self.db)?;
@ -150,41 +213,9 @@ impl<'a, DB: Database> DbTool<'a, DB> {
Ok(()) Ok(())
} }
/// Lists the given table data /// Grabs the contents of the table within a certain index range and places the
fn list(&mut self, args: &ListArgs) -> Result<()> { /// entries into a [HashMap].
macro_rules! list_tables { fn list<T: Table>(&mut self, start: usize, len: usize) -> Result<BTreeMap<T::Key, T::Value>> {
($arg:expr, $start:expr, $len:expr => [$($table:ident,)*]) => {
match $arg {
$(stringify!($table) => {
self.list_table::<tables::$table>($start, $len)?
},)*
_ => {
tracing::error!(target: "reth::cli", "Unknown table.");
return Ok(())
}
}
};
}
list_tables!(args.table.as_str(), args.start, args.len => [
CanonicalHeaders,
HeaderTD,
HeaderNumbers,
Headers,
BlockBodies,
BlockOmmers,
TxHashNumber,
PlainAccountState,
BlockTransitionIndex,
TxTransitionIndex,
SyncStage,
Transactions,
]);
Ok(())
}
fn list_table<T: Table>(&mut self, start: usize, len: usize) -> Result<()> {
let data = self.db.view(|tx| { let data = self.db.view(|tx| {
let mut cursor = tx.cursor_read::<T>().expect("Was not able to obtain a cursor."); let mut cursor = tx.cursor_read::<T>().expect("Was not able to obtain a cursor.");
@ -195,8 +226,9 @@ impl<'a, DB: Database> DbTool<'a, DB> {
walker.skip(start).take(len).collect::<Vec<_>>() walker.skip(start).take(len).collect::<Vec<_>>()
})?; })?;
println!("{data:?}"); data.into_iter()
Ok(()) .collect::<Result<BTreeMap<T::Key, T::Value>, _>>()
.map_err(|e| eyre::eyre!(e))
} }
fn drop(&mut self, path: &PlatformPath<DbPath>) -> Result<()> { fn drop(&mut self, path: &PlatformPath<DbPath>) -> Result<()> {

209
bin/reth/src/db/tui.rs Normal file
View File

@ -0,0 +1,209 @@
use crossterm::{
event::{self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode, MouseEventKind},
execute,
terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
};
use reth_db::table::Table;
use std::{
collections::BTreeMap,
io,
time::{Duration, Instant},
};
use tracing::error;
use tui::{
backend::{Backend, CrosstermBackend},
layout::{Alignment, Constraint, Corner, Direction, Layout},
style::{Color, Modifier, Style},
widgets::{Block, Borders, List, ListItem, ListState, Paragraph, Wrap},
Frame, Terminal,
};
/// Available keybindings for the [DbListTUI]
static CMDS: [(&str, &str); 3] = [("q", "Quit"), ("up", "Entry Above"), ("down", "Entry Below")];
/// Modified version of the [ListState] struct that exposes the `offset` field.
/// Used to make the [DbListTUI] keys clickable.
struct ExpListState {
pub(crate) offset: usize,
}
#[derive(Default)]
pub(crate) struct DbListTUI<T: Table> {
/// The state of the key list.
pub(crate) state: ListState,
/// The starting index of the key list in the DB.
pub(crate) start: usize,
/// The total number of entries in the database
pub(crate) total_entries: usize,
/// Entries to show in the TUI.
pub(crate) entries: BTreeMap<T::Key, T::Value>,
}
impl<T: Table> DbListTUI<T> {
fn new(entries: BTreeMap<T::Key, T::Value>, start: usize, total_entries: usize) -> Self {
Self { state: ListState::default(), start, total_entries, entries }
}
/// Move to the next list selection
fn next(&mut self) {
let i = match self.state.selected() {
Some(i) => {
if i >= self.entries.len() - 1 {
0
} else {
i + 1
}
}
None => 0,
};
self.state.select(Some(i));
}
/// Move to the previous list selection
fn previous(&mut self) {
let i = match self.state.selected() {
Some(i) => {
if i == 0 {
self.entries.len() - 1
} else {
i - 1
}
}
None => 0,
};
self.state.select(Some(i));
}
/// Show the [DbListTUI] in the terminal.
pub(crate) fn show_tui(
entries: BTreeMap<T::Key, T::Value>,
start: usize,
total_entries: usize,
) -> eyre::Result<()> {
// setup terminal
enable_raw_mode()?;
let mut stdout = io::stdout();
execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?;
let backend = CrosstermBackend::new(stdout);
let mut terminal = Terminal::new(backend)?;
// create app and run it
let tick_rate = Duration::from_millis(250);
let mut app = DbListTUI::<T>::new(entries, start, total_entries);
app.state.select(Some(0));
let res = run(&mut terminal, app, tick_rate);
// restore terminal
disable_raw_mode()?;
execute!(terminal.backend_mut(), LeaveAlternateScreen, DisableMouseCapture)?;
terminal.show_cursor()?;
if let Err(err) = res {
error!("{:?}", err)
}
Ok(())
}
}
fn run<B: Backend, T: Table>(
terminal: &mut Terminal<B>,
mut app: DbListTUI<T>,
tick_rate: Duration,
) -> io::Result<()> {
let mut last_tick = Instant::now();
loop {
terminal.draw(|f| ui(f, &mut app))?;
let timeout =
tick_rate.checked_sub(last_tick.elapsed()).unwrap_or_else(|| Duration::from_secs(0));
if crossterm::event::poll(timeout)? {
match event::read()? {
Event::Key(key) => match key.code {
KeyCode::Char('q') | KeyCode::Char('Q') => return Ok(()),
KeyCode::Down => app.next(),
KeyCode::Up => app.previous(),
_ => {}
},
Event::Mouse(e) => match e.kind {
MouseEventKind::ScrollDown => app.next(),
MouseEventKind::ScrollUp => app.previous(),
// TODO: This click event can be triggered outside of the list widget.
MouseEventKind::Down(_) => {
// SAFETY: The pointer to the app's state will always be valid for
// reads here, and the source is larger than the destination.
//
// This is technically unsafe, but because the alignment requirements
// in both the source and destination are the same and we can ensure
// that the pointer to `app.state` is valid for reads, this is safe.
let state: ExpListState = unsafe { std::mem::transmute_copy(&app.state) };
let new_idx = (e.row as usize + state.offset).saturating_sub(1);
if new_idx < app.entries.len() {
app.state.select(Some(new_idx));
}
}
_ => {}
},
_ => {}
}
}
if last_tick.elapsed() >= tick_rate {
last_tick = Instant::now();
}
}
}
fn ui<B: Backend, T: Table>(f: &mut Frame<'_, B>, app: &mut DbListTUI<T>) {
let outer_chunks = Layout::default()
.direction(Direction::Vertical)
.constraints([Constraint::Percentage(95), Constraint::Percentage(5)].as_ref())
.split(f.size());
// Columns
{
let inner_chunks = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Percentage(50), Constraint::Percentage(50)])
.split(outer_chunks[0]);
let formatted_keys = app
.entries
.keys()
.enumerate()
.map(|(i, k)| ListItem::new(format!("[{}] - {k:?}", i + app.start)))
.collect::<Vec<ListItem<'_>>>();
let key_list = List::new(formatted_keys)
.block(Block::default().borders(Borders::ALL).title(format!(
"Keys (Showing range [{}, {}] out of {} entries)",
app.start,
app.start + app.entries.len() - 1,
app.total_entries
)))
.style(Style::default().fg(Color::White))
.highlight_style(Style::default().fg(Color::Cyan).add_modifier(Modifier::ITALIC))
.highlight_symbol("")
.start_corner(Corner::TopLeft);
f.render_stateful_widget(key_list, inner_chunks[0], &mut app.state);
let value_display = Paragraph::new(
serde_json::to_string_pretty(
&app.entries.values().collect::<Vec<_>>()[app.state.selected().unwrap_or(0)],
)
.unwrap_or(String::from("Error serializing value!")),
)
.block(Block::default().borders(Borders::ALL).title("Value (JSON)"))
.wrap(Wrap { trim: false })
.alignment(Alignment::Left);
f.render_widget(value_display, inner_chunks[1]);
}
// Footer
let footer = Paragraph::new(
CMDS.iter().map(|(k, v)| format!("[{k}] {v}")).collect::<Vec<_>>().join(" | "),
)
.block(Block::default().borders(Borders::ALL))
.alignment(Alignment::Center)
.style(Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD));
f.render_widget(footer, outer_chunks[1]);
}

View File

@ -1,8 +1,9 @@
use super::{H256, U256}; use super::{H256, U256};
use reth_codecs::Compact; use reth_codecs::Compact;
use serde::Serialize;
/// Account storage entry. /// Account storage entry.
#[derive(Debug, Default, Clone, PartialEq, Eq)] #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)]
pub struct StorageEntry { pub struct StorageEntry {
/// Storage key. /// Storage key.
pub key: H256, pub key: H256,

View File

@ -25,9 +25,9 @@ pub trait Database: for<'a> DatabaseGAT<'a> {
/// Takes a function and passes a read-only transaction into it, making sure it's closed in the /// Takes a function and passes a read-only transaction into it, making sure it's closed in the
/// end of the execution. /// end of the execution.
fn view<T, F>(&self, f: F) -> Result<T, Error> fn view<T, F>(&self, mut f: F) -> Result<T, Error>
where where
F: Fn(&<Self as DatabaseGAT<'_>>::TX) -> T, F: FnMut(&<Self as DatabaseGAT<'_>>::TX) -> T,
{ {
let tx = self.tx()?; let tx = self.tx()?;

View File

@ -1,5 +1,6 @@
use crate::Error; use crate::Error;
use bytes::Bytes; use bytes::Bytes;
use serde::Serialize;
use std::{ use std::{
fmt::Debug, fmt::Debug,
marker::{Send, Sync}, marker::{Send, Sync},
@ -36,14 +37,14 @@ pub trait Decode: Send + Sync + Sized + Debug {
} }
/// Generic trait that enforces the database key to implement [`Encode`] and [`Decode`]. /// Generic trait that enforces the database key to implement [`Encode`] and [`Decode`].
pub trait Key: Encode + Decode {} pub trait Key: Encode + Decode + Ord {}
impl<T> Key for T where T: Encode + Decode {} impl<T> Key for T where T: Encode + Decode + Ord {}
/// Generic trait that enforces the database value to implement [`Compress`] and [`Decompress`]. /// Generic trait that enforces the database value to implement [`Compress`] and [`Decompress`].
pub trait Value: Compress + Decompress {} pub trait Value: Compress + Decompress + Serialize {}
impl<T> Value for T where T: Compress + Decompress {} impl<T> Value for T where T: Compress + Decompress + Serialize {}
/// Generic trait that a database table should follow. /// Generic trait that a database table should follow.
/// ///

View File

@ -78,12 +78,14 @@ macro_rules! table {
impl $table_name { impl $table_name {
#[doc=concat!("Return ", stringify!($table_name), " as it is present inside the database.")] #[doc=concat!("Return ", stringify!($table_name), " as it is present inside the database.")]
pub const fn const_name() -> &'static str { pub const fn const_name() -> &'static str {
stringify!($table_name) } stringify!($table_name)
}
} }
impl std::fmt::Display for $table_name { impl std::fmt::Display for $table_name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", stringify!($table_name)) } write!(f, "{}", stringify!($table_name))
}
} }
}; };
} }

View File

@ -11,7 +11,7 @@ use reth_primitives::{Account, Address, TransitionId};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
/// Account as it is saved inside [`AccountChangeSet`]. [`Address`] is the subkey. /// Account as it is saved inside [`AccountChangeSet`]. [`Address`] is the subkey.
#[derive(Debug, Default, Clone, Eq, PartialEq)] #[derive(Debug, Default, Clone, Eq, PartialEq, Serialize)]
pub struct AccountBeforeTx { pub struct AccountBeforeTx {
/// Address for the account. Acts as `DupSort::SubKey`. /// Address for the account. Acts as `DupSort::SubKey`.
pub address: Address, pub address: Address,
@ -42,7 +42,7 @@ impl Compact for AccountBeforeTx {
/// [`TxNumber`] concatenated with [`Address`]. Used as a key for [`StorageChangeSet`] /// [`TxNumber`] concatenated with [`Address`]. Used as a key for [`StorageChangeSet`]
/// ///
/// Since it's used as a key, it isn't compressed when encoding it. /// Since it's used as a key, it isn't compressed when encoding it.
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Ord, PartialOrd)]
pub struct TransitionIdAddress(pub (TransitionId, Address)); pub struct TransitionIdAddress(pub (TransitionId, Address));
impl TransitionIdAddress { impl TransitionIdAddress {

View File

@ -65,7 +65,7 @@ pub type HeaderHash = H256;
/// element as BlockNumber, helps out with querying/sorting. /// element as BlockNumber, helps out with querying/sorting.
/// ///
/// Since it's used as a key, the `BlockNumber` is not compressed when encoding it. /// Since it's used as a key, the `BlockNumber` is not compressed when encoding it.
#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, Ord, PartialOrd)]
pub struct BlockNumHash(pub (BlockNumber, BlockHash)); pub struct BlockNumHash(pub (BlockNumber, BlockHash));
impl std::fmt::Debug for BlockNumHash { impl std::fmt::Debug for BlockNumHash {

View File

@ -12,7 +12,7 @@ use reth_primitives::TxNumber;
/// `Address | 200` -> data is from transaction 0 to 200. /// `Address | 200` -> data is from transaction 0 to 200.
/// ///
/// `Address | 300` -> data is from transaction 201 to 300. /// `Address | 300` -> data is from transaction 201 to 300.
#[derive(Debug, Default, Clone, Eq, PartialEq)] #[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub struct ShardedKey<T> { pub struct ShardedKey<T> {
/// The key for this type. /// The key for this type.
pub key: T, pub key: T,

View File

@ -9,6 +9,7 @@ The database is a central component to Reth, enabling persistent storage for dat
Within Reth, the database is organized via "tables". A table is any struct that implements the `Table` trait. Within Reth, the database is organized via "tables". A table is any struct that implements the `Table` trait.
[File: crates/storage/db/src/abstraction/table.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/table.rs#L56-L65) [File: crates/storage/db/src/abstraction/table.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/table.rs#L56-L65)
```rust ignore ```rust ignore
pub trait Table: Send + Sync + Debug + 'static { pub trait Table: Send + Sync + Debug + 'static {
/// Return table name as it is present inside the MDBX. /// Return table name as it is present inside the MDBX.
@ -22,10 +23,10 @@ pub trait Table: Send + Sync + Debug + 'static {
} }
//--snip-- //--snip--
pub trait Key: Encode + Decode {} pub trait Key: Encode + Decode + Ord {}
//--snip-- //--snip--
pub trait Value: Compress + Decompress {} pub trait Value: Compress + Decompress + Serialize {}
``` ```
@ -61,9 +62,10 @@ There are many tables within the node, all used to store different types of data
## Database ## Database
Reth's database design revolves around it's main [Database trait](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/mod.rs#L33), which takes advantage of [generic associated types](https://blog.rust-lang.org/2022/10/28/gats-stabilization.html) and [a few design tricks](https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats) to implement the database's functionality across many types. Let's take a quick look at the `Database` trait and how it works. Reth's database design revolves around it's main [Database trait](https://github.com/paradigmxyz/reth/blob/0d9b9a392d4196793736522f3fc2ac804991b45d/crates/interfaces/src/db/mod.rs#L33), which takes advantage of [generic associated types](https://blog.rust-lang.org/2022/10/28/gats-stabilization.html) and [a few design tricks](https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats) to implement the database's functionality across many types. Let's take a quick look at the `Database` trait and how it works.
[File: crates/storage/db/src/abstraction/database.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/database.rs#L19) [File: crates/storage/db/src/abstraction/database.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/database.rs#L19)
```rust ignore ```rust ignore
/// Main Database trait that spawns transactions to be executed. /// Main Database trait that spawns transactions to be executed.
pub trait Database: for<'a> DatabaseGAT<'a> { pub trait Database: for<'a> DatabaseGAT<'a> {
@ -102,10 +104,11 @@ pub trait Database: for<'a> DatabaseGAT<'a> {
} }
} }
``` ```
Any type that implements the `Database` trait can create a database transaction, as well as view or update existing transactions. As an example, lets revisit the `Transaction` struct from the `stages` crate. This struct contains a field named `db` which is a reference to a generic type `DB` that implements the `Database` trait. The `Transaction` struct can use the `db` field to store new headers, bodies and senders in the database. In the code snippet below, you can see the `Transaction::open()` method, which uses the `Database::tx_mut()` function to create a mutable transaction.
Any type that implements the `Database` trait can create a database transaction, as well as view or update existing transactions. As an example, lets revisit the `Transaction` struct from the `stages` crate. This struct contains a field named `db` which is a reference to a generic type `DB` that implements the `Database` trait. The `Transaction` struct can use the `db` field to store new headers, bodies and senders in the database. In the code snippet below, you can see the `Transaction::open()` method, which uses the `Database::tx_mut()` function to create a mutable transaction.
[File: crates/stages/src/db.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/db.rs#L95-L98) [File: crates/stages/src/db.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/db.rs#L95-L98)
```rust ignore ```rust ignore
pub struct Transaction<'this, DB: Database> { pub struct Transaction<'this, DB: Database> {
/// A handle to the DB. /// A handle to the DB.
@ -131,6 +134,7 @@ where
The `Database` trait also implements the `DatabaseGAT` trait which defines two associated types `TX` and `TXMut`. The `Database` trait also implements the `DatabaseGAT` trait which defines two associated types `TX` and `TXMut`.
[File: crates/storage/db/src/abstraction/database.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/database.rs#L11) [File: crates/storage/db/src/abstraction/database.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/database.rs#L11)
```rust ignore ```rust ignore
/// Implements the GAT method from: /// Implements the GAT method from:
/// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats. /// https://sabrinajewson.org/blog/the-better-alternative-to-lifetime-gats#the-better-gats.
@ -144,13 +148,14 @@ pub trait DatabaseGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + S
} }
``` ```
In Rust, associated types are like generics in that they can be any type fitting the generic's definition, with the difference being that associated types are associated with a trait and can only be used in the context of that trait. In Rust, associated types are like generics in that they can be any type fitting the generic's definition, with the difference being that associated types are associated with a trait and can only be used in the context of that trait.
In the code snippet above, the `DatabaseGAT` trait has two associated types, `TX` and `TXMut`. In the code snippet above, the `DatabaseGAT` trait has two associated types, `TX` and `TXMut`.
The `TX` type can be any type that implements the `DbTx` trait, which provides a set of functions to interact with read only transactions. The `TX` type can be any type that implements the `DbTx` trait, which provides a set of functions to interact with read only transactions.
[File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L36) [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L36)
```rust ignore ```rust ignore
/// Read only transaction /// Read only transaction
pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> { pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> {
@ -169,6 +174,7 @@ pub trait DbTx<'tx>: for<'a> DbTxGAT<'a> {
The `TXMut` type can be any type that implements the `DbTxMut` trait, which provides a set of functions to interact with read/write transactions. The `TXMut` type can be any type that implements the `DbTxMut` trait, which provides a set of functions to interact with read/write transactions.
[File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L49) [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L49)
```rust ignore ```rust ignore
/// Read write transaction that allows writing to database /// Read write transaction that allows writing to database
pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> { pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> {
@ -178,9 +184,9 @@ pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> {
fn delete<T: Table>(&self, key: T::Key, value: Option<T::Value>) -> Result<bool, Error>; fn delete<T: Table>(&self, key: T::Key, value: Option<T::Value>) -> Result<bool, Error>;
/// Clears database. /// Clears database.
fn clear<T: Table>(&self) -> Result<(), Error>; fn clear<T: Table>(&self) -> Result<(), Error>;
/// Cursor for writing /// Cursor for writing
fn cursor_write<T: Table>(&self) -> Result<<Self as DbTxMutGAT<'_>>::CursorMut<T>, Error>; fn cursor_write<T: Table>(&self) -> Result<<Self as DbTxMutGAT<'_>>::CursorMut<T>, Error>;
/// DupCursor for writing /// DupCursor for writing
fn cursor_dup_write<T: DupSort>( fn cursor_dup_write<T: DupSort>(
&self, &self,
) -> Result<<Self as DbTxMutGAT<'_>>::DupCursorMut<T>, Error>; ) -> Result<<Self as DbTxMutGAT<'_>>::DupCursorMut<T>, Error>;
@ -190,6 +196,7 @@ pub trait DbTxMut<'tx>: for<'a> DbTxMutGAT<'a> {
Lets take a look at the `DbTx` and `DbTxMut` traits in action. Revisiting the `Transaction` struct as an example, the `Transaction::get_block_hash()` method uses the `DbTx::get()` function to get a block header hash in the form of `self.get::<tables::CanonicalHeaders>(number)`. Lets take a look at the `DbTx` and `DbTxMut` traits in action. Revisiting the `Transaction` struct as an example, the `Transaction::get_block_hash()` method uses the `DbTx::get()` function to get a block header hash in the form of `self.get::<tables::CanonicalHeaders>(number)`.
[File: crates/stages/src/db.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/db.rs#L106) [File: crates/stages/src/db.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/db.rs#L106)
```rust ignore ```rust ignore
impl<'this, DB> Transaction<'this, DB> impl<'this, DB> Transaction<'this, DB>
@ -219,20 +226,21 @@ impl<'a, DB: Database> Deref for Transaction<'a, DB> {
The `Transaction` struct implements the `Deref` trait, which returns a reference to its `tx` field, which is a `TxMut`. Recall that `TxMut` is a generic type on the `DatabaseGAT` trait, which is defined as `type TXMut: DbTxMut<'a> + DbTx<'a> + Send + Sync;`, giving it access to all of the functions available to `DbTx`, including the `DbTx::get()` function. The `Transaction` struct implements the `Deref` trait, which returns a reference to its `tx` field, which is a `TxMut`. Recall that `TxMut` is a generic type on the `DatabaseGAT` trait, which is defined as `type TXMut: DbTxMut<'a> + DbTx<'a> + Send + Sync;`, giving it access to all of the functions available to `DbTx`, including the `DbTx::get()` function.
Notice that the function uses a [turbofish](https://techblog.tonsser.com/posts/what-is-rusts-turbofish) to define which table to use when passing in the `key` to the `DbTx::get()` function. Taking a quick look at the function definition, a generic `T` is defined that implements the `Table` trait mentioned at the beginning of this chapter. Notice that the function uses a [turbofish](https://techblog.tonsser.com/posts/what-is-rusts-turbofish) to define which table to use when passing in the `key` to the `DbTx::get()` function. Taking a quick look at the function definition, a generic `T` is defined that implements the `Table` trait mentioned at the beginning of this chapter.
[File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L38) [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L38)
```rust ignore ```rust ignore
fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>, Error>; fn get<T: Table>(&self, key: T::Key) -> Result<Option<T::Value>, Error>;
``` ```
This design pattern is very powerful and allows Reth to use the methods available to the `DbTx` and `DbTxMut` traits without having to define implementation blocks for each table within the database. This design pattern is very powerful and allows Reth to use the methods available to the `DbTx` and `DbTxMut` traits without having to define implementation blocks for each table within the database.
Lets take a look at a couple examples before moving on. In the snippet below, the `DbTxMut::put()` method is used to insert values into the `CanonicalHeaders`, `Headers` and `HeaderNumbers` tables. Lets take a look at a couple examples before moving on. In the snippet below, the `DbTxMut::put()` method is used to insert values into the `CanonicalHeaders`, `Headers` and `HeaderNumbers` tables.
[File: crates/storage/provider/src/block.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/provider/src/block.rs#L121-L125) [File: crates/storage/provider/src/block.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/provider/src/block.rs#L121-L125)
```rust ignore
```rust ignore
let block_num_hash = BlockNumHash((block.number, block.hash())); let block_num_hash = BlockNumHash((block.number, block.hash()));
tx.put::<tables::CanonicalHeaders>(block.number, block.hash())?; tx.put::<tables::CanonicalHeaders>(block.number, block.hash())?;
// Put header with canonical hashes. // Put header with canonical hashes.
@ -240,12 +248,11 @@ Lets take a look at a couple examples before moving on. In the snippet below, th
tx.put::<tables::HeaderNumbers>(block.hash(), block.number)?; tx.put::<tables::HeaderNumbers>(block.hash(), block.number)?;
``` ```
This next example uses the `DbTx::cursor()` method to get a `Cursor`. The `Cursor` type provides a way to traverse through rows in a database table, one row at a time. A cursor enables the program to perform an operation (updating, deleting, etc) on each row in the table individually. The following code snippet gets a cursor for a few different tables in the database. This next example uses the `DbTx::cursor()` method to get a `Cursor`. The `Cursor` type provides a way to traverse through rows in a database table, one row at a time. A cursor enables the program to perform an operation (updating, deleting, etc) on each row in the table individually. The following code snippet gets a cursor for a few different tables in the database.
[File: crates/stages/src/stages/execution.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/stages/execution.rs#L93-L101) [File: crates/stages/src/stages/execution.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/stages/execution.rs#L93-L101)
```rust ignore
```rust ignore
// Get next canonical block hashes to execute. // Get next canonical block hashes to execute.
let mut canonicals = db_tx.cursor_read::<tables::CanonicalHeaders>()?; let mut canonicals = db_tx.cursor_read::<tables::CanonicalHeaders>()?;
// Get header with canonical hashes. // Get header with canonical hashes.
@ -262,6 +269,7 @@ This next example uses the `DbTx::cursor()` method to get a `Cursor`. The `Curso
We are almost at the last stop in the tour of the `db` crate. In addition to the methods provided by the `DbTx` and `DbTxMut` traits, `DbTx` also inherits the `DbTxGAT` trait, while `DbTxMut` inherits `DbTxMutGAT`. These next two traits provide various associated types related to cursors as well as methods to utilize the cursor types. We are almost at the last stop in the tour of the `db` crate. In addition to the methods provided by the `DbTx` and `DbTxMut` traits, `DbTx` also inherits the `DbTxGAT` trait, while `DbTxMut` inherits `DbTxMutGAT`. These next two traits provide various associated types related to cursors as well as methods to utilize the cursor types.
[File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L12-L17) [File: crates/storage/db/src/abstraction/transaction.rs](https://github.com/paradigmxyz/reth/blob/main/crates/storage/db/src/abstraction/transaction.rs#L12-L17)
```rust ignore ```rust ignore
pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync { pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync {
/// Cursor GAT /// Cursor GAT
@ -271,9 +279,10 @@ pub trait DbTxGAT<'a, __ImplicitBounds: Sealed = Bounds<&'a Self>>: Send + Sync
} }
``` ```
Lets look at an examples of how cursors are used. The code snippet below contains the `unwind` method from the `BodyStage` defined in the `stages` crate. This function is responsible for unwinding any changes to the database if there is an error when executing the body stage within the Reth pipeline. Lets look at an examples of how cursors are used. The code snippet below contains the `unwind` method from the `BodyStage` defined in the `stages` crate. This function is responsible for unwinding any changes to the database if there is an error when executing the body stage within the Reth pipeline.
[File: crates/stages/src/stages/bodies.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/stages/bodies.rs#L205-L238) [File: crates/stages/src/stages/bodies.rs](https://github.com/paradigmxyz/reth/blob/main/crates/stages/src/stages/bodies.rs#L205-L238)
```rust ignore ```rust ignore
/// Unwind the stage. /// Unwind the stage.
async fn unwind( async fn unwind(
@ -311,7 +320,7 @@ Lets look at an examples of how cursors are used. The code snippet below contain
``` ```
This function first grabs a mutable cursor for the `CumulativeTxCount`, `BlockOmmers` and `Transactions` tables. This function first grabs a mutable cursor for the `CumulativeTxCount`, `BlockOmmers` and `Transactions` tables.
The `tx_count_cursor` is used to get the last key value pair written to the `CumulativeTxCount` table and delete key value pair where the cursor is currently pointing. The `tx_count_cursor` is used to get the last key value pair written to the `CumulativeTxCount` table and delete key value pair where the cursor is currently pointing.
@ -324,10 +333,11 @@ While this is a brief look at how cursors work in the context of database tables
<br> <br>
## Summary ## Summary
This chapter was packed with information, so lets do a quick review. The database is comprised of tables, with each table being a collection of key-value pairs representing various pieces of data in the blockchain. Any struct that implements the `Database` trait can view, update or delete entries in the various tables. The database design leverages nested traits and generic associated types to provide methods to interact with each table in the database. This chapter was packed with information, so lets do a quick review. The database is comprised of tables, with each table being a collection of key-value pairs representing various pieces of data in the blockchain. Any struct that implements the `Database` trait can view, update or delete entries in the various tables. The database design leverages nested traits and generic associated types to provide methods to interact with each table in the database.
<br> <br>
# Next Chapter # Next Chapter
[Next Chapter]() [Next Chapter]()