jleony/nanoreth
1
0
Fork 0
mirror of https://github.com/hl-archive-node/nanoreth.git synced 2025-12-06 10:59:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(db): add search to reth db list command (#4165)

Browse Source
This commit is contained in:
joshieDo
2023-08-15 18:28:30 +01:00
committed by GitHub
parent 90d2a00a2d
commit c743acde77
15 changed files with 206 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Cargo.lock generated
View File

@ -846,6 +846,15 @@ name = "boa_profiler"
version = "0.17.0"
source = "git+https://github.com/boa-dev/boa#a3b46545a2a09f9ac81fd83ac6b180934c728f61"
[[package]]
name = "boyer-moore-magiclen"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c77eb6b3a37f71fcd40e49b56c028ea8795c0e550afd8021e3e6a2369653035"
dependencies = [
"debug-helper",
]
[[package]]
name = "brotli"
version = "3.3.4"
@ -1604,6 +1613,12 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
[[package]]
name = "debug-helper"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e"
[[package]]
name = "debugid"
version = "0.8.0"
@ -5202,6 +5217,7 @@ name = "reth"
version = "0.1.0-alpha.6"
dependencies = [
"backon",
"boyer-moore-magiclen",
"clap",
"comfy-table",
"confy",

1
bin/reth/Cargo.toml
View File

@ -89,6 +89,7 @@ thiserror.workspace = true
pretty_assertions = "1.3.0"
humantime = "2.1.0"
const-str = "0.5.6"
boyer-moore-magiclen = "0.2.16"
[target.'cfg(not(windows))'.dependencies]
jemallocator = { version = "0.5.0", optional = true }

59
bin/reth/src/db/list.rs
View File

@ -1,9 +1,9 @@
use crate::utils::DbTool;
use clap::Parser;
use super::tui::DbListTUI;
use crate::utils::{DbTool, ListFilter};
use clap::Parser;
use eyre::WrapErr;
use reth_db::{database::Database, table::Table, DatabaseEnvRO, TableType, TableViewer, Tables};
use std::cell::RefCell;
use tracing::error;
const DEFAULT_NUM_ITEMS: &str = "5";
@ -22,6 +22,16 @@ pub struct Command {
/// How many items to take from the walker
#[arg(long, short, default_value = DEFAULT_NUM_ITEMS)]
len: usize,
/// Search parameter for both keys and values. Prefix it with `0x` to search for binary data,
/// and text otherwise.
///
/// ATTENTION! For compressed tables (`Transactions` and `Receipts`), there might be
/// missing results since the search uses the raw uncompressed value from the database.
#[arg(long)]
search: Option<String>,
/// Returns the number of rows found.
#[arg(long, short)]
count: bool,
/// Dump as JSON instead of using TUI.
#[arg(long, short)]
json: bool,
@ -38,6 +48,28 @@ impl Command {
Ok(())
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
if let Some(search) = search.strip_prefix("0x") {
return hex::decode(search).unwrap()
}
search.as_bytes().to_vec()
})
.unwrap_or_default();
ListFilter {
skip: self.skip,
len: self.len,
search,
reverse: self.reverse,
only_count: self.count,
}
}
}
struct ListTableViewer<'a> {
@ -64,13 +96,24 @@ impl TableViewer<()> for ListTableViewer<'_> {
return Ok(());
}
if self.args.json {
let list_result = self.tool.list::<T>(self.args.skip, self.args.len, self.args.reverse)?.into_iter().collect::<Vec<_>>();
println!("{}", serde_json::to_string_pretty(&list_result)?);
let list_filter = self.args.list_filter();
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;
if self.args.count {
println!("{count} entries found.")
}else {
println!("{}", serde_json::to_string_pretty(&list)?);
}
Ok(())
} else {
DbListTUI::<_, T>::new(|skip, count| {
self.tool.list::<T>(skip, count, self.args.reverse).unwrap()
let list_filter = RefCell::new(list_filter);
DbListTUI::<_, T>::new(|skip, len| {
list_filter.borrow_mut().update_page(skip, len);
self.tool.list::<T>(&list_filter.borrow()).unwrap().0
}, self.args.skip, self.args.len, total_entries).run()
}
})??;

18
bin/reth/src/db/tui.rs
View File

@ -3,7 +3,7 @@ use crossterm::{
execute,
terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen},
};
use reth_db::table::Table;
use reth_db::table::{Table, TableRow};
use std::{
io,
time::{Duration, Instant},
@ -45,7 +45,7 @@ pub(crate) enum ViewMode {
#[derive(Default)]
pub(crate) struct DbListTUI<F, T: Table>
where
F: FnMut(usize, usize) -> Vec<(T::Key, T::Value)>,
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
{
/// Fetcher for the next page of items.
///
@ -65,12 +65,12 @@ where
/// The state of the key list.
list_state: ListState,
/// Entries to show in the TUI.
entries: Vec<(T::Key, T::Value)>,
entries: Vec<TableRow<T>>,
}
impl<F, T: Table> DbListTUI<F, T>
where
F: FnMut(usize, usize) -> Vec<(T::Key, T::Value)>,
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
{
/// Create a new database list TUI
pub(crate) fn new(fetch: F, skip: usize, count: usize, total_entries: usize) -> Self {
@ -188,7 +188,7 @@ fn event_loop<B: Backend, F, T: Table>(
tick_rate: Duration,
) -> io::Result<()>
where
F: FnMut(usize, usize) -> Vec<(T::Key, T::Value)>,
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
{
let mut last_tick = Instant::now();
let mut running = true;
@ -216,7 +216,7 @@ where
/// Handle incoming events
fn handle_event<F, T: Table>(app: &mut DbListTUI<F, T>, event: Event) -> io::Result<bool>
where
F: FnMut(usize, usize) -> Vec<(T::Key, T::Value)>,
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
{
if app.mode == ViewMode::GoToPage {
if let Event::Key(key) = event {
@ -282,7 +282,7 @@ where
/// Render the UI
fn ui<B: Backend, F, T: Table>(f: &mut Frame<'_, B>, app: &mut DbListTUI<F, T>)
where
F: FnMut(usize, usize) -> Vec<(T::Key, T::Value)>,
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
{
let outer_chunks = Layout::default()
.direction(Direction::Vertical)
@ -296,7 +296,7 @@ where
.constraints([Constraint::Percentage(50), Constraint::Percentage(50)])
.split(outer_chunks[0]);
let key_length = format!("{}", app.skip + app.count - 1).len();
let key_length = format!("{}", (app.skip + app.count).saturating_sub(1)).len();
let entries: Vec<_> = app.entries.iter().map(|(k, _)| k).collect();
@ -312,7 +312,7 @@ where
.block(Block::default().borders(Borders::ALL).title(format!(
"Keys (Showing entries {}-{} out of {} entries)",
app.skip,
app.skip + app.entries.len() - 1,
(app.skip + app.entries.len()).saturating_sub(1),
app.total_entries
)))
.style(Style::default().fg(Color::White))

4
bin/reth/src/init.rs
View File

@ -180,7 +180,7 @@ mod tests {
use reth_db::{
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
table::Table,
table::{Table, TableRow},
test_utils::create_test_rw_db,
DatabaseEnv,
};
@ -193,7 +193,7 @@ mod tests {
#[allow(clippy::type_complexity)]
fn collect_table_entries<DB, T>(
tx: &<DB as DatabaseGAT<'_>>::TX,
) -> Result<Vec<(T::Key, T::Value)>, InitDatabaseError>
) -> Result<Vec<TableRow<T>>, InitDatabaseError>
where
DB: Database,
T: Table,

6
bin/reth/src/test_vectors/tables.rs
View File

@ -8,7 +8,7 @@ use proptest::{
test_runner::TestRunner,
};
use reth_db::{
table::{DupSort, Table},
table::{DupSort, Table, TableRow},
tables,
};
use reth_primitives::fs;
@ -81,7 +81,7 @@ where
let mut rows = vec![];
let mut seen_keys = HashSet::new();
let strat = proptest::collection::vec(
any_with::<(T::Key, T::Value)>((
any_with::<TableRow<T>>((
<T::Key as Arbitrary>::Parameters::default(),
<T::Value as Arbitrary>::Parameters::default(),
)),
@ -154,7 +154,7 @@ where
}
/// Save rows to file.
fn save_to_file<T: Table>(rows: Vec<(T::Key, T::Value)>) -> eyre::Result<()>
fn save_to_file<T: Table>(rows: Vec<TableRow<T>>) -> eyre::Result<()>
where
T::Key: serde::Serialize,
T::Value: serde::Serialize,

104
bin/reth/src/utils.rs
View File

@ -1,12 +1,14 @@
//! Common CLI utility functions.
use boyer_moore_magiclen::BMByte;
use eyre::Result;
use reth_consensus_common::validation::validate_block_standalone;
use reth_db::{
cursor::DbCursorRO,
database::Database,
table::Table,
table::{Table, TableRow},
transaction::{DbTx, DbTxMut},
DatabaseError, RawTable, TableRawRow,
};
use reth_interfaces::p2p::{
bodies::client::BodiesClient,
@ -19,6 +21,7 @@ use reth_primitives::{
use std::{
env::VarError,
path::{Path, PathBuf},
rc::Rc,
sync::Arc,
};
use tracing::info;
@ -103,23 +106,65 @@ impl<'a, DB: Database> DbTool<'a, DB> {
/// Grabs the contents of the table within a certain index range and places the
/// entries into a [`HashMap`][std::collections::HashMap].
pub fn list<T: Table>(
&self,
skip: usize,
len: usize,
reverse: bool,
) -> Result<Vec<(T::Key, T::Value)>> {
let data = self.db.view(|tx| {
let mut cursor = tx.cursor_read::<T>().expect("Was not able to obtain a cursor.");
///
/// [`ListFilter`] can be used to further
/// filter down the desired results. (eg. List only rows which include `0xd3adbeef`)
pub fn list<T: Table>(&self, filter: &ListFilter) -> Result<(Vec<TableRow<T>>, usize)> {
let bmb = Rc::new(BMByte::from(&filter.search));
if bmb.is_none() && filter.has_search() {
eyre::bail!("Invalid search.")
}
if reverse {
cursor.walk_back(None)?.skip(skip).take(len).collect::<Result<_, _>>()
let mut hits = 0;
let data = self.db.view(|tx| {
let mut cursor =
tx.cursor_read::<RawTable<T>>().expect("Was not able to obtain a cursor.");
let map_filter = |row: Result<TableRawRow<T>, _>| {
if let Ok((k, v)) = row {
let result = || {
if filter.only_count {
return None
}
Some((k.key().unwrap(), v.value().unwrap()))
};
match &*bmb {
Some(searcher) => {
if searcher.find_first_in(v.raw_value()).is_some() ||
searcher.find_first_in(k.raw_key()).is_some()
{
hits += 1;
return result()
}
}
None => {
hits += 1;
return result()
}
}
}
None
};
if filter.reverse {
Ok(cursor
.walk_back(None)?
.skip(filter.skip)
.filter_map(map_filter)
.take(filter.len)
.collect::<Vec<(_, _)>>())
} else {
cursor.walk(None)?.skip(skip).take(len).collect::<Result<_, _>>()
Ok(cursor
.walk(None)?
.skip(filter.skip)
.filter_map(map_filter)
.take(filter.len)
.collect::<Vec<(_, _)>>())
}
})?;
data.map_err(|e| eyre::eyre!(e))
Ok((data.map_err(|e: DatabaseError| eyre::eyre!(e))?, hits))
}
/// Grabs the content of the table for the given key
@ -147,3 +192,36 @@ impl<'a, DB: Database> DbTool<'a, DB> {
pub fn parse_path(value: &str) -> Result<PathBuf, shellexpand::LookupError<VarError>> {
shellexpand::full(value).map(|path| PathBuf::from(path.into_owned()))
}
/// Filters the results coming from the database.
#[derive(Debug)]
pub struct ListFilter {
/// Skip first N entries.
pub skip: usize,
/// Take N entries.
pub len: usize,
/// Sequence of bytes that will be searched on values and keys from the database.
pub search: Vec<u8>,
/// Reverse order of entries.
pub reverse: bool,
/// Only counts the number of filtered entries without decoding and returning them.
pub only_count: bool,
}
impl ListFilter {
/// Creates a new [`ListFilter`].
pub fn new(skip: usize, len: usize, search: Vec<u8>, reverse: bool, only_count: bool) -> Self {
ListFilter { skip, len, search, reverse, only_count }
}
/// If `search` has a list of bytes, then filter for rows that have this sequence.
pub fn has_search(&self) -> bool {
!self.search.is_empty()
}
/// Updates the page with new `skip` and `len` values.
pub fn update_page(&mut self, skip: usize, len: usize) {
self.skip = skip;
self.len = len;
}
}

6
crates/stages/src/test_utils/test_db.rs
View File

@ -3,7 +3,7 @@ use reth_db::{
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
database::DatabaseGAT,
models::{AccountBeforeTx, StoredBlockBodyIndices},
table::Table,
table::{Table, TableRow},
tables,
test_utils::{create_test_rw_db, create_test_rw_db_with_path},
transaction::{DbTx, DbTxGAT, DbTxMut, DbTxMutGAT},
@ -122,7 +122,7 @@ impl TestTransaction {
where
T: Table,
S: Clone,
F: FnMut(&S) -> (T::Key, T::Value),
F: FnMut(&S) -> TableRow<T>,
{
self.commit(|tx| {
values.iter().try_for_each(|src| {
@ -147,7 +147,7 @@ impl TestTransaction {
T: Table,
<T as Table>::Value: Clone,
S: Clone,
F: FnMut(&Option<<T as Table>::Value>, &S) -> (T::Key, T::Value),
F: FnMut(&Option<<T as Table>::Value>, &S) -> TableRow<T>,
{
self.commit(|tx| {
let mut cursor = tx.cursor_write::<T>()?;

6
crates/storage/db/benches/hash_keys.rs
View File

@ -106,7 +106,7 @@ where
// Iteration to be benchmarked
let execution = |(input, db)| {
let mut input: Vec<(T::Key, T::Value)> = input;
let mut input: Vec<TableRow<T>> = input;
if scenario_str.contains("_sorted") || scenario_str.contains("append") {
input.sort_by(|a, b| a.0.cmp(&b.0));
}
@ -134,14 +134,14 @@ where
/// Generates two batches. The first is to be inserted into the database before running the
/// benchmark. The second is to be benchmarked with.
#[allow(clippy::type_complexity)]
fn generate_batches<T>(size: usize) -> (Vec<(T::Key, T::Value)>, Vec<(T::Key, T::Value)>)
fn generate_batches<T>(size: usize) -> (Vec<TableRow<T>>, Vec<TableRow<T>>)
where
T: Table + Default,
T::Key: std::hash::Hash + Arbitrary,
T::Value: Arbitrary,
{
let strat = proptest::collection::vec(
any_with::<(T::Key, T::Value)>((
any_with::<TableRow<T>>((
<T::Key as Arbitrary>::Parameters::default(),
<T::Value as Arbitrary>::Parameters::default(),
)),

2
crates/storage/db/benches/utils.rs
View File

@ -25,7 +25,7 @@ where
T::Key: Default + Clone + for<'de> serde::Deserialize<'de>,
T::Value: Default + Clone + for<'de> serde::Deserialize<'de>,
{
let list: Vec<(T::Key, T::Value)> = serde_json::from_reader(std::io::BufReader::new(
let list: Vec<TableRow<T>> = serde_json::from_reader(std::io::BufReader::new(
std::fs::File::open(format!(
"{}/../../../testdata/micro/db/{}.json",
env!("CARGO_MANIFEST_DIR"),

10
crates/storage/db/src/abstraction/cursor.rs
View File

@ -5,7 +5,7 @@ use std::{
use crate::{
common::{IterPairResult, PairResult, ValueOnlyResult},
table::{DupSort, Table},
table::{DupSort, Table, TableRow},
DatabaseError,
};
@ -151,7 +151,7 @@ pub struct Walker<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> {
impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> std::iter::Iterator
for Walker<'cursor, 'tx, T, CURSOR>
{
type Item = Result<(T::Key, T::Value), DatabaseError>;
type Item = Result<TableRow<T>, DatabaseError>;
fn next(&mut self) -> Option<Self::Item> {
let start = self.start.take();
if start.is_some() {
@ -220,7 +220,7 @@ impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRW<'tx, T> + DbCursorRO<'tx, T>>
impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> std::iter::Iterator
for ReverseWalker<'cursor, 'tx, T, CURSOR>
{
type Item = Result<(T::Key, T::Value), DatabaseError>;
type Item = Result<TableRow<T>, DatabaseError>;
fn next(&mut self) -> Option<Self::Item> {
let start = self.start.take();
@ -250,7 +250,7 @@ pub struct RangeWalker<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> {
impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> std::iter::Iterator
for RangeWalker<'cursor, 'tx, T, CURSOR>
{
type Item = Result<(T::Key, T::Value), DatabaseError>;
type Item = Result<TableRow<T>, DatabaseError>;
fn next(&mut self) -> Option<Self::Item> {
if self.is_done {
return None
@ -334,7 +334,7 @@ impl<'cursor, 'tx, T: DupSort, CURSOR: DbCursorRW<'tx, T> + DbDupCursorRO<'tx, T
impl<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> std::iter::Iterator
for DupWalker<'cursor, 'tx, T, CURSOR>
{
type Item = Result<(T::Key, T::Value), DatabaseError>;
type Item = Result<TableRow<T>, DatabaseError>;
fn next(&mut self) -> Option<Self::Item> {
let start = self.start.take();
if start.is_some() {

3
crates/storage/db/src/abstraction/table.rs
View File

@ -81,6 +81,9 @@ pub trait Table: Send + Sync + Debug + 'static {
type Value: Value;
}
/// Tuple with `T::Key` and `T::Value`.
pub type TableRow<T> = (<T as Table>::Key, <T as Table>::Value);
/// DupSort allows for keys to be repeated in the database.
///
/// Upstream docs: <https://libmdbx.dqdkfa.ru/usage.html#autotoc_md48>

2
crates/storage/db/src/tables/mod.rs
View File

@ -18,7 +18,7 @@ mod raw;
pub(crate) mod utils;
use crate::abstraction::table::Table;
pub use raw::{RawDupSort, RawKey, RawTable, RawValue};
pub use raw::{RawDupSort, RawKey, RawTable, RawValue, TableRawRow};
use std::{fmt::Display, str::FromStr};
/// Declaration of all Database tables.

17
crates/storage/db/src/tables/raw.rs
View File

@ -4,6 +4,9 @@ use crate::{
};
use serde::{Deserialize, Serialize};
/// Tuple with `RawKey<T::Key>` and `RawValue<T::Value>`.
pub type TableRawRow<T> = (RawKey<<T as Table>::Key>, RawValue<<T as Table>::Value>);
/// Raw table that can be used to access any table and its data in raw mode.
/// This is useful for delayed decoding/encoding of data.
#[derive(Default, Copy, Clone, Debug)]
@ -41,6 +44,7 @@ impl<T: DupSort> DupSort for RawDupSort<T> {
/// Raw table key.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct RawKey<K: Key> {
/// Inner encoded key
key: Vec<u8>,
_phantom: std::marker::PhantomData<K>,
}
@ -50,10 +54,14 @@ impl<K: Key> RawKey<K> {
pub fn new(key: K) -> Self {
Self { key: K::encode(key).as_ref().to_vec(), _phantom: std::marker::PhantomData }
}
/// Returns the raw key.
/// Returns the decoded value.
pub fn key(&self) -> Result<K, DatabaseError> {
K::decode(&self.key)
}
/// Returns the raw key as seen on the database.
pub fn raw_key(&self) -> &Vec<u8> {
&self.key
}
}
impl<K: Key> From<K> for RawKey<K> {
@ -87,6 +95,7 @@ impl<K: Key> Decode for RawKey<K> {
/// Raw table value.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Serialize, Ord, Hash)]
pub struct RawValue<V: Value> {
/// Inner compressed value
value: Vec<u8>,
_phantom: std::marker::PhantomData<V>,
}
@ -96,10 +105,14 @@ impl<V: Value> RawValue<V> {
pub fn new(value: V) -> Self {
Self { value: V::compress(value).as_ref().to_vec(), _phantom: std::marker::PhantomData }
}
/// Returns the raw value.
/// Returns the decompressed value.
pub fn value(&self) -> Result<V, DatabaseError> {
V::decompress(&self.value)
}
/// Returns the raw value as seen on the database.
pub fn raw_value(&self) -> &Vec<u8> {
&self.value
}
}
impl AsRef<[u8]> for RawValue<Vec<u8>> {

4
crates/storage/db/src/tables/utils.rs
View File

@ -1,6 +1,6 @@
//! Small database table utilities and helper functions.
use crate::{
table::{Decode, Decompress, Table},
table::{Decode, Decompress, Table, TableRow},
DatabaseError,
};
use std::borrow::Cow;
@ -42,7 +42,7 @@ macro_rules! impl_fixed_arbitrary {
/// Helper function to decode a `(key, value)` pair.
pub(crate) fn decoder<'a, T>(
kv: (Cow<'a, [u8]>, Cow<'a, [u8]>),
) -> Result<(T::Key, T::Value), DatabaseError>
) -> Result<TableRow<T>, DatabaseError>
where
T: Table,
T::Key: Decode,